From b012271d01b59759e4eefeab0308698cd9e7224c Mon Sep 17 00:00:00 2001 From: Simon Sawicki Date: Tue, 14 Nov 2023 22:40:38 +0100 Subject: [PATCH] [cleanup] Misc (#8510) Authored by: bashonly, coletdjnz, dirkf, gamer191, seproDev, Grub4K --- .github/PULL_REQUEST_TEMPLATE.md | 6 ------ README.md | 6 +++--- devscripts/changelog_override.json | 16 ++++++++++++++++ yt_dlp/extractor/la7.py | 4 ++-- yt_dlp/extractor/redtube.py | 2 +- yt_dlp/extractor/videoken.py | 15 ++++++++------- yt_dlp/extractor/youtube.py | 2 +- yt_dlp/networking/_requests.py | 3 ++- 8 files changed, 33 insertions(+), 21 deletions(-) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index cbed82173..c4d3e812e 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -40,10 +40,4 @@ ### What is the purpose of your *pull request*? - [ ] Core bug fix/improvement - [ ] New feature (It is strongly [recommended to open an issue first](https://github.com/yt-dlp/yt-dlp/blob/master/CONTRIBUTING.md#adding-new-feature-or-making-overarching-changes)) - - -
Copilot Summary - -copilot:all -
diff --git a/README.md b/README.md index 1b92c64d6..8b92f827b 100644 --- a/README.md +++ b/README.md @@ -163,10 +163,10 @@ ### Differences in default behavior For ease of use, a few more compat options are available: * `--compat-options all`: Use all compat options (Do NOT use) -* `--compat-options youtube-dl`: Same as `--compat-options all,-multistreams,-playlist-match-filter` -* `--compat-options youtube-dlc`: Same as `--compat-options all,-no-live-chat,-no-youtube-channel-redirect,-playlist-match-filter` +* `--compat-options youtube-dl`: Same as `--compat-options all,-multistreams,-playlist-match-filter,-manifest-filesize-approx` +* `--compat-options youtube-dlc`: Same as `--compat-options all,-no-live-chat,-no-youtube-channel-redirect,-playlist-match-filter,-manifest-filesize-approx` * `--compat-options 2021`: Same as `--compat-options 2022,no-certifi,filename-sanitization,no-youtube-prefer-utc-upload-date` -* `--compat-options 2022`: Same as `--compat-options playlist-match-filter,no-external-downloader-progress,prefer-legacy-http-handler`. Use this to enable all future compat options +* `--compat-options 2022`: Same as `--compat-options playlist-match-filter,no-external-downloader-progress,prefer-legacy-http-handler,manifest-filesize-approx`. Use this to enable all future compat options # INSTALLATION diff --git a/devscripts/changelog_override.json b/devscripts/changelog_override.json index fe0c82c66..010820295 100644 --- a/devscripts/changelog_override.json +++ b/devscripts/changelog_override.json @@ -98,5 +98,21 @@ "action": "add", "when": "61bdf15fc7400601c3da1aa7a43917310a5bf391", "short": "[priority] Security: [[CVE-2023-40581](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2023-40581)] [Prevent RCE when using `--exec` with `%q` on Windows](https://github.com/yt-dlp/yt-dlp/security/advisories/GHSA-42h4-v29r-42qg)\n - The shell escape function is now using `\"\"` instead of `\\\"`.\n - `utils.Popen` has been patched to properly quote commands." + }, + { + "action": "change", + "when": "8a8b54523addf46dfd50ef599761a81bc22362e6", + "short": "[rh:requests] Add handler for `requests` HTTP library (#3668)\n\n\tAdds support for HTTPS proxies and persistent connections (keep-alive)", + "authors": ["bashonly", "coletdjnz", "Grub4K"] + }, + { + "action": "add", + "when": "1d03633c5a1621b9f3a756f0a4f9dc61fab3aeaa", + "short": "[priority] **The release channels have been adjusted!**\n\t* [`master`](https://github.com/yt-dlp/yt-dlp-master-builds) builds are made after each push, containing the latest fixes (but also possibly bugs). This was previously the `nightly` channel.\n\t* [`nightly`](https://github.com/yt-dlp/yt-dlp-nightly-builds) builds are now made once a day, if there were any changes." + }, + { + "action": "add", + "when": "f04b5bedad7b281bee9814686bba1762bae092eb", + "short": "[priority] Security: [[CVE-2023-46121](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2023-46121)] Patch [Generic Extractor MITM Vulnerability via Arbitrary Proxy Injection](https://github.com/yt-dlp/yt-dlp/security/advisories/GHSA-3ch3-jhc6-5r8x)\n\t- Disallow smuggling of arbitary `http_headers`; extractors now only use specific headers" } ] diff --git a/yt_dlp/extractor/la7.py b/yt_dlp/extractor/la7.py index a3cd12b00..f5fd24134 100644 --- a/yt_dlp/extractor/la7.py +++ b/yt_dlp/extractor/la7.py @@ -208,9 +208,9 @@ class LA7PodcastIE(LA7PodcastEpisodeIE): # XXX: Do not subclass from concrete I 'url': 'https://www.la7.it/propagandalive/podcast', 'info_dict': { 'id': 'propagandalive', - 'title': "Propaganda Live", + 'title': 'Propaganda Live', }, - 'playlist_count_min': 10, + 'playlist_mincount': 10, }] def _real_extract(self, url): diff --git a/yt_dlp/extractor/redtube.py b/yt_dlp/extractor/redtube.py index 49076ccd8..172c31b39 100644 --- a/yt_dlp/extractor/redtube.py +++ b/yt_dlp/extractor/redtube.py @@ -39,7 +39,7 @@ class RedTubeIE(InfoExtractor): def _real_extract(self, url): video_id = self._match_id(url) webpage = self._download_webpage( - 'http://www.redtube.com/%s' % video_id, video_id) + f'https://www.redtube.com/{video_id}', video_id) ERRORS = ( (('video-deleted-info', '>This video has been removed'), 'has been removed'), diff --git a/yt_dlp/extractor/videoken.py b/yt_dlp/extractor/videoken.py index 560b41a6d..eaf0cc8ae 100644 --- a/yt_dlp/extractor/videoken.py +++ b/yt_dlp/extractor/videoken.py @@ -11,6 +11,7 @@ ExtractorError, InAdvancePagedList, int_or_none, + remove_start, traverse_obj, update_url_query, url_or_none, @@ -39,11 +40,11 @@ def _create_slideslive_url(self, video_url, video_id, referer): if not video_url and not video_id: return elif not video_url or 'embed/sign-in' in video_url: - video_url = f'https://slideslive.com/embed/{video_id.lstrip("slideslive-")}' + video_url = f'https://slideslive.com/embed/{remove_start(video_id, "slideslive-")}' if url_or_none(referer): return update_url_query(video_url, { 'embed_parent_url': referer, - 'embed_container_origin': f'https://{urllib.parse.urlparse(referer).netloc}', + 'embed_container_origin': f'https://{urllib.parse.urlparse(referer).hostname}', }) return video_url @@ -57,12 +58,12 @@ def _extract_videos(self, videos, url): video_url = video_id ie_key = 'Youtube' else: - video_url = traverse_obj(video, 'embed_url', 'embeddableurl') - if urllib.parse.urlparse(video_url).netloc == 'slideslive.com': + video_url = traverse_obj(video, 'embed_url', 'embeddableurl', expected_type=url_or_none) + if not video_url: + continue + elif urllib.parse.urlparse(video_url).hostname == 'slideslive.com': ie_key = SlidesLiveIE video_url = self._create_slideslive_url(video_url, video_id, url) - if not video_url: - continue yield self.url_result(video_url, ie_key, video_id) @@ -178,7 +179,7 @@ def _real_extract(self, url): return self.url_result( self._create_slideslive_url(None, video_id, url), SlidesLiveIE, video_id) elif re.match(r'^[\w-]{11}$', video_id): - self.url_result(video_id, 'Youtube', video_id) + return self.url_result(video_id, 'Youtube', video_id) else: raise ExtractorError('Unable to extract without VideoKen API response') diff --git a/yt_dlp/extractor/youtube.py b/yt_dlp/extractor/youtube.py index adbac8e95..f6caf0970 100644 --- a/yt_dlp/extractor/youtube.py +++ b/yt_dlp/extractor/youtube.py @@ -6687,7 +6687,7 @@ class YoutubePlaylistIE(InfoExtractor): 'uploader_url': 'https://www.youtube.com/@milan5503', 'availability': 'public', }, - 'expected_warnings': [r'[Uu]navailable videos? (is|are|will be) hidden'], + 'expected_warnings': [r'[Uu]navailable videos? (is|are|will be) hidden', 'Retrying', 'Giving up'], }, { 'url': 'http://www.youtube.com/embed/_xDOZElKyNU?list=PLsyOSbh5bs16vubvKePAQ1x3PhKavfBIl', 'playlist_mincount': 455, diff --git a/yt_dlp/networking/_requests.py b/yt_dlp/networking/_requests.py index fe3f60b0b..9fb1d75f4 100644 --- a/yt_dlp/networking/_requests.py +++ b/yt_dlp/networking/_requests.py @@ -255,7 +255,8 @@ def __init__(self, *args, **kwargs): handler.setFormatter(logging.Formatter('requests: %(message)s')) handler.addFilter(Urllib3LoggingFilter()) logger.addHandler(handler) - logger.setLevel(logging.WARNING) + # TODO: Use a logger filter to suppress pool reuse warning instead + logger.setLevel(logging.ERROR) if self.verbose: # Setting this globally is not ideal, but is easier than hacking with urllib3.