From c25228e5da88bb4eaba3034b1fb129c257f9a219 Mon Sep 17 00:00:00 2001 From: pukkandan Date: Fri, 5 Feb 2021 04:23:04 +0530 Subject: [PATCH] Release 2021.02.04 --- .github/workflows/quick-test.yml | 2 +- .gitignore | 3 +++ Changelog.md | 22 +++++++++++++++++++++- README.md | 21 ++++++++++++--------- docs/supportedsites.md | 3 ++- youtube_dlc/YoutubeDL.py | 23 ++++++++++++++--------- youtube_dlc/options.py | 8 +++++--- 7 files changed, 58 insertions(+), 24 deletions(-) diff --git a/.github/workflows/quick-test.yml b/.github/workflows/quick-test.yml index 06e9b03a1..fc8ca0b2f 100644 --- a/.github/workflows/quick-test.yml +++ b/.github/workflows/quick-test.yml @@ -2,7 +2,7 @@ name: Quick Test on: [push, pull_request] jobs: tests: - name: Core Tests + name: Core Test if: "!contains(github.event.head_commit.message, 'ci skip all')" runs-on: ubuntu-latest steps: diff --git a/.gitignore b/.gitignore index 60706f39e..c48bdb89c 100644 --- a/.gitignore +++ b/.gitignore @@ -57,6 +57,9 @@ youtube-dlc *.swp *.ogg *.opus +*.info.json +*.annotations.xml +*.description # Config *.conf diff --git a/Changelog.md b/Changelog.md index 1ce76cbd9..4154acc47 100644 --- a/Changelog.md +++ b/Changelog.md @@ -17,6 +17,26 @@ --> +### 2021.02.04 +* **Merge youtube-dl:** Upto [2021.02.04.1](https://github.com/ytdl-org/youtube-dl/releases/tag/2021.02.04.1) +* **Date/time formatting in output template:** You can now use [`strftime`](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes) to format date/time fields. Example: `%(upload_date>%Y-%m-%d)s` +* **Multiple output templates:** + * Seperate output templates can be given for the different metadata files by using `-o TYPE:TEMPLATE` + * The alowed types are: `subtitle|thumbnail|description|annotation|infojson|pl_description|pl_infojson` +* [youtube] More metadata extraction for channel/playlist URLs (channel, uploader, thumbnail, tags) +* New option `--no-write-playlist-metafiles` to prevent writing playlist metadata files +* [audius] Fix extractor +* [youtube_live_chat] Fix `parse_yt_initial_data` and add `fragment_retries` +* [postprocessor] Raise errors correctly +* [metadatafromtitle] Fix bug when extracting data from numeric fields +* Fix issue with overwriting files +* Fix "Default format spec" appearing in quiet mode +* [FormatSort] Allow user to prefer av01 over vp9 (The default is still vp9) +* [FormatSort] fix bug where `quality` had more priority than `hasvid` +* [pyinst] Automatically detect python architecture and working directory +* Strip out internal fields such as `_filename` from infojson + + ### 2021.01.29 * **Features from [animelover1984/youtube-dl](https://github.com/animelover1984/youtube-dl)**: Co-authored by @animelover1984 and @bbepis * Add `--get-comments` @@ -174,7 +194,7 @@ * Added `--no-ignore-dynamic-mpd`, `--no-allow-dynamic-mpd`, `--allow-dynamic-mpd`, `--youtube-include-hls-manifest`, `--no-youtube-include-hls-manifest`, `--no-youtube-skip-hls-manifest`, `--no-download`, `--no-download-archive`, `--resize-buffer`, `--part`, `--mtime`, `--no-keep-fragments`, `--no-cookies`, `--no-write-annotations`, `--no-write-info-json`, `--no-write-description`, `--no-write-thumbnail`, `--youtube-include-dash-manifest`, `--post-overwrites`, `--no-keep-video`, `--no-embed-subs`, `--no-embed-thumbnail`, `--no-add-metadata`, `--no-include-ads`, `--no-write-sub`, `--no-write-auto-sub`, `--no-playlist-reverse`, `--no-restrict-filenames`, `--youtube-include-dash-manifest`, `--no-format-sort-force`, `--flat-videos`, `--no-list-formats-as-table`, `--no-sponskrub`, `--no-sponskrub-cut`, `--no-sponskrub-force` * Renamed: `--write-subs`, `--no-write-subs`, `--no-write-auto-subs`, `--write-auto-subs`. Note that these can still be used without the ending "s" * Relaxed validation for format filters so that any arbitrary field can be used -* Fix for embedding thumbnail in mp3 by @pauldubois98 +* Fix for embedding thumbnail in mp3 by @pauldubois98 ([ytdl-org/youtube-dl#21569](https://github.com/ytdl-org/youtube-dl/pull/21569)) * Make Twitch Video ID output from Playlist and VOD extractor same. This is only a temporary fix * **Merge youtube-dl:** Upto [2021.01.03](https://github.com/ytdl-org/youtube-dl/commit/8e953dcbb10a1a42f4e12e4e132657cb0100a1f8) - See [blackjack4494/yt-dlc#280](https://github.com/blackjack4494/yt-dlc/pull/280) for details * Extractors [tiktok](https://github.com/ytdl-org/youtube-dl/commit/fb626c05867deab04425bad0c0b16b55473841a2) and [hotstar](https://github.com/ytdl-org/youtube-dl/commit/bb38a1215718cdf36d73ff0a7830a64cd9fa37cc) have not been merged diff --git a/README.md b/README.md index 60d32d6e3..bf52cf84a 100644 --- a/README.md +++ b/README.md @@ -54,7 +54,7 @@ The major new features from the latest release of [blackjack4494/yt-dlc](https:/ * **[Format Sorting](#sorting-formats)**: The default format sorting options have been changed so that higher resolution and better codecs will be now preferred instead of simply using larger bitrate. Furthermore, you can now specify the sort order using `-S`. This allows for much easier format selection that what is possible by simply using `--format` ([examples](#format-selection-examples)) -* **Merged with youtube-dl v2021.01.24.1**: You get all the latest features and patches of [youtube-dl](https://github.com/ytdl-org/youtube-dl) in addition to all the features of [youtube-dlc](https://github.com/blackjack4494/yt-dlc) +* **Merged with youtube-dl v2021.02.04.1**: You get all the latest features and patches of [youtube-dl](https://github.com/ytdl-org/youtube-dl) in addition to all the features of [youtube-dlc](https://github.com/blackjack4494/yt-dlc) * **Merged with animelover1984/youtube-dl**: You get most of the features and improvements from [animelover1984/youtube-dl](https://github.com/animelover1984/youtube-dl) including `--get-comments`, `BiliBiliSearch`, `BilibiliChannel`, Embedding thumbnail in mp4/ogg/opus, Playlist infojson etc. Note that the NicoNico improvements are not available. See [#31](https://github.com/pukkandan/yt-dlp/pull/31) for details. @@ -69,7 +69,7 @@ The major new features from the latest release of [blackjack4494/yt-dlc](https:/ * **Plugin support**: Extractors can be loaded from an external file. See [plugins](#plugins) for details -* **Multiple paths**: You can give different paths for different types of files. You can also set a temporary path where intermediary files are downloaded to. See [`--paths`](https://github.com/pukkandan/yt-dlp/#:~:text=-P,%20--paths%20TYPE:PATH) for details +* **Multiple paths and output templates**: You can give different [output templates](#output-template) and download paths for different types of files. You can also set a temporary path where intermediary files are downloaded to. See [`--paths`](https://github.com/pukkandan/yt-dlp/#:~:text=-P,%20--paths%20TYPE:PATH) for details @@ -77,7 +77,7 @@ The major new features from the latest release of [blackjack4494/yt-dlc](https:/ * **Other new options**: `--parse-metadata`, `--list-formats-as-table`, `--write-link`, `--force-download-archive`, `--force-overwrites`, `--break-on-reject` etc -* **Improvements**: Multiple `--postprocessor-args` and `--external-downloader-args`, `%(duration_string)s` in `-o`, faster archive checking, more [format selection options](#format-selection) etc +* **Improvements**: Multiple `--postprocessor-args` and `--external-downloader-args`, Date/time formatting in `-o`, faster archive checking, more [format selection options](#format-selection) etc See [changelog](Changelog.md) or [commits](https://github.com/pukkandan/yt-dlp/commits) for the full list of changes @@ -359,9 +359,11 @@ Then simply type this This option includes --no-continue --no-force-overwrites Do not overwrite the video, but overwrite related files (default) - -c, --continue Resume partially downloaded files (default) - --no-continue Restart download of partially downloaded - files from beginning + -c, --continue Resume partially downloaded files/fragments + (default) + --no-continue Do not resume partially downloaded + fragments. If the file is unfragmented, + restart download of the entire file --part Use .part files instead of writing directly into output file (default) --no-part Do not use .part files - write directly @@ -374,6 +376,7 @@ Then simply type this file --no-write-description Do not write video description (default) --write-info-json Write video metadata to a .info.json file + (this may contain personal information) --no-write-info-json Do not write video metadata (default) --write-annotations Write video annotations to a .annotations.xml file @@ -753,7 +756,7 @@ The `-o` option is used to indicate a template for the output file names while ` The basic usage of `-o` is not to set any template arguments when downloading a single file, like in `youtube-dlc -o funny_video.flv "https://some/video"`. However, it may contain special sequences that will be replaced when downloading each video. The special sequences may be formatted according to [python string formatting operations](https://docs.python.org/2/library/stdtypes.html#string-formatting). For example, `%(NAME)s` or `%(NAME)05d`. To clarify, that is a percent symbol followed by a name in parentheses, followed by formatting operations. Date/time fields can also be formatted according to [strftime formatting](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes) by specifying it inside the parantheses seperated from the field name using a `>`. For example, `%(duration>%H-%M-%S)s`. -Additionally, you can set different output templates for the various metadata files seperately from the general output template by specifying the type of file followed by the template seperated by a colon ":". The different filetypes supported are subtitle|thumbnail|description|annotation|infojson|pl_description|pl_infojson. For example, `-o '%(title)s.%(ext)s' -o 'thumbnail:%(title)s\%(title)s.%(ext)s'` will put the thumbnails in a folder with the same name as the video. +Additionally, you can set different output templates for the various metadata files seperately from the general output template by specifying the type of file followed by the template seperated by a colon ":". The different filetypes supported are `subtitle|thumbnail|description|annotation|infojson|pl_description|pl_infojson`. For example, `-o '%(title)s.%(ext)s' -o 'thumbnail:%(title)s\%(title)s.%(ext)s'` will put the thumbnails in a folder with the same name as the video. The available fields are: @@ -970,7 +973,7 @@ You can change the criteria for being considered the `best` by using `-S` (`--fo - `quality`: The quality of the format. This is a metadata field available in some websites - `source`: Preference of the source as given by the extractor - `proto`: Protocol used for download (`https`/`ftps` > `http`/`ftp` > `m3u8-native` > `m3u8` > `http-dash-segments` > other > `mms`/`rtsp` > unknown > `f4f`/`f4m`) - - `vcodec`: Video Codec (`vp9` > `h265` > `h264` > `vp8` > `h263` > `theora` > other > unknown) + - `vcodec`: Video Codec (`av01` > `vp9` > `h265` > `h264` > `vp8` > `h263` > `theora` > other > unknown) - `acodec`: Audio Codec (`opus` > `vorbis` > `aac` > `mp4a` > `mp3` > `ac3` > `dts` > other > unknown) - `codec`: Equivalent to `vcodec,acodec` - `vext`: Video Extension (`mp4` > `webm` > `flv` > other > unknown). If `--prefer-free-formats` is used, `webm` is prefered. @@ -991,7 +994,7 @@ You can change the criteria for being considered the `best` by using `-S` (`--fo Note that any other **numerical** field made available by the extractor can also be used. All fields, unless specified otherwise, are sorted in decending order. To reverse this, prefix the field with a `+`. Eg: `+res` prefers format with the smallest resolution. Additionally, you can suffix a prefered value for the fields, seperated by a `:`. Eg: `res:720` prefers larger videos, but no larger than 720p and the smallest video if there are no videos less than 720p. For `codec` and `ext`, you can provide two prefered values, the first for video and the second for audio. Eg: `+codec:avc:m4a` (equivalent to `+vcodec:avc,+acodec:m4a`) sets the video codec preference to `h264` > `h265` > `vp9` > `vp8` > `h263` > `theora` and audio codec preference to `mp4a` > `aac` > `vorbis` > `opus` > `mp3` > `ac3` > `dts`. You can also make the sorting prefer the nearest values to the provided by using `~` as the delimiter. Eg: `filesize~1G` prefers the format with filesize closest to 1 GiB. -The fields `hasvid`, `ie_pref`, `lang`, `quality` are always given highest priority in sorting, irrespective of the user-defined order. This behaviour can be changed by using `--force-format-sort`. Apart from these, the default order used is: `res,fps,codec,size,br,asr,proto,ext,hasaud,source,id`. Note that the extractors may override this default order, but they cannot override the user-provided order. +The fields `hasvid`, `ie_pref`, `lang`, `quality` are always given highest priority in sorting, irrespective of the user-defined order. This behaviour can be changed by using `--force-format-sort`. Apart from these, the default order used is: `res,fps,codec:vp9,size,br,asr,proto,ext,hasaud,source,id`. Note that the extractors may override this default order, but they cannot override the user-provided order. If your format selector is `worst`, the last item is selected after sorting. This means it will select the format that is worst in all repects. Most of the time, what you actually want is the video with the smallest filesize instead. So it is generally better to use `-f best -S +size,+br,+res,+fps`. diff --git a/docs/supportedsites.md b/docs/supportedsites.md index c422bf58f..00a34d9bf 100644 --- a/docs/supportedsites.md +++ b/docs/supportedsites.md @@ -559,6 +559,7 @@ - **mtv:video** - **mtvjapan** - **mtvservices:embedded** + - **MTVUutisetArticle** - **MuenchenTV**: münchen.tv - **mva**: Microsoft Virtual Academy videos - **mva:course**: Microsoft Virtual Academy courses @@ -1090,7 +1091,6 @@ - **vidme** - **vidme:user** - **vidme:user:likes** - - **Vidzi** - **vier**: vier.be and vijf.be - **vier:videos** - **viewlift** @@ -1135,6 +1135,7 @@ - **vrv** - **vrv:series** - **VShare** + - **VTM** - **VTXTV** - **vube**: Vube.com - **VuClip** diff --git a/youtube_dlc/YoutubeDL.py b/youtube_dlc/YoutubeDL.py index 9631745de..b0539493c 100644 --- a/youtube_dlc/YoutubeDL.py +++ b/youtube_dlc/YoutubeDL.py @@ -175,14 +175,18 @@ class YoutubeDL(object): forcejson: Force printing info_dict as JSON. dump_single_json: Force printing the info_dict of the whole playlist (or video) as a single JSON line. - force_write_download_archive: Force writing download archive regardless of - 'skip_download' or 'simulate'. + force_write_download_archive: Force writing download archive regardless + of 'skip_download' or 'simulate'. simulate: Do not download the video files. format: Video format code. see "FORMAT SELECTION" for more details. - format_sort: How to sort the video formats. see "Sorting Formats" for more details. - format_sort_force: Force the given format_sort. see "Sorting Formats" for more details. - allow_multiple_video_streams: Allow multiple video streams to be merged into a single file - allow_multiple_audio_streams: Allow multiple audio streams to be merged into a single file + format_sort: How to sort the video formats. see "Sorting Formats" + for more details. + format_sort_force: Force the given format_sort. see "Sorting Formats" + for more details. + allow_multiple_video_streams: Allow multiple video streams to be merged + into a single file + allow_multiple_audio_streams: Allow multiple audio streams to be merged + into a single file outtmpl: Dictionary of templates for output names. Allowed keys are 'default' and the keys of OUTTMPL_TYPES (in utils.py) outtmpl_na_placeholder: Placeholder for unavailable meta fields. @@ -210,7 +214,8 @@ class YoutubeDL(object): unless writeinfojson is also given writeannotations: Write the video annotations to a .annotations.xml file writethumbnail: Write the thumbnail image to a file - allow_playlist_files: Also write playlists' description, infojson etc in a seperate file + allow_playlist_files: Whether to write playlists' description, infojson etc + also to disk when using the 'write*' options write_all_thumbnails: Write all thumbnail formats to files writelink: Write an internet shortcut file, depending on the current platform (.url/.webloc/.desktop) @@ -727,9 +732,9 @@ class YoutubeDL(object): def report_file_delete(self, file_name): """Report that existing file will be deleted.""" try: - self.to_screen('Deleting already existent file %s' % file_name) + self.to_screen('Deleting existing file %s' % file_name) except UnicodeEncodeError: - self.to_screen('Deleting already existent file') + self.to_screen('Deleting existing file') def parse_outtmpl(self): outtmpl_dict = self.params.get('outtmpl', {}) diff --git a/youtube_dlc/options.py b/youtube_dlc/options.py index 06fbaa3cd..06273c20a 100644 --- a/youtube_dlc/options.py +++ b/youtube_dlc/options.py @@ -897,11 +897,13 @@ def parseOpts(overrideArguments=None): filesystem.add_option( '-c', '--continue', action='store_true', dest='continue_dl', default=True, - help='Resume partially downloaded files (default)') + help='Resume partially downloaded files/fragments (default)') filesystem.add_option( '--no-continue', action='store_false', dest='continue_dl', - help='Restart download of partially downloaded files from beginning') + help=( + 'Do not resume partially downloaded fragments. ' + 'If the file is unfragmented, restart download of the entire file')) filesystem.add_option( '--part', action='store_false', dest='nopart', default=False, @@ -929,7 +931,7 @@ def parseOpts(overrideArguments=None): filesystem.add_option( '--write-info-json', action='store_true', dest='writeinfojson', default=False, - help='Write video metadata to a .info.json file. Note that this may contain personal information') + help='Write video metadata to a .info.json file (this may contain personal information)') filesystem.add_option( '--no-write-info-json', action='store_false', dest='writeinfojson',