diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index cd7ead796..dcbb8c501 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -107,6 +107,8 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + with: + fetch-depth: 0 # Needed for changelog - uses: actions/setup-python@v5 with: python-version: "3.10" @@ -133,6 +135,7 @@ jobs: - name: Prepare run: | python devscripts/update-version.py -c "${{ inputs.channel }}" -r "${{ needs.process.outputs.origin }}" "${{ inputs.version }}" + python devscripts/update_changelog.py -vv python devscripts/make_lazy_extractors.py - name: Build Unix platform-independent binary run: | @@ -164,7 +167,7 @@ jobs: - name: Upload artifacts uses: actions/upload-artifact@v4 with: - name: build-${{ github.job }} + name: build-bin-${{ github.job }} path: | yt-dlp yt-dlp.tar.gz @@ -227,7 +230,7 @@ jobs: - name: Upload artifacts uses: actions/upload-artifact@v4 with: - name: build-linux_${{ matrix.architecture }} + name: build-bin-linux_${{ matrix.architecture }} path: | # run-on-arch-action designates armv7l as armv7 repo/dist/yt-dlp_linux_${{ (matrix.architecture == 'armv7' && 'armv7l') || matrix.architecture }} compression-level: 0 @@ -271,7 +274,7 @@ jobs: - name: Upload artifacts uses: actions/upload-artifact@v4 with: - name: build-${{ github.job }} + name: build-bin-${{ github.job }} path: | dist/yt-dlp_macos dist/yt-dlp_macos.zip @@ -324,7 +327,7 @@ jobs: - name: Upload artifacts uses: actions/upload-artifact@v4 with: - name: build-${{ github.job }} + name: build-bin-${{ github.job }} path: | dist/yt-dlp_macos_legacy compression-level: 0 @@ -373,7 +376,7 @@ jobs: - name: Upload artifacts uses: actions/upload-artifact@v4 with: - name: build-${{ github.job }} + name: build-bin-${{ github.job }} path: | dist/yt-dlp.exe dist/yt-dlp_min.exe @@ -421,7 +424,7 @@ jobs: - name: Upload artifacts uses: actions/upload-artifact@v4 with: - name: build-${{ github.job }} + name: build-bin-${{ github.job }} path: | dist/yt-dlp_x86.exe compression-level: 0 @@ -441,7 +444,7 @@ jobs: - uses: actions/download-artifact@v4 with: path: artifact - pattern: build-* + pattern: build-bin-* merge-multiple: true - name: Make SHA2-SUMS files @@ -484,3 +487,4 @@ jobs: _update_spec SHA*SUMS* compression-level: 0 + overwrite: true diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f5c6a793e..32268b32f 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -189,13 +189,8 @@ jobs: if: | !inputs.prerelease && env.target_repo == github.repository run: | + python devscripts/update_changelog.py -vv make doc - sed '/### /Q' Changelog.md >> ./CHANGELOG - echo '### ${{ env.version }}' >> ./CHANGELOG - python ./devscripts/make_changelog.py -vv -c >> ./CHANGELOG - echo >> ./CHANGELOG - grep -Poz '(?s)### \d+\.\d+\.\d+.+' 'Changelog.md' | head -n -1 >> ./CHANGELOG - cat ./CHANGELOG > Changelog.md - name: Push to release id: push_release @@ -266,6 +261,7 @@ jobs: pypi_project: ${{ needs.prepare.outputs.pypi_project }} run: | python devscripts/update-version.py -c "${{ env.channel }}" -r "${{ env.target_repo }}" -s "${{ env.suffix }}" "${{ env.version }}" + python devscripts/update_changelog.py -vv python devscripts/make_lazy_extractors.py sed -i -E '0,/(name = ")[^"]+(")/s//\1${{ env.pypi_project }}\2/' pyproject.toml @@ -312,19 +308,19 @@ jobs: target_tag: ${{ needs.prepare.outputs.target_tag }} run: | printf '%s' \ - '[![Installation](https://img.shields.io/badge/-Which%20file%20should%20I%20download%3F-white.svg?style=for-the-badge)]' \ + '[![Installation](https://img.shields.io/badge/-Which%20file%20to%20download%3F-white.svg?style=for-the-badge)]' \ '(https://github.com/${{ github.repository }}#installation "Installation instructions") ' \ + '[![Discord](https://img.shields.io/discord/807245652072857610?color=blue&labelColor=555555&label=&logo=discord&style=for-the-badge)]' \ + '(https://discord.gg/H5MNcFW63r "Discord") ' \ + '[![Donate](https://img.shields.io/badge/_-Donate-red.svg?logo=githubsponsors&labelColor=555555&style=for-the-badge)]' \ + '(https://github.com/yt-dlp/yt-dlp/blob/master/Collaborators.md#collaborators "Donate") ' \ '[![Documentation](https://img.shields.io/badge/-Docs-brightgreen.svg?style=for-the-badge&logo=GitBook&labelColor=555555)]' \ '(https://github.com/${{ github.repository }}' \ '${{ env.target_repo == github.repository && format('/tree/{0}', env.target_tag) || '' }}#readme "Documentation") ' \ - '[![Donate](https://img.shields.io/badge/_-Donate-red.svg?logo=githubsponsors&labelColor=555555&style=for-the-badge)]' \ - '(https://github.com/yt-dlp/yt-dlp/blob/master/Collaborators.md#collaborators "Donate") ' \ - '[![Discord](https://img.shields.io/discord/807245652072857610?color=blue&labelColor=555555&label=&logo=discord&style=for-the-badge)]' \ - '(https://discord.gg/H5MNcFW63r "Discord") ' \ ${{ env.target_repo == 'yt-dlp/yt-dlp' && '\ - "[![Nightly](https://img.shields.io/badge/Get%20nightly%20builds-purple.svg?style=for-the-badge)]" \ + "[![Nightly](https://img.shields.io/badge/Nightly%20builds-purple.svg?style=for-the-badge)]" \ "(https://github.com/yt-dlp/yt-dlp-nightly-builds/releases/latest \"Nightly builds\") " \ - "[![Master](https://img.shields.io/badge/Get%20master%20builds-lightblue.svg?style=for-the-badge)]" \ + "[![Master](https://img.shields.io/badge/Master%20builds-lightblue.svg?style=for-the-badge)]" \ "(https://github.com/yt-dlp/yt-dlp-master-builds/releases/latest \"Master builds\")"' || '' }} > ./RELEASE_NOTES printf '\n\n' >> ./RELEASE_NOTES cat >> ./RELEASE_NOTES << EOF diff --git a/.gitignore b/.gitignore index 0bd06d0d9..db322c4f0 100644 --- a/.gitignore +++ b/.gitignore @@ -33,6 +33,7 @@ cookies *.gif *.jpeg *.jpg +*.lrc *.m4a *.m4v *.mhtml @@ -40,6 +41,7 @@ cookies *.mov *.mp3 *.mp4 +*.mpg *.mpga *.oga *.ogg @@ -47,6 +49,7 @@ cookies *.png *.sbv *.srt +*.ssa *.swf *.swp *.tt diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 38ff67435..838cd76f5 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -79,7 +79,7 @@ ### Are you using the latest version? ### Is the issue already documented? -Make sure that someone has not already opened the issue you're trying to open. Search at the top of the window or browse the [GitHub Issues](https://github.com/yt-dlp/yt-dlp/search?type=Issues) of this repository. If there is an issue, subcribe to it to be notified when there is any progress. Unless you have something useful to add to the converation, please refrain from commenting. +Make sure that someone has not already opened the issue you're trying to open. Search at the top of the window or browse the [GitHub Issues](https://github.com/yt-dlp/yt-dlp/search?type=Issues) of this repository. If there is an issue, subscribe to it to be notified when there is any progress. Unless you have something useful to add to the conversation, please refrain from commenting. Additionally, it is also helpful to see if the issue has already been documented in the [youtube-dl issue tracker](https://github.com/ytdl-org/youtube-dl/issues). If similar issues have already been reported in youtube-dl (but not in our issue tracker), links to them can be included in your issue report here. @@ -169,7 +169,7 @@ # DEVELOPER INSTRUCTIONS ## Adding new feature or making overarching changes -Before you start writing code for implementing a new feature, open an issue explaining your feature request and atleast one use case. This allows the maintainers to decide whether such a feature is desired for the project in the first place, and will provide an avenue to discuss some implementation details. If you open a pull request for a new feature without discussing with us first, do not be surprised when we ask for large changes to the code, or even reject it outright. +Before you start writing code for implementing a new feature, open an issue explaining your feature request and at least one use case. This allows the maintainers to decide whether such a feature is desired for the project in the first place, and will provide an avenue to discuss some implementation details. If you open a pull request for a new feature without discussing with us first, do not be surprised when we ask for large changes to the code, or even reject it outright. The same applies for changes to the documentation, code style, or overarching changes to the architecture @@ -265,7 +265,7 @@ ## Adding support for a new site In any case, thank you very much for your contributions! -**Tip:** To test extractors that require login information, create a file `test/local_parameters.json` and add `"usenetrc": true` or your username and password in it: +**Tip:** To test extractors that require login information, create a file `test/local_parameters.json` and add `"usenetrc": true` or your `username`&`password` or `cookiefile`/`cookiesfrombrowser` in it: ```json { "username": "your user name", @@ -292,7 +292,7 @@ ### Mandatory and optional metafields For pornographic sites, appropriate `age_limit` must also be returned. -The extractor is allowed to return the info dict without url or formats in some special cases if it allows the user to extract usefull information with `--ignore-no-formats-error` - e.g. when the video is a live stream that has not started yet. +The extractor is allowed to return the info dict without url or formats in some special cases if it allows the user to extract useful information with `--ignore-no-formats-error` - e.g. when the video is a live stream that has not started yet. [Any field](yt_dlp/extractor/common.py#219-L426) apart from the aforementioned ones are considered **optional**. That means that extraction should be **tolerant** to situations when sources for these fields can potentially be unavailable (even if they are always available at the moment) and **future-proof** in order not to break the extraction of general purpose mandatory fields. diff --git a/CONTRIBUTORS b/CONTRIBUTORS index adcc92144..6ee3baa3d 100644 --- a/CONTRIBUTORS +++ b/CONTRIBUTORS @@ -542,3 +542,61 @@ prettykool S-Aarab sonmezberkay TSRBerry +114514ns +agibson-fl +alard +alien-developers +antonkesy +ArnauvGilotra +Arthurszzz +Bibhav48 +Bl4Cc4t +boredzo +Caesim404 +chkuendig +chtk +Danish-H +dasidiot +diman8 +divStar +DmitryScaletta +feederbox826 +gmes78 +gonzalezjo +hui1601 +infanf +jazz1611 +jingtra +jkmartindale +johnvictorfs +llistochek +marcdumais +martinxyz +michal-repo +mrmedieval +nbr23 +Nicals +Noor-5 +NurTasin +pompos02 +Pranaxcau +pwaldhauer +RaduManole +RalphORama +rrgomes +ruiminggu +rvsit +sefidel +shmohawk +Snack-X +src-tinkerer +stilor +syntaxsurge +t-nil +ufukk +vista-narvas +x11x +xpadev-net +Xpl0itU +YoshichikaAAA +zhijinwuu diff --git a/Changelog.md b/Changelog.md index 30de9072e..45a9cef3f 100644 --- a/Changelog.md +++ b/Changelog.md @@ -4,6 +4,228 @@ # Changelog # To create a release, dispatch the https://github.com/yt-dlp/yt-dlp/actions/workflows/release.yml workflow on master --> +### 2024.03.10 + +#### Core changes +- [Add `--compat-options 2023`](https://github.com/yt-dlp/yt-dlp/commit/3725b4f0c93ca3943e6300013a9670e4ab757fda) ([#9084](https://github.com/yt-dlp/yt-dlp/issues/9084)) by [Grub4K](https://github.com/Grub4K) (With fixes in [ffff1bc](https://github.com/yt-dlp/yt-dlp/commit/ffff1bc6598fc7a9258e51bc153cab812467f9f9) by [pukkandan](https://github.com/pukkandan)) +- [Create `ydl._request_director` when needed](https://github.com/yt-dlp/yt-dlp/commit/069b2aedae2279668b6051627a81fc4fbd9c146a) by [pukkandan](https://github.com/pukkandan) (With fixes in [dbd8b1b](https://github.com/yt-dlp/yt-dlp/commit/dbd8b1bff9afd8f05f982bcd52c20bc173c266ca) by [Grub4k](https://github.com/Grub4k)) +- [Don't select storyboard formats as fallback](https://github.com/yt-dlp/yt-dlp/commit/d63eae7e7ffb1f3e733e552b9e5e82355bfba214) by [bashonly](https://github.com/bashonly) +- [Handle `--load-info-json` format selection errors](https://github.com/yt-dlp/yt-dlp/commit/263a4b55ac17a796e8991ca8d2d86a3c349f8a60) ([#9392](https://github.com/yt-dlp/yt-dlp/issues/9392)) by [bashonly](https://github.com/bashonly) +- [Warn user when not launching through shell on Windows](https://github.com/yt-dlp/yt-dlp/commit/6a6cdcd1824a14e3b336332c8f31f65497b8c4b8) ([#9250](https://github.com/yt-dlp/yt-dlp/issues/9250)) by [Grub4K](https://github.com/Grub4K), [seproDev](https://github.com/seproDev) +- **cookies** + - [Fix `--cookies-from-browser` for `snap` Firefox](https://github.com/yt-dlp/yt-dlp/commit/cbed249aaa053a3f425b9bafc97f8dbd71c44487) ([#9016](https://github.com/yt-dlp/yt-dlp/issues/9016)) by [Grub4K](https://github.com/Grub4K) + - [Fix `--cookies-from-browser` with macOS Firefox profiles](https://github.com/yt-dlp/yt-dlp/commit/85b33f5c163f60dbd089a6b9bc2ba1366d3ddf93) ([#8909](https://github.com/yt-dlp/yt-dlp/issues/8909)) by [RalphORama](https://github.com/RalphORama) + - [Improve error message for Windows `--cookies-from-browser chrome` issue](https://github.com/yt-dlp/yt-dlp/commit/2792092afd367e39251ace1fb2819c855ab8919f) ([#9080](https://github.com/yt-dlp/yt-dlp/issues/9080)) by [Grub4K](https://github.com/Grub4K) +- **plugins**: [Handle `PermissionError`](https://github.com/yt-dlp/yt-dlp/commit/9a8afadd172b7cab143f0049959fa64973589d94) ([#9229](https://github.com/yt-dlp/yt-dlp/issues/9229)) by [pukkandan](https://github.com/pukkandan), [syntaxsurge](https://github.com/syntaxsurge) +- **utils** + - [Improve `repr` of `DateRange`, `match_filter_func`](https://github.com/yt-dlp/yt-dlp/commit/45491a2a30da4d1723cfa9288cb664813bb09afb) by [pukkandan](https://github.com/pukkandan) + - `traverse_obj`: [Support `xml.etree.ElementTree.Element`](https://github.com/yt-dlp/yt-dlp/commit/ffbd4f2a02fee387ea5e0a267ce32df5259111ac) ([#8911](https://github.com/yt-dlp/yt-dlp/issues/8911)) by [Grub4K](https://github.com/Grub4K) +- **webvtt**: [Don't parse single fragment files](https://github.com/yt-dlp/yt-dlp/commit/f24e44e8cbd88ce338d52f594a19330f64d38b50) ([#9034](https://github.com/yt-dlp/yt-dlp/issues/9034)) by [seproDev](https://github.com/seproDev) + +#### Extractor changes +- [Migrate commonly plural fields to lists](https://github.com/yt-dlp/yt-dlp/commit/104a7b5a46dc1805157fb4cc11c05876934d37c1) ([#8917](https://github.com/yt-dlp/yt-dlp/issues/8917)) by [llistochek](https://github.com/llistochek), [pukkandan](https://github.com/pukkandan) (With fixes in [b136e2a](https://github.com/yt-dlp/yt-dlp/commit/b136e2af341f7a88028aea4c5cd50efe2fa9b182) by [bashonly](https://github.com/bashonly)) +- [Support multi-period MPD streams](https://github.com/yt-dlp/yt-dlp/commit/4ce57d3b873c2887814cbec03d029533e82f7db5) ([#6654](https://github.com/yt-dlp/yt-dlp/issues/6654)) by [alard](https://github.com/alard), [pukkandan](https://github.com/pukkandan) +- **abematv** + - [Fix extraction with cache](https://github.com/yt-dlp/yt-dlp/commit/c51316f8a69fbd0080f2720777d42ab438e254a3) ([#8895](https://github.com/yt-dlp/yt-dlp/issues/8895)) by [sefidel](https://github.com/sefidel) + - [Support login for playlists](https://github.com/yt-dlp/yt-dlp/commit/8226a3818f804478c756cf460baa9bf3a3b062a5) ([#8901](https://github.com/yt-dlp/yt-dlp/issues/8901)) by [sefidel](https://github.com/sefidel) +- **adn** + - [Add support for German site](https://github.com/yt-dlp/yt-dlp/commit/5eb1458be4767385a9bf1d570ff08e46100cbaa2) ([#8708](https://github.com/yt-dlp/yt-dlp/issues/8708)) by [infanf](https://github.com/infanf) + - [Improve auth error handling](https://github.com/yt-dlp/yt-dlp/commit/9526b1f179d19f75284eceaa5e0ee381af18cf19) ([#9068](https://github.com/yt-dlp/yt-dlp/issues/9068)) by [infanf](https://github.com/infanf) +- **aenetworks**: [Rating should be optional for AP extraction](https://github.com/yt-dlp/yt-dlp/commit/014cb5774d7afe624b6eb4e07f7be924b9e5e186) ([#9005](https://github.com/yt-dlp/yt-dlp/issues/9005)) by [agibson-fl](https://github.com/agibson-fl) +- **altcensored**: channel: [Fix playlist extraction](https://github.com/yt-dlp/yt-dlp/commit/e28e135d6fd6a430fed3e20dfe1a8c8bbc5f9185) ([#9297](https://github.com/yt-dlp/yt-dlp/issues/9297)) by [marcdumais](https://github.com/marcdumais) +- **amadeustv**: [Add extractor](https://github.com/yt-dlp/yt-dlp/commit/e641aab7a61df7406df60ebfe0c77bd5186b2b41) ([#8744](https://github.com/yt-dlp/yt-dlp/issues/8744)) by [ArnauvGilotra](https://github.com/ArnauvGilotra) +- **ant1newsgrembed**: [Fix extractor](https://github.com/yt-dlp/yt-dlp/commit/1ed5ee2f045f717e814f84ba461dadc58e712266) ([#9191](https://github.com/yt-dlp/yt-dlp/issues/9191)) by [seproDev](https://github.com/seproDev) +- **archiveorg**: [Fix format URL encoding](https://github.com/yt-dlp/yt-dlp/commit/3894ab9574748188bbacbd925a3971eda6fa2bb0) ([#9279](https://github.com/yt-dlp/yt-dlp/issues/9279)) by [bashonly](https://github.com/bashonly) +- **ard** + - mediathek + - [Revert to using old id](https://github.com/yt-dlp/yt-dlp/commit/b6951271ac014761c9c317b9cecd5e8e139cfa7c) ([#8916](https://github.com/yt-dlp/yt-dlp/issues/8916)) by [Grub4K](https://github.com/Grub4K) + - [Support cookies to verify age](https://github.com/yt-dlp/yt-dlp/commit/c099ec9392b0283dde34b290d1a04158ad8eb882) ([#9037](https://github.com/yt-dlp/yt-dlp/issues/9037)) by [StefanLobbenmeier](https://github.com/StefanLobbenmeier) +- **art19**: [Add extractors](https://github.com/yt-dlp/yt-dlp/commit/999ea80beb053491089d256104c4188aced3110f) ([#9099](https://github.com/yt-dlp/yt-dlp/issues/9099)) by [seproDev](https://github.com/seproDev) +- **artetv**: [Separate closed captions](https://github.com/yt-dlp/yt-dlp/commit/393b487a4ea391c44e811505ec98531031d7e81e) ([#8231](https://github.com/yt-dlp/yt-dlp/issues/8231)) by [Nicals](https://github.com/Nicals), [seproDev](https://github.com/seproDev) +- **asobichannel**: [Add extractors](https://github.com/yt-dlp/yt-dlp/commit/12f042740550c06552819374e2251deb7a519bab) ([#8700](https://github.com/yt-dlp/yt-dlp/issues/8700)) by [Snack-X](https://github.com/Snack-X) +- **bigo**: [Fix JSON extraction](https://github.com/yt-dlp/yt-dlp/commit/85a2d07c1f82c2082b568963d1c32ad3fc848f61) ([#8893](https://github.com/yt-dlp/yt-dlp/issues/8893)) by [DmitryScaletta](https://github.com/DmitryScaletta) +- **bilibili** + - [Add referer header and fix metadata extraction](https://github.com/yt-dlp/yt-dlp/commit/1713c882730a928ac344c099874d2093fc2c8b51) ([#8832](https://github.com/yt-dlp/yt-dlp/issues/8832)) by [SirElderling](https://github.com/SirElderling) (With fixes in [f1570ab](https://github.com/yt-dlp/yt-dlp/commit/f1570ab84d5f49564256c620063d2d3e9ed4acf0) by [TobiX](https://github.com/TobiX)) + - [Support `--no-playlist`](https://github.com/yt-dlp/yt-dlp/commit/e439693f729daf6fb15457baea1bca10ef5da34d) ([#9139](https://github.com/yt-dlp/yt-dlp/issues/9139)) by [c-basalt](https://github.com/c-basalt) +- **bilibilisearch**: [Set cookie to fix extraction](https://github.com/yt-dlp/yt-dlp/commit/ffa017cfc5973b265c92248546fcf5020dc43eaf) ([#9119](https://github.com/yt-dlp/yt-dlp/issues/9119)) by [c-basalt](https://github.com/c-basalt) +- **biliintl**: [Fix and improve subtitles extraction](https://github.com/yt-dlp/yt-dlp/commit/cf6413e840476c15e5b166dc2f7cc2a90a4a9aad) ([#7077](https://github.com/yt-dlp/yt-dlp/issues/7077)) by [dirkf](https://github.com/dirkf), [HobbyistDev](https://github.com/HobbyistDev), [itachi-19](https://github.com/itachi-19), [seproDev](https://github.com/seproDev) +- **boosty**: [Add extractor](https://github.com/yt-dlp/yt-dlp/commit/540b68298192874c75ad5ee4589bed64d02a7d55) ([#9144](https://github.com/yt-dlp/yt-dlp/issues/9144)) by [un-def](https://github.com/un-def) +- **ccma**: [Extract 1080p DASH formats](https://github.com/yt-dlp/yt-dlp/commit/4253e3b7f483127bd812bdac02466f4a5b47ff34) ([#9130](https://github.com/yt-dlp/yt-dlp/issues/9130)) by [seproDev](https://github.com/seproDev) +- **cctv**: [Fix extraction](https://github.com/yt-dlp/yt-dlp/commit/6ad11fef65474bcf70f3a8556850d93c141e44a2) ([#9325](https://github.com/yt-dlp/yt-dlp/issues/9325)) by [src-tinkerer](https://github.com/src-tinkerer) +- **chzzk** + - [Add extractors](https://github.com/yt-dlp/yt-dlp/commit/ba6b0c8261e9f0a6373885736ff90a89dd1fb614) ([#8887](https://github.com/yt-dlp/yt-dlp/issues/8887)) by [DmitryScaletta](https://github.com/DmitryScaletta) + - live: [Support `--wait-for-video`](https://github.com/yt-dlp/yt-dlp/commit/804f2366117b7065552a1c3cddb9ec19b688a5c1) ([#9309](https://github.com/yt-dlp/yt-dlp/issues/9309)) by [hui1601](https://github.com/hui1601) +- **cineverse**: [Detect when login required](https://github.com/yt-dlp/yt-dlp/commit/fc2cc626f07328a6c71b5e21853e4cfa7b1e6256) ([#9081](https://github.com/yt-dlp/yt-dlp/issues/9081)) by [garret1317](https://github.com/garret1317) +- **cloudflarestream** + - [Extract subtitles](https://github.com/yt-dlp/yt-dlp/commit/4d9dc0abe24ad5d9d22a16f40fc61137dcd103f7) ([#9007](https://github.com/yt-dlp/yt-dlp/issues/9007)) by [Bibhav48](https://github.com/Bibhav48) + - [Improve `_VALID_URL`](https://github.com/yt-dlp/yt-dlp/commit/f3d5face83f948c24bcb91e06d4fa6e8622d7d79) ([#9280](https://github.com/yt-dlp/yt-dlp/issues/9280)) by [bashonly](https://github.com/bashonly) + - [Improve embed detection](https://github.com/yt-dlp/yt-dlp/commit/464c919ea82aefdf35f138a1ab2dd0bb8fb7fd0e) ([#9287](https://github.com/yt-dlp/yt-dlp/issues/9287)) by [bashonly](https://github.com/bashonly) +- **cloudycdn, lsm**: [Add extractors](https://github.com/yt-dlp/yt-dlp/commit/5dda3b291f59f388f953337e9fb09a94b64aaf34) ([#8643](https://github.com/yt-dlp/yt-dlp/issues/8643)) by [Caesim404](https://github.com/Caesim404) +- **cnbc**: [Overhaul extractors](https://github.com/yt-dlp/yt-dlp/commit/998dffb5a2343ec709b3d6bbf2bf019649080239) ([#8741](https://github.com/yt-dlp/yt-dlp/issues/8741)) by [gonzalezjo](https://github.com/gonzalezjo), [Noor-5](https://github.com/Noor-5), [ruiminggu](https://github.com/ruiminggu), [seproDev](https://github.com/seproDev), [zhijinwuu](https://github.com/zhijinwuu) +- **craftsy**: [Fix extractor](https://github.com/yt-dlp/yt-dlp/commit/96f3924bac174f2fd401f86f78e77d7e0c5ee008) ([#9384](https://github.com/yt-dlp/yt-dlp/issues/9384)) by [bashonly](https://github.com/bashonly) +- **crooksandliars**: [Fix extractor](https://github.com/yt-dlp/yt-dlp/commit/03536126d32bd861e38536371f0cd5f1b71dcb7a) ([#9192](https://github.com/yt-dlp/yt-dlp/issues/9192)) by [seproDev](https://github.com/seproDev) +- **crtvg**: [Fix `_VALID_URL`](https://github.com/yt-dlp/yt-dlp/commit/785ab1af7f131e73444634ad57b39478651a43d3) ([#9404](https://github.com/yt-dlp/yt-dlp/issues/9404)) by [Xpl0itU](https://github.com/Xpl0itU) +- **dailymotion**: [Support search](https://github.com/yt-dlp/yt-dlp/commit/11ffa92a61e5847b3dfa8975f91ecb3ac2178841) ([#8292](https://github.com/yt-dlp/yt-dlp/issues/8292)) by [drzraf](https://github.com/drzraf), [seproDev](https://github.com/seproDev) +- **douyin**: [Fix extractor](https://github.com/yt-dlp/yt-dlp/commit/9ff946645568e71046487571eefa9cb524a5189b) ([#9239](https://github.com/yt-dlp/yt-dlp/issues/9239)) by [114514ns](https://github.com/114514ns), [bashonly](https://github.com/bashonly) (With fixes in [e546e5d](https://github.com/yt-dlp/yt-dlp/commit/e546e5d3b33a50075e574a2e7b8eda7ea874d21e) by [bashonly](https://github.com/bashonly)) +- **duboku**: [Fix m3u8 formats extraction](https://github.com/yt-dlp/yt-dlp/commit/d3d4187da90a6b85f4ebae4bb07693cc9b412d75) ([#9161](https://github.com/yt-dlp/yt-dlp/issues/9161)) by [DmitryScaletta](https://github.com/DmitryScaletta) +- **dumpert**: [Improve `_VALID_URL`](https://github.com/yt-dlp/yt-dlp/commit/eedb38ce4093500e19279d50b708fb9c18bf4dbf) ([#9320](https://github.com/yt-dlp/yt-dlp/issues/9320)) by [rvsit](https://github.com/rvsit) +- **elementorembed**: [Add extractor](https://github.com/yt-dlp/yt-dlp/commit/6171b050d70435008e64fa06aa6f19c4e5bec75f) ([#8948](https://github.com/yt-dlp/yt-dlp/issues/8948)) by [pompos02](https://github.com/pompos02), [seproDev](https://github.com/seproDev) +- **eporner**: [Extract AV1 formats](https://github.com/yt-dlp/yt-dlp/commit/96d0f8c1cb8aec250c5614bfde6b5fb95f10819b) ([#9028](https://github.com/yt-dlp/yt-dlp/issues/9028)) by [michal-repo](https://github.com/michal-repo) +- **errjupiter** + - [Add extractor](https://github.com/yt-dlp/yt-dlp/commit/a514cc2feb1c3b265b19acab11487acad8bb3ab0) ([#8549](https://github.com/yt-dlp/yt-dlp/issues/8549)) by [glensc](https://github.com/glensc) + - [Improve `_VALID_URL`](https://github.com/yt-dlp/yt-dlp/commit/80ed8bdeba5a945f127ef9ab055a4823329a1210) ([#9218](https://github.com/yt-dlp/yt-dlp/issues/9218)) by [glensc](https://github.com/glensc) +- **facebook** + - [Add new ID format](https://github.com/yt-dlp/yt-dlp/commit/cf9af2c7f1fedd881a157b3fbe725e5494b00924) ([#3824](https://github.com/yt-dlp/yt-dlp/issues/3824)) by [kclauhk](https://github.com/kclauhk), [Wikidepia](https://github.com/Wikidepia) + - [Improve extraction](https://github.com/yt-dlp/yt-dlp/commit/2e30b5567b5c6113d46b39163db5b044aea8667e) by [jingtra](https://github.com/jingtra), [ringus1](https://github.com/ringus1) + - [Improve thumbnail extraction](https://github.com/yt-dlp/yt-dlp/commit/3c4d3ee491b0ec22ed3cade51d943d3d27141ba7) ([#9060](https://github.com/yt-dlp/yt-dlp/issues/9060)) by [kclauhk](https://github.com/kclauhk) + - [Set format HTTP chunk size](https://github.com/yt-dlp/yt-dlp/commit/5b68c478fb0b93ea6b8fac23f50e12217fa063db) ([#9058](https://github.com/yt-dlp/yt-dlp/issues/9058)) by [bashonly](https://github.com/bashonly), [kclauhk](https://github.com/kclauhk) + - [Support events](https://github.com/yt-dlp/yt-dlp/commit/9b5efaf86b99a2664fff9fc725d275f766c3221d) ([#9055](https://github.com/yt-dlp/yt-dlp/issues/9055)) by [kclauhk](https://github.com/kclauhk) + - [Support permalink URLs](https://github.com/yt-dlp/yt-dlp/commit/87286e93af949c4e6a0f8ba34af6a1ab5aa102b6) ([#9061](https://github.com/yt-dlp/yt-dlp/issues/9061)) by [kclauhk](https://github.com/kclauhk) + - ads: [Add extractor](https://github.com/yt-dlp/yt-dlp/commit/a40b0070c2a00d3ed839897462171a82323aa875) ([#8870](https://github.com/yt-dlp/yt-dlp/issues/8870)) by [kclauhk](https://github.com/kclauhk) +- **flextv**: [Add extractor](https://github.com/yt-dlp/yt-dlp/commit/4f043479090dc8a7e06e0bb53691e5414320dfb2) ([#9178](https://github.com/yt-dlp/yt-dlp/issues/9178)) by [DmitryScaletta](https://github.com/DmitryScaletta) +- **floatplane**: [Improve metadata extraction](https://github.com/yt-dlp/yt-dlp/commit/9cd90447907a59c8a2727583f4a755fb23ed8cd3) ([#8934](https://github.com/yt-dlp/yt-dlp/issues/8934)) by [chtk](https://github.com/chtk) +- **francetv** + - [Fix DAI livestreams](https://github.com/yt-dlp/yt-dlp/commit/e4fbe5f886a6693f2466877c12e99c30c5442ace) ([#9380](https://github.com/yt-dlp/yt-dlp/issues/9380)) by [bashonly](https://github.com/bashonly) + - [Fix extractors](https://github.com/yt-dlp/yt-dlp/commit/9749ac7fecbfda391afbadf2870797ce0e382622) ([#9333](https://github.com/yt-dlp/yt-dlp/issues/9333)) by [bashonly](https://github.com/bashonly) + - [Fix m3u8 formats extraction](https://github.com/yt-dlp/yt-dlp/commit/ede624d1db649f5a4b61f8abbb746f365322de27) ([#9347](https://github.com/yt-dlp/yt-dlp/issues/9347)) by [bashonly](https://github.com/bashonly) +- **funk**: [Fix extractor](https://github.com/yt-dlp/yt-dlp/commit/cd0443fb14e2ed805abb02792473457553a123d1) ([#9194](https://github.com/yt-dlp/yt-dlp/issues/9194)) by [seproDev](https://github.com/seproDev) +- **generic**: [Follow https redirects properly](https://github.com/yt-dlp/yt-dlp/commit/c8c9039e640495700f76a13496e3418bdd4382ba) ([#9121](https://github.com/yt-dlp/yt-dlp/issues/9121)) by [seproDev](https://github.com/seproDev) +- **getcourseru**: [Add extractors](https://github.com/yt-dlp/yt-dlp/commit/4310b6650eeb5630295f4591b37720877878c57a) ([#8873](https://github.com/yt-dlp/yt-dlp/issues/8873)) by [divStar](https://github.com/divStar), [seproDev](https://github.com/seproDev) +- **gofile**: [Fix extraction](https://github.com/yt-dlp/yt-dlp/commit/77c2472ca1ef9050a66aa68bc5fa1bee88706c66) ([#9074](https://github.com/yt-dlp/yt-dlp/issues/9074)) by [jazz1611](https://github.com/jazz1611) +- **googledrive**: [Fix source file extraction](https://github.com/yt-dlp/yt-dlp/commit/5498729c59b03a9511c64552da3ba2f802166f8d) ([#8990](https://github.com/yt-dlp/yt-dlp/issues/8990)) by [jazz1611](https://github.com/jazz1611) +- **goplay**: [Fix extractor](https://github.com/yt-dlp/yt-dlp/commit/7e90e34fa4617b53f8c8a9e69f460508cb1f51b0) ([#6654](https://github.com/yt-dlp/yt-dlp/issues/6654)) by [alard](https://github.com/alard) +- **gopro**: [Fix extractor](https://github.com/yt-dlp/yt-dlp/commit/4a07a455bbf7acf87550053bbba949c828e350ba) ([#9019](https://github.com/yt-dlp/yt-dlp/issues/9019)) by [stilor](https://github.com/stilor) +- **ilpost**: [Add extractor](https://github.com/yt-dlp/yt-dlp/commit/aa5dcc4ee65916a36cbe1b1b5b29b9110c3163ed) ([#9001](https://github.com/yt-dlp/yt-dlp/issues/9001)) by [CapacitorSet](https://github.com/CapacitorSet) +- **jiosaavnsong**: [Support more bitrates](https://github.com/yt-dlp/yt-dlp/commit/5154dc0a687528f995cde22b5ff63f82c740e98a) ([#8834](https://github.com/yt-dlp/yt-dlp/issues/8834)) by [alien-developers](https://github.com/alien-developers), [bashonly](https://github.com/bashonly) +- **kukululive**: [Add extractor](https://github.com/yt-dlp/yt-dlp/commit/20cdad5a2c0499d5a6746f5466a2ab0c97b75884) ([#8877](https://github.com/yt-dlp/yt-dlp/issues/8877)) by [DmitryScaletta](https://github.com/DmitryScaletta) +- **lefigarovideoembed**: [Fix extractor](https://github.com/yt-dlp/yt-dlp/commit/9401736fd08767c58af45a1e36ff5929c5fa1ac9) ([#9198](https://github.com/yt-dlp/yt-dlp/issues/9198)) by [seproDev](https://github.com/seproDev) +- **linkedin**: [Fix metadata and extract subtitles](https://github.com/yt-dlp/yt-dlp/commit/017adb28e7fe7b8c8fc472332d86740f31141519) ([#9056](https://github.com/yt-dlp/yt-dlp/issues/9056)) by [barsnick](https://github.com/barsnick) +- **magellantv**: [Support episodes](https://github.com/yt-dlp/yt-dlp/commit/3dc9232e1aa58fe3c2d8cafb50e8162d6f0e891e) ([#9199](https://github.com/yt-dlp/yt-dlp/issues/9199)) by [seproDev](https://github.com/seproDev) +- **magentamusik**: [Add extractor](https://github.com/yt-dlp/yt-dlp/commit/5e2e24b2c5795756d81785b06b10723ddb6db7b2) ([#7790](https://github.com/yt-dlp/yt-dlp/issues/7790)) by [pwaldhauer](https://github.com/pwaldhauer), [seproDev](https://github.com/seproDev) +- **medaltv**: [Fix extraction](https://github.com/yt-dlp/yt-dlp/commit/02e343f6ef6d7b3f9087ff69e4a1db0b4b4a5c5d) ([#9098](https://github.com/yt-dlp/yt-dlp/issues/9098)) by [Danish-H](https://github.com/Danish-H) +- **mlbarticle**: [Fix extractor](https://github.com/yt-dlp/yt-dlp/commit/50e06e21a68e336198198bda332b8e7d2314f201) ([#9021](https://github.com/yt-dlp/yt-dlp/issues/9021)) by [HobbyistDev](https://github.com/HobbyistDev) +- **motherless**: [Support uploader playlists](https://github.com/yt-dlp/yt-dlp/commit/9f1e9dab21bbe651544c8f4663b0e615dc450e4d) ([#8994](https://github.com/yt-dlp/yt-dlp/issues/8994)) by [dasidiot](https://github.com/dasidiot) +- **mujrozhlas**: [Fix extraction](https://github.com/yt-dlp/yt-dlp/commit/4170b3d7120e06db3391eef39c5add18a1ddf2c3) ([#9306](https://github.com/yt-dlp/yt-dlp/issues/9306)) by [bashonly](https://github.com/bashonly) +- **mx3**: [Add extractors](https://github.com/yt-dlp/yt-dlp/commit/5a63454b3637b3603434026cddfeac509218b90e) ([#8736](https://github.com/yt-dlp/yt-dlp/issues/8736)) by [martinxyz](https://github.com/martinxyz) +- **naver**: [Fix extractors](https://github.com/yt-dlp/yt-dlp/commit/a281beba8d8f007cf220f96dd1d9412bb070c7d8) ([#8883](https://github.com/yt-dlp/yt-dlp/issues/8883)) by [seproDev](https://github.com/seproDev) +- **nebula**: [Support podcasts](https://github.com/yt-dlp/yt-dlp/commit/0de09c5b9ed619d4a93d7c451c6ddff0381de808) ([#9140](https://github.com/yt-dlp/yt-dlp/issues/9140)) by [c-basalt](https://github.com/c-basalt), [seproDev](https://github.com/seproDev) +- **nerdcubedfeed**: [Overhaul extractor](https://github.com/yt-dlp/yt-dlp/commit/29a74a6126101aabaa1726ae41b1ca55cf26e7a7) ([#9269](https://github.com/yt-dlp/yt-dlp/issues/9269)) by [seproDev](https://github.com/seproDev) +- **newgrounds** + - [Fix login and clean up extraction](https://github.com/yt-dlp/yt-dlp/commit/0fcefb92f3ebfc5cada19c1e85a715f020d0f333) ([#9356](https://github.com/yt-dlp/yt-dlp/issues/9356)) by [Grub4K](https://github.com/Grub4K), [mrmedieval](https://github.com/mrmedieval) + - user: [Fix extractor](https://github.com/yt-dlp/yt-dlp/commit/3e083191cdc34dd8c482da9a9b4bc682f824cb9d) ([#9046](https://github.com/yt-dlp/yt-dlp/issues/9046)) by [u-spec-png](https://github.com/u-spec-png) +- **nfb**: [Add support for onf.ca and series](https://github.com/yt-dlp/yt-dlp/commit/4b8b0dded8c65cd5b2ab2e858058ba98c9bf49ff) ([#8997](https://github.com/yt-dlp/yt-dlp/issues/8997)) by [bashonly](https://github.com/bashonly), [rrgomes](https://github.com/rrgomes) +- **nhkradiru**: [Extract extended description](https://github.com/yt-dlp/yt-dlp/commit/4392447d9404e3c25cfeb8f5bdfff31b0448da39) ([#9162](https://github.com/yt-dlp/yt-dlp/issues/9162)) by [garret1317](https://github.com/garret1317) +- **nhkradirulive**: [Make metadata extraction non-fatal](https://github.com/yt-dlp/yt-dlp/commit/5af1f19787f7d652fce72dd3ab9536cdd980fe85) ([#8956](https://github.com/yt-dlp/yt-dlp/issues/8956)) by [garret1317](https://github.com/garret1317) +- **niconico** + - [Remove legacy danmaku extraction](https://github.com/yt-dlp/yt-dlp/commit/974d444039c8bbffb57265c6792cd52d169fe1b9) ([#9209](https://github.com/yt-dlp/yt-dlp/issues/9209)) by [pzhlkj6612](https://github.com/pzhlkj6612) + - [Support DMS formats](https://github.com/yt-dlp/yt-dlp/commit/aa13a8e3dd3b698cc40ec438988b1ad834e11a41) ([#9282](https://github.com/yt-dlp/yt-dlp/issues/9282)) by [pzhlkj6612](https://github.com/pzhlkj6612), [xpadev-net](https://github.com/xpadev-net) (With fixes in [40966e8](https://github.com/yt-dlp/yt-dlp/commit/40966e8da27bbf770dacf9be9363fcc3ad72cc9f) by [pzhlkj6612](https://github.com/pzhlkj6612)) +- **ninaprotocol**: [Add extractor](https://github.com/yt-dlp/yt-dlp/commit/62c65bfaf81e04e6746f6fdbafe384eb3edddfbc) ([#8946](https://github.com/yt-dlp/yt-dlp/issues/8946)) by [RaduManole](https://github.com/RaduManole), [seproDev](https://github.com/seproDev) +- **ninenews**: [Add extractor](https://github.com/yt-dlp/yt-dlp/commit/43694ce13c5a9f1afca8b02b8b2b9b1576d6503d) ([#8840](https://github.com/yt-dlp/yt-dlp/issues/8840)) by [SirElderling](https://github.com/SirElderling) +- **nova**: [Fix embed extraction](https://github.com/yt-dlp/yt-dlp/commit/c168d8791d0974a8a8fcb3b4a4bc2d830df51622) ([#9221](https://github.com/yt-dlp/yt-dlp/issues/9221)) by [seproDev](https://github.com/seproDev) +- **ntvru**: [Fix extraction](https://github.com/yt-dlp/yt-dlp/commit/7a29cbbd5fd7363e7e8535ee1506b7052465d13f) ([#9276](https://github.com/yt-dlp/yt-dlp/issues/9276)) by [bashonly](https://github.com/bashonly), [dirkf](https://github.com/dirkf) +- **nuum**: [Add extractors](https://github.com/yt-dlp/yt-dlp/commit/acaf806c15f0a802ba286c23af02a10cf4bd4731) ([#8868](https://github.com/yt-dlp/yt-dlp/issues/8868)) by [DmitryScaletta](https://github.com/DmitryScaletta), [seproDev](https://github.com/seproDev) +- **nytimes** + - [Extract timestamp](https://github.com/yt-dlp/yt-dlp/commit/05420227aaab60a39c0f9ade069c5862be36b1fa) ([#9142](https://github.com/yt-dlp/yt-dlp/issues/9142)) by [SirElderling](https://github.com/SirElderling) + - [Overhaul extractors](https://github.com/yt-dlp/yt-dlp/commit/07256b9fee23960799024b95d5972abc7174aa81) ([#9075](https://github.com/yt-dlp/yt-dlp/issues/9075)) by [SirElderling](https://github.com/SirElderling) +- **onefootball**: [Fix extractor](https://github.com/yt-dlp/yt-dlp/commit/644738ddaa45428cb0babd41ead22454e5a2545e) ([#9222](https://github.com/yt-dlp/yt-dlp/issues/9222)) by [seproDev](https://github.com/seproDev) +- **openrec**: [Pass referer for m3u8 formats](https://github.com/yt-dlp/yt-dlp/commit/f591e605dfee4085ec007d6d056c943cbcacc429) ([#9253](https://github.com/yt-dlp/yt-dlp/issues/9253)) by [fireattack](https://github.com/fireattack) +- **orf**: on: [Add extractor](https://github.com/yt-dlp/yt-dlp/commit/a0d50aabc5462aee302bd3f2663d3a3554875789) ([#9113](https://github.com/yt-dlp/yt-dlp/issues/9113)) by [HobbyistDev](https://github.com/HobbyistDev) +- **patreon**: [Fix embedded HLS extraction](https://github.com/yt-dlp/yt-dlp/commit/f0e8bc7c60b61fe18b63116c975609d76b904771) ([#8993](https://github.com/yt-dlp/yt-dlp/issues/8993)) by [johnvictorfs](https://github.com/johnvictorfs) +- **peertube**: [Update instances](https://github.com/yt-dlp/yt-dlp/commit/35d96982f1033e36215d323317981ee17e8ab0d5) ([#9070](https://github.com/yt-dlp/yt-dlp/issues/9070)) by [Chocobozzz](https://github.com/Chocobozzz) +- **piapro**: [Improve `_VALID_URL`](https://github.com/yt-dlp/yt-dlp/commit/8e6e3651727b0b85764857fc6329fe5e0a3f00de) ([#8999](https://github.com/yt-dlp/yt-dlp/issues/8999)) by [FinnRG](https://github.com/FinnRG) +- **playsuisse**: [Add login support](https://github.com/yt-dlp/yt-dlp/commit/cae6e461073fb7c32fd32052a3e6721447c469bc) ([#9077](https://github.com/yt-dlp/yt-dlp/issues/9077)) by [chkuendig](https://github.com/chkuendig) +- **pornhub**: [Fix login support](https://github.com/yt-dlp/yt-dlp/commit/de954c1b4d3a6db8a6525507e65303c7bb03f39f) ([#9227](https://github.com/yt-dlp/yt-dlp/issues/9227)) by [feederbox826](https://github.com/feederbox826) +- **pr0gramm**: [Enable POL filter and provide tags without login](https://github.com/yt-dlp/yt-dlp/commit/5f25f348f9eb5db842b1ec6799f95bebb7ba35a7) ([#9051](https://github.com/yt-dlp/yt-dlp/issues/9051)) by [Grub4K](https://github.com/Grub4K) +- **prankcastpost**: [Add extractor](https://github.com/yt-dlp/yt-dlp/commit/a2bac6b7adb7b0e955125838e20bb39eece630ce) ([#8933](https://github.com/yt-dlp/yt-dlp/issues/8933)) by [columndeeply](https://github.com/columndeeply) +- **radiko**: [Extract more metadata](https://github.com/yt-dlp/yt-dlp/commit/e3ce2b385ec1f03fac9d4210c57fda77134495fc) ([#9115](https://github.com/yt-dlp/yt-dlp/issues/9115)) by [YoshichikaAAA](https://github.com/YoshichikaAAA) +- **rai** + - [Filter unavailable formats](https://github.com/yt-dlp/yt-dlp/commit/f78814923748277e7067b796f25870686fb46205) ([#9189](https://github.com/yt-dlp/yt-dlp/issues/9189)) by [nixxo](https://github.com/nixxo) + - [Fix m3u8 formats extraction](https://github.com/yt-dlp/yt-dlp/commit/8f423cf8051fbfeedd57cca00d106012e6e86a97) ([#9291](https://github.com/yt-dlp/yt-dlp/issues/9291)) by [nixxo](https://github.com/nixxo) +- **redcdnlivx, sejm**: [Add extractors](https://github.com/yt-dlp/yt-dlp/commit/fcaa2e735b00b15a2b0d9f55f4187c654b4b5b39) ([#8676](https://github.com/yt-dlp/yt-dlp/issues/8676)) by [selfisekai](https://github.com/selfisekai) +- **redtube** + - [Fix formats extraction](https://github.com/yt-dlp/yt-dlp/commit/c91d8b1899403daff6fc15206ad32de8db17fb8f) ([#9076](https://github.com/yt-dlp/yt-dlp/issues/9076)) by [jazz1611](https://github.com/jazz1611) + - [Support redtube.com.br URLs](https://github.com/yt-dlp/yt-dlp/commit/4a6ff0b47a700dee3ee5c54804c31965308479ae) ([#9103](https://github.com/yt-dlp/yt-dlp/issues/9103)) by [jazz1611](https://github.com/jazz1611) +- **ridehome**: [Add extractor](https://github.com/yt-dlp/yt-dlp/commit/cd7086c0d54ec1d7e02a30bd5bd934bdb2c54642) ([#8875](https://github.com/yt-dlp/yt-dlp/issues/8875)) by [SirElderling](https://github.com/SirElderling) +- **rinsefmartistplaylist**: [Add extractor](https://github.com/yt-dlp/yt-dlp/commit/1a36dbad712d359ec1c5b73d9bbbe562c03e9660) ([#8794](https://github.com/yt-dlp/yt-dlp/issues/8794)) by [SirElderling](https://github.com/SirElderling) +- **roosterteeth** + - [Add Brightcove fallback](https://github.com/yt-dlp/yt-dlp/commit/b2cc150ad83ba20ceb2d6e73d09854eed3c2d05c) ([#9403](https://github.com/yt-dlp/yt-dlp/issues/9403)) by [bashonly](https://github.com/bashonly) + - [Extract ad-free streams](https://github.com/yt-dlp/yt-dlp/commit/dd29e6e5fdf0f3758cb0829e73749832768f1a4e) ([#9355](https://github.com/yt-dlp/yt-dlp/issues/9355)) by [jkmartindale](https://github.com/jkmartindale) + - [Extract release date and timestamp](https://github.com/yt-dlp/yt-dlp/commit/dfd8c0b69683b1c11beea039a96dd2949026c1d7) ([#9393](https://github.com/yt-dlp/yt-dlp/issues/9393)) by [bashonly](https://github.com/bashonly) + - [Support bonus features](https://github.com/yt-dlp/yt-dlp/commit/8993721ecb34867b52b79f6e92b233008d1cbe78) ([#9406](https://github.com/yt-dlp/yt-dlp/issues/9406)) by [Bl4Cc4t](https://github.com/Bl4Cc4t) +- **rule34video** + - [Extract `creators`](https://github.com/yt-dlp/yt-dlp/commit/3d9dc2f3590e10abf1561ebdaed96734a740587c) ([#9258](https://github.com/yt-dlp/yt-dlp/issues/9258)) by [gmes78](https://github.com/gmes78) + - [Extract more metadata](https://github.com/yt-dlp/yt-dlp/commit/fee2d8d9c38f9b5f0a8df347c1e698983339c34d) ([#7416](https://github.com/yt-dlp/yt-dlp/issues/7416)) by [gmes78](https://github.com/gmes78) + - [Fix `_VALID_URL`](https://github.com/yt-dlp/yt-dlp/commit/c0ecceeefe6ebd27452d9d8f20658f83ae121d04) ([#9044](https://github.com/yt-dlp/yt-dlp/issues/9044)) by [gmes78](https://github.com/gmes78) +- **rumblechannel**: [Fix extractor](https://github.com/yt-dlp/yt-dlp/commit/0023af81fbce01984f35b34ecaf8562739831227) ([#9092](https://github.com/yt-dlp/yt-dlp/issues/9092)) by [Pranaxcau](https://github.com/Pranaxcau), [vista-narvas](https://github.com/vista-narvas) +- **screencastify**: [Update `_VALID_URL`](https://github.com/yt-dlp/yt-dlp/commit/0bee29493ca8f91a0055a3706c7c94f5860188df) ([#9232](https://github.com/yt-dlp/yt-dlp/issues/9232)) by [seproDev](https://github.com/seproDev) +- **svtpage**: [Fix extractor](https://github.com/yt-dlp/yt-dlp/commit/ddd4b5e10a653bee78e656107710021c1b82934c) ([#8938](https://github.com/yt-dlp/yt-dlp/issues/8938)) by [diman8](https://github.com/diman8) +- **swearnet**: [Raise for login required](https://github.com/yt-dlp/yt-dlp/commit/b05640d532c43a52c0a0da096bb2dbd51e105ec0) ([#9281](https://github.com/yt-dlp/yt-dlp/issues/9281)) by [bashonly](https://github.com/bashonly) +- **tiktok**: [Fix webpage extraction](https://github.com/yt-dlp/yt-dlp/commit/d9b4154cbcb979d7e30af3a73b1bee422aae5aa3) ([#9327](https://github.com/yt-dlp/yt-dlp/issues/9327)) by [bashonly](https://github.com/bashonly) +- **trtworld**: [Add extractor](https://github.com/yt-dlp/yt-dlp/commit/8ab84650837e58046430c9f4b615c56a8886e071) ([#8701](https://github.com/yt-dlp/yt-dlp/issues/8701)) by [ufukk](https://github.com/ufukk) +- **tvp**: [Support livestreams](https://github.com/yt-dlp/yt-dlp/commit/882e3b753c79c7799ce135c3a5edb72494b576af) ([#8860](https://github.com/yt-dlp/yt-dlp/issues/8860)) by [selfisekai](https://github.com/selfisekai) +- **twitch**: [Fix m3u8 extraction](https://github.com/yt-dlp/yt-dlp/commit/5b8c69ae04444a4c80a5a99917e40f75a116c3b8) ([#8960](https://github.com/yt-dlp/yt-dlp/issues/8960)) by [DmitryScaletta](https://github.com/DmitryScaletta) +- **twitter** + - [Extract bitrate for HLS audio formats](https://github.com/yt-dlp/yt-dlp/commit/28e53d60df9b8aadd52a93504e30e885c9c35262) ([#9257](https://github.com/yt-dlp/yt-dlp/issues/9257)) by [bashonly](https://github.com/bashonly) + - [Extract numeric `channel_id`](https://github.com/yt-dlp/yt-dlp/commit/55f1833376505ed1e4be0516b09bb3ea4425e8a4) ([#9263](https://github.com/yt-dlp/yt-dlp/issues/9263)) by [bashonly](https://github.com/bashonly) +- **txxx**: [Extract thumbnails](https://github.com/yt-dlp/yt-dlp/commit/d79c7e9937c388c68b722ab7450960e43ef776d6) ([#9063](https://github.com/yt-dlp/yt-dlp/issues/9063)) by [shmohawk](https://github.com/shmohawk) +- **utreon**: [Support playeur.com](https://github.com/yt-dlp/yt-dlp/commit/41d6b61e9852a5b97f47cc8a7718b31fb23f0aea) ([#9182](https://github.com/yt-dlp/yt-dlp/issues/9182)) by [DmitryScaletta](https://github.com/DmitryScaletta) +- **vbox7**: [Fix extractor](https://github.com/yt-dlp/yt-dlp/commit/67bb70cd700c8d4c3149cd9e0539a5f32c3d1ce6) ([#9100](https://github.com/yt-dlp/yt-dlp/issues/9100)) by [seproDev](https://github.com/seproDev) +- **viewlift**: [Add support for chorki.com](https://github.com/yt-dlp/yt-dlp/commit/41b6cdb4197aaf7ad82bdad6885eb5d5c64acd74) ([#9095](https://github.com/yt-dlp/yt-dlp/issues/9095)) by [NurTasin](https://github.com/NurTasin) +- **vimeo** + - [Extract `live_status` and `release_timestamp`](https://github.com/yt-dlp/yt-dlp/commit/f0426e9ca57dd14b82e6c13afc17947614f1e8eb) ([#9290](https://github.com/yt-dlp/yt-dlp/issues/9290)) by [pzhlkj6612](https://github.com/pzhlkj6612) + - [Fix API headers](https://github.com/yt-dlp/yt-dlp/commit/8e765755f7f4909e1b535e61b7376b2d66e1ba6a) ([#9125](https://github.com/yt-dlp/yt-dlp/issues/9125)) by [bashonly](https://github.com/bashonly) + - [Fix login](https://github.com/yt-dlp/yt-dlp/commit/2e8de097ad82da378e97005e8f1ff7e5aebca585) ([#9274](https://github.com/yt-dlp/yt-dlp/issues/9274)) by [bashonly](https://github.com/bashonly) +- **viously**: [Add extractor](https://github.com/yt-dlp/yt-dlp/commit/95e82347b398d8bb160767cdd975edecd62cbabd) ([#8927](https://github.com/yt-dlp/yt-dlp/issues/8927)) by [nbr23](https://github.com/nbr23), [seproDev](https://github.com/seproDev) +- **youtube** + - [Better error when all player responses are skipped](https://github.com/yt-dlp/yt-dlp/commit/5eedc208ec89d6284777060c94aadd06502338b9) ([#9083](https://github.com/yt-dlp/yt-dlp/issues/9083)) by [Grub4K](https://github.com/Grub4K), [pukkandan](https://github.com/pukkandan) + - [Bump Android and iOS client versions](https://github.com/yt-dlp/yt-dlp/commit/413d3675804599bc8fe419c19e36490fd8f0b30f) ([#9317](https://github.com/yt-dlp/yt-dlp/issues/9317)) by [bashonly](https://github.com/bashonly) + - [Further bump client versions](https://github.com/yt-dlp/yt-dlp/commit/7aad06541e543fa3452d3d2513e6f079aad1f99b) ([#9395](https://github.com/yt-dlp/yt-dlp/issues/9395)) by [bashonly](https://github.com/bashonly) + - tab: [Fix `tags` extraction](https://github.com/yt-dlp/yt-dlp/commit/8828f4576bd862438d4fbf634f1d6ab18a217b0e) ([#9413](https://github.com/yt-dlp/yt-dlp/issues/9413)) by [x11x](https://github.com/x11x) +- **zenporn**: [Add extractor](https://github.com/yt-dlp/yt-dlp/commit/f00c0def7434fac3c88503c2a77c4b2419b8e5ca) ([#8509](https://github.com/yt-dlp/yt-dlp/issues/8509)) by [SirElderling](https://github.com/SirElderling) +- **zetland**: [Add extractor](https://github.com/yt-dlp/yt-dlp/commit/2f4b57594673035a59d72f7667588da848820034) ([#9116](https://github.com/yt-dlp/yt-dlp/issues/9116)) by [HobbyistDev](https://github.com/HobbyistDev) + +#### Downloader changes +- **http**: [Reset resume length to handle `FileNotFoundError`](https://github.com/yt-dlp/yt-dlp/commit/2d91b9845621639c53dca7ee9d3d954f3624ba18) ([#8399](https://github.com/yt-dlp/yt-dlp/issues/8399)) by [boredzo](https://github.com/boredzo) + +#### Networking changes +- [Remove `_CompatHTTPError`](https://github.com/yt-dlp/yt-dlp/commit/811d298b231cfa29e75c321b23a91d1c2b17602c) ([#8871](https://github.com/yt-dlp/yt-dlp/issues/8871)) by [coletdjnz](https://github.com/coletdjnz) +- **Request Handler** + - [Remove additional logging handlers on close](https://github.com/yt-dlp/yt-dlp/commit/0085e2bab8465ee7d46d16fcade3ed5e96cc8a48) ([#9032](https://github.com/yt-dlp/yt-dlp/issues/9032)) by [coletdjnz](https://github.com/coletdjnz) + - requests: [Apply `remove_dot_segments` to absolute redirect locations](https://github.com/yt-dlp/yt-dlp/commit/35f4f764a786685ea45d84abe1cf1ad3847f4c97) by [coletdjnz](https://github.com/coletdjnz) + +#### Misc. changes +- **build** + - [Add `default` optional dependency group](https://github.com/yt-dlp/yt-dlp/commit/cf91400a1dd6cc99b11a6d163e1af73b64d618c9) ([#9295](https://github.com/yt-dlp/yt-dlp/issues/9295)) by [bashonly](https://github.com/bashonly), [Grub4K](https://github.com/Grub4K) + - [Add transitional `setup.py` and `pyinst.py`](https://github.com/yt-dlp/yt-dlp/commit/0abf2f1f153ab47990edbeee3477dc55f74c7f89) ([#9296](https://github.com/yt-dlp/yt-dlp/issues/9296)) by [bashonly](https://github.com/bashonly), [Grub4K](https://github.com/Grub4K), [pukkandan](https://github.com/pukkandan) + - [Bump `actions/upload-artifact` to v4 and adjust workflows](https://github.com/yt-dlp/yt-dlp/commit/3876429d72afb35247f4b2531eb9b16cfc7e0968) by [bashonly](https://github.com/bashonly) + - [Bump `conda-incubator/setup-miniconda` to v3](https://github.com/yt-dlp/yt-dlp/commit/b0059f0413a6ba6ab0a3aec1f00188ce083cd8bf) by [bashonly](https://github.com/bashonly) + - [Fix `secretstorage` for ARM builds](https://github.com/yt-dlp/yt-dlp/commit/920397634d1e84e76d2cb897bd6d69ba0c6bd5ca) by [bashonly](https://github.com/bashonly) + - [Migrate to `pyproject.toml` and `hatchling`](https://github.com/yt-dlp/yt-dlp/commit/775cde82dc5b1dc64ab0539a92dd8c7ba6c0ad33) by [bashonly](https://github.com/bashonly) (With fixes in [43cfd46](https://github.com/yt-dlp/yt-dlp/commit/43cfd462c0d01eff22c1d4290aeb96eb1ea2c0e1)) + - [Move bundle scripts into `bundle` submodule](https://github.com/yt-dlp/yt-dlp/commit/a1b778428991b1779203bac243ef4e9b6baea90c) by [bashonly](https://github.com/bashonly) + - [Support failed build job re-runs](https://github.com/yt-dlp/yt-dlp/commit/eabbccc439720fba381919a88be4fe4d96464cbd) ([#9277](https://github.com/yt-dlp/yt-dlp/issues/9277)) by [bashonly](https://github.com/bashonly) + - Makefile + - [Add automated `CODE_FOLDERS` and `CODE_FILES`](https://github.com/yt-dlp/yt-dlp/commit/868d2f60a7cb59b410c8cbfb452cbdb072687b81) by [bashonly](https://github.com/bashonly) + - [Ensure compatibility with BSD `make`](https://github.com/yt-dlp/yt-dlp/commit/beaa1a44554d04d9fe63a743a5bb4431ca778f28) ([#9210](https://github.com/yt-dlp/yt-dlp/issues/9210)) by [bashonly](https://github.com/bashonly) (With fixes in [73fcfa3](https://github.com/yt-dlp/yt-dlp/commit/73fcfa39f59113a8728249de2c4cee3025f17dc2)) + - [Fix man pages generated by `pandoc>=3`](https://github.com/yt-dlp/yt-dlp/commit/fb44020fa98e47620b3aa1dab94b4c5b7bfb40bd) ([#7047](https://github.com/yt-dlp/yt-dlp/issues/7047)) by [t-nil](https://github.com/t-nil) +- **ci**: [Bump `actions/setup-python` to v5](https://github.com/yt-dlp/yt-dlp/commit/b14e818b37f62e3224da157b3ad768b3f0815fcd) by [bashonly](https://github.com/bashonly) +- **cleanup** + - [Build files cleanup](https://github.com/yt-dlp/yt-dlp/commit/867f637b95b342e1cb9f1dc3c6cf0ffe727187ce) by [bashonly](https://github.com/bashonly) + - [Fix infodict returned fields](https://github.com/yt-dlp/yt-dlp/commit/f4f9f6d00edcac6d4eb2b3fb78bf81326235d492) ([#8906](https://github.com/yt-dlp/yt-dlp/issues/8906)) by [seproDev](https://github.com/seproDev) + - [Fix typo in README.md](https://github.com/yt-dlp/yt-dlp/commit/292d60b1ed3b9fe5bcb2775a894cca99b0f9473e) ([#8894](https://github.com/yt-dlp/yt-dlp/issues/8894)) by [antonkesy](https://github.com/antonkesy) + - [Mark broken and remove dead extractors](https://github.com/yt-dlp/yt-dlp/commit/df773c3d5d1cc1f877cf8582f0072e386fc49318) ([#9238](https://github.com/yt-dlp/yt-dlp/issues/9238)) by [seproDev](https://github.com/seproDev) + - [Match both `http` and `https` in `_VALID_URL`](https://github.com/yt-dlp/yt-dlp/commit/a687226b48f71b874fa18b0165ec528d591f53fb) ([#8968](https://github.com/yt-dlp/yt-dlp/issues/8968)) by [seproDev](https://github.com/seproDev) + - [Remove unused code](https://github.com/yt-dlp/yt-dlp/commit/ed3bb2b0a12c44334e0d09481752dabf2ca1dc13) ([#8968](https://github.com/yt-dlp/yt-dlp/issues/8968)) by [pukkandan](https://github.com/pukkandan), [seproDev](https://github.com/seproDev) + - Miscellaneous + - [93240fc](https://github.com/yt-dlp/yt-dlp/commit/93240fc1848de4a94f25844c96e0dcd282ef1d3b) by [bashonly](https://github.com/bashonly), [Grub4k](https://github.com/Grub4k), [pukkandan](https://github.com/pukkandan), [seproDev](https://github.com/seproDev) + - [615a844](https://github.com/yt-dlp/yt-dlp/commit/615a84447e8322720be77a0e64298d7f42848693) by [bashonly](https://github.com/bashonly), [pukkandan](https://github.com/pukkandan), [seproDev](https://github.com/seproDev) +- **devscripts** + - `install_deps`: [Add script and migrate to it](https://github.com/yt-dlp/yt-dlp/commit/b8a433aaca86b15cb9f1a451b0f69371d2fc22a9) by [bashonly](https://github.com/bashonly) + - `tomlparse`: [Add makeshift toml parser](https://github.com/yt-dlp/yt-dlp/commit/fd647775e27e030ab17387c249e2ebeba68f8ff0) by [Grub4K](https://github.com/Grub4K) +- **docs**: [Misc Cleanup](https://github.com/yt-dlp/yt-dlp/commit/47ab66db0f083a76c7fba0f6e136b21dd5a93e3b) ([#8977](https://github.com/yt-dlp/yt-dlp/issues/8977)) by [Arthurszzz](https://github.com/Arthurszzz), [bashonly](https://github.com/bashonly), [Grub4k](https://github.com/Grub4k), [pukkandan](https://github.com/pukkandan), [seproDev](https://github.com/seproDev) +- **test** + - [Skip source address tests if the address cannot be bound to](https://github.com/yt-dlp/yt-dlp/commit/69d31914952dd33082ac7019c6f76b43c45b9d06) ([#8900](https://github.com/yt-dlp/yt-dlp/issues/8900)) by [coletdjnz](https://github.com/coletdjnz) + - websockets: [Fix timeout test on Windows](https://github.com/yt-dlp/yt-dlp/commit/ac340d0745a9de5d494033e3507ef624ba25add3) ([#9344](https://github.com/yt-dlp/yt-dlp/issues/9344)) by [seproDev](https://github.com/seproDev) + ### 2023.12.30 #### Core changes @@ -1936,7 +2158,7 @@ ### 2022.04.08 * [utils] `format_decimal_suffix`: Fix for very large numbers by [s0u1h](https://github.com/s0u1h) * [utils] `traverse_obj`: Allow filtering by value * [utils] Add `filter_dict`, `get_first`, `try_call` -* [utils] ExtractorError: Fix for older python versions +* [utils] ExtractorError: Fix for older Python versions * [utils] WebSocketsWrapper: Allow omitting `__enter__` invocation by [Lesmiscore](https://github.com/Lesmiscore) * [docs] Add an `.editorconfig` file by [fstirlitz](https://github.com/fstirlitz) * [docs] Clarify the exact `BSD` license of dependencies by [MrRawes](https://github.com/MrRawes) @@ -3400,7 +3622,7 @@ ### 2021.05.20 * [cleanup] code formatting, youtube tests and readme ### 2021.05.11 -* **Deprecate support for python versions < 3.6** +* **Deprecate support for Python versions < 3.6** * **Subtitle extraction from manifests** by [fstirlitz](https://github.com/fstirlitz). See [be6202f](https://github.com/yt-dlp/yt-dlp/commit/be6202f12b97858b9d716e608394b51065d0419f) for details * **Improve output template:** * Allow slicing lists/strings using `field.start:end:step` @@ -3690,7 +3912,7 @@ ### 2021.02.19 * Remove unnecessary `field_preference` and misuse of `preference` from extractors * Build improvements: * Fix hash output by [shirt](https://github.com/shirt-dev) - * Lock python package versions for x86 and use `wheels` by [shirt](https://github.com/shirt-dev) + * Lock Python package versions for x86 and use `wheels` by [shirt](https://github.com/shirt-dev) * Exclude `vcruntime140.dll` from UPX by [jbruchon](https://github.com/jbruchon) * Set version number based on UTC time, not local time * Publish on PyPi only if token is set @@ -3757,7 +3979,7 @@ ### 2021.02.04 * Fix "Default format spec" appearing in quiet mode * [FormatSort] Allow user to prefer av01 over vp9 (The default is still vp9) * [FormatSort] fix bug where `quality` had more priority than `hasvid` -* [pyinst] Automatically detect python architecture and working directory +* [pyinst] Automatically detect Python architecture and working directory * Strip out internal fields such as `_filename` from infojson diff --git a/Makefile b/Makefile index d1087a3b4..57443af9d 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ all: lazy-extractors yt-dlp doc pypi-files clean: clean-test clean-dist clean-all: clean clean-cache completions: completion-bash completion-fish completion-zsh -doc: README.md CONTRIBUTING.md issuetemplates supportedsites +doc: README.md CONTRIBUTING.md CONTRIBUTORS issuetemplates supportedsites ot: offlinetest tar: yt-dlp.tar.gz @@ -17,8 +17,8 @@ pypi-files: AUTHORS Changelog.md LICENSE README.md README.txt supportedsites \ clean-test: rm -rf test/testdata/sigs/player-*.js tmp/ *.annotations.xml *.aria2 *.description *.dump *.frag \ *.frag.aria2 *.frag.urls *.info.json *.live_chat.json *.meta *.part* *.tmp *.temp *.unknown_video *.ytdl \ - *.3gp *.ape *.ass *.avi *.desktop *.f4v *.flac *.flv *.gif *.jpeg *.jpg *.m4a *.m4v *.mhtml *.mkv *.mov *.mp3 \ - *.mp4 *.mpga *.oga *.ogg *.opus *.png *.sbv *.srt *.swf *.swp *.tt *.ttml *.url *.vtt *.wav *.webloc *.webm *.webp + *.3gp *.ape *.ass *.avi *.desktop *.f4v *.flac *.flv *.gif *.jpeg *.jpg *.lrc *.m4a *.m4v *.mhtml *.mkv *.mov *.mp3 *.mp4 \ + *.mpg *.mpga *.oga *.ogg *.opus *.png *.sbv *.srt *.ssa *.swf *.swp *.tt *.ttml *.url *.vtt *.wav *.webloc *.webm *.webp clean-dist: rm -rf yt-dlp.1.temp.md yt-dlp.1 README.txt MANIFEST build/ dist/ .coverage cover/ yt-dlp.tar.gz completions/ \ yt_dlp/extractor/lazy_extractors.py *.spec CONTRIBUTING.md.tmp yt-dlp yt-dlp.exe yt_dlp.egg-info/ AUTHORS @@ -37,14 +37,15 @@ BINDIR ?= $(PREFIX)/bin MANDIR ?= $(PREFIX)/man SHAREDIR ?= $(PREFIX)/share PYTHON ?= /usr/bin/env python3 - -# $(shell) and $(error) are no-ops in BSD Make and the != variable assignment operator is not supported by GNU Make <4.0 -VERSION_CHECK != echo supported -VERSION_CHECK ?= $(error GNU Make 4+ or BSD Make is required) -CHECK_VERSION := $(VERSION_CHECK) +GNUTAR ?= tar # set markdown input format to "markdown-smart" for pandoc version 2+ and to "markdown" for pandoc prior to version 2 -MARKDOWN != if [ "`pandoc -v | head -n1 | cut -d' ' -f2 | head -c1`" -ge "2" ]; then echo markdown-smart; else echo markdown; fi +PANDOC_VERSION_CMD = pandoc -v 2>/dev/null | head -n1 | cut -d' ' -f2 | head -c1 +PANDOC_VERSION != $(PANDOC_VERSION_CMD) +PANDOC_VERSION ?= $(shell $(PANDOC_VERSION_CMD)) +MARKDOWN_CMD = if [ "$(PANDOC_VERSION)" = "1" -o "$(PANDOC_VERSION)" = "0" ]; then echo markdown; else echo markdown-smart; fi +MARKDOWN != $(MARKDOWN_CMD) +MARKDOWN ?= $(shell $(MARKDOWN_CMD)) install: lazy-extractors yt-dlp yt-dlp.1 completions mkdir -p $(DESTDIR)$(BINDIR) @@ -76,8 +77,12 @@ test: offlinetest: codetest $(PYTHON) -m pytest -k "not download" -CODE_FOLDERS != find yt_dlp -type f -name '__init__.py' -exec dirname {} \+ | grep -v '/__' | sort -CODE_FILES != for f in $(CODE_FOLDERS) ; do echo "$$f" | sed 's,$$,/*.py,' ; done +CODE_FOLDERS_CMD = find yt_dlp -type f -name '__init__.py' | sed 's,/__init__.py,,' | grep -v '/__' | sort +CODE_FOLDERS != $(CODE_FOLDERS_CMD) +CODE_FOLDERS ?= $(shell $(CODE_FOLDERS_CMD)) +CODE_FILES_CMD = for f in $(CODE_FOLDERS) ; do echo "$$f" | sed 's,$$,/*.py,' ; done +CODE_FILES != $(CODE_FILES_CMD) +CODE_FILES ?= $(shell $(CODE_FILES_CMD)) yt-dlp: $(CODE_FILES) mkdir -p zip for d in $(CODE_FOLDERS) ; do \ @@ -130,12 +135,14 @@ completions/fish/yt-dlp.fish: $(CODE_FILES) devscripts/fish-completion.in mkdir -p completions/fish $(PYTHON) devscripts/fish-completion.py -_EXTRACTOR_FILES != find yt_dlp/extractor -name '*.py' -and -not -name 'lazy_extractors.py' +_EXTRACTOR_FILES_CMD = find yt_dlp/extractor -name '*.py' -and -not -name 'lazy_extractors.py' +_EXTRACTOR_FILES != $(_EXTRACTOR_FILES_CMD) +_EXTRACTOR_FILES ?= $(shell $(_EXTRACTOR_FILES_CMD)) yt_dlp/extractor/lazy_extractors.py: devscripts/make_lazy_extractors.py devscripts/lazy_load_template.py $(_EXTRACTOR_FILES) $(PYTHON) devscripts/make_lazy_extractors.py $@ yt-dlp.tar.gz: all - @tar -czf yt-dlp.tar.gz --transform "s|^|yt-dlp/|" --owner 0 --group 0 \ + @$(GNUTAR) -czf yt-dlp.tar.gz --transform "s|^|yt-dlp/|" --owner 0 --group 0 \ --exclude '*.DS_Store' \ --exclude '*.kate-swp' \ --exclude '*.pyc' \ @@ -144,12 +151,20 @@ yt-dlp.tar.gz: all --exclude '__pycache__' \ --exclude '.*_cache' \ --exclude '.git' \ - --exclude '__pyinstaller' \ -- \ README.md supportedsites.md Changelog.md LICENSE \ CONTRIBUTING.md Collaborators.md CONTRIBUTORS AUTHORS \ Makefile yt-dlp.1 README.txt completions .gitignore \ yt-dlp yt_dlp pyproject.toml devscripts test -AUTHORS: - git shortlog -s -n HEAD | cut -f2 | sort > AUTHORS +AUTHORS: Changelog.md + @if [ -d '.git' ] && command -v git > /dev/null ; then \ + echo 'Generating $@ from git commit history' ; \ + git shortlog -s -n HEAD | cut -f2 | sort > $@ ; \ + fi + +CONTRIBUTORS: Changelog.md + @if [ -d '.git' ] && command -v git > /dev/null ; then \ + echo 'Updating $@ from git commit history' ; \ + $(PYTHON) devscripts/make_changelog.py -v -c > /dev/null ; \ + fi diff --git a/README.md b/README.md index 2fcb09917..1e108a29c 100644 --- a/README.md +++ b/README.md @@ -22,12 +22,10 @@ -* [NEW FEATURES](#new-features) - * [Differences in default behavior](#differences-in-default-behavior) * [INSTALLATION](#installation) * [Detailed instructions](https://github.com/yt-dlp/yt-dlp/wiki/Installation) - * [Update](#update) * [Release Files](#release-files) + * [Update](#update) * [Dependencies](#dependencies) * [Compile](#compile) * [USAGE AND OPTIONS](#usage-and-options) @@ -65,7 +63,10 @@ * [Developing Plugins](#developing-plugins) * [EMBEDDING YT-DLP](#embedding-yt-dlp) * [Embedding examples](#embedding-examples) -* [DEPRECATED OPTIONS](#deprecated-options) +* [CHANGES FROM YOUTUBE-DL](#changes-from-youtube-dl) + * [New features](#new-features) + * [Differences in default behavior](#differences-in-default-behavior) + * [Deprecated options](#deprecated-options) * [CONTRIBUTING](CONTRIBUTING.md#contributing-to-yt-dlp) * [Opening an Issue](CONTRIBUTING.md#opening-an-issue) * [Developer Instructions](CONTRIBUTING.md#developer-instructions) @@ -74,103 +75,6 @@ -# NEW FEATURES - -* Forked from [**yt-dlc@f9401f2**](https://github.com/blackjack4494/yt-dlc/commit/f9401f2a91987068139c5f757b12fc711d4c0cee) and merged with [**youtube-dl@be008e6**](https://github.com/ytdl-org/youtube-dl/commit/be008e657d79832642e2158557c899249c9e31cd) ([exceptions](https://github.com/yt-dlp/yt-dlp/issues/21)) - -* **[SponsorBlock Integration](#sponsorblock-options)**: You can mark/remove sponsor sections in YouTube videos by utilizing the [SponsorBlock](https://sponsor.ajay.app) API - -* **[Format Sorting](#sorting-formats)**: The default format sorting options have been changed so that higher resolution and better codecs will be now preferred instead of simply using larger bitrate. Furthermore, you can now specify the sort order using `-S`. This allows for much easier format selection than what is possible by simply using `--format` ([examples](#format-selection-examples)) - -* **Merged with animelover1984/youtube-dl**: You get most of the features and improvements from [animelover1984/youtube-dl](https://github.com/animelover1984/youtube-dl) including `--write-comments`, `BiliBiliSearch`, `BilibiliChannel`, Embedding thumbnail in mp4/ogg/opus, playlist infojson etc. Note that NicoNico livestreams are not available. See [#31](https://github.com/yt-dlp/yt-dlp/pull/31) for details. - -* **YouTube improvements**: - * Supports Clips, Stories (`ytstories:`), Search (including filters)**\***, YouTube Music Search, Channel-specific search, Search prefixes (`ytsearch:`, `ytsearchdate:`)**\***, Mixes, and Feeds (`:ytfav`, `:ytwatchlater`, `:ytsubs`, `:ythistory`, `:ytrec`, `:ytnotif`) - * Fix for [n-sig based throttling](https://github.com/ytdl-org/youtube-dl/issues/29326) **\*** - * Supports some (but not all) age-gated content without cookies - * Download livestreams from the start using `--live-from-start` (*experimental*) - * Channel URLs download all uploads of the channel, including shorts and live - -* **Cookies from browser**: Cookies can be automatically extracted from all major web browsers using `--cookies-from-browser BROWSER[+KEYRING][:PROFILE][::CONTAINER]` - -* **Download time range**: Videos can be downloaded partially based on either timestamps or chapters using `--download-sections` - -* **Split video by chapters**: Videos can be split into multiple files based on chapters using `--split-chapters` - -* **Multi-threaded fragment downloads**: Download multiple fragments of m3u8/mpd videos in parallel. Use `--concurrent-fragments` (`-N`) option to set the number of threads used - -* **Aria2c with HLS/DASH**: You can use `aria2c` as the external downloader for DASH(mpd) and HLS(m3u8) formats - -* **New and fixed extractors**: Many new extractors have been added and a lot of existing ones have been fixed. See the [changelog](Changelog.md) or the [list of supported sites](supportedsites.md) - -* **New MSOs**: Philo, Spectrum, SlingTV, Cablevision, RCN etc. - -* **Subtitle extraction from manifests**: Subtitles can be extracted from streaming media manifests. See [commit/be6202f](https://github.com/yt-dlp/yt-dlp/commit/be6202f12b97858b9d716e608394b51065d0419f) for details - -* **Multiple paths and output templates**: You can give different [output templates](#output-template) and download paths for different types of files. You can also set a temporary path where intermediary files are downloaded to using `--paths` (`-P`) - -* **Portable Configuration**: Configuration files are automatically loaded from the home and root directories. See [CONFIGURATION](#configuration) for details - -* **Output template improvements**: Output templates can now have date-time formatting, numeric offsets, object traversal etc. See [output template](#output-template) for details. Even more advanced operations can also be done with the help of `--parse-metadata` and `--replace-in-metadata` - -* **Other new options**: Many new options have been added such as `--alias`, `--print`, `--concat-playlist`, `--wait-for-video`, `--retry-sleep`, `--sleep-requests`, `--convert-thumbnails`, `--force-download-archive`, `--force-overwrites`, `--break-match-filter` etc - -* **Improvements**: Regex and other operators in `--format`/`--match-filter`, multiple `--postprocessor-args` and `--downloader-args`, faster archive checking, more [format selection options](#format-selection), merge multi-video/audio, multiple `--config-locations`, `--exec` at different stages, etc - -* **Plugins**: Extractors and PostProcessors can be loaded from an external file. See [plugins](#plugins) for details - -* **Self updater**: The releases can be updated using `yt-dlp -U`, and downgraded using `--update-to` if required - -* **Automated builds**: [Nightly/master builds](#update-channels) can be used with `--update-to nightly` and `--update-to master` - -See [changelog](Changelog.md) or [commits](https://github.com/yt-dlp/yt-dlp/commits) for the full list of changes - -Features marked with a **\*** have been back-ported to youtube-dl - -### Differences in default behavior - -Some of yt-dlp's default options are different from that of youtube-dl and youtube-dlc: - -* yt-dlp supports only [Python 3.8+](## "Windows 7"), and *may* remove support for more versions as they [become EOL](https://devguide.python.org/versions/#python-release-cycle); while [youtube-dl still supports Python 2.6+ and 3.2+](https://github.com/ytdl-org/youtube-dl/issues/30568#issue-1118238743) -* The options `--auto-number` (`-A`), `--title` (`-t`) and `--literal` (`-l`), no longer work. See [removed options](#Removed) for details -* `avconv` is not supported as an alternative to `ffmpeg` -* yt-dlp stores config files in slightly different locations to youtube-dl. See [CONFIGURATION](#configuration) for a list of correct locations -* The default [output template](#output-template) is `%(title)s [%(id)s].%(ext)s`. There is no real reason for this change. This was changed before yt-dlp was ever made public and now there are no plans to change it back to `%(title)s-%(id)s.%(ext)s`. Instead, you may use `--compat-options filename` -* The default [format sorting](#sorting-formats) is different from youtube-dl and prefers higher resolution and better codecs rather than higher bitrates. You can use the `--format-sort` option to change this to any order you prefer, or use `--compat-options format-sort` to use youtube-dl's sorting order -* The default format selector is `bv*+ba/b`. This means that if a combined video + audio format that is better than the best video-only format is found, the former will be preferred. Use `-f bv+ba/b` or `--compat-options format-spec` to revert this -* Unlike youtube-dlc, yt-dlp does not allow merging multiple audio/video streams into one file by default (since this conflicts with the use of `-f bv*+ba`). If needed, this feature must be enabled using `--audio-multistreams` and `--video-multistreams`. You can also use `--compat-options multistreams` to enable both -* `--no-abort-on-error` is enabled by default. Use `--abort-on-error` or `--compat-options abort-on-error` to abort on errors instead -* When writing metadata files such as thumbnails, description or infojson, the same information (if available) is also written for playlists. Use `--no-write-playlist-metafiles` or `--compat-options no-playlist-metafiles` to not write these files -* `--add-metadata` attaches the `infojson` to `mkv` files in addition to writing the metadata when used with `--write-info-json`. Use `--no-embed-info-json` or `--compat-options no-attach-info-json` to revert this -* Some metadata are embedded into different fields when using `--add-metadata` as compared to youtube-dl. Most notably, `comment` field contains the `webpage_url` and `synopsis` contains the `description`. You can [use `--parse-metadata`](#modifying-metadata) to modify this to your liking or use `--compat-options embed-metadata` to revert this -* `playlist_index` behaves differently when used with options like `--playlist-reverse` and `--playlist-items`. See [#302](https://github.com/yt-dlp/yt-dlp/issues/302) for details. You can use `--compat-options playlist-index` if you want to keep the earlier behavior -* The output of `-F` is listed in a new format. Use `--compat-options list-formats` to revert this -* Live chats (if available) are considered as subtitles. Use `--sub-langs all,-live_chat` to download all subtitles except live chat. You can also use `--compat-options no-live-chat` to prevent any live chat/danmaku from downloading -* YouTube channel URLs download all uploads of the channel. To download only the videos in a specific tab, pass the tab's URL. If the channel does not show the requested tab, an error will be raised. Also, `/live` URLs raise an error if there are no live videos instead of silently downloading the entire channel. You may use `--compat-options no-youtube-channel-redirect` to revert all these redirections -* Unavailable videos are also listed for YouTube playlists. Use `--compat-options no-youtube-unavailable-videos` to remove this -* The upload dates extracted from YouTube are in UTC [when available](https://github.com/yt-dlp/yt-dlp/blob/89e4d86171c7b7c997c77d4714542e0383bf0db0/yt_dlp/extractor/youtube.py#L3898-L3900). Use `--compat-options no-youtube-prefer-utc-upload-date` to prefer the non-UTC upload date. -* If `ffmpeg` is used as the downloader, the downloading and merging of formats happen in a single step when possible. Use `--compat-options no-direct-merge` to revert this -* Thumbnail embedding in `mp4` is done with mutagen if possible. Use `--compat-options embed-thumbnail-atomicparsley` to force the use of AtomicParsley instead -* Some internal metadata such as filenames are removed by default from the infojson. Use `--no-clean-infojson` or `--compat-options no-clean-infojson` to revert this -* When `--embed-subs` and `--write-subs` are used together, the subtitles are written to disk and also embedded in the media file. You can use just `--embed-subs` to embed the subs and automatically delete the separate file. See [#630 (comment)](https://github.com/yt-dlp/yt-dlp/issues/630#issuecomment-893659460) for more info. `--compat-options no-keep-subs` can be used to revert this -* `certifi` will be used for SSL root certificates, if installed. If you want to use system certificates (e.g. self-signed), use `--compat-options no-certifi` -* yt-dlp's sanitization of invalid characters in filenames is different/smarter than in youtube-dl. You can use `--compat-options filename-sanitization` to revert to youtube-dl's behavior -* yt-dlp tries to parse the external downloader outputs into the standard progress output if possible (Currently implemented: [~~aria2c~~](https://github.com/yt-dlp/yt-dlp/issues/5931)). You can use `--compat-options no-external-downloader-progress` to get the downloader output as-is -* yt-dlp versions between 2021.09.01 and 2023.01.02 applies `--match-filter` to nested playlists. This was an unintentional side-effect of [8f18ac](https://github.com/yt-dlp/yt-dlp/commit/8f18aca8717bb0dd49054555af8d386e5eda3a88) and is fixed in [d7b460](https://github.com/yt-dlp/yt-dlp/commit/d7b460d0e5fc710950582baed2e3fc616ed98a80). Use `--compat-options playlist-match-filter` to revert this -* yt-dlp versions between 2021.11.10 and 2023.06.21 estimated `filesize_approx` values for fragmented/manifest formats. This was added for convenience in [f2fe69](https://github.com/yt-dlp/yt-dlp/commit/f2fe69c7b0d208bdb1f6292b4ae92bc1e1a7444a), but was reverted in [0dff8e](https://github.com/yt-dlp/yt-dlp/commit/0dff8e4d1e6e9fb938f4256ea9af7d81f42fd54f) due to the potentially extreme inaccuracy of the estimated values. Use `--compat-options manifest-filesize-approx` to keep extracting the estimated values -* yt-dlp uses modern http client backends such as `requests`. Use `--compat-options prefer-legacy-http-handler` to prefer the legacy http handler (`urllib`) to be used for standard http requests. -* The sub-module `swfinterp` is removed. - -For ease of use, a few more compat options are available: - -* `--compat-options all`: Use all compat options (Do NOT use) -* `--compat-options youtube-dl`: Same as `--compat-options all,-multistreams,-playlist-match-filter,-manifest-filesize-approx` -* `--compat-options youtube-dlc`: Same as `--compat-options all,-no-live-chat,-no-youtube-channel-redirect,-playlist-match-filter,-manifest-filesize-approx` -* `--compat-options 2021`: Same as `--compat-options 2022,no-certifi,filename-sanitization,no-youtube-prefer-utc-upload-date` -* `--compat-options 2022`: Same as `--compat-options 2023,playlist-match-filter,no-external-downloader-progress` -* `--compat-options 2023`: Same as `--compat-options prefer-legacy-http-handler,manifest-filesize-approx`. Use this to enable all future compat options - - # INSTALLATION @@ -186,41 +90,6 @@ # INSTALLATION You can install yt-dlp using [the binaries](#release-files), [pip](https://pypi.org/project/yt-dlp) or one using a third-party package manager. See [the wiki](https://github.com/yt-dlp/yt-dlp/wiki/Installation) for detailed instructions -## UPDATE -You can use `yt-dlp -U` to update if you are using the [release binaries](#release-files) - -If you [installed with pip](https://github.com/yt-dlp/yt-dlp/wiki/Installation#with-pip), simply re-run the same command that was used to install the program - -For other third-party package managers, see [the wiki](https://github.com/yt-dlp/yt-dlp/wiki/Installation#third-party-package-managers) or refer their documentation - - - -There are currently three release channels for binaries: `stable`, `nightly` and `master`. - -* `stable` is the default channel, and many of its changes have been tested by users of the `nightly` and `master` channels. -* The `nightly` channel has releases scheduled to build every day around midnight UTC, for a snapshot of the project's new patches and changes. This is the **recommended channel for regular users** of yt-dlp. The `nightly` releases are available from [yt-dlp/yt-dlp-nightly-builds](https://github.com/yt-dlp/yt-dlp-nightly-builds/releases) or as development releases of the `yt-dlp` PyPI package (which can be installed with pip's `--pre` flag). -* The `master` channel features releases that are built after each push to the master branch, and these will have the very latest fixes and additions, but may also be more prone to regressions. They are available from [yt-dlp/yt-dlp-master-builds](https://github.com/yt-dlp/yt-dlp-master-builds/releases). - -When using `--update`/`-U`, a release binary will only update to its current channel. -`--update-to CHANNEL` can be used to switch to a different channel when a newer version is available. `--update-to [CHANNEL@]TAG` can also be used to upgrade or downgrade to specific tags from a channel. - -You may also use `--update-to ` (`/`) to update to a channel on a completely different repository. Be careful with what repository you are updating to though, there is no verification done for binaries from different repositories. - -Example usage: -* `yt-dlp --update-to master` switch to the `master` channel and update to its latest release -* `yt-dlp --update-to stable@2023.07.06` upgrade/downgrade to release to `stable` channel tag `2023.07.06` -* `yt-dlp --update-to 2023.10.07` upgrade/downgrade to tag `2023.10.07` if it exists on the current channel -* `yt-dlp --update-to example/yt-dlp@2023.09.24` upgrade/downgrade to the release from the `example/yt-dlp` repository, tag `2023.09.24` - -**Important**: Any user experiencing an issue with the `stable` release should install or update to the `nightly` release before submitting a bug report: -``` -# To update to nightly from stable executable/binary: -yt-dlp --update-to nightly - -# To install nightly with pip: -python -m pip install -U --pre yt-dlp -``` - ## RELEASE FILES @@ -236,7 +105,7 @@ #### Alternatives File|Description :---|:--- -[yt-dlp_x86.exe](https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp_x86.exe)|Windows (Vista SP2+) standalone x86 (32-bit) binary +[yt-dlp_x86.exe](https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp_x86.exe)|Windows (Win7 SP1+) standalone x86 (32-bit) binary [yt-dlp_min.exe](https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp_min.exe)|Windows (Win7 SP1+) standalone x64 binary built with `py2exe`
([Not recommended](#standalone-py2exe-builds-windows)) [yt-dlp_linux](https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp_linux)|Linux standalone x64 binary [yt-dlp_linux.zip](https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp_linux.zip)|Unpackaged Linux executable (no auto-update) @@ -267,6 +136,42 @@ #### Misc **Note**: The manpages, shell completion (autocomplete) files etc. are available inside the [source tarball](https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp.tar.gz) + +## UPDATE +You can use `yt-dlp -U` to update if you are using the [release binaries](#release-files) + +If you [installed with pip](https://github.com/yt-dlp/yt-dlp/wiki/Installation#with-pip), simply re-run the same command that was used to install the program + +For other third-party package managers, see [the wiki](https://github.com/yt-dlp/yt-dlp/wiki/Installation#third-party-package-managers) or refer their documentation + + + +There are currently three release channels for binaries: `stable`, `nightly` and `master`. + +* `stable` is the default channel, and many of its changes have been tested by users of the `nightly` and `master` channels. +* The `nightly` channel has releases scheduled to build every day around midnight UTC, for a snapshot of the project's new patches and changes. This is the **recommended channel for regular users** of yt-dlp. The `nightly` releases are available from [yt-dlp/yt-dlp-nightly-builds](https://github.com/yt-dlp/yt-dlp-nightly-builds/releases) or as development releases of the `yt-dlp` PyPI package (which can be installed with pip's `--pre` flag). +* The `master` channel features releases that are built after each push to the master branch, and these will have the very latest fixes and additions, but may also be more prone to regressions. They are available from [yt-dlp/yt-dlp-master-builds](https://github.com/yt-dlp/yt-dlp-master-builds/releases). + +When using `--update`/`-U`, a release binary will only update to its current channel. +`--update-to CHANNEL` can be used to switch to a different channel when a newer version is available. `--update-to [CHANNEL@]TAG` can also be used to upgrade or downgrade to specific tags from a channel. + +You may also use `--update-to ` (`/`) to update to a channel on a completely different repository. Be careful with what repository you are updating to though, there is no verification done for binaries from different repositories. + +Example usage: +* `yt-dlp --update-to master` switch to the `master` channel and update to its latest release +* `yt-dlp --update-to stable@2023.07.06` upgrade/downgrade to release to `stable` channel tag `2023.07.06` +* `yt-dlp --update-to 2023.10.07` upgrade/downgrade to tag `2023.10.07` if it exists on the current channel +* `yt-dlp --update-to example/yt-dlp@2023.09.24` upgrade/downgrade to the release from the `example/yt-dlp` repository, tag `2023.09.24` + +**Important**: Any user experiencing an issue with the `stable` release should install or update to the `nightly` release before submitting a bug report: +``` +# To update to nightly from stable executable/binary: +yt-dlp --update-to nightly + +# To install nightly with pip: +python3 -m pip install -U --pre yt-dlp[default] +``` + ## DEPENDENCIES Python versions 3.8+ (CPython and PyPy) are supported. Other versions and implementations may or may not work correctly. @@ -283,7 +188,7 @@ ### Strongly recommended There are bugs in ffmpeg that cause various issues when used alongside yt-dlp. Since ffmpeg is such an important dependency, we provide [custom builds](https://github.com/yt-dlp/FFmpeg-Builds#ffmpeg-static-auto-builds) with patches for some of these issues at [yt-dlp/FFmpeg-Builds](https://github.com/yt-dlp/FFmpeg-Builds). See [the readme](https://github.com/yt-dlp/FFmpeg-Builds#patches-applied) for details on the specific issues solved by these builds - **Important**: What you need is ffmpeg *binary*, **NOT** [the python package of the same name](https://pypi.org/project/ffmpeg) + **Important**: What you need is ffmpeg *binary*, **NOT** [the Python package of the same name](https://pypi.org/project/ffmpeg) ### Networking * [**certifi**](https://github.com/certifi/python-certifi)\* - Provides Mozilla's root certificate bundle. Licensed under [MPLv2](https://github.com/certifi/python-certifi/blob/master/LICENSE) @@ -321,7 +226,9 @@ ### Deprecated ## COMPILE ### Standalone PyInstaller Builds -To build the standalone executable, you must have Python and `pyinstaller` (plus any of yt-dlp's [optional dependencies](#dependencies) if needed). The executable will be built for the same architecture (x86/ARM, 32/64 bit) as the Python used. You can run the following commands: +To build the standalone executable, you must have Python and `pyinstaller` (plus any of yt-dlp's [optional dependencies](#dependencies) if needed). The executable will be built for the same CPU architecture as the Python used. + +You can run the following commands: ``` python3 devscripts/install_deps.py --include pyinstaller @@ -331,11 +238,11 @@ ### Standalone PyInstaller Builds On some systems, you may need to use `py` or `python` instead of `python3`. -`bundle/pyinstaller.py` accepts any arguments that can be passed to `pyinstaller`, such as `--onefile/-F` or `--onedir/-D`, which is further [documented here](https://pyinstaller.org/en/stable/usage.html#what-to-generate). +`python -m bundle.pyinstaller` accepts any arguments that can be passed to `pyinstaller`, such as `--onefile/-F` or `--onedir/-D`, which is further [documented here](https://pyinstaller.org/en/stable/usage.html#what-to-generate). **Note**: Pyinstaller versions below 4.4 [do not support](https://github.com/pyinstaller/pyinstaller#requirements-and-tested-platforms) Python installed from the Windows store without using a virtual environment. -**Important**: Running `pyinstaller` directly **without** using `bundle/pyinstaller.py` is **not** officially supported. This may or may not work correctly. +**Important**: Running `pyinstaller` directly **instead of** using `python -m bundle.pyinstaller` is **not** officially supported. This may or may not work correctly. ### Platform-independent Binary (UNIX) You will need the build tools `python` (3.8+), `zip`, `make` (GNU), `pandoc`\* and `pytest`\*. @@ -418,7 +325,7 @@ ## General Options: URLs, but emits an error if this is not possible instead of searching --ignore-config Don't load any more configuration files - except those given by --config-locations. + except those given to --config-locations. For backward compatibility, if this option is found inside the system configuration file, the user configuration is not loaded. @@ -683,7 +590,7 @@ ## Filesystem Options: -o, --output [TYPES:]TEMPLATE Output filename template; see "OUTPUT TEMPLATE" for details --output-na-placeholder TEXT Placeholder for unavailable fields in - "OUTPUT TEMPLATE" (default: "NA") + --output (default: "NA") --restrict-filenames Restrict filenames to only ASCII characters, and avoid "&" and spaces in filenames --no-restrict-filenames Allow Unicode characters, "&" and spaces in @@ -1172,12 +1079,12 @@ # CONFIGURATION You can configure yt-dlp by placing any supported command line option to a configuration file. The configuration is loaded from the following locations: 1. **Main Configuration**: - * The file given by `--config-location` + * The file given to `--config-location` 1. **Portable Configuration**: (Recommended for portable installations) * If using a binary, `yt-dlp.conf` in the same directory as the binary * If running from source-code, `yt-dlp.conf` in the parent directory of `yt_dlp` 1. **Home Configuration**: - * `yt-dlp.conf` in the home path given by `-P` + * `yt-dlp.conf` in the home path given to `-P` * If `-P` is not given, the current directory is searched 1. **User Configuration**: * `${XDG_CONFIG_HOME}/yt-dlp.conf` @@ -1296,7 +1203,7 @@ # OUTPUT TEMPLATE Additionally, you can set different output templates for the various metadata files separately from the general output template by specifying the type of file followed by the template separated by a colon `:`. The different file types supported are `subtitle`, `thumbnail`, `description`, `annotation` (deprecated), `infojson`, `link`, `pl_thumbnail`, `pl_description`, `pl_infojson`, `chapter`, `pl_video`. E.g. `-o "%(title)s.%(ext)s" -o "thumbnail:%(title)s\%(title)s.%(ext)s"` will put the thumbnails in a folder with the same name as the video. If any of the templates is empty, that type of file will not be written. E.g. `--write-thumbnail -o "thumbnail:"` will write thumbnails only for playlists and not for video. - + **Note**: Due to post-processing (i.e. merging etc.), the actual output filename might differ. Use `--print after_move:filepath` to get the name after all post-processing is complete. @@ -1310,8 +1217,11 @@ # OUTPUT TEMPLATE - `description` (string): The description of the video - `display_id` (string): An alternative identifier for the video - `uploader` (string): Full name of the video uploader + - `uploader_id` (string): Nickname or id of the video uploader + - `uploader_url` (string): URL to the video uploader's profile - `license` (string): License name the video is licensed under - - `creator` (string): The creator of the video + - `creators` (list): The creators of the video + - `creator` (string): The creators of the video; comma-separated - `timestamp` (numeric): UNIX timestamp of the moment the video became available - `upload_date` (string): Video upload date in UTC (YYYYMMDD) - `release_timestamp` (numeric): UNIX timestamp of the moment the video was released @@ -1319,9 +1229,9 @@ # OUTPUT TEMPLATE - `release_year` (numeric): Year (YYYY) when the video or album was released - `modified_timestamp` (numeric): UNIX timestamp of the moment the video was last modified - `modified_date` (string): The date (YYYYMMDD) when the video was last modified in UTC - - `uploader_id` (string): Nickname or id of the video uploader - `channel` (string): Full name of the channel the video is uploaded on - `channel_id` (string): Id of the channel + - `channel_url` (string): URL of the channel - `channel_follower_count` (numeric): Number of followers of the channel - `channel_is_verified` (boolean): Whether the channel is verified on the platform - `location` (string): Physical location where the video was filmed @@ -1361,7 +1271,10 @@ # OUTPUT TEMPLATE - `webpage_url_basename` (string): The basename of the webpage URL - `webpage_url_domain` (string): The domain of the webpage URL - `original_url` (string): The URL given by the user (or same as `webpage_url` for playlist entries) - + - `categories` (list): List of categories the video belongs to + - `tags` (list): List of tags assigned to the video + - `cast` (list): List of cast members + All the fields in [Filtering Formats](#filtering-formats) can also be used Available for the video that belongs to some logical chapter or section: @@ -1373,6 +1286,7 @@ # OUTPUT TEMPLATE Available for the video that is an episode of some series or programme: - `series` (string): Title of the series or programme the video episode belongs to + - `series_id` (string): Id of the series or programme the video episode belongs to - `season` (string): Title of the season the video episode belongs to - `season_number` (numeric): Number of the season the video episode belongs to - `season_id` (string): Id of the season the video episode belongs to @@ -1385,11 +1299,16 @@ # OUTPUT TEMPLATE - `track` (string): Title of the track - `track_number` (numeric): Number of the track within an album or a disc - `track_id` (string): Id of the track - - `artist` (string): Artist(s) of the track - - `genre` (string): Genre(s) of the track + - `artists` (list): Artist(s) of the track + - `artist` (string): Artist(s) of the track; comma-separated + - `genres` (list): Genre(s) of the track + - `genre` (string): Genre(s) of the track; comma-separated + - `composers` (list): Composer(s) of the piece + - `composer` (string): Composer(s) of the piece; comma-separated - `album` (string): Title of the album the track belongs to - `album_type` (string): Type of the album - - `album_artist` (string): List of all artists appeared on the album + - `album_artists` (list): All artists appeared on the album + - `album_artist` (string): All artists appeared on the album; comma-separated - `disc_number` (numeric): Number of the disc or other physical medium the track belongs to Available only when using `--download-sections` and for `chapter:` prefix when using `--split-chapters` for videos with internal chapters: @@ -1744,9 +1663,9 @@ # MODIFYING METADATA The metadata obtained by the extractors can be modified by using `--parse-metadata` and `--replace-in-metadata` -`--replace-in-metadata FIELDS REGEX REPLACE` is used to replace text in any metadata field using [python regular expression](https://docs.python.org/3/library/re.html#regular-expression-syntax). [Backreferences](https://docs.python.org/3/library/re.html?highlight=backreferences#re.sub) can be used in the replace string for advanced use. +`--replace-in-metadata FIELDS REGEX REPLACE` is used to replace text in any metadata field using [Python regular expression](https://docs.python.org/3/library/re.html#regular-expression-syntax). [Backreferences](https://docs.python.org/3/library/re.html?highlight=backreferences#re.sub) can be used in the replace string for advanced use. -The general syntax of `--parse-metadata FROM:TO` is to give the name of a field or an [output template](#output-template) to extract data from, and the format to interpret it as, separated by a colon `:`. Either a [python regular expression](https://docs.python.org/3/library/re.html#regular-expression-syntax) with named capture groups, a single field name, or a similar syntax to the [output template](#output-template) (only `%(field)s` formatting is supported) can be used for `TO`. The option can be used multiple times to parse and modify various fields. +The general syntax of `--parse-metadata FROM:TO` is to give the name of a field or an [output template](#output-template) to extract data from, and the format to interpret it as, separated by a colon `:`. Either a [Python regular expression](https://docs.python.org/3/library/re.html#regular-expression-syntax) with named capture groups, a single field name, or a similar syntax to the [output template](#output-template) (only `%(field)s` formatting is supported) can be used for `TO`. The option can be used multiple times to parse and modify various fields. Note that these options preserve their relative order, allowing replacements to be made in parsed fields and viceversa. Also, any field thus created can be used in the [output template](#output-template) and will also affect the media file's metadata added when using `--embed-metadata`. @@ -1767,10 +1686,11 @@ # MODIFYING METADATA `description`, `synopsis` | `description` `purl`, `comment` | `webpage_url` `track` | `track_number` -`artist` | `artist`, `creator`, `uploader` or `uploader_id` -`genre` | `genre` +`artist` | `artist`, `artists`, `creator`, `creators`, `uploader` or `uploader_id` +`composer` | `composer` or `composers` +`genre` | `genre` or `genres` `album` | `album` -`album_artist` | `album_artist` +`album_artist` | `album_artist` or `album_artists` `disc` | `disc_number` `show` | `series` `season_number` | `season_number` @@ -2167,9 +2087,106 @@ #### Use a custom format selector ydl.download(URLS) ``` - -# DEPRECATED OPTIONS +# CHANGES FROM YOUTUBE-DL + +### New features + +* Forked from [**yt-dlc@f9401f2**](https://github.com/blackjack4494/yt-dlc/commit/f9401f2a91987068139c5f757b12fc711d4c0cee) and merged with [**youtube-dl@be008e6**](https://github.com/ytdl-org/youtube-dl/commit/be008e657d79832642e2158557c899249c9e31cd) ([exceptions](https://github.com/yt-dlp/yt-dlp/issues/21)) + +* **[SponsorBlock Integration](#sponsorblock-options)**: You can mark/remove sponsor sections in YouTube videos by utilizing the [SponsorBlock](https://sponsor.ajay.app) API + +* **[Format Sorting](#sorting-formats)**: The default format sorting options have been changed so that higher resolution and better codecs will be now preferred instead of simply using larger bitrate. Furthermore, you can now specify the sort order using `-S`. This allows for much easier format selection than what is possible by simply using `--format` ([examples](#format-selection-examples)) + +* **Merged with animelover1984/youtube-dl**: You get most of the features and improvements from [animelover1984/youtube-dl](https://github.com/animelover1984/youtube-dl) including `--write-comments`, `BiliBiliSearch`, `BilibiliChannel`, Embedding thumbnail in mp4/ogg/opus, playlist infojson etc. Note that NicoNico livestreams are not available. See [#31](https://github.com/yt-dlp/yt-dlp/pull/31) for details. + +* **YouTube improvements**: + * Supports Clips, Stories (`ytstories:`), Search (including filters)**\***, YouTube Music Search, Channel-specific search, Search prefixes (`ytsearch:`, `ytsearchdate:`)**\***, Mixes, and Feeds (`:ytfav`, `:ytwatchlater`, `:ytsubs`, `:ythistory`, `:ytrec`, `:ytnotif`) + * Fix for [n-sig based throttling](https://github.com/ytdl-org/youtube-dl/issues/29326) **\*** + * Supports some (but not all) age-gated content without cookies + * Download livestreams from the start using `--live-from-start` (*experimental*) + * Channel URLs download all uploads of the channel, including shorts and live + +* **Cookies from browser**: Cookies can be automatically extracted from all major web browsers using `--cookies-from-browser BROWSER[+KEYRING][:PROFILE][::CONTAINER]` + +* **Download time range**: Videos can be downloaded partially based on either timestamps or chapters using `--download-sections` + +* **Split video by chapters**: Videos can be split into multiple files based on chapters using `--split-chapters` + +* **Multi-threaded fragment downloads**: Download multiple fragments of m3u8/mpd videos in parallel. Use `--concurrent-fragments` (`-N`) option to set the number of threads used + +* **Aria2c with HLS/DASH**: You can use `aria2c` as the external downloader for DASH(mpd) and HLS(m3u8) formats + +* **New and fixed extractors**: Many new extractors have been added and a lot of existing ones have been fixed. See the [changelog](Changelog.md) or the [list of supported sites](supportedsites.md) + +* **New MSOs**: Philo, Spectrum, SlingTV, Cablevision, RCN etc. + +* **Subtitle extraction from manifests**: Subtitles can be extracted from streaming media manifests. See [commit/be6202f](https://github.com/yt-dlp/yt-dlp/commit/be6202f12b97858b9d716e608394b51065d0419f) for details + +* **Multiple paths and output templates**: You can give different [output templates](#output-template) and download paths for different types of files. You can also set a temporary path where intermediary files are downloaded to using `--paths` (`-P`) + +* **Portable Configuration**: Configuration files are automatically loaded from the home and root directories. See [CONFIGURATION](#configuration) for details + +* **Output template improvements**: Output templates can now have date-time formatting, numeric offsets, object traversal etc. See [output template](#output-template) for details. Even more advanced operations can also be done with the help of `--parse-metadata` and `--replace-in-metadata` + +* **Other new options**: Many new options have been added such as `--alias`, `--print`, `--concat-playlist`, `--wait-for-video`, `--retry-sleep`, `--sleep-requests`, `--convert-thumbnails`, `--force-download-archive`, `--force-overwrites`, `--break-match-filter` etc + +* **Improvements**: Regex and other operators in `--format`/`--match-filter`, multiple `--postprocessor-args` and `--downloader-args`, faster archive checking, more [format selection options](#format-selection), merge multi-video/audio, multiple `--config-locations`, `--exec` at different stages, etc + +* **Plugins**: Extractors and PostProcessors can be loaded from an external file. See [plugins](#plugins) for details + +* **Self updater**: The releases can be updated using `yt-dlp -U`, and downgraded using `--update-to` if required + +* **Automated builds**: [Nightly/master builds](#update-channels) can be used with `--update-to nightly` and `--update-to master` + +See [changelog](Changelog.md) or [commits](https://github.com/yt-dlp/yt-dlp/commits) for the full list of changes + +Features marked with a **\*** have been back-ported to youtube-dl + +### Differences in default behavior + +Some of yt-dlp's default options are different from that of youtube-dl and youtube-dlc: + +* yt-dlp supports only [Python 3.8+](## "Windows 7"), and *may* remove support for more versions as they [become EOL](https://devguide.python.org/versions/#python-release-cycle); while [youtube-dl still supports Python 2.6+ and 3.2+](https://github.com/ytdl-org/youtube-dl/issues/30568#issue-1118238743) +* The options `--auto-number` (`-A`), `--title` (`-t`) and `--literal` (`-l`), no longer work. See [removed options](#Removed) for details +* `avconv` is not supported as an alternative to `ffmpeg` +* yt-dlp stores config files in slightly different locations to youtube-dl. See [CONFIGURATION](#configuration) for a list of correct locations +* The default [output template](#output-template) is `%(title)s [%(id)s].%(ext)s`. There is no real reason for this change. This was changed before yt-dlp was ever made public and now there are no plans to change it back to `%(title)s-%(id)s.%(ext)s`. Instead, you may use `--compat-options filename` +* The default [format sorting](#sorting-formats) is different from youtube-dl and prefers higher resolution and better codecs rather than higher bitrates. You can use the `--format-sort` option to change this to any order you prefer, or use `--compat-options format-sort` to use youtube-dl's sorting order +* The default format selector is `bv*+ba/b`. This means that if a combined video + audio format that is better than the best video-only format is found, the former will be preferred. Use `-f bv+ba/b` or `--compat-options format-spec` to revert this +* Unlike youtube-dlc, yt-dlp does not allow merging multiple audio/video streams into one file by default (since this conflicts with the use of `-f bv*+ba`). If needed, this feature must be enabled using `--audio-multistreams` and `--video-multistreams`. You can also use `--compat-options multistreams` to enable both +* `--no-abort-on-error` is enabled by default. Use `--abort-on-error` or `--compat-options abort-on-error` to abort on errors instead +* When writing metadata files such as thumbnails, description or infojson, the same information (if available) is also written for playlists. Use `--no-write-playlist-metafiles` or `--compat-options no-playlist-metafiles` to not write these files +* `--add-metadata` attaches the `infojson` to `mkv` files in addition to writing the metadata when used with `--write-info-json`. Use `--no-embed-info-json` or `--compat-options no-attach-info-json` to revert this +* Some metadata are embedded into different fields when using `--add-metadata` as compared to youtube-dl. Most notably, `comment` field contains the `webpage_url` and `synopsis` contains the `description`. You can [use `--parse-metadata`](#modifying-metadata) to modify this to your liking or use `--compat-options embed-metadata` to revert this +* `playlist_index` behaves differently when used with options like `--playlist-reverse` and `--playlist-items`. See [#302](https://github.com/yt-dlp/yt-dlp/issues/302) for details. You can use `--compat-options playlist-index` if you want to keep the earlier behavior +* The output of `-F` is listed in a new format. Use `--compat-options list-formats` to revert this +* Live chats (if available) are considered as subtitles. Use `--sub-langs all,-live_chat` to download all subtitles except live chat. You can also use `--compat-options no-live-chat` to prevent any live chat/danmaku from downloading +* YouTube channel URLs download all uploads of the channel. To download only the videos in a specific tab, pass the tab's URL. If the channel does not show the requested tab, an error will be raised. Also, `/live` URLs raise an error if there are no live videos instead of silently downloading the entire channel. You may use `--compat-options no-youtube-channel-redirect` to revert all these redirections +* Unavailable videos are also listed for YouTube playlists. Use `--compat-options no-youtube-unavailable-videos` to remove this +* The upload dates extracted from YouTube are in UTC [when available](https://github.com/yt-dlp/yt-dlp/blob/89e4d86171c7b7c997c77d4714542e0383bf0db0/yt_dlp/extractor/youtube.py#L3898-L3900). Use `--compat-options no-youtube-prefer-utc-upload-date` to prefer the non-UTC upload date. +* If `ffmpeg` is used as the downloader, the downloading and merging of formats happen in a single step when possible. Use `--compat-options no-direct-merge` to revert this +* Thumbnail embedding in `mp4` is done with mutagen if possible. Use `--compat-options embed-thumbnail-atomicparsley` to force the use of AtomicParsley instead +* Some internal metadata such as filenames are removed by default from the infojson. Use `--no-clean-infojson` or `--compat-options no-clean-infojson` to revert this +* When `--embed-subs` and `--write-subs` are used together, the subtitles are written to disk and also embedded in the media file. You can use just `--embed-subs` to embed the subs and automatically delete the separate file. See [#630 (comment)](https://github.com/yt-dlp/yt-dlp/issues/630#issuecomment-893659460) for more info. `--compat-options no-keep-subs` can be used to revert this +* `certifi` will be used for SSL root certificates, if installed. If you want to use system certificates (e.g. self-signed), use `--compat-options no-certifi` +* yt-dlp's sanitization of invalid characters in filenames is different/smarter than in youtube-dl. You can use `--compat-options filename-sanitization` to revert to youtube-dl's behavior +* ~~yt-dlp tries to parse the external downloader outputs into the standard progress output if possible (Currently implemented: [aria2c](https://github.com/yt-dlp/yt-dlp/issues/5931)). You can use `--compat-options no-external-downloader-progress` to get the downloader output as-is~~ +* yt-dlp versions between 2021.09.01 and 2023.01.02 applies `--match-filter` to nested playlists. This was an unintentional side-effect of [8f18ac](https://github.com/yt-dlp/yt-dlp/commit/8f18aca8717bb0dd49054555af8d386e5eda3a88) and is fixed in [d7b460](https://github.com/yt-dlp/yt-dlp/commit/d7b460d0e5fc710950582baed2e3fc616ed98a80). Use `--compat-options playlist-match-filter` to revert this +* yt-dlp versions between 2021.11.10 and 2023.06.21 estimated `filesize_approx` values for fragmented/manifest formats. This was added for convenience in [f2fe69](https://github.com/yt-dlp/yt-dlp/commit/f2fe69c7b0d208bdb1f6292b4ae92bc1e1a7444a), but was reverted in [0dff8e](https://github.com/yt-dlp/yt-dlp/commit/0dff8e4d1e6e9fb938f4256ea9af7d81f42fd54f) due to the potentially extreme inaccuracy of the estimated values. Use `--compat-options manifest-filesize-approx` to keep extracting the estimated values +* yt-dlp uses modern http client backends such as `requests`. Use `--compat-options prefer-legacy-http-handler` to prefer the legacy http handler (`urllib`) to be used for standard http requests. +* The sub-modules `swfinterp`, `casefold` are removed. + +For ease of use, a few more compat options are available: + +* `--compat-options all`: Use all compat options (Do NOT use) +* `--compat-options youtube-dl`: Same as `--compat-options all,-multistreams,-playlist-match-filter,-manifest-filesize-approx` +* `--compat-options youtube-dlc`: Same as `--compat-options all,-no-live-chat,-no-youtube-channel-redirect,-playlist-match-filter,-manifest-filesize-approx` +* `--compat-options 2021`: Same as `--compat-options 2022,no-certifi,filename-sanitization,no-youtube-prefer-utc-upload-date` +* `--compat-options 2022`: Same as `--compat-options 2023,playlist-match-filter,no-external-downloader-progress,prefer-legacy-http-handler,manifest-filesize-approx` +* `--compat-options 2023`: Currently does nothing. Use this to enable all future compat options + +### Deprecated options These are all the deprecated options and the current alternative to achieve the same effect @@ -2205,7 +2222,6 @@ #### Redundant options --no-playlist-reverse Default --no-colors --color no_color - #### Not recommended While these options still work, their use is not recommended since there are other alternatives to achieve the same @@ -2232,7 +2248,6 @@ #### Not recommended --geo-bypass-country CODE --xff CODE --geo-bypass-ip-block IP_BLOCK --xff IP_BLOCK - #### Developer options These options are not intended to be used by the end-user @@ -2242,7 +2257,6 @@ #### Developer options --allow-unplayable-formats List unplayable formats also --no-allow-unplayable-formats Default - #### Old aliases These are aliases that are no longer documented for various reasons @@ -2295,6 +2309,7 @@ #### Removed -A, --auto-number -o "%(autonumber)s-%(id)s.%(ext)s" -t, -l, --title, --literal -o "%(title)s-%(id)s.%(ext)s" + # CONTRIBUTING See [CONTRIBUTING.md](CONTRIBUTING.md#contributing-to-yt-dlp) for instructions on [Opening an Issue](CONTRIBUTING.md#opening-an-issue) and [Contributing code to the project](CONTRIBUTING.md#developer-instructions) diff --git a/bundle/__init__.py b/bundle/__init__.py index 932b79829..e69de29bb 100644 --- a/bundle/__init__.py +++ b/bundle/__init__.py @@ -1 +0,0 @@ -# Empty file diff --git a/bundle/py2exe.py b/bundle/py2exe.py index a7e4113f1..ccb52eaa2 100755 --- a/bundle/py2exe.py +++ b/bundle/py2exe.py @@ -20,7 +20,7 @@ def main(): 'py2exe builds do not support pycryptodomex and needs VC++14 to run. ' 'It is recommended to run "pyinst.py" to build using pyinstaller instead') - return freeze( + freeze( console=[{ 'script': './yt_dlp/__main__.py', 'dest_base': 'yt-dlp', diff --git a/devscripts/SizeOfImage.patch b/devscripts/SizeOfImage.patch deleted file mode 100644 index d5845af46..000000000 Binary files a/devscripts/SizeOfImage.patch and /dev/null differ diff --git a/devscripts/SizeOfImage_w.patch b/devscripts/SizeOfImage_w.patch deleted file mode 100644 index c1a338ff3..000000000 Binary files a/devscripts/SizeOfImage_w.patch and /dev/null differ diff --git a/devscripts/__init__.py b/devscripts/__init__.py index 750dbdca7..e69de29bb 100644 --- a/devscripts/__init__.py +++ b/devscripts/__init__.py @@ -1 +0,0 @@ -# Empty file needed to make devscripts.utils properly importable from outside diff --git a/devscripts/changelog_override.json b/devscripts/changelog_override.json index 8c5286432..2a34ad071 100644 --- a/devscripts/changelog_override.json +++ b/devscripts/changelog_override.json @@ -120,5 +120,11 @@ "when": "15f22b4880b6b3f71f350c64d70976ae65b9f1ca", "short": "[webvtt] Allow spaces before newlines for CueBlock (#7681)", "authors": ["TSRBerry"] + }, + { + "action": "change", + "when": "4ce57d3b873c2887814cbec03d029533e82f7db5", + "short": "[ie] Support multi-period MPD streams (#6654)", + "authors": ["alard", "pukkandan"] } ] diff --git a/devscripts/install_deps.py b/devscripts/install_deps.py index 715e5b044..889d9abeb 100755 --- a/devscripts/install_deps.py +++ b/devscripts/install_deps.py @@ -19,7 +19,7 @@ def parse_args(): parser.add_argument( 'input', nargs='?', metavar='TOMLFILE', default='pyproject.toml', help='Input file (default: %(default)s)') parser.add_argument( - '-e', '--exclude', metavar='REQUIREMENT', action='append', help='Exclude a required dependency') + '-e', '--exclude', metavar='DEPENDENCY', action='append', help='Exclude a dependency') parser.add_argument( '-i', '--include', metavar='GROUP', action='append', help='Include an optional dependency group') parser.add_argument( @@ -33,21 +33,28 @@ def parse_args(): def main(): args = parse_args() - toml_data = parse_toml(read_file(args.input)) - deps = toml_data['project']['dependencies'] - targets = deps.copy() if not args.only_optional else [] + project_table = parse_toml(read_file(args.input))['project'] + optional_groups = project_table['optional-dependencies'] + excludes = args.exclude or [] - for exclude in args.exclude or []: - for dep in deps: - simplified_dep = re.match(r'[\w-]+', dep)[0] - if dep in targets and (exclude.lower() == simplified_dep.lower() or exclude == dep): - targets.remove(dep) + deps = [] + if not args.only_optional: # `-o` should exclude 'dependencies' and the 'default' group + deps.extend(project_table['dependencies']) + if 'default' not in excludes: # `--exclude default` should exclude entire 'default' group + deps.extend(optional_groups['default']) - optional_deps = toml_data['project']['optional-dependencies'] - for include in args.include or []: - group = optional_deps.get(include) - if group: - targets.extend(group) + def name(dependency): + return re.match(r'[\w-]+', dependency)[0].lower() + + target_map = {name(dep): dep for dep in deps} + + for include in filter(None, map(optional_groups.get, args.include or [])): + target_map.update(zip(map(name, include), include)) + + for exclude in map(name, excludes): + target_map.pop(exclude, None) + + targets = list(target_map.values()) if args.print: for target in targets: diff --git a/devscripts/make_changelog.py b/devscripts/make_changelog.py index 123eebc2a..8e199e7d0 100644 --- a/devscripts/make_changelog.py +++ b/devscripts/make_changelog.py @@ -253,7 +253,7 @@ class CommitRange: ''', re.VERBOSE | re.DOTALL) EXTRACTOR_INDICATOR_RE = re.compile(r'(?:Fix|Add)\s+Extractors?', re.IGNORECASE) REVERT_RE = re.compile(r'(?:\[[^\]]+\]\s+)?(?i:Revert)\s+([\da-f]{40})') - FIXES_RE = re.compile(r'(?i:Fix(?:es)?(?:\s+bugs?)?(?:\s+in|\s+for)?|Revert)\s+([\da-f]{40})') + FIXES_RE = re.compile(r'(?i:Fix(?:es)?(?:\s+bugs?)?(?:\s+in|\s+for)?|Revert|Improve)\s+([\da-f]{40})') UPSTREAM_MERGE_RE = re.compile(r'Update to ytdl-commit-([\da-f]+)') def __init__(self, start, end, default_author=None): @@ -445,7 +445,32 @@ def get_new_contributors(contributors_path, commits): return sorted(new_contributors, key=str.casefold) -if __name__ == '__main__': +def create_changelog(args): + logging.basicConfig( + datefmt='%Y-%m-%d %H-%M-%S', format='{asctime} | {levelname:<8} | {message}', + level=logging.WARNING - 10 * args.verbosity, style='{', stream=sys.stderr) + + commits = CommitRange(None, args.commitish, args.default_author) + + if not args.no_override: + if args.override_path.exists(): + overrides = json.loads(read_file(args.override_path)) + commits.apply_overrides(overrides) + else: + logger.warning(f'File {args.override_path.as_posix()} does not exist') + + logger.info(f'Loaded {len(commits)} commits') + + new_contributors = get_new_contributors(args.contributors_path, commits) + if new_contributors: + if args.contributors: + write_file(args.contributors_path, '\n'.join(new_contributors) + '\n', mode='a') + logger.info(f'New contributors: {", ".join(new_contributors)}') + + return Changelog(commits.groups(), args.repo, args.collapsible) + + +def create_parser(): import argparse parser = argparse.ArgumentParser( @@ -477,27 +502,9 @@ def get_new_contributors(contributors_path, commits): parser.add_argument( '--collapsible', action='store_true', help='make changelog collapsible (default: %(default)s)') - args = parser.parse_args() - logging.basicConfig( - datefmt='%Y-%m-%d %H-%M-%S', format='{asctime} | {levelname:<8} | {message}', - level=logging.WARNING - 10 * args.verbosity, style='{', stream=sys.stderr) + return parser - commits = CommitRange(None, args.commitish, args.default_author) - if not args.no_override: - if args.override_path.exists(): - overrides = json.loads(read_file(args.override_path)) - commits.apply_overrides(overrides) - else: - logger.warning(f'File {args.override_path.as_posix()} does not exist') - - logger.info(f'Loaded {len(commits)} commits') - - new_contributors = get_new_contributors(args.contributors_path, commits) - if new_contributors: - if args.contributors: - write_file(args.contributors_path, '\n'.join(new_contributors) + '\n', mode='a') - logger.info(f'New contributors: {", ".join(new_contributors)}') - - print(Changelog(commits.groups(), args.repo, args.collapsible)) +if __name__ == '__main__': + print(create_changelog(create_parser().parse_args())) diff --git a/devscripts/update_changelog.py b/devscripts/update_changelog.py new file mode 100755 index 000000000..36b9a8e86 --- /dev/null +++ b/devscripts/update_changelog.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python3 + +# Allow direct execution +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from pathlib import Path + +from devscripts.make_changelog import create_changelog, create_parser +from devscripts.utils import read_file, read_version, write_file + +# Always run after devscripts/update-version.py, and run before `make doc|pypi-files|tar|all` + +if __name__ == '__main__': + parser = create_parser() + parser.description = 'Update an existing changelog file with an entry for a new release' + parser.add_argument( + '--changelog-path', type=Path, default=Path(__file__).parent.parent / 'Changelog.md', + help='path to the Changelog file') + args = parser.parse_args() + new_entry = create_changelog(args) + + header, sep, changelog = read_file(args.changelog_path).partition('\n### ') + write_file(args.changelog_path, f'{header}{sep}{read_version()}\n{new_entry}\n{sep}{changelog}') diff --git a/pyinst.py b/pyinst.py new file mode 100755 index 000000000..4a8ed2d34 --- /dev/null +++ b/pyinst.py @@ -0,0 +1,17 @@ +#!/usr/bin/env python3 + +# Allow execution from anywhere +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +import warnings + +from bundle.pyinstaller import main + +warnings.warn(DeprecationWarning('`pyinst.py` is deprecated and will be removed in a future version. ' + 'Use `bundle.pyinstaller` instead')) + +if __name__ == '__main__': + main() diff --git a/pyproject.toml b/pyproject.toml index 652ded952..e860e1e58 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,6 +8,7 @@ maintainers = [ {name = "pukkandan", email = "pukkandan.ytdlp@gmail.com"}, {name = "Grub4K", email = "contact@grub4k.xyz"}, {name = "bashonly", email = "bashonly@protonmail.com"}, + {name = "coletdjnz", email = "coletdjnz@protonmail.com"}, ] description = "A youtube-dl fork with additional features and patches" readme = "README.md" @@ -51,6 +52,7 @@ dependencies = [ ] [project.optional-dependencies] +default = [] secretstorage = [ "cffi", "secretstorage", @@ -94,7 +96,6 @@ include = [ "/README.md", # included as readme "/supportedsites.md", ] -exclude = ["/yt_dlp/__pyinstaller"] artifacts = [ "/yt_dlp/extractor/lazy_extractors.py", "/completions", @@ -105,7 +106,6 @@ artifacts = [ [tool.hatch.build.targets.wheel] packages = ["yt_dlp"] -exclude = ["/yt_dlp/__pyinstaller"] artifacts = ["/yt_dlp/extractor/lazy_extractors.py"] [tool.hatch.build.targets.wheel.shared-data] diff --git a/setup.py b/setup.py new file mode 100755 index 000000000..8d1e6d10b --- /dev/null +++ b/setup.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python3 + +# Allow execution from anywhere +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +import warnings + + +if sys.argv[1:2] == ['py2exe']: + warnings.warn(DeprecationWarning('`setup.py py2exe` is deprecated and will be removed in a future version. ' + 'Use `bundle.py2exe` instead')) + + import bundle.py2exe + + bundle.py2exe.main() + +elif 'build_lazy_extractors' in sys.argv: + warnings.warn(DeprecationWarning('`setup.py build_lazy_extractors` is deprecated and will be removed in a future version. ' + 'Use `devscripts.make_lazy_extractors` instead')) + + import subprocess + + os.chdir(sys.path[0]) + print('running build_lazy_extractors') + subprocess.run([sys.executable, 'devscripts/make_lazy_extractors.py']) + +else: + + print( + 'ERROR: Building by calling `setup.py` is deprecated. ' + 'Use a build frontend like `build` instead. ', + 'Refer to https://build.pypa.io for more info', file=sys.stderr) + sys.exit(1) diff --git a/supportedsites.md b/supportedsites.md index 96681c16b..a4b2d5799 100644 --- a/supportedsites.md +++ b/supportedsites.md @@ -5,7 +5,7 @@ # Supported sites - **1tv**: Первый канал - **20min** - **23video** - - **247sports** + - **247sports**: (**Currently broken**) - **24tv.ua** - **3qsdn**: 3Q SDN - **3sat** @@ -17,6 +17,7 @@ # Supported sites - **91porn** - **9c9media** - **9gag**: 9GAG + - **9News** - **9now.com.au** - **abc.net.au** - **abc.net.au:iview** @@ -26,13 +27,14 @@ # Supported sites - **abcotvs**: ABC Owned Television Stations - **abcotvs:clips** - **AbemaTV**: [*abematv*](## "netrc machine") - - **AbemaTVTitle** + - **AbemaTVTitle**: [*abematv*](## "netrc machine") - **AcademicEarth:Course** - **acast** - **acast:channel** - **AcFunBangumi** - **AcFunVideo** - **ADN**: [*animationdigitalnetwork*](## "netrc machine") Animation Digital Network + - **ADNSeason**: [*animationdigitalnetwork*](## "netrc machine") Animation Digital Network - **AdobeConnect** - **adobetv** - **adobetv:channel** @@ -61,6 +63,7 @@ # Supported sites - **altcensored:channel** - **Alura**: [*alura*](## "netrc machine") - **AluraCourse**: [*aluracourse*](## "netrc machine") + - **AmadeusTV** - **Amara** - **AmazonMiniTV** - **amazonminitv:season**: Amazon MiniTV Season, "minitv:season:" prefix @@ -93,11 +96,15 @@ # Supported sites - **ARDMediathek** - **ARDMediathekCollection** - **Arkena** + - **Art19** + - **Art19Show** - **arte.sky.it** - **ArteTV** - **ArteTVCategory** - **ArteTVEmbed** - **ArteTVPlaylist** + - **asobichannel**: ASOBI CHANNEL + - **asobichannel:tag**: ASOBI CHANNEL - **AtresPlayer**: [*atresplayer*](## "netrc machine") - **AtScaleConfEvent** - **ATVAt** @@ -180,13 +187,14 @@ # Supported sites - **BitChute** - **BitChuteChannel** - **BlackboardCollaborate** - - **BleacherReport** - - **BleacherReportCMS** + - **BleacherReport**: (**Currently broken**) + - **BleacherReportCMS**: (**Currently broken**) - **blerp** - **blogger.com** - **Bloomberg** - **BokeCC** - **BongaCams** + - **Boosty** - **BostonGlobe** - **Box** - **BoxCastVideo** @@ -231,8 +239,7 @@ # Supported sites - **cbc.ca** - **cbc.ca:player** - **cbc.ca:​player:playlist** - - **CBS** - - **CBSInteractive** + - **CBS**: (**Currently broken**) - **CBSLocal** - **CBSLocalArticle** - **CBSLocalLive** @@ -240,8 +247,8 @@ # Supported sites - **cbsnews:embed** - **cbsnews:live**: CBS News Livestream - **cbsnews:livevideo**: CBS News Live Videos - - **cbssports** - - **cbssports:embed** + - **cbssports**: (**Currently broken**) + - **cbssports:embed**: (**Currently broken**) - **CCMA** - **CCTV**: 央视网 - **CDA**: [*cdapl*](## "netrc machine") @@ -251,10 +258,10 @@ # Supported sites - **CharlieRose** - **Chaturbate** - **Chilloutzone** - - **Chingari** - - **ChingariUser** + - **chzzk:live** + - **chzzk:video** - **cielotv.it** - - **Cinemax** + - **Cinemax**: (**Currently broken**) - **CinetecaMilano** - **Cineverse** - **CineverseDetails** @@ -263,16 +270,15 @@ # Supported sites - **ciscowebex**: Cisco Webex - **CJSW** - **Clipchamp** - - **cliphunter** - **Clippit** - - **ClipRs** + - **ClipRs**: (**Currently broken**) - **ClipYouEmbed** - - **CloserToTruth** + - **CloserToTruth**: (**Currently broken**) - **CloudflareStream** + - **CloudyCDN** - **Clubic**: (**Currently broken**) - **Clyp** - **cmt.com**: (**Currently broken**) - - **CNBC** - **CNBCVideo** - **CNN** - **CNNArticle** @@ -320,6 +326,7 @@ # Supported sites - **DailyMail** - **dailymotion**: [*dailymotion*](## "netrc machine") - **dailymotion:playlist**: [*dailymotion*](## "netrc machine") + - **dailymotion:search**: [*dailymotion*](## "netrc machine") - **dailymotion:user**: [*dailymotion*](## "netrc machine") - **DailyWire** - **DailyWirePodcast** @@ -340,7 +347,6 @@ # Supported sites - **DeuxM** - **DeuxMNews** - **DHM**: Filmarchiv - Deutsches Historisches Museum (**Currently broken**) - - **Digg** - **DigitalConcertHall**: [*digitalconcerthall*](## "netrc machine") DigitalConcertHall extractor - **DigitallySpeaking** - **Digiteka** @@ -373,14 +379,14 @@ # Supported sites - **drtv:live** - **drtv:season** - **drtv:series** - - **DTube** + - **DTube**: (**Currently broken**) - **duboku**: www.duboku.io - **duboku:list**: www.duboku.io entire series - **Dumpert** - **Duoplay** - **dvtv**: http://video.aktualne.cz/ - - **dw** - - **dw:article** + - **dw**: (**Currently broken**) + - **dw:article**: (**Currently broken**) - **EaglePlatform** - **EbaumsWorld** - **Ebay** @@ -391,6 +397,7 @@ # Supported sites - **EinsUndEinsTVRecordings**: [*1und1tv*](## "netrc machine") - **Einthusan** - **eitb.tv** + - **ElementorEmbed** - **Elonet** - **ElPais**: El País - **ElTreceTV**: El Trece TV (Argentina) @@ -405,6 +412,7 @@ # Supported sites - **Erocast** - **EroProfile**: [*eroprofile*](## "netrc machine") - **EroProfile:album** + - **ERRJupiter** - **ertflix**: ERTFLIX videos - **ertflix:codename**: ERTFLIX videos by codename - **ertwebtv:embed**: ert.gr webtv embedded videos @@ -412,7 +420,7 @@ # Supported sites - **ESPNArticle** - **ESPNCricInfo** - **EttuTv** - - **Europa** + - **Europa**: (**Currently broken**) - **EuroParlWebstream** - **EuropeanTour** - **Eurosport** @@ -423,22 +431,23 @@ # Supported sites - **Expressen** - **EyedoTV** - **facebook**: [*facebook*](## "netrc machine") + - **facebook:ads** - **facebook:reel** - **FacebookPluginsVideo** - - **fancode:live**: [*fancode*](## "netrc machine") - - **fancode:vod**: [*fancode*](## "netrc machine") + - **fancode:live**: [*fancode*](## "netrc machine") (**Currently broken**) + - **fancode:vod**: [*fancode*](## "netrc machine") (**Currently broken**) - **faz.net** - **fc2**: [*fc2*](## "netrc machine") - **fc2:embed** - **fc2:live** - **Fczenit** - **Fifa** - - **Filmmodu** - **filmon** - **filmon:channel** - **Filmweb** - **FiveThirtyEight** - **FiveTV** + - **FlexTV** - **Flickr** - **Floatplane** - **FloatplaneChannel** @@ -477,7 +486,6 @@ # Supported sites - **Gab** - **GabTV** - **Gaia**: [*gaia*](## "netrc machine") - - **GameInformer** - **GameJolt** - **GameJoltCommunity** - **GameJoltGame** @@ -487,18 +495,19 @@ # Supported sites - **GameSpot** - **GameStar** - **Gaskrank** - - **Gazeta** - - **GDCVault**: [*gdcvault*](## "netrc machine") + - **Gazeta**: (**Currently broken**) + - **GDCVault**: [*gdcvault*](## "netrc machine") (**Currently broken**) - **GediDigital** - **gem.cbc.ca**: [*cbcgem*](## "netrc machine") - **gem.cbc.ca:live** - **gem.cbc.ca:playlist** - **Genius** - **GeniusLyrics** + - **GetCourseRu**: [*getcourseru*](## "netrc machine") + - **GetCourseRuPlayer** - **Gettr** - **GettrStreaming** - **GiantBomb** - - **Giga** - **GlattvisionTV**: [*glattvisiontv*](## "netrc machine") - **GlattvisionTVLive**: [*glattvisiontv*](## "netrc machine") - **GlattvisionTVRecordings**: [*glattvisiontv*](## "netrc machine") @@ -516,7 +525,7 @@ # Supported sites - **GMANetworkVideo** - **Go** - **GoDiscovery** - - **GodTube** + - **GodTube**: (**Currently broken**) - **Gofile** - **Golem** - **goodgame:stream** @@ -551,7 +560,7 @@ # Supported sites - **HollywoodReporter** - **HollywoodReporterPlaylist** - **Holodex** - - **HotNewHipHop** + - **HotNewHipHop**: (**Currently broken**) - **hotstar** - **hotstar:playlist** - **hotstar:season** @@ -579,6 +588,7 @@ # Supported sites - **IGNVideo** - **iheartradio** - **iheartradio:podcast** + - **IlPost** - **Iltalehti** - **imdb**: Internet Movie Database trailers - **imdb:list**: Internet Movie Database lists @@ -592,7 +602,7 @@ # Supported sites - **Instagram**: [*instagram*](## "netrc machine") - **instagram:story**: [*instagram*](## "netrc machine") - **instagram:tag**: [*instagram*](## "netrc machine") Instagram hashtag search URLs - - **instagram:user**: [*instagram*](## "netrc machine") Instagram user profile + - **instagram:user**: [*instagram*](## "netrc machine") Instagram user profile (**Currently broken**) - **InstagramIOS**: IOS instagram:// URL - **Internazionale** - **InternetVideoArchive** @@ -622,7 +632,7 @@ # Supported sites - **JablePlaylist** - **Jamendo** - **JamendoAlbum** - - **JeuxVideo** + - **JeuxVideo**: (**Currently broken**) - **JioSaavnAlbum** - **JioSaavnSong** - **Joj** @@ -634,12 +644,10 @@ # Supported sites - **JWPlatform** - **Kakao** - **Kaltura** - - **Kanal2** - - **KankaNews** + - **KankaNews**: (**Currently broken**) - **Karaoketv** - - **KarriereVideos** - - **Katsomo** - - **KelbyOne** + - **Katsomo**: (**Currently broken**) + - **KelbyOne**: (**Currently broken**) - **Ketnet** - **khanacademy** - **khanacademy:unit** @@ -651,18 +659,17 @@ # Supported sites - **KinoPoisk** - **Kommunetv** - **KompasVideo** - - **KonserthusetPlay** - - **Koo** - - **KrasView**: Красвью + - **Koo**: (**Currently broken**) + - **KrasView**: Красвью (**Currently broken**) - **KTH** - **Ku6** - - **KUSI** - - **kuwo:album**: 酷我音乐 - 专辑 - - **kuwo:category**: 酷我音乐 - 分类 - - **kuwo:chart**: 酷我音乐 - 排行榜 - - **kuwo:mv**: 酷我音乐 - MV - - **kuwo:singer**: 酷我音乐 - 歌手 - - **kuwo:song**: 酷我音乐 + - **KukuluLive** + - **kuwo:album**: 酷我音乐 - 专辑 (**Currently broken**) + - **kuwo:category**: 酷我音乐 - 分类 (**Currently broken**) + - **kuwo:chart**: 酷我音乐 - 排行榜 (**Currently broken**) + - **kuwo:mv**: 酷我音乐 - MV (**Currently broken**) + - **kuwo:singer**: 酷我音乐 - 歌手 (**Currently broken**) + - **kuwo:song**: 酷我音乐 (**Currently broken**) - **la7.it** - **la7.it:​pod:episode** - **la7.it:podcast** @@ -677,7 +684,7 @@ # Supported sites - **Lcp** - **LcpPlay** - **Le**: 乐视网 - - **Lecture2Go** + - **Lecture2Go**: (**Currently broken**) - **Lecturio**: [*lecturio*](## "netrc machine") - **LecturioCourse**: [*lecturio*](## "netrc machine") - **LecturioDeCourse**: [*lecturio*](## "netrc machine") @@ -685,7 +692,7 @@ # Supported sites - **LeFigaroVideoSection** - **LEGO** - **Lemonde** - - **Lenta** + - **Lenta**: (**Currently broken**) - **LePlaylist** - **LetvCloud**: 乐视云 - **Libsyn** @@ -709,31 +716,32 @@ # Supported sites - **Lnk** - **LnkGo** - **loc**: Library of Congress - - **LocalNews8** - **LoveHomePorn** - **LRTStream** - **LRTVOD** + - **LSMLREmbed** + - **LSMLTVEmbed** + - **LSMReplay** - **Lumni** - **lynda**: [*lynda*](## "netrc machine") lynda.com videos - **lynda:course**: [*lynda*](## "netrc machine") lynda.com online courses - **maariv.co.il** - **MagellanTV** - - **MagentaMusik360** + - **MagentaMusik** - **mailru**: Видео@Mail.Ru - **mailru:music**: Музыка@Mail.Ru - **mailru:​music:search**: Музыка@Mail.Ru - **MainStreaming**: MainStreaming Player - - **MallTV** - **mangomolo:live** - **mangomolo:video** - **MangoTV**: 芒果TV - **ManotoTV**: Manoto TV (Episode) - **ManotoTVLive**: Manoto TV (Live) - **ManotoTVShow**: Manoto TV (Show) - - **ManyVids** + - **ManyVids**: (**Currently broken**) - **MaoriTV** - - **Markiza** - - **MarkizaPage** + - **Markiza**: (**Currently broken**) + - **MarkizaPage**: (**Currently broken**) - **massengeschmack.tv** - **Masters** - **MatchTV** @@ -760,7 +768,6 @@ # Supported sites - **MelonVOD** - **Metacritic** - **mewatch** - - **MiaoPai** - **MicrosoftEmbed** - **microsoftstream**: Microsoft Stream - **mildom**: Record ongoing live by specific user in Mildom @@ -770,7 +777,6 @@ # Supported sites - **minds** - **minds:channel** - **minds:group** - - **MinistryGrid** - **Minoto** - **mirrativ** - **mirrativ:user** @@ -793,11 +799,11 @@ # Supported sites - **Mojvideo** - **Monstercat** - **MonsterSirenHypergryphMusic** - - **Morningstar**: morningstar.com - **Motherless** - **MotherlessGallery** - **MotherlessGroup** - - **Motorsport**: motorsport.com + - **MotherlessUploader** + - **Motorsport**: motorsport.com (**Currently broken**) - **MotorTrend** - **MotorTrendOnDemand** - **MovieFap** @@ -808,17 +814,17 @@ # Supported sites - **MSN**: (**Currently broken**) - **mtg**: MTG services - **mtv** - - **mtv.de** + - **mtv.de**: (**Currently broken**) - **mtv.it** - **mtv.it:programma** - **mtv:video** - **mtvjapan** - **mtvservices:embedded** - - **MTVUutisetArticle** - - **MuenchenTV**: münchen.tv + - **MTVUutisetArticle**: (**Currently broken**) + - **MuenchenTV**: münchen.tv (**Currently broken**) - **MujRozhlas** - - **Murrtube** - - **MurrtubeUser**: Murrtube user profile + - **Murrtube**: (**Currently broken**) + - **MurrtubeUser**: Murrtube user profile (**Currently broken**) - **MuseAI** - **MuseScore** - **MusicdexAlbum** @@ -827,6 +833,9 @@ # Supported sites - **MusicdexSong** - **mva**: Microsoft Virtual Academy videos - **mva:course**: Microsoft Virtual Academy courses + - **Mx3** + - **Mx3Neo** + - **Mx3Volksmusik** - **Mxplayer** - **MxplayerShow** - **MySpace** @@ -862,11 +871,11 @@ # Supported sites - **ndr**: NDR.de - Norddeutscher Rundfunk - **ndr:embed** - **ndr:​embed:base** - - **NDTV** - - **Nebula**: [*watchnebula*](## "netrc machine") + - **NDTV**: (**Currently broken**) - **nebula:channel**: [*watchnebula*](## "netrc machine") - - **nebula:class**: [*watchnebula*](## "netrc machine") + - **nebula:media**: [*watchnebula*](## "netrc machine") - **nebula:subscriptions**: [*watchnebula*](## "netrc machine") + - **nebula:video**: [*watchnebula*](## "netrc machine") - **NekoHacker** - **NerdCubedFeed** - **netease:album**: 网易云音乐 - 专辑 @@ -882,18 +891,19 @@ # Supported sites - **Netverse** - **NetversePlaylist** - **NetverseSearch**: "netsearch:" prefix - - **Netzkino** - - **Newgrounds** + - **Netzkino**: (**Currently broken**) + - **Newgrounds**: [*newgrounds*](## "netrc machine") - **Newgrounds:playlist** - **Newgrounds:user** - **NewsPicks** - **Newsy** - **NextMedia**: 蘋果日報 - **NextMediaActionNews**: 蘋果日報 - 動新聞 - - **NextTV**: 壹電視 + - **NextTV**: 壹電視 (**Currently broken**) - **Nexx** - **NexxEmbed** - - **NFB** + - **nfb**: nfb.ca and onf.ca films and episodes + - **nfb:series**: nfb.ca and onf.ca series - **NFHSNetwork** - **nfl.com** - **nfl.com:article** @@ -925,11 +935,12 @@ # Supported sites - **nicovideo:search**: Nico video search; "nicosearch:" prefix - **nicovideo:​search:date**: Nico video search, newest first; "nicosearchdate:" prefix - **nicovideo:search_url**: Nico video search URLs + - **NinaProtocol** - **Nintendo** - **Nitter** - **njoy**: N-JOY - **njoy:embed** - - **NobelPrize** + - **NobelPrize**: (**Currently broken**) - **NoicePodcast** - **NonkTube** - **NoodleMagazine** @@ -941,7 +952,7 @@ # Supported sites - **nowness** - **nowness:playlist** - **nowness:series** - - **Noz** + - **Noz**: (**Currently broken**) - **npo**: npo.nl, ntr.nl, omroepwnl.nl, zapp.nl and npo3.nl - **npo.nl:live** - **npo.nl:radio** @@ -960,15 +971,18 @@ # Supported sites - **NRLTV**: (**Currently broken**) - **ntv.ru** - **NubilesPorn**: [*nubiles-porn*](## "netrc machine") + - **nuum:live** + - **nuum:media** + - **nuum:tab** - **Nuvid** - **NYTimes** - **NYTimesArticle** - - **NYTimesCooking** + - **NYTimesCookingGuide** + - **NYTimesCookingRecipe** - **nzherald** - **NZOnScreen** - **NZZ** - **ocw.mit.edu** - - **OdaTV** - **Odnoklassniki** - **OfTV** - **OfTVPlaylist** @@ -993,6 +1007,7 @@ # Supported sites - **OraTV** - **orf:​fm4:story**: fm4.orf.at stories - **orf:iptv**: iptv.ORF.at + - **orf:on** - **orf:podcast** - **orf:radio** - **orf:tvthek**: ORF TVthek @@ -1015,7 +1030,7 @@ # Supported sites - **ParamountPressExpress** - **Parler**: Posts on parler.com - **parliamentlive.tv**: UK parliament videos - - **Parlview** + - **Parlview**: (**Currently broken**) - **Patreon** - **PatreonCampaign** - **pbs**: Public Broadcasting Service (PBS) and member stations: PBS: Public Broadcasting Service, APT - Alabama Public Television (WBIQ), GPB/Georgia Public Broadcasting (WGTV), Mississippi Public Broadcasting (WMPN), Nashville Public Television (WNPT), WFSU-TV (WFSU), WSRE (WSRE), WTCI (WTCI), WPBA/Channel 30 (WPBA), Alaska Public Media (KAKM), Arizona PBS (KAET), KNME-TV/Channel 5 (KNME), Vegas PBS (KLVX), AETN/ARKANSAS ETV NETWORK (KETS), KET (WKLE), WKNO/Channel 10 (WKNO), LPB/LOUISIANA PUBLIC BROADCASTING (WLPB), OETA (KETA), Ozarks Public Television (KOZK), WSIU Public Broadcasting (WSIU), KEET TV (KEET), KIXE/Channel 9 (KIXE), KPBS San Diego (KPBS), KQED (KQED), KVIE Public Television (KVIE), PBS SoCal/KOCE (KOCE), ValleyPBS (KVPT), CONNECTICUT PUBLIC TELEVISION (WEDH), KNPB Channel 5 (KNPB), SOPTV (KSYS), Rocky Mountain PBS (KRMA), KENW-TV3 (KENW), KUED Channel 7 (KUED), Wyoming PBS (KCWC), Colorado Public Television / KBDI 12 (KBDI), KBYU-TV (KBYU), Thirteen/WNET New York (WNET), WGBH/Channel 2 (WGBH), WGBY (WGBY), NJTV Public Media NJ (WNJT), WLIW21 (WLIW), mpt/Maryland Public Television (WMPB), WETA Television and Radio (WETA), WHYY (WHYY), PBS 39 (WLVT), WVPT - Your Source for PBS and More! (WVPT), Howard University Television (WHUT), WEDU PBS (WEDU), WGCU Public Media (WGCU), WPBT2 (WPBT), WUCF TV (WUCF), WUFT/Channel 5 (WUFT), WXEL/Channel 42 (WXEL), WLRN/Channel 17 (WLRN), WUSF Public Broadcasting (WUSF), ETV (WRLK), UNC-TV (WUNC), PBS Hawaii - Oceanic Cable Channel 10 (KHET), Idaho Public Television (KAID), KSPS (KSPS), OPB (KOPB), KWSU/Channel 10 & KTNW/Channel 31 (KWSU), WILL-TV (WILL), Network Knowledge - WSEC/Springfield (WSEC), WTTW11 (WTTW), Iowa Public Television/IPTV (KDIN), Nine Network (KETC), PBS39 Fort Wayne (WFWA), WFYI Indianapolis (WFYI), Milwaukee Public Television (WMVS), WNIN (WNIN), WNIT Public Television (WNIT), WPT (WPNE), WVUT/Channel 22 (WVUT), WEIU/Channel 51 (WEIU), WQPT-TV (WQPT), WYCC PBS Chicago (WYCC), WIPB-TV (WIPB), WTIU (WTIU), CET (WCET), ThinkTVNetwork (WPTD), WBGU-TV (WBGU), WGVU TV (WGVU), NET1 (KUON), Pioneer Public Television (KWCM), SDPB Television (KUSD), TPT (KTCA), KSMQ (KSMQ), KPTS/Channel 8 (KPTS), KTWU/Channel 11 (KTWU), East Tennessee PBS (WSJK), WCTE-TV (WCTE), WLJT, Channel 11 (WLJT), WOSU TV (WOSU), WOUB/WOUC (WOUB), WVPB (WVPB), WKYU-PBS (WKYU), KERA 13 (KERA), MPBN (WCBB), Mountain Lake PBS (WCFE), NHPTV (WENH), Vermont PBS (WETK), witf (WITF), WQED Multimedia (WQED), WMHT Educational Telecommunications (WMHT), Q-TV (WDCQ), WTVS Detroit Public TV (WTVS), CMU Public Television (WCMU), WKAR-TV (WKAR), WNMU-TV Public TV 13 (WNMU), WDSE - WRPT (WDSE), WGTE TV (WGTE), Lakeland Public Television (KAWE), KMOS-TV - Channels 6.1, 6.2 and 6.3 (KMOS), MontanaPBS (KUSM), KRWG/Channel 22 (KRWG), KACV (KACV), KCOS/Channel 13 (KCOS), WCNY/Channel 24 (WCNY), WNED (WNED), WPBS (WPBS), WSKG Public TV (WSKG), WXXI (WXXI), WPSU (WPSU), WVIA Public Media Studios (WVIA), WTVI (WTVI), Western Reserve PBS (WNEO), WVIZ/PBS ideastream (WVIZ), KCTS 9 (KCTS), Basin PBS (KPBT), KUHT / Channel 8 (KUHT), KLRN (KLRN), KLRU (KLRU), WTJX Channel 12 (WTJX), WCVE PBS (WCVE), KBTC Public Television (KBTC) @@ -1049,19 +1064,19 @@ # Supported sites - **Platzi**: [*platzi*](## "netrc machine") - **PlatziCourse**: [*platzi*](## "netrc machine") - **player.sky.it** + - **playeur** - **PlayPlusTV**: [*playplustv*](## "netrc machine") - - **PlayStuff** - - **PlaySuisse** + - **PlaySuisse**: [*playsuisse*](## "netrc machine") - **Playtvak**: Playtvak.cz, iDNES.cz and Lidovky.cz - **PlayVids** - **Playwire** - **pluralsight**: [*pluralsight*](## "netrc machine") - **pluralsight:course** - - **PlutoTV** + - **PlutoTV**: (**Currently broken**) - **PodbayFM** - **PodbayFMChannel** - **Podchaser** - - **podomatic** + - **podomatic**: (**Currently broken**) - **Pokemon** - **PokemonWatch** - **PokerGo**: [*pokergo*](## "netrc machine") @@ -1085,15 +1100,16 @@ # Supported sites - **PornHubUser**: [*pornhub*](## "netrc machine") - **PornHubUserVideosUpload**: [*pornhub*](## "netrc machine") - **Pornotube** - - **PornoVoisines** - - **PornoXO** + - **PornoVoisines**: (**Currently broken**) + - **PornoXO**: (**Currently broken**) - **PornTop** - **PornTube** - **Pr0gramm** - **PrankCast** + - **PrankCastPost** - **PremiershipRugby** - **PressTV** - - **ProjectVeritas** + - **ProjectVeritas**: (**Currently broken**) - **prosiebensat1**: ProSiebenSat.1 Digital - **PRXAccount** - **PRXSeries** @@ -1115,11 +1131,11 @@ # Supported sites - **QuantumTVLive**: [*quantumtv*](## "netrc machine") - **QuantumTVRecordings**: [*quantumtv*](## "netrc machine") - **Qub** - - **R7** - - **R7Article** + - **R7**: (**Currently broken**) + - **R7Article**: (**Currently broken**) - **Radiko** - **RadikoRadio** - - **radio.de** + - **radio.de**: (**Currently broken**) - **radiocanada** - **radiocanada:audiovideo** - **RadioComercial** @@ -1129,7 +1145,7 @@ # Supported sites - **RadioFrancePodcast** - **RadioFranceProfile** - **RadioFranceProgramSchedule** - - **RadioJavan** + - **RadioJavan**: (**Currently broken**) - **radiokapital** - **radiokapital:show** - **RadioZetPodcast** @@ -1151,33 +1167,34 @@ # Supported sites - **RbgTum** - **RbgTumCourse** - **RbgTumNewCourse** - - **RBMARadio** - **RCS** - **RCSEmbeds** - **RCSVarious** - **RCTIPlus** - **RCTIPlusSeries** - **RCTIPlusTV** - - **RDS**: RDS.ca + - **RDS**: RDS.ca (**Currently broken**) - **RedBull** - **RedBullEmbed** - **RedBullTV** - **RedBullTVRrnContent** + - **redcdnlivx** - **Reddit**: [*reddit*](## "netrc machine") - **RedGifs** - **RedGifsSearch**: Redgifs search - **RedGifsUser**: Redgifs user - **RedTube** - - **RegioTV** - - **RENTV** - - **RENTVArticle** - - **Restudy** - - **Reuters** + - **RENTV**: (**Currently broken**) + - **RENTVArticle**: (**Currently broken**) + - **Restudy**: (**Currently broken**) + - **Reuters**: (**Currently broken**) - **ReverbNation** - **RheinMainTV** + - **RideHome** - **RinseFM** + - **RinseFMArtistPlaylist** - **RMCDecouverte** - - **RockstarGames** + - **RockstarGames**: (**Currently broken**) - **Rokfin**: [*rokfin*](## "netrc machine") - **rokfin:channel**: Rokfin Channels - **rokfin:search**: Rokfin Search; "rkfnsearch:" prefix @@ -1187,7 +1204,7 @@ # Supported sites - **RottenTomatoes** - **Rozhlas** - **RozhlasVltava** - - **RTBF**: [*rtbf*](## "netrc machine") + - **RTBF**: [*rtbf*](## "netrc machine") (**Currently broken**) - **RTDocumentry** - **RTDocumentryPlaylist** - **rte**: Raidió Teilifís Éireann TV @@ -1201,7 +1218,7 @@ # Supported sites - **RTNews** - **RTP** - **RTRFM** - - **RTS**: RTS.ch + - **RTS**: RTS.ch (**Currently broken**) - **RTVCKaltura** - **RTVCPlay** - **RTVCPlayEmbed** @@ -1234,7 +1251,7 @@ # Supported sites - **safari**: [*safari*](## "netrc machine") safaribooksonline.com online video - **safari:api**: [*safari*](## "netrc machine") - **safari:course**: [*safari*](## "netrc machine") safaribooksonline.com online courses - - **Saitosan** + - **Saitosan**: (**Currently broken**) - **SAKTV**: [*saktv*](## "netrc machine") - **SAKTVLive**: [*saktv*](## "netrc machine") - **SAKTVRecordings**: [*saktv*](## "netrc machine") @@ -1244,7 +1261,6 @@ # Supported sites - **SampleFocus** - **Sangiin**: 参議院インターネット審議中継 (archive) - **Sapo**: SAPO Vídeos - - **savefrom.net** - **SBS**: sbs.com.au - **sbs.co.kr** - **sbs.co.kr:allvod_program** @@ -1261,13 +1277,13 @@ # Supported sites - **Scrolller** - **SCTE**: [*scte*](## "netrc machine") (**Currently broken**) - **SCTECourse**: [*scte*](## "netrc machine") (**Currently broken**) - - **Seeker** - - **SenalColombiaLive** + - **sejm** + - **SenalColombiaLive**: (**Currently broken**) - **SenateGov** - **SenateISVP** - - **SendtoNews** + - **SendtoNews**: (**Currently broken**) - **Servus** - - **Sexu** + - **Sexu**: (**Currently broken**) - **SeznamZpravy** - **SeznamZpravyArticle** - **Shahid**: [*shahid*](## "netrc machine") @@ -1289,9 +1305,9 @@ # Supported sites - **sky:​news:story** - **sky:sports** - **sky:​sports:news** - - **SkylineWebcams** - - **skynewsarabia:article** - - **skynewsarabia:video** + - **SkylineWebcams**: (**Currently broken**) + - **skynewsarabia:article**: (**Currently broken**) + - **skynewsarabia:video**: (**Currently broken**) - **SkyNewsAU** - **Slideshare** - **SlidesLive** @@ -1342,7 +1358,7 @@ # Supported sites - **StacommuVOD**: [*stacommu*](## "netrc machine") - **StagePlusVODConcert**: [*stageplus*](## "netrc machine") - **stanfordoc**: Stanford Open ClassRoom - - **StarTrek** + - **StarTrek**: (**Currently broken**) - **startv** - **Steam** - **SteamCommunityBroadcast** @@ -1353,7 +1369,6 @@ # Supported sites - **StoryFireUser** - **Streamable** - **StreamCZ** - - **StreamFF** - **StreetVoice** - **StretchInternet** - **Stripchat** @@ -1367,22 +1382,21 @@ # Supported sites - **SVTPlay**: SVT Play and Öppet arkiv - **SVTSeries** - **SwearnetEpisode** - - **Syfy** + - **Syfy**: (**Currently broken**) - **SYVDK** - **SztvHu** - - **t-online.de** - - **Tagesschau** - - **Tass** + - **t-online.de**: (**Currently broken**) + - **Tagesschau**: (**Currently broken**) + - **Tass**: (**Currently broken**) - **TBS** - **TBSJPEpisode** - **TBSJPPlaylist** - **TBSJPProgram** - - **TDSLifeway** - - **Teachable**: [*teachable*](## "netrc machine") + - **Teachable**: [*teachable*](## "netrc machine") (**Currently broken**) - **TeachableCourse**: [*teachable*](## "netrc machine") - - **teachertube**: teachertube.com videos - - **teachertube:​user:collection**: teachertube.com user and collection videos - - **TeachingChannel** + - **teachertube**: teachertube.com videos (**Currently broken**) + - **teachertube:​user:collection**: teachertube.com user and collection videos (**Currently broken**) + - **TeachingChannel**: (**Currently broken**) - **Teamcoco** - **TeamTreeHouse**: [*teamtreehouse*](## "netrc machine") - **techtv.mit.edu** @@ -1391,20 +1405,20 @@ # Supported sites - **TedSeries** - **TedTalk** - **Tele13** - - **Tele5** + - **Tele5**: (**Currently broken**) - **TeleBruxelles** - **TelecaribePlay** - **Telecinco**: telecinco.es, cuatro.com and mediaset.es - **Telegraaf** - **telegram:embed** - - **TeleMB** - - **Telemundo** + - **TeleMB**: (**Currently broken**) + - **Telemundo**: (**Currently broken**) - **TeleQuebec** - **TeleQuebecEmission** - **TeleQuebecLive** - **TeleQuebecSquat** - **TeleQuebecVideo** - - **TeleTask** + - **TeleTask**: (**Currently broken**) - **Telewebion** - **Tempo** - **TennisTV**: [*tennistv*](## "netrc machine") @@ -1458,6 +1472,7 @@ # Supported sites - **TrovoChannelVod**: All VODs of a trovo.live channel; "trovovod:" prefix - **TrovoVod** - **TrtCocukVideo** + - **TrtWorld** - **TrueID** - **TruNews** - **Truth** @@ -1471,7 +1486,6 @@ # Supported sites - **TuneInPodcast** - **TuneInPodcastEpisode** - **TuneInStation** - - **Turbo** - **tv.dfb.de** - **TV2** - **TV2Article** @@ -1493,8 +1507,8 @@ # Supported sites - **tvigle**: Интернет-телевидение Tvigle.ru - **TVIPlayer** - **tvland.com** - - **TVN24** - - **TVNoe** + - **TVN24**: (**Currently broken**) + - **TVNoe**: (**Currently broken**) - **tvopengr:embed**: tvopen.gr embedded videos - **tvopengr:watch**: tvopen.gr (and ethnos.gr) videos - **tvp**: Telewizja Polska @@ -1527,15 +1541,15 @@ # Supported sites - **UDNEmbed**: 聯合影音 - **UFCArabia**: [*ufcarabia*](## "netrc machine") - **UFCTV**: [*ufctv*](## "netrc machine") - - **ukcolumn** + - **ukcolumn**: (**Currently broken**) - **UKTVPlay** - - **umg:de**: Universal Music Deutschland + - **umg:de**: Universal Music Deutschland (**Currently broken**) - **Unistra** - - **Unity** + - **Unity**: (**Currently broken**) - **uol.com.br** - **uplynk** - **uplynk:preplay** - - **Urort**: NRK P3 Urørt + - **Urort**: NRK P3 Urørt (**Currently broken**) - **URPlay** - **USANetwork** - **USAToday** @@ -1543,13 +1557,12 @@ # Supported sites - **ustream:channel** - **ustudio** - **ustudio:embed** - - **Utreon** - - **Varzesh3** + - **Varzesh3**: (**Currently broken**) - **Vbox7** - **Veo** - **Veoh** - **veoh:user** - - **Vesti**: Вести.Ru + - **Vesti**: Вести.Ru (**Currently broken**) - **Vevo** - **VevoPlaylist** - **VGTV**: VGTV, BTTV, FTV, Aftenposten and Aftonbladet @@ -1565,7 +1578,7 @@ # Supported sites - **video.sky.it** - **video.sky.it:live** - **VideoDetective** - - **videofy.me** + - **videofy.me**: (**Currently broken**) - **VideoKen** - **VideoKenCategory** - **VideoKenPlayer** @@ -1601,7 +1614,8 @@ # Supported sites - **ViMP:Playlist** - **Vine** - **vine:user** - - **Viqeo** + - **Viously** + - **Viqeo**: (**Currently broken**) - **Viu** - **viu:ott**: [*viu*](## "netrc machine") - **viu:playlist** @@ -1615,8 +1629,8 @@ # Supported sites - **Vocaroo** - **VODPl** - **VODPlatform** - - **voicy** - - **voicy:channel** + - **voicy**: (**Currently broken**) + - **voicy:channel**: (**Currently broken**) - **VolejTV** - **Voot**: [*voot*](## "netrc machine") (**Currently broken**) - **VootSeries**: [*voot*](## "netrc machine") (**Currently broken**) @@ -1627,7 +1641,7 @@ # Supported sites - **vqq:video** - **VRT**: VRT NWS, Flanders News, Flandern Info and Sporza - **VrtNU**: [*vrtnu*](## "netrc machine") VRT MAX - - **VTM** + - **VTM**: (**Currently broken**) - **VTXTV**: [*vtxtv*](## "netrc machine") - **VTXTVLive**: [*vtxtv*](## "netrc machine") - **VTXTVRecordings**: [*vtxtv*](## "netrc machine") @@ -1638,9 +1652,6 @@ # Supported sites - **WalyTV**: [*walytv*](## "netrc machine") - **WalyTVLive**: [*walytv*](## "netrc machine") - **WalyTVRecordings**: [*walytv*](## "netrc machine") - - **wasdtv:clip** - - **wasdtv:record** - - **wasdtv:stream** - **washingtonpost** - **washingtonpost:article** - **wat.tv** @@ -1658,7 +1669,7 @@ # Supported sites - **Weibo** - **WeiboUser** - **WeiboVideo** - - **WeiqiTV**: WQTV + - **WeiqiTV**: WQTV (**Currently broken**) - **wetv:episode** - **WeTvSeries** - **Weverse**: [*weverse*](## "netrc machine") @@ -1703,8 +1714,8 @@ # Supported sites - **XHamsterUser** - **ximalaya**: 喜马拉雅FM - **ximalaya:album**: 喜马拉雅FM 专辑 - - **xinpianchang**: xinpianchang.com - - **XMinus** + - **xinpianchang**: xinpianchang.com (**Currently broken**) + - **XMinus**: (**Currently broken**) - **XNXX** - **Xstream** - **XVideos** @@ -1720,8 +1731,8 @@ # Supported sites - **yandexmusic:track**: Яндекс.Музыка - Трек - **YandexVideo** - **YandexVideoPreview** - - **YapFiles** - - **Yappy** + - **YapFiles**: (**Currently broken**) + - **Yappy**: (**Currently broken**) - **YappyProfile** - **YleAreena** - **YouJizz** @@ -1762,9 +1773,11 @@ # Supported sites - **ZDFChannel** - **Zee5**: [*zee5*](## "netrc machine") - **zee5:series** - - **ZeeNews** + - **ZeeNews**: (**Currently broken**) + - **ZenPorn** - **ZenYandex** - **ZenYandexChannel** + - **ZetlandDKArticle** - **Zhihu** - **zingmp3**: zingmp3.vn - **zingmp3:album** diff --git a/test/helper.py b/test/helper.py index 4aca47025..7760fd8d7 100644 --- a/test/helper.py +++ b/test/helper.py @@ -223,6 +223,10 @@ def sanitize(key, value): if test_info_dict.get('display_id') == test_info_dict.get('id'): test_info_dict.pop('display_id') + # Remove deprecated fields + for old in YoutubeDL._deprecated_multivalue_fields.keys(): + test_info_dict.pop(old, None) + # release_year may be generated from release_date if try_call(lambda: test_info_dict['release_year'] == int(test_info_dict['release_date'][:4])): test_info_dict.pop('release_year') diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py index 0087cbc94..6be47af97 100644 --- a/test/test_YoutubeDL.py +++ b/test/test_YoutubeDL.py @@ -941,7 +941,7 @@ def test_match_filter(self): def get_videos(filter_=None): ydl = YDL({'match_filter': filter_, 'simulate': True}) for v in videos: - ydl.process_ie_result(v, download=True) + ydl.process_ie_result(v.copy(), download=True) return [v['id'] for v in ydl.downloaded_info_dicts] res = get_videos() diff --git a/test/test_execution.py b/test/test_execution.py index fb2f6e2e9..c6ee9cf9d 100644 --- a/test/test_execution.py +++ b/test/test_execution.py @@ -45,7 +45,7 @@ def test_lazy_extractors(self): self.assertTrue(os.path.exists(LAZY_EXTRACTORS)) _, stderr = self.run_yt_dlp(opts=('-s', 'test:')) - # `MIN_RECOMMENDED` emits a deprecated feature warning for deprecated python versions + # `MIN_RECOMMENDED` emits a deprecated feature warning for deprecated Python versions if stderr and stderr.startswith('Deprecated Feature: Support for Python'): stderr = '' self.assertFalse(stderr) diff --git a/test/test_networking.py b/test/test_networking.py index 8cadd86f5..628f1f171 100644 --- a/test/test_networking.py +++ b/test/test_networking.py @@ -13,6 +13,7 @@ import http.cookiejar import http.server import io +import logging import pathlib import random import ssl @@ -68,7 +69,7 @@ def do_GET(self): self.send_response(200) self.send_header('Content-Type', 'text/plain; charset=utf-8') self.end_headers() - self.wfile.write('{self.proxy_name}: {self.path}'.format(self=self).encode()) + self.wfile.write(f'{self.proxy_name}: {self.path}'.encode()) return HTTPTestRequestHandler @@ -752,6 +753,25 @@ def test_certificate_nocombined_pass(self, handler): }) +class TestRequestHandlerMisc: + """Misc generic tests for request handlers, not related to request or validation testing""" + @pytest.mark.parametrize('handler,logger_name', [ + ('Requests', 'urllib3'), + ('Websockets', 'websockets.client'), + ('Websockets', 'websockets.server') + ], indirect=['handler']) + def test_remove_logging_handler(self, handler, logger_name): + # Ensure any logging handlers, which may contain a YoutubeDL instance, + # are removed when we close the request handler + # See: https://github.com/yt-dlp/yt-dlp/issues/8922 + logging_handlers = logging.getLogger(logger_name).handlers + before_count = len(logging_handlers) + rh = handler() + assert len(logging_handlers) == before_count + 1 + rh.close() + assert len(logging_handlers) == before_count + + class TestUrllibRequestHandler(TestRequestHandlerBase): @pytest.mark.parametrize('handler', ['Urllib'], indirect=True) def test_file_urls(self, handler): @@ -827,6 +847,7 @@ def test_httplib_validation_errors(self, handler, req, match, version_check): assert not isinstance(exc_info.value, TransportError) +@pytest.mark.parametrize('handler', ['Requests'], indirect=True) class TestRequestsRequestHandler(TestRequestHandlerBase): @pytest.mark.parametrize('raised,expected', [ (lambda: requests.exceptions.ConnectTimeout(), TransportError), @@ -843,7 +864,6 @@ class TestRequestsRequestHandler(TestRequestHandlerBase): (lambda: requests.exceptions.RequestException(), RequestError) # (lambda: requests.exceptions.TooManyRedirects(), HTTPError) - Needs a response object ]) - @pytest.mark.parametrize('handler', ['Requests'], indirect=True) def test_request_error_mapping(self, handler, monkeypatch, raised, expected): with handler() as rh: def mock_get_instance(*args, **kwargs): @@ -877,7 +897,6 @@ def request(self, *args, **kwargs): '3 bytes read, 5 more expected' ), ]) - @pytest.mark.parametrize('handler', ['Requests'], indirect=True) def test_response_error_mapping(self, handler, monkeypatch, raised, expected, match): from requests.models import Response as RequestsResponse from urllib3.response import HTTPResponse as Urllib3Response @@ -896,6 +915,21 @@ def mock_read(*args, **kwargs): assert exc_info.type is expected + def test_close(self, handler, monkeypatch): + rh = handler() + session = rh._get_instance(cookiejar=rh.cookiejar) + called = False + original_close = session.close + + def mock_close(*args, **kwargs): + nonlocal called + called = True + return original_close(*args, **kwargs) + + monkeypatch.setattr(session, 'close', mock_close) + rh.close() + assert called + def run_validation(handler, error, req, **handler_kwargs): with handler(**handler_kwargs) as rh: @@ -1205,6 +1239,19 @@ def some_preference(rh, request): assert director.send(Request('http://')).read() == b'' assert director.send(Request('http://', headers={'prefer': '1'})).read() == b'supported' + def test_close(self, monkeypatch): + director = RequestDirector(logger=FakeLogger()) + director.add_handler(FakeRH(logger=FakeLogger())) + called = False + + def mock_close(*args, **kwargs): + nonlocal called + called = True + + monkeypatch.setattr(director.handlers[FakeRH.RH_KEY], 'close', mock_close) + director.close() + assert called + # XXX: do we want to move this to test_YoutubeDL.py? class TestYoutubeDLNetworking: diff --git a/test/test_utils.py b/test/test_utils.py index 09c648cf8..a3073f0e0 100644 --- a/test/test_utils.py +++ b/test/test_utils.py @@ -2386,7 +2386,7 @@ def test_traverse_obj(self): self.assertEqual(traverse_obj(etree, '//year/text()'), ['2008', '2011', '2011'], msg='`text()` at end of path should give the inner text') self.assertEqual(traverse_obj(etree, '//*[@direction]/@direction'), ['E', 'W', 'N', 'W', 'E'], - msg='full python xpath features should be supported') + msg='full Python xpath features should be supported') self.assertEqual(traverse_obj(etree, (0, '@name')), 'Liechtenstein', msg='special transformations should act on current element') self.assertEqual(traverse_obj(etree, ('country', 0, ..., 'text()', {int_or_none})), [1, 2008, 141100], diff --git a/test/test_websockets.py b/test/test_websockets.py index 91bac3442..13b3a1e76 100644 --- a/test/test_websockets.py +++ b/test/test_websockets.py @@ -192,8 +192,8 @@ def test_raise_http_error(self, handler, status): @pytest.mark.parametrize('handler', ['Websockets'], indirect=True) @pytest.mark.parametrize('params,extensions', [ - ({'timeout': 0.00001}, {}), - ({}, {'timeout': 0.00001}), + ({'timeout': sys.float_info.min}, {}), + ({}, {'timeout': sys.float_info.min}), ]) def test_timeout(self, handler, params, extensions): with handler(**params) as rh: diff --git a/yt_dlp/YoutubeDL.py b/yt_dlp/YoutubeDL.py index e7d654d0f..c34d97bba 100644 --- a/yt_dlp/YoutubeDL.py +++ b/yt_dlp/YoutubeDL.py @@ -575,11 +575,18 @@ class YoutubeDL: 'url', 'manifest_url', 'manifest_stream_number', 'ext', 'format', 'format_id', 'format_note', 'width', 'height', 'aspect_ratio', 'resolution', 'dynamic_range', 'tbr', 'abr', 'acodec', 'asr', 'audio_channels', 'vbr', 'fps', 'vcodec', 'container', 'filesize', 'filesize_approx', 'rows', 'columns', - 'player_url', 'protocol', 'fragment_base_url', 'fragments', 'is_from_start', + 'player_url', 'protocol', 'fragment_base_url', 'fragments', 'is_from_start', 'is_dash_periods', 'request_data', 'preference', 'language', 'language_preference', 'quality', 'source_preference', 'cookies', 'http_headers', 'stretched_ratio', 'no_resume', 'has_drm', 'extra_param_to_segment_url', 'hls_aes', 'downloader_options', 'page_url', 'app', 'play_path', 'tc_url', 'flash_version', 'rtmp_live', 'rtmp_conn', 'rtmp_protocol', 'rtmp_real_time' } + _deprecated_multivalue_fields = { + 'album_artist': 'album_artists', + 'artist': 'artists', + 'composer': 'composers', + 'creator': 'creators', + 'genre': 'genres', + } _format_selection_exts = { 'audio': set(MEDIA_EXTENSIONS.common_audio), 'video': set(MEDIA_EXTENSIONS.common_video + ('3gp', )), @@ -683,7 +690,6 @@ def process_color_policy(stream): self.params['http_headers'] = HTTPHeaderDict(std_headers, self.params.get('http_headers')) self._load_cookies(self.params['http_headers'].get('Cookie')) # compat self.params['http_headers'].pop('Cookie', None) - self._request_director = self.build_request_director(_REQUEST_HANDLERS.values(), _RH_PREFERENCES) if auto_init and auto_init != 'no_verbose_header': self.print_debug_header() @@ -956,7 +962,9 @@ def __exit__(self, *args): def close(self): self.save_cookies() - self._request_director.close() + if '_request_director' in self.__dict__: + self._request_director.close() + del self._request_director def trouble(self, message=None, tb=None, is_error=True): """Determine action to take when a download problem appears. @@ -2219,7 +2227,7 @@ def _parse_format_selection(tokens, inside_merge=False, inside_choice=False, ins selectors = [] current_selector = None for type, string_, start, _, _ in tokens: - # ENCODING is only defined in python 3.x + # ENCODING is only defined in Python 3.x if type == getattr(tokenize, 'ENCODING', None): continue elif type in [tokenize.NAME, tokenize.NUMBER]: @@ -2640,6 +2648,15 @@ def _fill_common_fields(self, info_dict, final=True): if final and info_dict.get('%s_number' % field) is not None and not info_dict.get(field): info_dict[field] = '%s %d' % (field.capitalize(), info_dict['%s_number' % field]) + for old_key, new_key in self._deprecated_multivalue_fields.items(): + if new_key in info_dict and old_key in info_dict: + if '_version' not in info_dict: # HACK: Do not warn when using --load-info-json + self.deprecation_warning(f'Do not return {old_key!r} when {new_key!r} is present') + elif old_value := info_dict.get(old_key): + info_dict[new_key] = old_value.split(', ') + elif new_value := info_dict.get(new_key): + info_dict[old_key] = ', '.join(v.replace(',', '\N{FULLWIDTH COMMA}') for v in new_value) + def _raise_pending_errors(self, info): err = info.pop('__pending_error', None) if err: @@ -3483,7 +3500,8 @@ def ffmpeg_fixup(cndn, msg, cls): or info_dict.get('is_live') and self.params.get('hls_use_mpegts') is None, 'Possible MPEG-TS in MP4 container or malformed AAC timestamps', FFmpegFixupM3u8PP) - ffmpeg_fixup(info_dict.get('is_live') and downloader == 'dashsegments', + ffmpeg_fixup(downloader == 'dashsegments' + and (info_dict.get('is_live') or info_dict.get('is_dash_periods')), 'Possible duplicate MOOV atoms', FFmpegFixupDuplicateMoovPP) ffmpeg_fixup(downloader == 'web_socket_fragment', 'Malformed timestamps detected', FFmpegFixupTimestampPP) @@ -3560,6 +3578,8 @@ def download_with_info_file(self, info_filename): raise self.report_warning(f'The info failed to download: {e}; trying with URL {webpage_url}') self.download([webpage_url]) + except ExtractorError as e: + self.report_error(e) return self._download_retcode @staticmethod @@ -4144,6 +4164,10 @@ def build_request_director(self, handlers, preferences=None): director.preferences.add(lambda rh, _: 500 if rh.RH_KEY == 'Urllib' else 0) return director + @functools.cached_property + def _request_director(self): + return self.build_request_director(_REQUEST_HANDLERS.values(), _RH_PREFERENCES) + def encode(self, s): if isinstance(s, bytes): return s # Already encoded diff --git a/yt_dlp/__init__.py b/yt_dlp/__init__.py index 57a487157..aeea2625e 100644 --- a/yt_dlp/__init__.py +++ b/yt_dlp/__init__.py @@ -4,7 +4,7 @@ raise ImportError( f'You are using an unsupported version of Python. Only Python versions 3.8 and above are supported by yt-dlp') # noqa: F541 -__license__ = 'Public Domain' +__license__ = 'The Unlicense' import collections import getpass @@ -14,7 +14,7 @@ import re import traceback -from .compat import compat_shlex_quote +from .compat import compat_os_name, compat_shlex_quote from .cookies import SUPPORTED_BROWSERS, SUPPORTED_KEYRINGS from .downloader.external import get_external_downloader from .extractor import list_extractor_classes @@ -984,7 +984,28 @@ def _real_main(argv=None): if pre_process: return ydl._download_retcode - ydl.warn_if_short_id(sys.argv[1:] if argv is None else argv) + args = sys.argv[1:] if argv is None else argv + ydl.warn_if_short_id(args) + + # Show a useful error message and wait for keypress if not launched from shell on Windows + if not args and compat_os_name == 'nt' and getattr(sys, 'frozen', False): + import ctypes.wintypes + import msvcrt + + kernel32 = ctypes.WinDLL('Kernel32') + + buffer = (1 * ctypes.wintypes.DWORD)() + attached_processes = kernel32.GetConsoleProcessList(buffer, 1) + # If we only have a single process attached, then the executable was double clicked + # When using `pyinstaller` with `--onefile`, two processes get attached + is_onefile = hasattr(sys, '_MEIPASS') and os.path.basename(sys._MEIPASS).startswith('_MEI') + if attached_processes == 1 or is_onefile and attached_processes == 2: + print(parser._generate_error_message( + 'Do not double-click the executable, instead call it from a command line.\n' + 'Please read the README for further information on how to use yt-dlp: ' + 'https://github.com/yt-dlp/yt-dlp#readme')) + msvcrt.getch() + _exit(2) parser.error( 'You must provide at least one URL.\n' 'Type yt-dlp --help to see a list of all options.') diff --git a/yt_dlp/__main__.py b/yt_dlp/__main__.py index 78701df8d..06c392039 100644 --- a/yt_dlp/__main__.py +++ b/yt_dlp/__main__.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # Execute with -# $ python -m yt_dlp +# $ python3 -m yt_dlp import sys diff --git a/yt_dlp/__pyinstaller/hook-yt_dlp.py b/yt_dlp/__pyinstaller/hook-yt_dlp.py index 20f037d32..7c3dbfb66 100644 --- a/yt_dlp/__pyinstaller/hook-yt_dlp.py +++ b/yt_dlp/__pyinstaller/hook-yt_dlp.py @@ -10,7 +10,7 @@ def pycryptodome_module(): try: import Crypto # noqa: F401 print('WARNING: Using Crypto since Cryptodome is not available. ' - 'Install with: pip install pycryptodomex', file=sys.stderr) + 'Install with: python3 -m pip install pycryptodomex', file=sys.stderr) return 'Crypto' except ImportError: pass @@ -31,4 +31,4 @@ def get_hidden_imports(): hiddenimports = list(get_hidden_imports()) print(f'Adding imports: {hiddenimports}') -excludedimports = ['youtube_dl', 'youtube_dlc', 'test', 'ytdlp_plugins', 'devscripts'] +excludedimports = ['youtube_dl', 'youtube_dlc', 'test', 'ytdlp_plugins', 'devscripts', 'bundle'] diff --git a/yt_dlp/casefold.py b/yt_dlp/casefold.py deleted file mode 100644 index 41a53e5b6..000000000 --- a/yt_dlp/casefold.py +++ /dev/null @@ -1,5 +0,0 @@ -import warnings - -warnings.warn(DeprecationWarning(f'{__name__} is deprecated')) - -casefold = str.casefold diff --git a/yt_dlp/compat/urllib/request.py b/yt_dlp/compat/urllib/request.py index ff63b2f0e..ad9fa83c8 100644 --- a/yt_dlp/compat/urllib/request.py +++ b/yt_dlp/compat/urllib/request.py @@ -10,10 +10,10 @@ from .. import compat_os_name if compat_os_name == 'nt': - # On older python versions, proxies are extracted from Windows registry erroneously. [1] + # On older Python versions, proxies are extracted from Windows registry erroneously. [1] # If the https proxy in the registry does not have a scheme, urllib will incorrectly add https:// to it. [2] # It is unlikely that the user has actually set it to be https, so we should be fine to safely downgrade - # it to http on these older python versions to avoid issues + # it to http on these older Python versions to avoid issues # This also applies for ftp proxy type, as ftp:// proxy scheme is not supported. # 1: https://github.com/python/cpython/issues/86793 # 2: https://github.com/python/cpython/blob/51f1ae5ceb0673316c4e4b0175384e892e33cc6e/Lib/urllib/request.py#L2683-L2698 diff --git a/yt_dlp/cookies.py b/yt_dlp/cookies.py index deb2e35f2..28d174a09 100644 --- a/yt_dlp/cookies.py +++ b/yt_dlp/cookies.py @@ -121,7 +121,7 @@ def _extract_firefox_cookies(profile, container, logger): logger.info('Extracting cookies from firefox') if not sqlite3: logger.warning('Cannot extract cookies from firefox without sqlite3 support. ' - 'Please use a python interpreter compiled with sqlite3 support') + 'Please use a Python interpreter compiled with sqlite3 support') return YoutubeDLCookieJar() if profile is None: @@ -264,7 +264,7 @@ def _extract_chrome_cookies(browser_name, profile, keyring, logger): if not sqlite3: logger.warning(f'Cannot extract cookies from {browser_name} without sqlite3 support. ' - 'Please use a python interpreter compiled with sqlite3 support') + 'Please use a Python interpreter compiled with sqlite3 support') return YoutubeDLCookieJar() config = _get_chromium_based_browser_settings(browser_name) diff --git a/yt_dlp/dependencies/__init__.py b/yt_dlp/dependencies/__init__.py index ef83739a3..9e3f90724 100644 --- a/yt_dlp/dependencies/__init__.py +++ b/yt_dlp/dependencies/__init__.py @@ -46,16 +46,14 @@ # We need to get the underlying `sqlite` version, see https://github.com/yt-dlp/yt-dlp/issues/8152 sqlite3._yt_dlp__version = sqlite3.sqlite_version except ImportError: - # although sqlite3 is part of the standard library, it is possible to compile python without + # although sqlite3 is part of the standard library, it is possible to compile Python without # sqlite support. See: https://github.com/yt-dlp/yt-dlp/issues/544 sqlite3 = None try: import websockets -except (ImportError, SyntaxError): - # websockets 3.10 on python 3.6 causes SyntaxError - # See https://github.com/yt-dlp/yt-dlp/issues/2633 +except ImportError: websockets = None try: diff --git a/yt_dlp/downloader/http.py b/yt_dlp/downloader/http.py index f5237443e..693828b6e 100644 --- a/yt_dlp/downloader/http.py +++ b/yt_dlp/downloader/http.py @@ -237,8 +237,13 @@ def download(): def retry(e): close_stream() - ctx.resume_len = (byte_counter if ctx.tmpfilename == '-' - else os.path.getsize(encodeFilename(ctx.tmpfilename))) + if ctx.tmpfilename == '-': + ctx.resume_len = byte_counter + else: + try: + ctx.resume_len = os.path.getsize(encodeFilename(ctx.tmpfilename)) + except FileNotFoundError: + ctx.resume_len = 0 raise RetryDownload(e) while True: diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py index 5d1dd6038..c75365536 100644 --- a/yt_dlp/extractor/_extractors.py +++ b/yt_dlp/extractor/_extractors.py @@ -320,7 +320,6 @@ CBSIE, ParamountPressExpressIE, ) -from .cbsinteractive import CBSInteractiveIE from .cbsnews import ( CBSNewsEmbedIE, CBSNewsIE, @@ -348,10 +347,6 @@ from .charlierose import CharlieRoseIE from .chaturbate import ChaturbateIE from .chilloutzone import ChilloutzoneIE -from .chingari import ( - ChingariIE, - ChingariUserIE, -) from .chzzk import ( CHZZKLiveIE, CHZZKVideoIE, @@ -369,7 +364,6 @@ from .ciscowebex import CiscoWebexIE from .cjsw import CJSWIE from .clipchamp import ClipchampIE -from .cliphunter import CliphunterIE from .clippit import ClippitIE from .cliprs import ClipRsIE from .closertotruth import CloserToTruthIE @@ -379,7 +373,6 @@ from .clyp import ClypIE from .cmt import CMTIE from .cnbc import ( - CNBCIE, CNBCVideoIE, ) from .cnn import ( @@ -445,6 +438,7 @@ from .dailymotion import ( DailymotionIE, DailymotionPlaylistIE, + DailymotionSearchIE, DailymotionUserIE, ) from .dailywire import ( @@ -476,7 +470,6 @@ ) from .dfb import DFBIE from .dhm import DHMIE -from .digg import DiggIE from .douyutv import ( DouyuShowIE, DouyuTVIE, @@ -610,7 +603,6 @@ ) from .fczenit import FczenitIE from .fifa import FifaIE -from .filmmodu import FilmmoduIE from .filmon import ( FilmOnIE, FilmOnChannelIE, @@ -618,6 +610,7 @@ from .filmweb import FilmwebIE from .firsttv import FirstTVIE from .fivetv import FiveTVIE +from .flextv import FlexTVIE from .flickr import FlickrIE from .floatplane import ( FloatplaneIE, @@ -675,7 +668,6 @@ GabIE, ) from .gaia import GaiaIE -from .gameinformer import GameInformerIE from .gamejolt import ( GameJoltIE, GameJoltUserIE, @@ -704,7 +696,6 @@ GettrStreamingIE, ) from .giantbomb import GiantBombIE -from .giga import GigaIE from .glide import GlideIE from .globalplayer import ( GlobalPlayerLiveIE, @@ -895,10 +886,8 @@ from .jwplatform import JWPlatformIE from .kakao import KakaoIE from .kaltura import KalturaIE -from .kanal2 import Kanal2IE from .kankanews import KankaNewsIE from .karaoketv import KaraoketvIE -from .karrierevideos import KarriereVideosIE from .kelbyone import KelbyOneIE from .khanacademy import ( KhanAcademyIE, @@ -914,13 +903,11 @@ from .kinopoisk import KinoPoiskIE from .kommunetv import KommunetvIE from .kompas import KompasVideoIE -from .konserthusetplay import KonserthusetPlayIE from .koo import KooIE from .kth import KTHIE from .krasview import KrasViewIE from .ku6 import Ku6IE from .kukululive import KukuluLiveIE -from .kusi import KUSIIE from .kuwo import ( KuwoIE, KuwoAlbumIE, @@ -1002,7 +989,6 @@ LnkGoIE, LnkIE, ) -from .localnews8 import LocalNews8IE from .lovehomeporn import LoveHomePornIE from .lrt import ( LRTVODIE, @@ -1029,7 +1015,6 @@ MailRuMusicSearchIE, ) from .mainstreaming import MainStreamingIE -from .malltv import MallTVIE from .mangomolo import ( MangomoloVideoIE, MangomoloLiveIE, @@ -1073,7 +1058,6 @@ from .melonvod import MelonVODIE from .metacritic import MetacriticIE from .mgtv import MGTVIE -from .miaopai import MiaoPaiIE from .microsoftstream import MicrosoftStreamIE from .microsoftvirtualacademy import ( MicrosoftVirtualAcademyIE, @@ -1091,7 +1075,6 @@ MindsChannelIE, MindsGroupIE, ) -from .ministrygrid import MinistryGridIE from .minoto import MinotoIE from .mirrativ import ( MirrativIE, @@ -1119,7 +1102,6 @@ from .mocha import MochaVideoIE from .mojvideo import MojvideoIE from .monstercat import MonstercatIE -from .morningstar import MorningstarIE from .motherless import ( MotherlessIE, MotherlessGroupIE, @@ -1364,7 +1346,6 @@ from .nzherald import NZHeraldIE from .nzonscreen import NZOnScreenIE from .nzz import NZZIE -from .odatv import OdaTVIE from .odkmedia import OnDemandChinaEpisodeIE from .odnoklassniki import OdnoklassnikiIE from .oftv import ( @@ -1476,7 +1457,6 @@ PlatziCourseIE, ) from .playplustv import PlayPlusTVIE -from .playstuff import PlayStuffIE from .playsuisse import PlaySuisseIE from .playtvak import PlaytvakIE from .playwire import PlaywireIE @@ -1598,7 +1578,6 @@ RayWenderlichIE, RayWenderlichCourseIE, ) -from .rbmaradio import RBMARadioIE from .rbgtum import ( RbgTumIE, RbgTumCourseIE, @@ -1630,7 +1609,6 @@ RedGifsUserIE, ) from .redtube import RedTubeIE -from .regiotv import RegioTVIE from .rentv import ( RENTVIE, RENTVArticleIE, @@ -1639,6 +1617,7 @@ from .reuters import ReutersIE from .reverbnation import ReverbNationIE from .rheinmaintv import RheinMainTVIE +from .ridehome import RideHomeIE from .rinsefm import ( RinseFMIE, RinseFMArtistPlaylistIE, @@ -1737,7 +1716,6 @@ from .saitosan import SaitosanIE from .samplefocus import SampleFocusIE from .sapo import SapoIE -from .savefrom import SaveFromIE from .sbs import SBSIE from .sbscokr import ( SBSCoKrIE, @@ -1757,7 +1735,6 @@ SCTECourseIE, ) from .scrolller import ScrolllerIE -from .seeker import SeekerIE from .sejmpl import SejmIE from .senalcolombia import SenalColombiaLiveIE from .senategov import SenateISVPIE, SenateGovIE @@ -1900,7 +1877,6 @@ ) from .streamable import StreamableIE from .streamcz import StreamCZIE -from .streamff import StreamFFIE from .streetvoice import StreetVoiceIE from .stretchinternet import StretchInternetIE from .stripchat import StripchatIE @@ -1929,7 +1905,6 @@ TBSJPProgramIE, TBSJPPlaylistIE, ) -from .tdslifeway import TDSLifewayIE from .teachable import ( TeachableIE, TeachableCourseIE, @@ -2499,6 +2474,7 @@ Zee5SeriesIE, ) from .zeenews import ZeeNewsIE +from .zenporn import ZenPornIE from .zetland import ZetlandDKArticleIE from .zhihu import ZhihuIE from .zingmp3 import ( diff --git a/yt_dlp/extractor/abc.py b/yt_dlp/extractor/abc.py index a7b614ca1..b21742281 100644 --- a/yt_dlp/extractor/abc.py +++ b/yt_dlp/extractor/abc.py @@ -245,7 +245,6 @@ class ABCIViewIE(InfoExtractor): 'episode_id': 'NC2203H039S00', 'season_number': 2022, 'season': 'Season 2022', - 'episode_number': None, 'episode': 'Locking Up Kids', 'thumbnail': 'https://cdn.iview.abc.net.au/thumbs/i/nc/NC2203H039S00_636d8a0944a22_1920.jpg', 'timestamp': 1668460497, @@ -271,8 +270,6 @@ class ABCIViewIE(InfoExtractor): 'episode_id': 'RF2004Q043S00', 'season_number': 2021, 'season': 'Season 2021', - 'episode_number': None, - 'episode': None, 'thumbnail': 'https://cdn.iview.abc.net.au/thumbs/i/rf/RF2004Q043S00_61a950639dbc0_1920.jpg', 'timestamp': 1638710705, diff --git a/yt_dlp/extractor/abematv.py b/yt_dlp/extractor/abematv.py index 6453dde97..fee7375ea 100644 --- a/yt_dlp/extractor/abematv.py +++ b/yt_dlp/extractor/abematv.py @@ -53,7 +53,7 @@ def __init__(self, ie: 'AbemaTVIE'): # the protocol that this should really handle is 'abematv-license://' # abematv_license_open is just a placeholder for development purposes # ref. https://github.com/python/cpython/blob/f4c03484da59049eb62a9bf7777b963e2267d187/Lib/urllib/request.py#L510 - setattr(self, 'abematv-license_open', getattr(self, 'abematv_license_open')) + setattr(self, 'abematv-license_open', getattr(self, 'abematv_license_open', None)) self.ie = ie def _get_videokey_from_ticket(self, ticket): @@ -259,7 +259,7 @@ class AbemaTVIE(AbemaTVBaseIE): 'title': 'ゆるキャン△ SEASON２ 全話一挙【無料ビデオ72時間】', 'series': 'ゆるキャン△ SEASON２', 'episode': 'ゆるキャン△ SEASON２ 全話一挙【無料ビデオ72時間】', - 'series_number': 2, + 'season_number': 2, 'episode_number': 1, 'description': 'md5:9c5a3172ae763278f9303922f0ea5b17', }, diff --git a/yt_dlp/extractor/acfun.py b/yt_dlp/extractor/acfun.py index dc5792944..c3b4f432e 100644 --- a/yt_dlp/extractor/acfun.py +++ b/yt_dlp/extractor/acfun.py @@ -3,6 +3,7 @@ float_or_none, format_field, int_or_none, + str_or_none, traverse_obj, parse_codecs, parse_qs, @@ -129,7 +130,7 @@ class AcFunBangumiIE(AcFunVideoBaseIE): 'title': '红孩儿之趴趴蛙寻石记 第5话 ', 'duration': 760.0, 'season': '红孩儿之趴趴蛙寻石记', - 'season_id': 5023171, + 'season_id': '5023171', 'season_number': 1, # series has only 1 season 'episode': 'Episode 5', 'episode_number': 5, @@ -146,7 +147,7 @@ class AcFunBangumiIE(AcFunVideoBaseIE): 'title': '叽歪老表（第二季） 第5话 坚不可摧', 'season': '叽歪老表（第二季）', 'season_number': 2, - 'season_id': 6065485, + 'season_id': '6065485', 'episode': '坚不可摧', 'episode_number': 5, 'upload_date': '20220324', @@ -191,7 +192,7 @@ def _real_extract(self, url): 'title': json_bangumi_data.get('showTitle'), 'thumbnail': json_bangumi_data.get('image'), 'season': json_bangumi_data.get('bangumiTitle'), - 'season_id': season_id, + 'season_id': str_or_none(season_id), 'season_number': season_number, 'episode': json_bangumi_data.get('title'), 'episode_number': episode_number, diff --git a/yt_dlp/extractor/adultswim.py b/yt_dlp/extractor/adultswim.py index daaeddeb6..d807c4181 100644 --- a/yt_dlp/extractor/adultswim.py +++ b/yt_dlp/extractor/adultswim.py @@ -107,7 +107,6 @@ def _real_extract(self, url): title tvRating }''' % episode_path - ['getVideoBySlug'] else: query = query % '''metaDescription title diff --git a/yt_dlp/extractor/altcensored.py b/yt_dlp/extractor/altcensored.py index 0e1627bfd..6878918a0 100644 --- a/yt_dlp/extractor/altcensored.py +++ b/yt_dlp/extractor/altcensored.py @@ -4,6 +4,7 @@ from .common import InfoExtractor from ..utils import ( InAdvancePagedList, + clean_html, int_or_none, orderedSet, str_to_int, @@ -22,7 +23,7 @@ class AltCensoredIE(InfoExtractor): 'title': "QUELLES SONT LES CONSÉQUENCES DE L'HYPERSEXUALISATION DE LA SOCIÉTÉ ?", 'display_id': 'k0srjLSkga8.webm', 'release_date': '20180403', - 'creator': 'Virginie Vota', + 'creators': ['Virginie Vota'], 'release_year': 2018, 'upload_date': '20230318', 'uploader': 'admin@altcensored.com', @@ -39,6 +40,8 @@ class AltCensoredIE(InfoExtractor): def _real_extract(self, url): video_id = self._match_id(url) webpage = self._download_webpage(url, video_id) + category = clean_html(self._html_search_regex( + r'([^<]+)', webpage, 'category', default=None)) return { '_type': 'url_transparent', @@ -46,9 +49,7 @@ def _real_extract(self, url): 'ie_key': ArchiveOrgIE.ie_key(), 'view_count': str_to_int(self._html_search_regex( r'YouTube Views:(?:\s| )*([\d,]+)', webpage, 'view count', default=None)), - 'categories': self._html_search_regex( - r'\s*\n?\s*([^<]+)', - webpage, 'category', default='').split() or None, + 'categories': [category] if category else None, } @@ -62,14 +63,21 @@ class AltCensoredChannelIE(InfoExtractor): 'title': 'Virginie Vota', 'id': 'UCFPTO55xxHqFqkzRZHu4kcw', }, - 'playlist_count': 91 + 'playlist_count': 85, }, { 'url': 'https://altcensored.com/channel/UC9CcJ96HKMWn0LZlcxlpFTw', 'info_dict': { 'title': 'yukikaze775', 'id': 'UC9CcJ96HKMWn0LZlcxlpFTw', }, - 'playlist_count': 4 + 'playlist_count': 4, + }, { + 'url': 'https://altcensored.com/channel/UCfYbb7nga6-icsFWWgS-kWw', + 'info_dict': { + 'title': 'Mister Metokur', + 'id': 'UCfYbb7nga6-icsFWWgS-kWw', + }, + 'playlist_count': 121, }] def _real_extract(self, url): @@ -78,7 +86,7 @@ def _real_extract(self, url): url, channel_id, 'Download channel webpage', 'Unable to get channel webpage') title = self._html_search_meta('altcen_title', webpage, 'title', fatal=False) page_count = int_or_none(self._html_search_regex( - r']+href="/channel/\w+/page/(\d+)">(?:\1)', + r']+href="/channel/[\w-]+/page/(\d+)">(?:\1)', webpage, 'page count', default='1')) def page_func(page_num): diff --git a/yt_dlp/extractor/antenna.py b/yt_dlp/extractor/antenna.py index 17a4b6900..2929d6550 100644 --- a/yt_dlp/extractor/antenna.py +++ b/yt_dlp/extractor/antenna.py @@ -67,7 +67,7 @@ def _real_extract(self, url): webpage = self._download_webpage(url, video_id) info = self._download_and_extract_api_data(video_id, netloc) info['description'] = self._og_search_description(webpage, default=None) - info['_old_archive_ids'] = [make_archive_id('Ant1NewsGrWatch', video_id)], + info['_old_archive_ids'] = [make_archive_id('Ant1NewsGrWatch', video_id)] return info diff --git a/yt_dlp/extractor/archiveorg.py b/yt_dlp/extractor/archiveorg.py index 3bb6f2e31..41f3a4ff2 100644 --- a/yt_dlp/extractor/archiveorg.py +++ b/yt_dlp/extractor/archiveorg.py @@ -31,6 +31,7 @@ unified_timestamp, url_or_none, urlhandle_detect_ext, + variadic, ) @@ -49,7 +50,7 @@ class ArchiveOrgIE(InfoExtractor): 'release_date': '19681210', 'timestamp': 1268695290, 'upload_date': '20100315', - 'creator': 'SRI International', + 'creators': ['SRI International'], 'uploader': 'laura@archive.org', 'thumbnail': r're:https://archive\.org/download/.*\.jpg', 'display_id': 'XD300-23_68HighlightsAResearchCntAugHumanIntellect.cdr', @@ -109,7 +110,7 @@ class ArchiveOrgIE(InfoExtractor): 'title': 'Turning', 'ext': 'flac', 'track': 'Turning', - 'creator': 'Grateful Dead', + 'creators': ['Grateful Dead'], 'display_id': 'gd1977-05-08d01t01.flac', 'track_number': 1, 'album': '1977-05-08 - Barton Hall - Cornell University', @@ -129,7 +130,7 @@ class ArchiveOrgIE(InfoExtractor): 'location': 'Barton Hall - Cornell University', 'duration': 438.68, 'track': 'Deal', - 'creator': 'Grateful Dead', + 'creators': ['Grateful Dead'], 'album': '1977-05-08 - Barton Hall - Cornell University', 'release_date': '19770508', 'display_id': 'gd1977-05-08d01t07.flac', @@ -167,7 +168,7 @@ class ArchiveOrgIE(InfoExtractor): 'upload_date': '20160610', 'description': 'md5:f70956a156645a658a0dc9513d9e78b7', 'uploader': 'dimitrios@archive.org', - 'creator': ['British Broadcasting Corporation', 'Time-Life Films'], + 'creators': ['British Broadcasting Corporation', 'Time-Life Films'], 'timestamp': 1465594947, }, 'playlist': [ @@ -257,7 +258,7 @@ def _real_extract(self, url): 'title': m['title'], 'description': clean_html(m.get('description')), 'uploader': dict_get(m, ['uploader', 'adder']), - 'creator': m.get('creator'), + 'creators': traverse_obj(m, ('creator', {variadic}, {lambda x: x[0] and list(x)})), 'license': m.get('licenseurl'), 'release_date': unified_strdate(m.get('date')), 'timestamp': unified_timestamp(dict_get(m, ['publicdate', 'addeddate'])), @@ -272,7 +273,7 @@ def _real_extract(self, url): 'title': f.get('title') or f['name'], 'display_id': f['name'], 'description': clean_html(f.get('description')), - 'creator': f.get('creator'), + 'creators': traverse_obj(f, ('creator', {variadic}, {lambda x: x[0] and list(x)})), 'duration': parse_duration(f.get('length')), 'track_number': int_or_none(f.get('track')), 'album': f.get('album'), @@ -300,7 +301,7 @@ def _real_extract(self, url): is_logged_in = bool(self._get_cookies('https://archive.org').get('logged-in-sig')) if extension in KNOWN_EXTENSIONS and (not f.get('private') or is_logged_in): entry['formats'].append({ - 'url': 'https://archive.org/download/' + identifier + '/' + f['name'], + 'url': 'https://archive.org/download/' + identifier + '/' + urllib.parse.quote(f['name']), 'format': f.get('format'), 'width': int_or_none(f.get('width')), 'height': int_or_none(f.get('height')), diff --git a/yt_dlp/extractor/arte.py b/yt_dlp/extractor/arte.py index 92b4900f9..1c180b1fd 100644 --- a/yt_dlp/extractor/arte.py +++ b/yt_dlp/extractor/arte.py @@ -142,10 +142,10 @@ class ArteTVIE(ArteTVBaseIE): def _fix_accessible_subs_locale(subs): updated_subs = {} for lang, sub_formats in subs.items(): - for format in sub_formats: - if format.get('url', '').endswith('-MAL.m3u8'): + for fmt in sub_formats: + if fmt.get('url', '').endswith('-MAL.m3u8'): lang += '-acc' - updated_subs.setdefault(lang, []).append(format) + updated_subs.setdefault(lang, []).append(fmt) return updated_subs def _real_extract(self, url): diff --git a/yt_dlp/extractor/axs.py b/yt_dlp/extractor/axs.py index 4b263725f..7e9166771 100644 --- a/yt_dlp/extractor/axs.py +++ b/yt_dlp/extractor/axs.py @@ -24,7 +24,8 @@ class AxsIE(InfoExtractor): 'timestamp': 1685729564, 'duration': 1284.216, 'series': 'Rock & Roll Road Trip with Sammy Hagar', - 'season': 2, + 'season': 'Season 2', + 'season_number': 2, 'episode': '3', 'thumbnail': 'https://images.dotstudiopro.com/5f4e9d330a0c3b295a7e8394', }, @@ -41,7 +42,8 @@ class AxsIE(InfoExtractor): 'timestamp': 1676403615, 'duration': 2570.668, 'series': 'The Big Interview with Dan Rather', - 'season': 3, + 'season': 'Season 3', + 'season_number': 3, 'episode': '5', 'thumbnail': 'https://images.dotstudiopro.com/5f4d1901f340b50d937cec32', }, @@ -77,7 +79,7 @@ def _real_extract(self, url): 'title': ('title', {str}), 'description': ('description', {str}), 'series': ('seriestitle', {str}), - 'season': ('season', {int}), + 'season_number': ('season', {int}), 'episode': ('episode', {str}), 'duration': ('duration', {float_or_none}), 'timestamp': ('updated_at', {parse_iso8601}), diff --git a/yt_dlp/extractor/beatbump.py b/yt_dlp/extractor/beatbump.py index f48566b2d..777a1b326 100644 --- a/yt_dlp/extractor/beatbump.py +++ b/yt_dlp/extractor/beatbump.py @@ -3,7 +3,7 @@ class BeatBumpVideoIE(InfoExtractor): - _VALID_URL = r'https://beatbump\.(?:ml|io)/listen\?id=(?P[\w-]+)' + _VALID_URL = r'https?://beatbump\.(?:ml|io)/listen\?id=(?P[\w-]+)' _TESTS = [{ 'url': 'https://beatbump.ml/listen?id=MgNrAu2pzNs', 'md5': '5ff3fff41d3935b9810a9731e485fe66', @@ -48,7 +48,7 @@ def _real_extract(self, url): class BeatBumpPlaylistIE(InfoExtractor): - _VALID_URL = r'https://beatbump\.(?:ml|io)/(?:release\?id=|artist/|playlist/)(?P[\w-]+)' + _VALID_URL = r'https?://beatbump\.(?:ml|io)/(?:release\?id=|artist/|playlist/)(?P[\w-]+)' _TESTS = [{ 'url': 'https://beatbump.ml/release?id=MPREb_gTAcphH99wE', 'playlist_count': 50, diff --git a/yt_dlp/extractor/beeg.py b/yt_dlp/extractor/beeg.py index 52ee68eca..042b3220b 100644 --- a/yt_dlp/extractor/beeg.py +++ b/yt_dlp/extractor/beeg.py @@ -2,6 +2,7 @@ from ..utils import ( int_or_none, + str_or_none, traverse_obj, try_get, unified_timestamp, @@ -22,7 +23,7 @@ class BeegIE(InfoExtractor): 'age_limit': 18, 'upload_date': '20220131', 'timestamp': 1643656455, - 'display_id': 2540839, + 'display_id': '2540839', } }, { 'url': 'https://beeg.com/-0599050563103750?t=4-861', @@ -36,7 +37,7 @@ class BeegIE(InfoExtractor): 'age_limit': 18, 'description': 'md5:b4fc879a58ae6c604f8f259155b7e3b9', 'timestamp': 1643623200, - 'display_id': 2569965, + 'display_id': '2569965', 'upload_date': '20220131', } }, { @@ -78,7 +79,7 @@ def _real_extract(self, url): return { 'id': video_id, - 'display_id': first_fact.get('id'), + 'display_id': str_or_none(first_fact.get('id')), 'title': traverse_obj(video, ('file', 'stuff', 'sf_name')), 'description': traverse_obj(video, ('file', 'stuff', 'sf_story')), 'timestamp': unified_timestamp(first_fact.get('fc_created')), diff --git a/yt_dlp/extractor/bellmedia.py b/yt_dlp/extractor/bellmedia.py index 5ae4b917a..677680b42 100644 --- a/yt_dlp/extractor/bellmedia.py +++ b/yt_dlp/extractor/bellmedia.py @@ -32,7 +32,7 @@ class BellMediaIE(InfoExtractor): 'description': 'md5:810f7f8c6a83ad5b48677c3f8e5bb2c3', 'upload_date': '20180525', 'timestamp': 1527288600, - 'season_id': 73997, + 'season_id': '73997', 'season': '2018', 'thumbnail': 'http://images2.9c9media.com/image_asset/2018_5_25_baf30cbd-b28d-4a18-9903-4bb8713b00f5_PNG_956x536.jpg', 'tags': [], diff --git a/yt_dlp/extractor/bfmtv.py b/yt_dlp/extractor/bfmtv.py index 5d0c73ff3..c4621ca82 100644 --- a/yt_dlp/extractor/bfmtv.py +++ b/yt_dlp/extractor/bfmtv.py @@ -93,7 +93,6 @@ class BFMTVArticleIE(BFMTVBaseIE): 'id': '6318445464112', 'ext': 'mp4', 'title': 'Le plein de bioéthanol fait de plus en plus mal à la pompe', - 'description': None, 'uploader_id': '876630703001', 'upload_date': '20230110', 'timestamp': 1673341692, diff --git a/yt_dlp/extractor/bilibili.py b/yt_dlp/extractor/bilibili.py index c138bde3a..fee4b2994 100644 --- a/yt_dlp/extractor/bilibili.py +++ b/yt_dlp/extractor/bilibili.py @@ -1965,6 +1965,7 @@ class BiliIntlIE(BiliIntlBaseIE): 'only_matching': True, }] + @staticmethod def _make_url(video_id, series_id=None): if series_id: return f'https://www.bilibili.tv/en/play/{series_id}/{video_id}' @@ -1996,7 +1997,7 @@ def _extract_video_metadata(self, url, video_id, season_id): 'title': get_element_by_class( 'bstar-meta__title', webpage) or self._html_search_meta('og:title', webpage), 'description': get_element_by_class( - 'bstar-meta__desc', webpage) or self._html_search_meta('og:description'), + 'bstar-meta__desc', webpage) or self._html_search_meta('og:description', webpage), }, self._search_json_ld(webpage, video_id, default={})) def _get_comments_reply(self, root_id, next_id=0, display_id=None): diff --git a/yt_dlp/extractor/bitchute.py b/yt_dlp/extractor/bitchute.py index 41367c5b9..194bf1f46 100644 --- a/yt_dlp/extractor/bitchute.py +++ b/yt_dlp/extractor/bitchute.py @@ -185,7 +185,6 @@ class BitChuteChannelIE(InfoExtractor): 'info_dict': { 'id': 'UGlrF9o9b-Q', 'ext': 'mp4', - 'filesize': None, 'title': 'This is the first video on #BitChute !', 'description': 'md5:a0337e7b1fe39e32336974af8173a034', 'thumbnail': r're:^https?://.*\.jpg$', diff --git a/yt_dlp/extractor/bleacherreport.py b/yt_dlp/extractor/bleacherreport.py index 5e5155af2..e875957cf 100644 --- a/yt_dlp/extractor/bleacherreport.py +++ b/yt_dlp/extractor/bleacherreport.py @@ -4,10 +4,12 @@ ExtractorError, int_or_none, parse_iso8601, + str_or_none, ) class BleacherReportIE(InfoExtractor): + _WORKING = False _VALID_URL = r'https?://(?:www\.)?bleacherreport\.com/articles/(?P\d+)' _TESTS = [{ 'url': 'http://bleacherreport.com/articles/2496438-fsu-stat-projections-is-jalen-ramsey-best-defensive-player-in-college-football', @@ -16,7 +18,7 @@ class BleacherReportIE(InfoExtractor): 'id': '2496438', 'ext': 'mp4', 'title': 'FSU Stat Projections: Is Jalen Ramsey Best Defensive Player in College Football?', - 'uploader_id': 3992341, + 'uploader_id': '3992341', 'description': 'CFB, ACC, Florida State', 'timestamp': 1434380212, 'upload_date': '20150615', @@ -33,7 +35,7 @@ class BleacherReportIE(InfoExtractor): 'timestamp': 1446839961, 'uploader': 'Sean Fay', 'description': 'md5:b1601e2314c4d8eec23b6eafe086a757', - 'uploader_id': 6466954, + 'uploader_id': '6466954', 'upload_date': '20151011', }, 'add_ie': ['Youtube'], @@ -58,7 +60,7 @@ def _real_extract(self, url): 'id': article_id, 'title': article_data['title'], 'uploader': article_data.get('author', {}).get('name'), - 'uploader_id': article_data.get('authorId'), + 'uploader_id': str_or_none(article_data.get('authorId')), 'timestamp': parse_iso8601(article_data.get('createdAt')), 'thumbnails': thumbnails, 'comment_count': int_or_none(article_data.get('commentsCount')), @@ -82,6 +84,7 @@ def _real_extract(self, url): class BleacherReportCMSIE(AMPIE): + _WORKING = False _VALID_URL = r'https?://(?:www\.)?bleacherreport\.com/video_embed\?id=(?P[0-9a-f-]{36}|\d{5})' _TESTS = [{ 'url': 'http://bleacherreport.com/video_embed?id=8fd44c2f-3dc5-4821-9118-2c825a98c0e1&library=video-cms', diff --git a/yt_dlp/extractor/cableav.py b/yt_dlp/extractor/cableav.py index 2e374e5eb..4a221414e 100644 --- a/yt_dlp/extractor/cableav.py +++ b/yt_dlp/extractor/cableav.py @@ -2,7 +2,7 @@ class CableAVIE(InfoExtractor): - _VALID_URL = r'https://cableav\.tv/(?P[a-zA-Z0-9]+)' + _VALID_URL = r'https?://cableav\.tv/(?P[a-zA-Z0-9]+)' _TESTS = [{ 'url': 'https://cableav.tv/lS4iR9lWjN8/', 'md5': '7e3fe5e49d61c4233b7f5b0f69b15e18', diff --git a/yt_dlp/extractor/camfm.py b/yt_dlp/extractor/camfm.py index a9850f46e..11dafa4a2 100644 --- a/yt_dlp/extractor/camfm.py +++ b/yt_dlp/extractor/camfm.py @@ -13,7 +13,7 @@ class CamFMShowIE(InfoExtractor): - _VALID_URL = r'https://(?:www\.)?camfm\.co\.uk/shows/(?P[^/]+)' + _VALID_URL = r'https?://(?:www\.)?camfm\.co\.uk/shows/(?P[^/]+)' _TESTS = [{ 'playlist_mincount': 5, 'url': 'https://camfm.co.uk/shows/soul-mining/', @@ -42,7 +42,7 @@ def _real_extract(self, url): class CamFMEpisodeIE(InfoExtractor): - _VALID_URL = r'https://(?:www\.)?camfm\.co\.uk/player/(?P[^/]+)' + _VALID_URL = r'https?://(?:www\.)?camfm\.co\.uk/player/(?P[^/]+)' _TESTS = [{ 'url': 'https://camfm.co.uk/player/43336', 'skip': 'Episode will expire - don\'t actually know when, but it will go eventually', diff --git a/yt_dlp/extractor/cbs.py b/yt_dlp/extractor/cbs.py index d97fbd758..cf830210f 100644 --- a/yt_dlp/extractor/cbs.py +++ b/yt_dlp/extractor/cbs.py @@ -76,6 +76,7 @@ def _real_extract(self, url): class CBSIE(CBSBaseIE): + _WORKING = False _VALID_URL = r'''(?x) (?: cbs:| diff --git a/yt_dlp/extractor/cbsinteractive.py b/yt_dlp/extractor/cbsinteractive.py deleted file mode 100644 index b09e9823e..000000000 --- a/yt_dlp/extractor/cbsinteractive.py +++ /dev/null @@ -1,98 +0,0 @@ -from .cbs import CBSIE -from ..utils import int_or_none - - -class CBSInteractiveIE(CBSIE): # XXX: Do not subclass from concrete IE - _VALID_URL = r'https?://(?:www\.)?(?Pcnet|zdnet)\.com/(?:videos|video(?:/share)?)/(?P[^/?]+)' - _TESTS = [{ - 'url': 'http://www.cnet.com/videos/hands-on-with-microsofts-windows-8-1-update/', - 'info_dict': { - 'id': 'R49SYt__yAfmlXR85z4f7gNmCBDcN_00', - 'display_id': 'hands-on-with-microsofts-windows-8-1-update', - 'ext': 'mp4', - 'title': 'Hands-on with Microsoft Windows 8.1 Update', - 'description': 'The new update to the Windows 8 OS brings improved performance for mouse and keyboard users.', - 'uploader_id': '6085384d-619e-11e3-b231-14feb5ca9861', - 'uploader': 'Sarah Mitroff', - 'duration': 70, - 'timestamp': 1396479627, - 'upload_date': '20140402', - }, - 'params': { - # m3u8 download - 'skip_download': True, - }, - }, { - 'url': 'http://www.cnet.com/videos/whiny-pothole-tweets-at-local-government-when-hit-by-cars-tomorrow-daily-187/', - 'md5': 'f11d27b2fa18597fbf92444d2a9ed386', - 'info_dict': { - 'id': 'kjOJd_OoVJqbg_ZD8MZCOk8Wekb9QccK', - 'display_id': 'whiny-pothole-tweets-at-local-government-when-hit-by-cars-tomorrow-daily-187', - 'ext': 'mp4', - 'title': 'Whiny potholes tweet at local government when hit by cars (Tomorrow Daily 187)', - 'description': 'md5:d2b9a95a5ffe978ae6fbd4cf944d618f', - 'uploader_id': 'b163284d-6b73-44fc-b3e6-3da66c392d40', - 'uploader': 'Ashley Esqueda', - 'duration': 1482, - 'timestamp': 1433289889, - 'upload_date': '20150603', - }, - }, { - 'url': 'http://www.zdnet.com/video/share/video-keeping-android-smartphones-and-tablets-secure/', - 'info_dict': { - 'id': 'k0r4T_ehht4xW_hAOqiVQPuBDPZ8SRjt', - 'display_id': 'video-keeping-android-smartphones-and-tablets-secure', - 'ext': 'mp4', - 'title': 'Video: Keeping Android smartphones and tablets secure', - 'description': 'Here\'s the best way to keep Android devices secure, and what you do when they\'ve come to the end of their lives.', - 'uploader_id': 'f2d97ea2-8175-11e2-9d12-0018fe8a00b0', - 'uploader': 'Adrian Kingsley-Hughes', - 'duration': 731, - 'timestamp': 1449129925, - 'upload_date': '20151203', - }, - 'params': { - # m3u8 download - 'skip_download': True, - }, - }, { - 'url': 'http://www.zdnet.com/video/huawei-matebook-x-video/', - 'only_matching': True, - }] - - MPX_ACCOUNTS = { - 'cnet': 2198311517, - 'zdnet': 2387448114, - } - - def _real_extract(self, url): - site, display_id = self._match_valid_url(url).groups() - webpage = self._download_webpage(url, display_id) - - data_json = self._html_search_regex( - r"data(?:-(?:cnet|zdnet))?-video(?:-(?:uvp(?:js)?|player))?-options='([^']+)'", - webpage, 'data json') - data = self._parse_json(data_json, display_id) - vdata = data.get('video') or (data.get('videos') or data.get('playlist'))[0] - - video_id = vdata['mpxRefId'] - - title = vdata['title'] - author = vdata.get('author') - if author: - uploader = '%s %s' % (author['firstName'], author['lastName']) - uploader_id = author.get('id') - else: - uploader = None - uploader_id = None - - info = self._extract_video_info(video_id, site, self.MPX_ACCOUNTS[site]) - info.update({ - 'id': video_id, - 'display_id': display_id, - 'title': title, - 'duration': int_or_none(vdata.get('duration')), - 'uploader': uploader, - 'uploader_id': uploader_id, - }) - return info diff --git a/yt_dlp/extractor/cbssports.py b/yt_dlp/extractor/cbssports.py index b5d85af12..b9c82dab6 100644 --- a/yt_dlp/extractor/cbssports.py +++ b/yt_dlp/extractor/cbssports.py @@ -8,6 +8,7 @@ # class CBSSportsEmbedIE(CBSBaseIE): class CBSSportsEmbedIE(InfoExtractor): + _WORKING = False IE_NAME = 'cbssports:embed' _VALID_URL = r'''(?ix)https?://(?:(?:www\.)?cbs|embed\.247)sports\.com/player/embed.+? (?: @@ -75,6 +76,7 @@ def _real_extract(self, url): class CBSSportsIE(CBSSportsBaseIE): + _WORKING = False IE_NAME = 'cbssports' _VALID_URL = r'https?://(?:www\.)?cbssports\.com/[^/]+/video/(?P[^/?#&]+)' _TESTS = [{ @@ -92,6 +94,7 @@ class CBSSportsIE(CBSSportsBaseIE): class TwentyFourSevenSportsIE(CBSSportsBaseIE): + _WORKING = False IE_NAME = '247sports' _VALID_URL = r'https?://(?:www\.)?247sports\.com/Video/(?:[^/?#&]+-)?(?P\d+)' _TESTS = [{ diff --git a/yt_dlp/extractor/cctv.py b/yt_dlp/extractor/cctv.py index 466bdfb7c..8552ee511 100644 --- a/yt_dlp/extractor/cctv.py +++ b/yt_dlp/extractor/cctv.py @@ -88,6 +88,20 @@ class CCTVIE(InfoExtractor): 'params': { 'skip_download': True, }, + }, { + # videoCenterId: "id" + 'url': 'http://news.cctv.com/2024/02/21/ARTIcU5tKIOIF2myEGCATkLo240221.shtml', + 'info_dict': { + 'id': '5c846c0518444308ba32c4159df3b3e0', + 'ext': 'mp4', + 'title': '《平“语”近人——习近平喜欢的典故》第三季 第5集：风物长宜放眼量', + 'uploader': 'yangjuan', + 'timestamp': 1708554940, + 'upload_date': '20240221', + }, + 'params': { + 'skip_download': True, + }, }, { # var ids = ["id"] 'url': 'http://www.ncpa-classic.com/clt/more/416/index.shtml', @@ -128,7 +142,7 @@ def _real_extract(self, url): video_id = self._search_regex( [r'var\s+guid\s*=\s*["\']([\da-fA-F]+)', - r'videoCenterId["\']\s*,\s*["\']([\da-fA-F]+)', + r'videoCenterId(?:["\']\s*,|:)\s*["\']([\da-fA-F]+)', r'changePlayer\s*\(\s*["\']([\da-fA-F]+)', r'load[Vv]ideo\s*\(\s*["\']([\da-fA-F]+)', r'var\s+initMyAray\s*=\s*["\']([\da-fA-F]+)', diff --git a/yt_dlp/extractor/ceskatelevize.py b/yt_dlp/extractor/ceskatelevize.py index 8390160a0..156b6a324 100644 --- a/yt_dlp/extractor/ceskatelevize.py +++ b/yt_dlp/extractor/ceskatelevize.py @@ -51,7 +51,7 @@ class CeskaTelevizeIE(InfoExtractor): 'url': 'http://www.ceskatelevize.cz/ivysilani/zive/ct4/', 'only_matching': True, 'info_dict': { - 'id': 402, + 'id': '402', 'ext': 'mp4', 'title': r're:^ČT Sport \d{4}-\d{2}-\d{2} \d{2}:\d{2}$', 'is_live': True, diff --git a/yt_dlp/extractor/cgtn.py b/yt_dlp/extractor/cgtn.py index aaafa02d1..5d9d9bcde 100644 --- a/yt_dlp/extractor/cgtn.py +++ b/yt_dlp/extractor/cgtn.py @@ -17,6 +17,7 @@ class CGTNIE(InfoExtractor): 'thumbnail': r're:^https?://.*\.jpg$', 'timestamp': 1615295940, 'upload_date': '20210309', + 'categories': ['Video'], }, 'params': { 'skip_download': True @@ -29,8 +30,8 @@ class CGTNIE(InfoExtractor): 'title': 'China, Indonesia vow to further deepen maritime cooperation', 'thumbnail': r're:^https?://.*\.png$', 'description': 'China and Indonesia vowed to upgrade their cooperation into the maritime sector and also for political security, economy, and cultural and people-to-people exchanges.', - 'author': 'CGTN', - 'category': 'China', + 'creators': ['CGTN'], + 'categories': ['China'], 'timestamp': 1622950200, 'upload_date': '20210606', }, @@ -45,7 +46,12 @@ def _real_extract(self, url): webpage = self._download_webpage(url, video_id) download_url = self._html_search_regex(r'data-video ="(?P.+m3u8)"', webpage, 'download_url') - datetime_str = self._html_search_regex(r'\s*(.+?)\s*', webpage, 'datetime_str', fatal=False) + datetime_str = self._html_search_regex( + r'\s*(.+?)\s*', webpage, 'datetime_str', fatal=False) + category = self._html_search_regex( + r'\s*(.+?)\s*', webpage, 'category', fatal=False) + author = self._search_regex( + r'
\s*(.+?)\s*
', webpage, 'author', default=None) return { 'id': video_id, @@ -53,9 +59,7 @@ def _real_extract(self, url): 'description': self._og_search_description(webpage, default=None), 'thumbnail': self._og_search_thumbnail(webpage), 'formats': self._extract_m3u8_formats(download_url, video_id, 'mp4', 'm3u8_native', m3u8_id='hls'), - 'category': self._html_search_regex(r'\s*(.+?)\s*', - webpage, 'category', fatal=False), - 'author': self._html_search_regex(r'
\s*(.+?)\s*
', - webpage, 'author', default=None, fatal=False), + 'categories': [category] if category else None, + 'creators': [author] if author else None, 'timestamp': try_get(unified_timestamp(datetime_str), lambda x: x - 8 * 3600), } diff --git a/yt_dlp/extractor/chingari.py b/yt_dlp/extractor/chingari.py deleted file mode 100644 index 48091dd65..000000000 --- a/yt_dlp/extractor/chingari.py +++ /dev/null @@ -1,207 +0,0 @@ -import itertools -import json -import urllib.parse - -from .common import InfoExtractor -from ..utils import ( - ExtractorError, - clean_html, - int_or_none, - str_to_int, - url_or_none, -) - - -class ChingariBaseIE(InfoExtractor): - def _get_post(self, id, post_data): - media_data = post_data['mediaLocation'] - base_url = media_data['base'] - author_data = post_data.get('authorData', {}) - song_data = post_data.get('song', {}) # revist this in future for differentiating b/w 'art' and 'author' - - formats = [{ - 'format_id': frmt, - 'width': str_to_int(frmt[1:]), - 'url': base_url + frmt_path, - } for frmt, frmt_path in media_data.get('transcoded', {}).items()] - - if media_data.get('path'): - formats.append({ - 'format_id': 'original', - 'format_note': 'Direct video.', - 'url': base_url + '/apipublic' + media_data['path'], - 'quality': 10, - }) - timestamp = str_to_int(post_data.get('created_at')) - if timestamp: - timestamp = int_or_none(timestamp, 1000) - - thumbnail, uploader_url = None, None - if media_data.get('thumbnail'): - thumbnail = base_url + media_data.get('thumbnail') - if author_data.get('username'): - uploader_url = 'https://chingari.io/' + author_data.get('username') - - return { - 'id': id, - 'extractor_key': ChingariIE.ie_key(), - 'extractor': 'Chingari', - 'title': urllib.parse.unquote_plus(clean_html(post_data.get('caption'))), - 'description': urllib.parse.unquote_plus(clean_html(post_data.get('caption'))), - 'duration': media_data.get('duration'), - 'thumbnail': url_or_none(thumbnail), - 'like_count': post_data.get('likeCount'), - 'view_count': post_data.get('viewsCount'), - 'comment_count': post_data.get('commentCount'), - 'repost_count': post_data.get('shareCount'), - 'timestamp': timestamp, - 'uploader_id': post_data.get('userId') or author_data.get('_id'), - 'uploader': author_data.get('name'), - 'uploader_url': url_or_none(uploader_url), - 'track': song_data.get('title'), - 'artist': song_data.get('author'), - 'formats': formats, - } - - -class ChingariIE(ChingariBaseIE): - _VALID_URL = r'https?://(?:www\.)?chingari\.io/share/post\?id=(?P[^&/#?]+)' - _TESTS = [{ - 'url': 'https://chingari.io/share/post?id=612f8f4ce1dc57090e8a7beb', - 'info_dict': { - 'id': '612f8f4ce1dc57090e8a7beb', - 'ext': 'mp4', - 'title': 'Happy birthday Srila Prabhupada', - 'description': 'md5:c7080ebfdfeb06016e638c286d6bc3fa', - 'duration': 0, - 'thumbnail': 'https://media.chingari.io/uploads/c41d30e2-06b6-4e3b-9b4b-edbb929cec06-1630506826911/thumbnail/198f993f-ce87-4623-82c6-cd071bd6d4f4-1630506828016.jpg', - 'like_count': int, - 'view_count': int, - 'comment_count': int, - 'repost_count': int, - 'timestamp': 1630506828, - 'upload_date': '20210901', - 'uploader_id': '5f0403982c8bd344f4813f8c', - 'uploader': 'ISKCON,Inc.', - 'uploader_url': 'https://chingari.io/iskcon,inc', - 'track': None, - 'artist': None, - }, - 'params': {'skip_download': True} - }] - - def _real_extract(self, url): - id = self._match_id(url) - post_json = self._download_json(f'https://api.chingari.io/post/post_details/{id}', id) - if post_json['code'] != 200: - raise ExtractorError(post_json['message'], expected=True) - post_data = post_json['data'] - return self._get_post(id, post_data) - - -class ChingariUserIE(ChingariBaseIE): - _VALID_URL = r'https?://(?:www\.)?chingari\.io/(?!share/post)(?P[^/?]+)' - _TESTS = [{ - 'url': 'https://chingari.io/dada1023', - 'info_dict': { - 'id': 'dada1023', - }, - 'params': {'playlistend': 3}, - 'playlist': [{ - 'url': 'https://chingari.io/share/post?id=614781f3ade60b3a0bfff42a', - 'info_dict': { - 'id': '614781f3ade60b3a0bfff42a', - 'ext': 'mp4', - 'title': '#chingaribappa ', - 'description': 'md5:d1df21d84088770468fa63afe3b17857', - 'duration': 7, - 'thumbnail': 'https://media.chingari.io/uploads/346d86d4-abb2-474e-a164-ffccf2bbcb72-1632076273717/thumbnail/b0b3aac2-2b86-4dd1-909d-9ed6e57cf77c-1632076275552.jpg', - 'like_count': int, - 'view_count': int, - 'comment_count': int, - 'repost_count': int, - 'timestamp': 1632076275, - 'upload_date': '20210919', - 'uploader_id': '5efc4b12cca35c3d1794c2d3', - 'uploader': 'dada (girish) dhawale', - 'uploader_url': 'https://chingari.io/dada1023', - 'track': None, - 'artist': None - }, - 'params': {'skip_download': True} - }, { - 'url': 'https://chingari.io/share/post?id=6146b132bcbf860959e12cba', - 'info_dict': { - 'id': '6146b132bcbf860959e12cba', - 'ext': 'mp4', - 'title': 'Tactor harvesting', - 'description': 'md5:8403f12dce68828b77ecee7eb7e887b7', - 'duration': 59.3, - 'thumbnail': 'https://media.chingari.io/uploads/b353ca70-7a87-400d-93a6-fa561afaec86-1632022814584/thumbnail/c09302e3-2043-41b1-a2fe-77d97e5bd676-1632022834260.jpg', - 'like_count': int, - 'view_count': int, - 'comment_count': int, - 'repost_count': int, - 'timestamp': 1632022834, - 'upload_date': '20210919', - 'uploader_id': '5efc4b12cca35c3d1794c2d3', - 'uploader': 'dada (girish) dhawale', - 'uploader_url': 'https://chingari.io/dada1023', - 'track': None, - 'artist': None - }, - 'params': {'skip_download': True} - }, { - 'url': 'https://chingari.io/share/post?id=6145651b74cb030a64c40b82', - 'info_dict': { - 'id': '6145651b74cb030a64c40b82', - 'ext': 'mp4', - 'title': '#odiabhajan ', - 'description': 'md5:687ea36835b9276cf2af90f25e7654cb', - 'duration': 56.67, - 'thumbnail': 'https://media.chingari.io/uploads/6cbf216b-babc-4cce-87fe-ceaac8d706ac-1631937782708/thumbnail/8855754f-6669-48ce-b269-8cc0699ed6da-1631937819522.jpg', - 'like_count': int, - 'view_count': int, - 'comment_count': int, - 'repost_count': int, - 'timestamp': 1631937819, - 'upload_date': '20210918', - 'uploader_id': '5efc4b12cca35c3d1794c2d3', - 'uploader': 'dada (girish) dhawale', - 'uploader_url': 'https://chingari.io/dada1023', - 'track': None, - 'artist': None - }, - 'params': {'skip_download': True} - }], - }, { - 'url': 'https://chingari.io/iskcon%2Cinc', - 'playlist_mincount': 1025, - 'info_dict': { - 'id': 'iskcon%2Cinc', - }, - }] - - def _entries(self, id): - skip = 0 - has_more = True - for page in itertools.count(): - posts = self._download_json('https://api.chingari.io/users/getPosts', id, - data=json.dumps({'userId': id, 'ownerId': id, 'skip': skip, 'limit': 20}).encode(), - headers={'content-type': 'application/json;charset=UTF-8'}, - note='Downloading page %s' % page) - for post in posts.get('data', []): - post_data = post['post'] - yield self._get_post(post_data['_id'], post_data) - skip += 20 - has_more = posts['hasMoreData'] - if not has_more: - break - - def _real_extract(self, url): - alt_id = self._match_id(url) - post_json = self._download_json(f'https://api.chingari.io/user/{alt_id}', alt_id) - if post_json['code'] != 200: - raise ExtractorError(post_json['message'], expected=True) - id = post_json['data']['_id'] - return self.playlist_result(self._entries(id), playlist_id=alt_id) diff --git a/yt_dlp/extractor/chzzk.py b/yt_dlp/extractor/chzzk.py index 6894baea5..420fe0514 100644 --- a/yt_dlp/extractor/chzzk.py +++ b/yt_dlp/extractor/chzzk.py @@ -2,7 +2,7 @@ from .common import InfoExtractor from ..utils import ( - ExtractorError, + UserNotLive, float_or_none, int_or_none, parse_iso8601, @@ -40,7 +40,7 @@ def _real_extract(self, url): note='Downloading channel info', errnote='Unable to download channel info')['content'] if live_detail.get('status') == 'CLOSE': - raise ExtractorError('The channel is not currently live', expected=True) + raise UserNotLive(video_id=channel_id) live_playback = self._parse_json(live_detail['livePlaybackJson'], channel_id) diff --git a/yt_dlp/extractor/cinemax.py b/yt_dlp/extractor/cinemax.py index 54cab2285..706ec8553 100644 --- a/yt_dlp/extractor/cinemax.py +++ b/yt_dlp/extractor/cinemax.py @@ -2,6 +2,7 @@ class CinemaxIE(HBOBaseIE): + _WORKING = False _VALID_URL = r'https?://(?:www\.)?cinemax\.com/(?P[^/]+/video/[0-9a-z-]+-(?P\d+))' _TESTS = [{ 'url': 'https://www.cinemax.com/warrior/video/s1-ep-1-recap-20126903', diff --git a/yt_dlp/extractor/cineverse.py b/yt_dlp/extractor/cineverse.py index 032c4334b..4405297c6 100644 --- a/yt_dlp/extractor/cineverse.py +++ b/yt_dlp/extractor/cineverse.py @@ -13,7 +13,7 @@ class CineverseBaseIE(InfoExtractor): - _VALID_URL_BASE = r'https://www\.(?P%s)' % '|'.join(map(re.escape, ( + _VALID_URL_BASE = r'https?://www\.(?P%s)' % '|'.join(map(re.escape, ( 'cineverse.com', 'asiancrush.com', 'dovechannel.com', diff --git a/yt_dlp/extractor/cliphunter.py b/yt_dlp/extractor/cliphunter.py deleted file mode 100644 index 2b907dc80..000000000 --- a/yt_dlp/extractor/cliphunter.py +++ /dev/null @@ -1,76 +0,0 @@ -from .common import InfoExtractor -from ..utils import ( - int_or_none, - url_or_none, -) - - -class CliphunterIE(InfoExtractor): - IE_NAME = 'cliphunter' - - _VALID_URL = r'''(?x)https?://(?:www\.)?cliphunter\.com/w/ - (?P[0-9]+)/ - (?P.+?)(?:$|[#\?]) - ''' - _TESTS = [{ - 'url': 'http://www.cliphunter.com/w/1012420/Fun_Jynx_Maze_solo', - 'md5': 'b7c9bbd4eb3a226ab91093714dcaa480', - 'info_dict': { - 'id': '1012420', - 'ext': 'flv', - 'title': 'Fun Jynx Maze solo', - 'thumbnail': r're:^https?://.*\.jpg$', - 'age_limit': 18, - }, - 'skip': 'Video gone', - }, { - 'url': 'http://www.cliphunter.com/w/2019449/ShesNew__My_booty_girlfriend_Victoria_Paradices_pussy_filled_with_jizz', - 'md5': '55a723c67bfc6da6b0cfa00d55da8a27', - 'info_dict': { - 'id': '2019449', - 'ext': 'mp4', - 'title': 'ShesNew - My booty girlfriend, Victoria Paradice\'s pussy filled with jizz', - 'thumbnail': r're:^https?://.*\.jpg$', - 'age_limit': 18, - }, - }] - - def _real_extract(self, url): - video_id = self._match_id(url) - webpage = self._download_webpage(url, video_id) - - video_title = self._search_regex( - r'mediaTitle = "([^"]+)"', webpage, 'title') - - gexo_files = self._parse_json( - self._search_regex( - r'var\s+gexoFiles\s*=\s*({.+?});', webpage, 'gexo files'), - video_id) - - formats = [] - for format_id, f in gexo_files.items(): - video_url = url_or_none(f.get('url')) - if not video_url: - continue - fmt = f.get('fmt') - height = f.get('h') - format_id = '%s_%sp' % (fmt, height) if fmt and height else format_id - formats.append({ - 'url': video_url, - 'format_id': format_id, - 'width': int_or_none(f.get('w')), - 'height': int_or_none(height), - 'tbr': int_or_none(f.get('br')), - }) - - thumbnail = self._search_regex( - r"var\s+mov_thumb\s*=\s*'([^']+)';", - webpage, 'thumbnail', fatal=False) - - return { - 'id': video_id, - 'title': video_title, - 'formats': formats, - 'age_limit': self._rta_search(webpage), - 'thumbnail': thumbnail, - } diff --git a/yt_dlp/extractor/cliprs.py b/yt_dlp/extractor/cliprs.py index 567f77b94..c2add02da 100644 --- a/yt_dlp/extractor/cliprs.py +++ b/yt_dlp/extractor/cliprs.py @@ -2,6 +2,7 @@ class ClipRsIE(OnetBaseIE): + _WORKING = False _VALID_URL = r'https?://(?:www\.)?clip\.rs/(?P[^/]+)/\d+' _TEST = { 'url': 'http://www.clip.rs/premijera-frajle-predstavljaju-novi-spot-za-pesmu-moli-me-moli/3732', diff --git a/yt_dlp/extractor/closertotruth.py b/yt_dlp/extractor/closertotruth.py index e78e26a11..1f9a5f611 100644 --- a/yt_dlp/extractor/closertotruth.py +++ b/yt_dlp/extractor/closertotruth.py @@ -4,6 +4,7 @@ class CloserToTruthIE(InfoExtractor): + _WORKING = False _VALID_URL = r'https?://(?:www\.)?closertotruth\.com/(?:[^/]+/)*(?P[^/?#&]+)' _TESTS = [{ 'url': 'http://closertotruth.com/series/solutions-the-mind-body-problem#video-3688', diff --git a/yt_dlp/extractor/cloudflarestream.py b/yt_dlp/extractor/cloudflarestream.py index c4c7d66a5..a812c24af 100644 --- a/yt_dlp/extractor/cloudflarestream.py +++ b/yt_dlp/extractor/cloudflarestream.py @@ -4,27 +4,25 @@ class CloudflareStreamIE(InfoExtractor): + _SUBDOMAIN_RE = r'(?:(?:watch|iframe|customer-\w+)\.)?' _DOMAIN_RE = r'(?:cloudflarestream\.com|(?:videodelivery|bytehighway)\.net)' - _EMBED_RE = r'embed\.%s/embed/[^/]+\.js\?.*?\bvideo=' % _DOMAIN_RE + _EMBED_RE = rf'embed\.{_DOMAIN_RE}/embed/[^/]+\.js\?.*?\bvideo=' _ID_RE = r'[\da-f]{32}|[\w-]+\.[\w-]+\.[\w-]+' - _VALID_URL = r'''(?x) - https?:// - (?: - (?:watch\.)?%s/| - %s - ) - (?P%s) - ''' % (_DOMAIN_RE, _EMBED_RE, _ID_RE) - _EMBED_REGEX = [fr']+\bsrc=(["\'])(?P(?:https?:)?//{_EMBED_RE}(?:{_ID_RE}).*?)\1'] + _VALID_URL = rf'https?://(?:{_SUBDOMAIN_RE}{_DOMAIN_RE}/|{_EMBED_RE})(?P{_ID_RE})' + _EMBED_REGEX = [ + rf']+\bsrc=(["\'])(?P(?:https?:)?//{_EMBED_RE}(?:{_ID_RE}).*?)\1', + rf']+\bsrc=["\'](?Phttps?://{_SUBDOMAIN_RE}{_DOMAIN_RE}/[\da-f]{{32}})', + ] _TESTS = [{ 'url': 'https://embed.cloudflarestream.com/embed/we4g.fla9.latest.js?video=31c9291ab41fac05471db4e73aa11717', 'info_dict': { 'id': '31c9291ab41fac05471db4e73aa11717', 'ext': 'mp4', 'title': '31c9291ab41fac05471db4e73aa11717', + 'thumbnail': 'https://videodelivery.net/31c9291ab41fac05471db4e73aa11717/thumbnails/thumbnail.jpg', }, 'params': { - 'skip_download': True, + 'skip_download': 'm3u8', }, }, { 'url': 'https://watch.cloudflarestream.com/9df17203414fd1db3e3ed74abbe936c1', @@ -35,6 +33,21 @@ class CloudflareStreamIE(InfoExtractor): }, { 'url': 'https://embed.videodelivery.net/embed/r4xu.fla9.latest.js?video=81d80727f3022488598f68d323c1ad5e', 'only_matching': True, + }, { + 'url': 'https://customer-aw5py76sw8wyqzmh.cloudflarestream.com/2463f6d3e06fa29710a337f5f5389fd8/iframe', + 'only_matching': True, + }] + _WEBPAGE_TESTS = [{ + 'url': 'https://upride.cc/incident/shoulder-pass-at-light/', + 'info_dict': { + 'id': 'eaef9dea5159cf968be84241b5cedfe7', + 'ext': 'mp4', + 'title': 'eaef9dea5159cf968be84241b5cedfe7', + 'thumbnail': 'https://videodelivery.net/eaef9dea5159cf968be84241b5cedfe7/thumbnails/thumbnail.jpg', + }, + 'params': { + 'skip_download': 'm3u8', + }, }] def _real_extract(self, url): diff --git a/yt_dlp/extractor/cnbc.py b/yt_dlp/extractor/cnbc.py index 7d209b6d9..cedfd3ef9 100644 --- a/yt_dlp/extractor/cnbc.py +++ b/yt_dlp/extractor/cnbc.py @@ -1,68 +1,97 @@ from .common import InfoExtractor -from ..utils import smuggle_url - - -class CNBCIE(InfoExtractor): - _VALID_URL = r'https?://video\.cnbc\.com/gallery/\?video=(?P[0-9]+)' - _TEST = { - 'url': 'http://video.cnbc.com/gallery/?video=3000503714', - 'info_dict': { - 'id': '3000503714', - 'ext': 'mp4', - 'title': 'Fighting zombies is big business', - 'description': 'md5:0c100d8e1a7947bd2feec9a5550e519e', - 'timestamp': 1459332000, - 'upload_date': '20160330', - 'uploader': 'NBCU-CNBC', - }, - 'params': { - # m3u8 download - 'skip_download': True, - }, - 'skip': 'Dead link', - } - - def _real_extract(self, url): - video_id = self._match_id(url) - return { - '_type': 'url_transparent', - 'ie_key': 'ThePlatform', - 'url': smuggle_url( - 'http://link.theplatform.com/s/gZWlPC/media/guid/2408950221/%s?mbr=true&manifest=m3u' % video_id, - {'force_smil_url': True}), - 'id': video_id, - } +from ..utils import int_or_none, parse_iso8601, str_or_none, url_or_none +from ..utils.traversal import traverse_obj class CNBCVideoIE(InfoExtractor): - _VALID_URL = r'https?://(?:www\.)?cnbc\.com(?P/video/(?:[^/]+/)+(?P[^./?#&]+)\.html)' - _TEST = { - 'url': 'https://www.cnbc.com/video/2018/07/19/trump-i-dont-necessarily-agree-with-raising-rates.html', + _VALID_URL = r'https?://(?:www\.)?cnbc\.com/video/(?:[^/?#]+/)+(?P[^./?#&]+)\.html' + + _TESTS = [{ + 'url': 'https://www.cnbc.com/video/2023/12/07/mcdonalds-just-unveiled-cosmcsits-new-spinoff-brand.html', 'info_dict': { - 'id': '7000031301', 'ext': 'mp4', - 'title': "Trump: I don't necessarily agree with raising rates", - 'description': 'md5:878d8f0b4ebb5bb1dda3514b91b49de3', - 'timestamp': 1531958400, - 'upload_date': '20180719', - 'uploader': 'NBCU-CNBC', + 'id': '107344774', + 'display_id': 'mcdonalds-just-unveiled-cosmcsits-new-spinoff-brand', + 'modified_timestamp': 1702053483, + 'timestamp': 1701977810, + 'channel': 'News Videos', + 'upload_date': '20231207', + 'description': 'md5:882c001d85cb43d7579b514307b3e78b', + 'release_timestamp': 1701977375, + 'modified_date': '20231208', + 'release_date': '20231207', + 'duration': 65, + 'creators': ['Sean Conlon'], + 'title': 'Here\'s a first look at McDonald\'s new spinoff brand, CosMc\'s', + 'thumbnail': 'https://image.cnbcfm.com/api/v1/image/107344192-1701894812493-CosMcsskyHero_2336x1040_hero-desktop.jpg?v=1701894855', }, - 'params': { - 'skip_download': True, + 'expected_warnings': ['Unable to download f4m manifest'], + }, { + 'url': 'https://www.cnbc.com/video/2023/12/08/jim-cramer-shares-his-take-on-seattles-tech-scene.html', + 'info_dict': { + 'creators': ['Jim Cramer'], + 'channel': 'Mad Money with Jim Cramer', + 'description': 'md5:72925be21b952e95eba51178dddf4e3e', + 'duration': 299.0, + 'ext': 'mp4', + 'id': '107345451', + 'display_id': 'jim-cramer-shares-his-take-on-seattles-tech-scene', + 'thumbnail': 'https://image.cnbcfm.com/api/v1/image/107345481-1702079431MM-B-120823.jpg?v=1702079430', + 'timestamp': 1702080139, + 'title': 'Jim Cramer shares his take on Seattle\'s tech scene', + 'release_date': '20231208', + 'upload_date': '20231209', + 'modified_timestamp': 1702080139, + 'modified_date': '20231209', + 'release_timestamp': 1702073551, }, - 'skip': 'Dead link', - } + 'expected_warnings': ['Unable to download f4m manifest'], + }, { + 'url': 'https://www.cnbc.com/video/2023/12/08/the-epicenter-of-ai-is-in-seattle-says-jim-cramer.html', + 'info_dict': { + 'creators': ['Jim Cramer'], + 'channel': 'Mad Money with Jim Cramer', + 'description': 'md5:72925be21b952e95eba51178dddf4e3e', + 'duration': 113.0, + 'ext': 'mp4', + 'id': '107345474', + 'display_id': 'the-epicenter-of-ai-is-in-seattle-says-jim-cramer', + 'thumbnail': 'https://image.cnbcfm.com/api/v1/image/107345486-Screenshot_2023-12-08_at_70339_PM.png?v=1702080248', + 'timestamp': 1702080535, + 'title': 'The epicenter of AI is in Seattle, says Jim Cramer', + 'release_timestamp': 1702077347, + 'modified_timestamp': 1702080535, + 'release_date': '20231208', + 'upload_date': '20231209', + 'modified_date': '20231209', + }, + 'expected_warnings': ['Unable to download f4m manifest'], + }] def _real_extract(self, url): - path, display_id = self._match_valid_url(url).groups() - video_id = self._download_json( - 'https://webql-redesign.cnbcfm.com/graphql', display_id, query={ - 'query': '''{ - page(path: "%s") { - vcpsId - } -}''' % path, - })['data']['page']['vcpsId'] - return self.url_result( - 'http://video.cnbc.com/gallery/?video=%d' % video_id, - CNBCIE.ie_key()) + display_id = self._match_id(url) + webpage = self._download_webpage(url, display_id) + data = self._search_json(r'window\.__s_data=', webpage, 'video data', display_id) + + player_data = traverse_obj(data, ( + 'page', 'page', 'layout', ..., 'columns', ..., 'modules', + lambda _, v: v['name'] == 'clipPlayer', 'data', {dict}), get_all=False) + + return { + 'id': display_id, + 'display_id': display_id, + 'formats': self._extract_akamai_formats(player_data['playbackURL'], display_id), + **self._search_json_ld(webpage, display_id, fatal=False), + **traverse_obj(player_data, { + 'id': ('id', {str_or_none}), + 'title': ('title', {str}), + 'description': ('description', {str}), + 'creators': ('author', ..., 'name', {str}), + 'timestamp': ('datePublished', {parse_iso8601}), + 'release_timestamp': ('uploadDate', {parse_iso8601}), + 'modified_timestamp': ('dateLastPublished', {parse_iso8601}), + 'thumbnail': ('thumbnail', {url_or_none}), + 'duration': ('duration', {int_or_none}), + 'channel': ('section', 'title', {str}), + }), + } diff --git a/yt_dlp/extractor/common.py b/yt_dlp/extractor/common.py index af534775f..e776ccae9 100644 --- a/yt_dlp/extractor/common.py +++ b/yt_dlp/extractor/common.py @@ -247,6 +247,8 @@ class InfoExtractor: (For internal use only) * http_chunk_size Chunk size for HTTP downloads * ffmpeg_args Extra arguments for ffmpeg downloader + * is_dash_periods Whether the format is a result of merging + multiple DASH periods. RTMP formats can also have the additional fields: page_url, app, play_path, tc_url, flash_version, rtmp_live, rtmp_conn, rtmp_protocol, rtmp_real_time @@ -260,7 +262,7 @@ class InfoExtractor: direct: True if a direct video file was given (must only be set by GenericIE) alt_title: A secondary title of the video. - display_id An alternative identifier for the video, not necessarily + display_id: An alternative identifier for the video, not necessarily unique, but available before title. Typically, id is something like "4234987", title "Dancing naked mole rats", and display_id "dancing-naked-mole-rats" @@ -278,7 +280,7 @@ class InfoExtractor: description: Full video description. uploader: Full name of the video uploader. license: License name the video is licensed under. - creator: The creator of the video. + creators: List of creators of the video. timestamp: UNIX timestamp of the moment the video was uploaded upload_date: Video upload date in UTC (YYYYMMDD). If not explicitly set, calculated from timestamp @@ -422,16 +424,16 @@ class InfoExtractor: track_number: Number of the track within an album or a disc, as an integer. track_id: Id of the track (useful in case of custom indexing, e.g. 6.iii), as a unicode string. - artist: Artist(s) of the track. - genre: Genre(s) of the track. + artists: List of artists of the track. + composers: List of composers of the piece. + genres: List of genres of the track. album: Title of the album the track belongs to. album_type: Type of the album (e.g. "Demo", "Full-length", "Split", "Compilation", etc). - album_artist: List of all artists appeared on the album (e.g. - "Ash Borer / Fell Voices" or "Various Artists", useful for splits - and compilations). + album_artists: List of all artists appeared on the album. + E.g. ["Ash Borer", "Fell Voices"] or ["Various Artists"]. + Useful for splits and compilations. disc_number: Number of the disc or other physical medium the track belongs to, as an integer. - composer: Composer of the piece The following fields should only be set for clips that should be cut from the original video: @@ -442,6 +444,18 @@ class InfoExtractor: rows: Number of rows in each storyboard fragment, as an integer columns: Number of columns in each storyboard fragment, as an integer + The following fields are deprecated and should not be set by new code: + composer: Use "composers" instead. + Composer(s) of the piece, comma-separated. + artist: Use "artists" instead. + Artist(s) of the track, comma-separated. + genre: Use "genres" instead. + Genre(s) of the track, comma-separated. + album_artist: Use "album_artists" instead. + All artists appeared on the album, comma-separated. + creator: Use "creators" instead. + The creator of the video. + Unless mentioned otherwise, the fields should be Unicode strings. Unless mentioned otherwise, None is equivalent to absence of information. @@ -733,7 +747,7 @@ def extract(self, url): raise except ExtractorError as e: e.video_id = e.video_id or self.get_temp_id(url) - e.ie = e.ie or self.IE_NAME, + e.ie = e.ie or self.IE_NAME e.traceback = e.traceback or sys.exc_info()[2] raise except IncompleteRead as e: @@ -1325,7 +1339,10 @@ def _get_netrc_login_info(self, netrc_machine=None): else: return None, None if not info: - raise netrc.NetrcParseError(f'No authenticators for {netrc_machine}') + self.to_screen(f'No authenticators for {netrc_machine}') + return None, None + + self.write_debug(f'Using netrc for {netrc_machine} authentication') return info[0], info[2] def _get_login_info(self, username_option='username', password_option='password', netrc_machine=None): @@ -2530,7 +2547,11 @@ def _extract_mpd_formats(self, *args, **kwargs): self._report_ignoring_subs('DASH') return fmts - def _extract_mpd_formats_and_subtitles( + def _extract_mpd_formats_and_subtitles(self, *args, **kwargs): + periods = self._extract_mpd_periods(*args, **kwargs) + return self._merge_mpd_periods(periods) + + def _extract_mpd_periods( self, mpd_url, video_id, mpd_id=None, note=None, errnote=None, fatal=True, data=None, headers={}, query={}): @@ -2543,17 +2564,16 @@ def _extract_mpd_formats_and_subtitles( errnote='Failed to download MPD manifest' if errnote is None else errnote, fatal=fatal, data=data, headers=headers, query=query) if res is False: - return [], {} + return [] mpd_doc, urlh = res if mpd_doc is None: - return [], {} + return [] # We could have been redirected to a new url when we retrieved our mpd file. mpd_url = urlh.url mpd_base_url = base_url(mpd_url) - return self._parse_mpd_formats_and_subtitles( - mpd_doc, mpd_id, mpd_base_url, mpd_url) + return self._parse_mpd_periods(mpd_doc, mpd_id, mpd_base_url, mpd_url) def _parse_mpd_formats(self, *args, **kwargs): fmts, subs = self._parse_mpd_formats_and_subtitles(*args, **kwargs) @@ -2561,8 +2581,39 @@ def _parse_mpd_formats(self, *args, **kwargs): self._report_ignoring_subs('DASH') return fmts - def _parse_mpd_formats_and_subtitles( - self, mpd_doc, mpd_id=None, mpd_base_url='', mpd_url=None): + def _parse_mpd_formats_and_subtitles(self, *args, **kwargs): + periods = self._parse_mpd_periods(*args, **kwargs) + return self._merge_mpd_periods(periods) + + def _merge_mpd_periods(self, periods): + """ + Combine all formats and subtitles from an MPD manifest into a single list, + by concatenate streams with similar formats. + """ + formats, subtitles = {}, {} + for period in periods: + for f in period['formats']: + assert 'is_dash_periods' not in f, 'format already processed' + f['is_dash_periods'] = True + format_key = tuple(v for k, v in f.items() if k not in ( + ('format_id', 'fragments', 'manifest_stream_number'))) + if format_key not in formats: + formats[format_key] = f + elif 'fragments' in f: + formats[format_key].setdefault('fragments', []).extend(f['fragments']) + + if subtitles and period['subtitles']: + self.report_warning(bug_reports_message( + 'Found subtitles in multiple periods in the DASH manifest; ' + 'if part of the subtitles are missing,' + ), only_once=True) + + for sub_lang, sub_info in period['subtitles'].items(): + subtitles.setdefault(sub_lang, []).extend(sub_info) + + return list(formats.values()), subtitles + + def _parse_mpd_periods(self, mpd_doc, mpd_id=None, mpd_base_url='', mpd_url=None): """ Parse formats from MPD manifest. References: @@ -2641,9 +2692,13 @@ def extract_Initialization(source): return ms_info mpd_duration = parse_duration(mpd_doc.get('mediaPresentationDuration')) - formats, subtitles = [], {} stream_numbers = collections.defaultdict(int) - for period in mpd_doc.findall(_add_ns('Period')): + for period_idx, period in enumerate(mpd_doc.findall(_add_ns('Period'))): + period_entry = { + 'id': period.get('id', f'period-{period_idx}'), + 'formats': [], + 'subtitles': collections.defaultdict(list), + } period_duration = parse_duration(period.get('duration')) or mpd_duration period_ms_info = extract_multisegment_info(period, { 'start_number': 1, @@ -2893,11 +2948,10 @@ def add_segment_url(): if content_type in ('video', 'audio', 'image/jpeg'): f['manifest_stream_number'] = stream_numbers[f['url']] stream_numbers[f['url']] += 1 - formats.append(f) + period_entry['formats'].append(f) elif content_type == 'text': - subtitles.setdefault(lang or 'und', []).append(f) - - return formats, subtitles + period_entry['subtitles'][lang or 'und'].append(f) + yield period_entry def _extract_ism_formats(self, *args, **kwargs): fmts, subs = self._extract_ism_formats_and_subtitles(*args, **kwargs) diff --git a/yt_dlp/extractor/cpac.py b/yt_dlp/extractor/cpac.py index 0f23f2be2..32bba1e5a 100644 --- a/yt_dlp/extractor/cpac.py +++ b/yt_dlp/extractor/cpac.py @@ -65,7 +65,7 @@ def is_live(v_type): 'title': title, 'description': str_or_none(content['details'].get('description_%s_t' % (url_lang, ))), 'timestamp': unified_timestamp(content['details'].get('liveDateTime')), - 'category': [category] if category else None, + 'categories': [category] if category else None, 'thumbnail': urljoin(url, str_or_none(content['details'].get('image_%s_s' % (url_lang, )))), 'is_live': is_live(content['details'].get('type')), } diff --git a/yt_dlp/extractor/craftsy.py b/yt_dlp/extractor/craftsy.py index 5d3733143..3a05ed48a 100644 --- a/yt_dlp/extractor/craftsy.py +++ b/yt_dlp/extractor/craftsy.py @@ -1,12 +1,13 @@ +import json + from .brightcove import BrightcoveNewIE from .common import InfoExtractor - from ..utils import ( - dict_get, - get_element_by_id, - js_to_json, - traverse_obj, + extract_attributes, + get_element_html_by_class, + get_element_text_and_html_by_tag, ) +from ..utils.traversal import traverse_obj class CraftsyIE(InfoExtractor): @@ -41,28 +42,34 @@ def _real_extract(self, url): video_id = self._match_id(url) webpage = self._download_webpage(url, video_id) - video_data = self._parse_json(self._search_regex( - r'class_video_player_vars\s*=\s*({.*})\s*;', - get_element_by_id('vidstore-classes_class-video-player-js-extra', webpage), - 'video data'), video_id, transform_source=js_to_json) + video_player = get_element_html_by_class('class-video-player', webpage) + video_data = traverse_obj(video_player, ( + {extract_attributes}, 'wire:snapshot', {json.loads}, 'data', {dict})) or {} + video_js = traverse_obj(video_player, ( + {lambda x: get_element_text_and_html_by_tag('video-js', x)}, 1, {extract_attributes})) or {} - account_id = traverse_obj(video_data, ('video_player', 'bc_account_id')) + has_access = video_data.get('userHasAccess') + lessons = traverse_obj(video_data, ('lessons', ..., ..., lambda _, v: v['video_id'])) - entries = [] - class_preview = traverse_obj(video_data, ('video_player', 'class_preview')) - if class_preview: - v_id = class_preview.get('video_id') - entries.append(self.url_result( - f'http://players.brightcove.net/{account_id}/default_default/index.html?videoId={v_id}', - BrightcoveNewIE, v_id, class_preview.get('title'))) + preview_id = video_js.get('data-video-id') + if preview_id and preview_id not in traverse_obj(lessons, (..., 'video_id')): + if not lessons and not has_access: + self.report_warning( + 'Only extracting preview. For the full class, pass cookies ' + + f'from an account that has access. {self._login_hint()}') + lessons.append({'video_id': preview_id}) - if dict_get(video_data, ('is_free', 'user_has_access')): - entries += [ - self.url_result( + if not lessons and not has_access: + self.raise_login_required('You do not have access to this class') + + account_id = video_data.get('accountId') or video_js['data-account'] + + def entries(lessons): + for lesson in lessons: + yield self.url_result( f'http://players.brightcove.net/{account_id}/default_default/index.html?videoId={lesson["video_id"]}', BrightcoveNewIE, lesson['video_id'], lesson.get('title')) - for lesson in video_data['lessons']] return self.playlist_result( - entries, video_id, video_data.get('class_title'), + entries(lessons), video_id, self._html_search_meta(('og:title', 'twitter:title'), webpage), self._html_search_meta(('og:description', 'description'), webpage, default=None)) diff --git a/yt_dlp/extractor/crtvg.py b/yt_dlp/extractor/crtvg.py index 1aa8d7705..21325e331 100644 --- a/yt_dlp/extractor/crtvg.py +++ b/yt_dlp/extractor/crtvg.py @@ -1,18 +1,32 @@ +import re + from .common import InfoExtractor -from ..utils import remove_end +from ..utils import make_archive_id, remove_end class CrtvgIE(InfoExtractor): - _VALID_URL = r'https?://(?:www\.)?crtvg\.es/tvg/a-carta/[^/#?]+-(?P\d+)' + _VALID_URL = r'https?://(?:www\.)?crtvg\.es/tvg/a-carta/(?P[^/#?]+)' _TESTS = [{ 'url': 'https://www.crtvg.es/tvg/a-carta/os-caimans-do-tea-5839623', 'md5': 'c0958d9ff90e4503a75544358758921d', 'info_dict': { - 'id': '5839623', + 'id': 'os-caimans-do-tea-5839623', 'title': 'Os caimáns do Tea', 'ext': 'mp4', 'description': 'md5:f71cfba21ae564f0a6f415b31de1f842', 'thumbnail': r're:^https?://.*\.(?:jpg|png)', + '_old_archive_ids': ['crtvg 5839623'], + }, + 'params': {'skip_download': 'm3u8'} + }, { + 'url': 'https://www.crtvg.es/tvg/a-carta/a-parabolica-love-story', + 'md5': '9a47b95a1749db7b7eb3214904624584', + 'info_dict': { + 'id': 'a-parabolica-love-story', + 'title': 'A parabólica / Trabuco, o can mordedor / Love Story', + 'ext': 'mp4', + 'description': 'md5:f71cfba21ae564f0a6f415b31de1f842', + 'thumbnail': r're:^https?://.*\.(?:jpg|png)', }, 'params': {'skip_download': 'm3u8'} }] @@ -24,8 +38,13 @@ def _real_extract(self, url): formats = self._extract_m3u8_formats(video_url + '/playlist.m3u8', video_id, fatal=False) formats.extend(self._extract_mpd_formats(video_url + '/manifest.mpd', video_id, fatal=False)) + old_video_id = None + if mobj := re.fullmatch(r'[^/#?]+-(?P\d{7})', video_id): + old_video_id = [make_archive_id(self, mobj.group('old_id'))] + return { 'id': video_id, + '_old_archive_ids': old_video_id, 'formats': formats, 'title': remove_end(self._html_search_meta( ['og:title', 'twitter:title'], webpage, 'title', default=None), ' | CRTVG'), diff --git a/yt_dlp/extractor/crunchyroll.py b/yt_dlp/extractor/crunchyroll.py index ee34aced5..d35e9995a 100644 --- a/yt_dlp/extractor/crunchyroll.py +++ b/yt_dlp/extractor/crunchyroll.py @@ -136,7 +136,7 @@ def _call_api(self, path, internal_id, lang, note='api', query={}): return result def _extract_formats(self, stream_response, display_id=None): - requested_formats = self._configuration_arg('format') or ['adaptive_hls'] + requested_formats = self._configuration_arg('format') or ['vo_adaptive_hls'] available_formats = {} for stream_type, streams in traverse_obj( stream_response, (('streams', ('data', 0)), {dict.items}, ...)): @@ -514,7 +514,7 @@ class CrunchyrollMusicIE(CrunchyrollBaseIE): 'track': 'Egaono Hana', 'artist': 'Goose house', 'thumbnail': r're:(?i)^https://www.crunchyroll.com/imgsrv/.*\.jpeg?$', - 'genre': ['J-Pop'], + 'genres': ['J-Pop'], }, 'params': {'skip_download': 'm3u8'}, }, { @@ -527,7 +527,7 @@ class CrunchyrollMusicIE(CrunchyrollBaseIE): 'track': 'Crossing Field', 'artist': 'LiSA', 'thumbnail': r're:(?i)^https://www.crunchyroll.com/imgsrv/.*\.jpeg?$', - 'genre': ['Anime'], + 'genres': ['Anime'], }, 'params': {'skip_download': 'm3u8'}, }, { @@ -541,7 +541,7 @@ class CrunchyrollMusicIE(CrunchyrollBaseIE): 'artist': 'LiSA', 'thumbnail': r're:(?i)^https://www.crunchyroll.com/imgsrv/.*\.jpeg?$', 'description': 'md5:747444e7e6300907b7a43f0a0503072e', - 'genre': ['J-Pop'], + 'genres': ['J-Pop'], }, 'params': {'skip_download': 'm3u8'}, }, { @@ -594,7 +594,7 @@ def _transform_music_response(data): 'width': ('width', {int_or_none}), 'height': ('height', {int_or_none}), }), - 'genre': ('genres', ..., 'displayValue'), + 'genres': ('genres', ..., 'displayValue'), 'age_limit': ('maturity_ratings', -1, {parse_age_limit}), }), } @@ -611,7 +611,7 @@ class CrunchyrollArtistIE(CrunchyrollBaseIE): 'info_dict': { 'id': 'MA179CB50D', 'title': 'LiSA', - 'genre': ['J-Pop', 'Anime', 'Rock'], + 'genres': ['J-Pop', 'Anime', 'Rock'], 'description': 'md5:16d87de61a55c3f7d6c454b73285938e', }, 'playlist_mincount': 83, @@ -645,6 +645,6 @@ def _transform_artist_response(data): 'width': ('width', {int_or_none}), 'height': ('height', {int_or_none}), }), - 'genre': ('genres', ..., 'displayValue'), + 'genres': ('genres', ..., 'displayValue'), }), } diff --git a/yt_dlp/extractor/cybrary.py b/yt_dlp/extractor/cybrary.py index c4c78ee1b..c6995b25b 100644 --- a/yt_dlp/extractor/cybrary.py +++ b/yt_dlp/extractor/cybrary.py @@ -110,11 +110,11 @@ def _real_extract(self, url): class CybraryCourseIE(CybraryBaseIE): - _VALID_URL = r'https://app\.cybrary\.it/browse/course/(?P[\w-]+)/?(?:$|[#?])' + _VALID_URL = r'https?://app\.cybrary\.it/browse/course/(?P[\w-]+)/?(?:$|[#?])' _TESTS = [{ 'url': 'https://app.cybrary.it/browse/course/az-500-microsoft-azure-security-technologies', 'info_dict': { - 'id': 898, + 'id': '898', 'title': 'AZ-500: Microsoft Azure Security Technologies', 'description': 'md5:69549d379c0fc1dec92926d4e8b6fbd4' }, @@ -122,7 +122,7 @@ class CybraryCourseIE(CybraryBaseIE): }, { 'url': 'https://app.cybrary.it/browse/course/cybrary-orientation', 'info_dict': { - 'id': 1245, + 'id': '1245', 'title': 'Cybrary Orientation', 'description': 'md5:9e69ff66b32fe78744e0ad4babe2e88e' }, diff --git a/yt_dlp/extractor/dailymotion.py b/yt_dlp/extractor/dailymotion.py index 708d6fed2..c570a4f52 100644 --- a/yt_dlp/extractor/dailymotion.py +++ b/yt_dlp/extractor/dailymotion.py @@ -1,6 +1,7 @@ import functools import json import re +import urllib.parse from .common import InfoExtractor from ..networking.exceptions import HTTPError @@ -44,36 +45,41 @@ def _real_initialize(self): self._FAMILY_FILTER = ff == 'on' if ff else age_restricted(18, self.get_param('age_limit')) self._set_dailymotion_cookie('ff', 'on' if self._FAMILY_FILTER else 'off') + def _get_token(self, xid): + cookies = self._get_dailymotion_cookies() + token = self._get_cookie_value(cookies, 'access_token') or self._get_cookie_value(cookies, 'client_token') + if token: + return token + + data = { + 'client_id': 'f1a362d288c1b98099c7', + 'client_secret': 'eea605b96e01c796ff369935357eca920c5da4c5', + } + username, password = self._get_login_info() + if username: + data.update({ + 'grant_type': 'password', + 'password': password, + 'username': username, + }) + else: + data['grant_type'] = 'client_credentials' + try: + token = self._download_json( + 'https://graphql.api.dailymotion.com/oauth/token', + None, 'Downloading Access Token', + data=urlencode_postdata(data))['access_token'] + except ExtractorError as e: + if isinstance(e.cause, HTTPError) and e.cause.status == 400: + raise ExtractorError(self._parse_json( + e.cause.response.read().decode(), xid)['error_description'], expected=True) + raise + self._set_dailymotion_cookie('access_token' if username else 'client_token', token) + return token + def _call_api(self, object_type, xid, object_fields, note, filter_extra=None): if not self._HEADERS.get('Authorization'): - cookies = self._get_dailymotion_cookies() - token = self._get_cookie_value(cookies, 'access_token') or self._get_cookie_value(cookies, 'client_token') - if not token: - data = { - 'client_id': 'f1a362d288c1b98099c7', - 'client_secret': 'eea605b96e01c796ff369935357eca920c5da4c5', - } - username, password = self._get_login_info() - if username: - data.update({ - 'grant_type': 'password', - 'password': password, - 'username': username, - }) - else: - data['grant_type'] = 'client_credentials' - try: - token = self._download_json( - 'https://graphql.api.dailymotion.com/oauth/token', - None, 'Downloading Access Token', - data=urlencode_postdata(data))['access_token'] - except ExtractorError as e: - if isinstance(e.cause, HTTPError) and e.cause.status == 400: - raise ExtractorError(self._parse_json( - e.cause.response.read().decode(), xid)['error_description'], expected=True) - raise - self._set_dailymotion_cookie('access_token' if username else 'client_token', token) - self._HEADERS['Authorization'] = 'Bearer ' + token + self._HEADERS['Authorization'] = f'Bearer {self._get_token(xid)}' resp = self._download_json( 'https://graphql.api.dailymotion.com/', xid, note, data=json.dumps({ @@ -393,9 +399,55 @@ def _extract_embed_urls(cls, url, webpage): yield '//dailymotion.com/playlist/%s' % p +class DailymotionSearchIE(DailymotionPlaylistBaseIE): + IE_NAME = 'dailymotion:search' + _VALID_URL = r'https?://(?:www\.)?dailymotion\.[a-z]{2,3}/search/(?P[^/?#]+)/videos' + _PAGE_SIZE = 20 + _TESTS = [{ + 'url': 'http://www.dailymotion.com/search/king of turtles/videos', + 'info_dict': { + 'id': 'king of turtles', + 'title': 'king of turtles', + }, + 'playlist_mincount': 90, + }] + _SEARCH_QUERY = 'query SEARCH_QUERY( $query: String! $page: Int $limit: Int ) { search { videos( query: $query first: $limit page: $page ) { edges { node { xid } } } } } ' + + def _call_search_api(self, term, page, note): + if not self._HEADERS.get('Authorization'): + self._HEADERS['Authorization'] = f'Bearer {self._get_token(term)}' + resp = self._download_json( + 'https://graphql.api.dailymotion.com/', None, note, data=json.dumps({ + 'operationName': 'SEARCH_QUERY', + 'query': self._SEARCH_QUERY, + 'variables': { + 'limit': 20, + 'page': page, + 'query': term, + } + }).encode(), headers=self._HEADERS) + obj = traverse_obj(resp, ('data', 'search', {dict})) + if not obj: + raise ExtractorError( + traverse_obj(resp, ('errors', 0, 'message', {str})) or 'Could not fetch search data') + + return obj + + def _fetch_page(self, term, page): + page += 1 + response = self._call_search_api(term, page, f'Searching "{term}" page {page}') + for xid in traverse_obj(response, ('videos', 'edges', ..., 'node', 'xid')): + yield self.url_result(f'https://www.dailymotion.com/video/{xid}', DailymotionIE, xid) + + def _real_extract(self, url): + term = urllib.parse.unquote_plus(self._match_id(url)) + return self.playlist_result( + OnDemandPagedList(functools.partial(self._fetch_page, term), self._PAGE_SIZE), term, term) + + class DailymotionUserIE(DailymotionPlaylistBaseIE): IE_NAME = 'dailymotion:user' - _VALID_URL = r'https?://(?:www\.)?dailymotion\.[a-z]{2,3}/(?!(?:embed|swf|#|video|playlist)/)(?:(?:old/)?user/)?(?P[^/]+)' + _VALID_URL = r'https?://(?:www\.)?dailymotion\.[a-z]{2,3}/(?!(?:embed|swf|#|video|playlist|search)/)(?:(?:old/)?user/)?(?P[^/?#]+)' _TESTS = [{ 'url': 'https://www.dailymotion.com/user/nqtv', 'info_dict': { diff --git a/yt_dlp/extractor/damtomo.py b/yt_dlp/extractor/damtomo.py index 0e08e4f65..5e14d6aff 100644 --- a/yt_dlp/extractor/damtomo.py +++ b/yt_dlp/extractor/damtomo.py @@ -83,7 +83,6 @@ class DamtomoRecordIE(DamtomoBaseIE): 'info_dict': { 'id': '27376862', 'title': 'イカSUMMER [良音]', - 'description': None, 'uploader': 'ＮＡＮＡ', 'uploader_id': 'MzAyMDExNTY', 'upload_date': '20210721', diff --git a/yt_dlp/extractor/daum.py b/yt_dlp/extractor/daum.py index 3ef514065..24c520855 100644 --- a/yt_dlp/extractor/daum.py +++ b/yt_dlp/extractor/daum.py @@ -27,7 +27,7 @@ class DaumIE(DaumBaseIE): 'duration': 2117, 'view_count': int, 'comment_count': int, - 'uploader_id': 186139, + 'uploader_id': '186139', 'uploader': '콘간지', 'timestamp': 1387310323, }, @@ -44,7 +44,7 @@ class DaumIE(DaumBaseIE): 'view_count': int, 'comment_count': int, 'uploader': 'MBC 예능', - 'uploader_id': 132251, + 'uploader_id': '132251', 'timestamp': 1421604228, }, }, { @@ -63,7 +63,7 @@ class DaumIE(DaumBaseIE): 'view_count': int, 'comment_count': int, 'uploader': '까칠한 墮落始祖 황비홍님의', - 'uploader_id': 560824, + 'uploader_id': '560824', 'timestamp': 1203770745, }, }, { @@ -77,7 +77,7 @@ class DaumIE(DaumBaseIE): 'description': '러블리즈 - Destiny (나의 지구) (Lovelyz - Destiny)\r\n\r\n[쇼! 음악중심] 20160611, 507회', 'upload_date': '20170129', 'uploader': '쇼! 음악중심', - 'uploader_id': 2653210, + 'uploader_id': '2653210', 'timestamp': 1485684628, }, }] @@ -107,7 +107,7 @@ class DaumClipIE(DaumBaseIE): 'duration': 3868, 'view_count': int, 'uploader': 'GOMeXP', - 'uploader_id': 6667, + 'uploader_id': '6667', 'timestamp': 1377911092, }, }, { diff --git a/yt_dlp/extractor/digg.py b/yt_dlp/extractor/digg.py deleted file mode 100644 index 86e8a6fac..000000000 --- a/yt_dlp/extractor/digg.py +++ /dev/null @@ -1,54 +0,0 @@ -from .common import InfoExtractor -from ..utils import js_to_json - - -class DiggIE(InfoExtractor): - _VALID_URL = r'https?://(?:www\.)?digg\.com/video/(?P[^/?#&]+)' - _TESTS = [{ - # JWPlatform via provider - 'url': 'http://digg.com/video/sci-fi-short-jonah-daniel-kaluuya-get-out', - 'info_dict': { - 'id': 'LcqvmS0b', - 'ext': 'mp4', - 'title': "'Get Out' Star Daniel Kaluuya Goes On 'Moby Dick'-Like Journey In Sci-Fi Short 'Jonah'", - 'description': 'md5:541bb847648b6ee3d6514bc84b82efda', - 'upload_date': '20180109', - 'timestamp': 1515530551, - }, - 'params': { - 'skip_download': True, - }, - }, { - # Youtube via provider - 'url': 'http://digg.com/video/dog-boat-seal-play', - 'only_matching': True, - }, { - # vimeo as regular embed - 'url': 'http://digg.com/video/dream-girl-short-film', - 'only_matching': True, - }] - - def _real_extract(self, url): - display_id = self._match_id(url) - - webpage = self._download_webpage(url, display_id) - - info = self._parse_json( - self._search_regex( - r'(?s)video_info\s*=\s*({.+?});\n', webpage, 'video info', - default='{}'), display_id, transform_source=js_to_json, - fatal=False) - - video_id = info.get('video_id') - - if video_id: - provider = info.get('provider_name') - if provider == 'youtube': - return self.url_result( - video_id, ie='Youtube', video_id=video_id) - elif provider == 'jwplayer': - return self.url_result( - 'jwplatform:%s' % video_id, ie='JWPlatform', - video_id=video_id) - - return self.url_result(url, 'Generic') diff --git a/yt_dlp/extractor/dtube.py b/yt_dlp/extractor/dtube.py index 25a98f625..bb06c42be 100644 --- a/yt_dlp/extractor/dtube.py +++ b/yt_dlp/extractor/dtube.py @@ -9,6 +9,7 @@ class DTubeIE(InfoExtractor): + _WORKING = False _VALID_URL = r'https?://(?:www\.)?d\.tube/(?:#!/)?v/(?P[0-9a-z.-]+)/(?P[0-9a-z]{8})' _TEST = { 'url': 'https://d.tube/#!/v/broncnutz/x380jtr1', diff --git a/yt_dlp/extractor/duboku.py b/yt_dlp/extractor/duboku.py index fc9564cef..626e577e7 100644 --- a/yt_dlp/extractor/duboku.py +++ b/yt_dlp/extractor/duboku.py @@ -1,4 +1,6 @@ +import base64 import re +import urllib.parse from .common import InfoExtractor from ..compat import compat_urlparse @@ -129,11 +131,15 @@ def _real_extract(self, url): data_url = player_data.get('url') if not data_url: raise ExtractorError('Cannot find url in player_data') - data_from = player_data.get('from') + player_encrypt = player_data.get('encrypt') + if player_encrypt == 1: + data_url = urllib.parse.unquote(data_url) + elif player_encrypt == 2: + data_url = urllib.parse.unquote(base64.b64decode(data_url).decode('ascii')) # if it is an embedded iframe, maybe it's an external source headers = {'Referer': webpage_url} - if data_from == 'iframe': + if player_data.get('from') == 'iframe': # use _type url_transparent to retain the meaningful details # of the video. return { diff --git a/yt_dlp/extractor/dumpert.py b/yt_dlp/extractor/dumpert.py index 0cf84263c..5e7aef0c5 100644 --- a/yt_dlp/extractor/dumpert.py +++ b/yt_dlp/extractor/dumpert.py @@ -8,9 +8,9 @@ class DumpertIE(InfoExtractor): _VALID_URL = r'''(?x) - (?Phttps?)://(?:(?:www|legacy)\.)?dumpert\.nl(?: - /(?:mediabase|embed|item)/| - (?:/toppers|/latest|/?)\?selectedId= + (?Phttps?)://(?:(?:www|legacy)\.)?dumpert\.nl/(?: + (?:mediabase|embed|item)/| + [^#]*[?&]selectedId= )(?P[0-9]+[/_][0-9a-zA-Z]+)''' _TESTS = [{ 'url': 'https://www.dumpert.nl/item/6646981_951bc60f', @@ -56,6 +56,9 @@ class DumpertIE(InfoExtractor): }, { 'url': 'https://www.dumpert.nl/?selectedId=100031688_b317a185', 'only_matching': True, + }, { + 'url': 'https://www.dumpert.nl/toppers/dag?selectedId=100086074_f5cef3ac', + 'only_matching': True, }] def _real_extract(self, url): diff --git a/yt_dlp/extractor/duoplay.py b/yt_dlp/extractor/duoplay.py index 7d3f39942..18642fea3 100644 --- a/yt_dlp/extractor/duoplay.py +++ b/yt_dlp/extractor/duoplay.py @@ -13,7 +13,7 @@ class DuoplayIE(InfoExtractor): - _VALID_URL = r'https://duoplay\.ee/(?P\d+)/[\w-]+/?(?:\?(?:[^#]+&)?ep=(?P\d+))?' + _VALID_URL = r'https?://duoplay\.ee/(?P\d+)/[\w-]+/?(?:\?(?:[^#]+&)?ep=(?P\d+))?' _TESTS = [{ 'note': 'Siberi võmm S02E12', 'url': 'https://duoplay.ee/4312/siberi-vomm?ep=24', @@ -32,7 +32,7 @@ class DuoplayIE(InfoExtractor): 'season_number': 2, 'episode': 'Operatsioon "Öö"', 'episode_number': 12, - 'episode_id': 24, + 'episode_id': '24', }, }, { 'note': 'Empty title', @@ -50,7 +50,7 @@ class DuoplayIE(InfoExtractor): 'series_id': '17', 'season': 'Season 2', 'season_number': 2, - 'episode_id': 14, + 'episode_id': '14', 'release_year': 2010, }, }, { @@ -99,6 +99,6 @@ def _real_extract(self, url): 'season_number': ('season_id', {int_or_none}), 'episode': 'subtitle', 'episode_number': ('episode_nr', {int_or_none}), - 'episode_id': ('episode_id', {int_or_none}), + 'episode_id': ('episode_id', {str_or_none}), }, get_all=False) if episode_attr.get('category') != 'movies' else {}), } diff --git a/yt_dlp/extractor/dw.py b/yt_dlp/extractor/dw.py index 9c4a08e54..f7b852076 100644 --- a/yt_dlp/extractor/dw.py +++ b/yt_dlp/extractor/dw.py @@ -8,6 +8,8 @@ class DWIE(InfoExtractor): + _WORKING = False + _ENABLED = None # XXX: pass through to GenericIE IE_NAME = 'dw' _VALID_URL = r'https?://(?:www\.)?dw\.com/(?:[^/]+/)+(?:av|e)-(?P\d+)' _TESTS = [{ @@ -82,6 +84,8 @@ def _real_extract(self, url): class DWArticleIE(InfoExtractor): + _WORKING = False + _ENABLED = None # XXX: pass through to GenericIE IE_NAME = 'dw:article' _VALID_URL = r'https?://(?:www\.)?dw\.com/(?:[^/]+/)+a-(?P\d+)' _TEST = { diff --git a/yt_dlp/extractor/egghead.py b/yt_dlp/extractor/egghead.py index a4b2a12f6..c94f3f81f 100644 --- a/yt_dlp/extractor/egghead.py +++ b/yt_dlp/extractor/egghead.py @@ -19,7 +19,7 @@ def _call_api(self, path, video_id, resource, fatal=True): class EggheadCourseIE(EggheadBaseIE): IE_DESC = 'egghead.io course' IE_NAME = 'egghead:course' - _VALID_URL = r'https://(?:app\.)?egghead\.io/(?:course|playlist)s/(?P[^/?#&]+)' + _VALID_URL = r'https?://(?:app\.)?egghead\.io/(?:course|playlist)s/(?P[^/?#&]+)' _TESTS = [{ 'url': 'https://egghead.io/courses/professor-frisby-introduces-composable-functional-javascript', 'playlist_count': 29, @@ -65,7 +65,7 @@ def _real_extract(self, url): class EggheadLessonIE(EggheadBaseIE): IE_DESC = 'egghead.io lesson' IE_NAME = 'egghead:lesson' - _VALID_URL = r'https://(?:app\.)?egghead\.io/(?:api/v1/)?lessons/(?P[^/?#&]+)' + _VALID_URL = r'https?://(?:app\.)?egghead\.io/(?:api/v1/)?lessons/(?P[^/?#&]+)' _TESTS = [{ 'url': 'https://egghead.io/lessons/javascript-linear-data-flow-with-container-style-types-box', 'info_dict': { diff --git a/yt_dlp/extractor/eplus.py b/yt_dlp/extractor/eplus.py index 6383691a1..88a8d5a94 100644 --- a/yt_dlp/extractor/eplus.py +++ b/yt_dlp/extractor/eplus.py @@ -42,7 +42,6 @@ class EplusIbIE(InfoExtractor): 'live_status': 'was_live', 'release_date': '20210719', 'release_timestamp': 1626703200, - 'description': None, }, 'params': { 'skip_download': True, diff --git a/yt_dlp/extractor/err.py b/yt_dlp/extractor/err.py index 129f39ad6..abd00f2d5 100644 --- a/yt_dlp/extractor/err.py +++ b/yt_dlp/extractor/err.py @@ -9,7 +9,7 @@ class ERRJupiterIE(InfoExtractor): - _VALID_URL = r'https?://jupiter(?:pluss)?\.err\.ee/(?P\d+)' + _VALID_URL = r'https?://(?:jupiter(?:pluss)?|lasteekraan)\.err\.ee/(?P\d+)' _TESTS = [{ 'note': 'Jupiter: Movie: siin-me-oleme', 'url': 'https://jupiter.err.ee/1211107/siin-me-oleme', @@ -145,6 +145,31 @@ class ERRJupiterIE(InfoExtractor): 'season_number': 0, 'series': 'Лесные истории | Аисты', 'series_id': '1037497', + } + }, { + 'note': 'Lasteekraan: Pätu', + 'url': 'https://lasteekraan.err.ee/1092243/patu', + 'md5': 'a67eb9b9bcb3d201718c15d1638edf77', + 'info_dict': { + 'id': '1092243', + 'ext': 'mp4', + 'title': 'Pätu', + 'alt_title': '', + 'description': 'md5:64a7b5a80afd7042d3f8ec48c77befd9', + 'release_date': '20230614', + 'upload_date': '20200520', + 'modified_date': '20200520', + 'release_timestamp': 1686745800, + 'timestamp': 1589975640, + 'modified_timestamp': 1589975640, + 'release_year': 1990, + 'episode': 'Episode 1', + 'episode_id': '1092243', + 'episode_number': 1, + 'season': 'Season 1', + 'season_number': 1, + 'series': 'Pätu', + 'series_id': '1092236', }, }] diff --git a/yt_dlp/extractor/europa.py b/yt_dlp/extractor/europa.py index f3da95f5c..191a4361a 100644 --- a/yt_dlp/extractor/europa.py +++ b/yt_dlp/extractor/europa.py @@ -13,6 +13,7 @@ class EuropaIE(InfoExtractor): + _WORKING = False _VALID_URL = r'https?://ec\.europa\.eu/avservices/(?:video/player|audio/audioDetails)\.cfm\?.*?\bref=(?P[A-Za-z0-9-]+)' _TESTS = [{ 'url': 'http://ec.europa.eu/avservices/video/player.cfm?ref=I107758', diff --git a/yt_dlp/extractor/facebook.py b/yt_dlp/extractor/facebook.py index 830bbcc3c..834b1df18 100644 --- a/yt_dlp/extractor/facebook.py +++ b/yt_dlp/extractor/facebook.py @@ -500,6 +500,7 @@ def extract_metadata(webpage): webpage, 'description', default=None) uploader_data = ( get_first(media, ('owner', {dict})) + or get_first(post, ('video', 'creation_story', 'attachments', ..., 'media', lambda k, v: k == 'owner' and v['name'])) or get_first(post, (..., 'video', lambda k, v: k == 'owner' and v['name'])) or get_first(post, ('node', 'actors', ..., {dict})) or get_first(post, ('event', 'event_creator', {dict})) or {}) @@ -583,8 +584,8 @@ def extract_relay_data(_filter): def extract_relay_prefetched_data(_filter): return traverse_obj(extract_relay_data(_filter), ( 'require', (None, (..., ..., ..., '__bbox', 'require')), - lambda _, v: 'RelayPrefetchedStreamCache' in v, ..., ..., - '__bbox', 'result', 'data', {dict}), get_all=False) or {} + lambda _, v: any(key.startswith('RelayPrefetchedStreamCache') for key in v), + ..., ..., '__bbox', 'result', 'data', {dict}), get_all=False) or {} if not video_data: server_js_data = self._parse_json(self._search_regex([ diff --git a/yt_dlp/extractor/fancode.py b/yt_dlp/extractor/fancode.py index 1b5db818a..cddf25497 100644 --- a/yt_dlp/extractor/fancode.py +++ b/yt_dlp/extractor/fancode.py @@ -10,6 +10,7 @@ class FancodeVodIE(InfoExtractor): + _WORKING = False IE_NAME = 'fancode:vod' _VALID_URL = r'https?://(?:www\.)?fancode\.com/video/(?P[0-9]+)\b' @@ -126,6 +127,7 @@ def _real_extract(self, url): class FancodeLiveIE(FancodeVodIE): # XXX: Do not subclass from concrete IE + _WORKING = False IE_NAME = 'fancode:live' _VALID_URL = r'https?://(www\.)?fancode\.com/match/(?P[0-9]+).+' diff --git a/yt_dlp/extractor/filmmodu.py b/yt_dlp/extractor/filmmodu.py deleted file mode 100644 index 1e793560d..000000000 --- a/yt_dlp/extractor/filmmodu.py +++ /dev/null @@ -1,69 +0,0 @@ -from .common import InfoExtractor -from ..utils import int_or_none - - -class FilmmoduIE(InfoExtractor): - _VALID_URL = r'https?://(?:www\.)?filmmodu\.org/(?P[^/]+-(?:turkce-dublaj-izle|altyazili-izle))' - _TESTS = [{ - 'url': 'https://www.filmmodu.org/f9-altyazili-izle', - 'md5': 'aeefd955c2a508a5bdaa3bcec8eeb0d4', - 'info_dict': { - 'id': '10804', - 'ext': 'mp4', - 'title': 'F9', - 'description': 'md5:2713f584a4d65afa2611e2948d0b953c', - 'subtitles': { - 'tr': [{ - 'ext': 'vtt', - }], - }, - 'thumbnail': r're:https://s[0-9]+.filmmodu.org/uploads/movie/cover/10804/xXHZeb1yhJvnSHPzZDqee0zfMb6.jpg', - }, - }, { - 'url': 'https://www.filmmodu.org/the-godfather-turkce-dublaj-izle', - 'md5': '109f2fcb9c941330eed133971c035c00', - 'info_dict': { - 'id': '3646', - 'ext': 'mp4', - 'title': 'Baba', - 'description': 'md5:d43fd651937cd75cc650883ebd8d8461', - 'thumbnail': r're:https://s[0-9]+.filmmodu.org/uploads/movie/cover/3646/6xKCYgH16UuwEGAyroLU6p8HLIn.jpg', - }, - }] - - def _real_extract(self, url): - video_id = self._match_id(url) - webpage = self._download_webpage(url, video_id) - title = self._og_search_title(webpage, fatal=True) - description = self._og_search_description(webpage) - thumbnail = self._og_search_thumbnail(webpage) - real_video_id = self._search_regex(r'var\s*videoId\s*=\s*\'([0-9]+)\'', webpage, 'video_id') - video_type = self._search_regex(r'var\s*videoType\s*=\s*\'([a-z]+)\'', webpage, 'video_type') - data = self._download_json('https://www.filmmodu.org/get-source', real_video_id, query={ - 'movie_id': real_video_id, - 'type': video_type, - }) - formats = [{ - 'url': source['src'], - 'ext': 'mp4', - 'format_id': source['label'], - 'height': int_or_none(source.get('res')), - 'protocol': 'm3u8_native', - } for source in data['sources']] - - subtitles = {} - - if data.get('subtitle'): - subtitles['tr'] = [{ - 'url': data['subtitle'], - }] - - return { - 'id': real_video_id, - 'display_id': video_id, - 'title': title, - 'description': description, - 'formats': formats, - 'subtitles': subtitles, - 'thumbnail': thumbnail, - } diff --git a/yt_dlp/extractor/flextv.py b/yt_dlp/extractor/flextv.py new file mode 100644 index 000000000..f3d3eff85 --- /dev/null +++ b/yt_dlp/extractor/flextv.py @@ -0,0 +1,62 @@ +from .common import InfoExtractor +from ..networking.exceptions import HTTPError +from ..utils import ( + ExtractorError, + UserNotLive, + parse_iso8601, + str_or_none, + traverse_obj, + url_or_none, +) + + +class FlexTVIE(InfoExtractor): + _VALID_URL = r'https?://(?:www\.)?flextv\.co\.kr/channels/(?P\d+)/live' + _TESTS = [{ + 'url': 'https://www.flextv.co.kr/channels/231638/live', + 'info_dict': { + 'id': '231638', + 'ext': 'mp4', + 'title': r're:^214하나만\.\.\. ', + 'thumbnail': r're:^https?://.+\.jpg', + 'upload_date': r're:\d{8}', + 'timestamp': int, + 'live_status': 'is_live', + 'channel': 'Hi별', + 'channel_id': '244396', + }, + 'skip': 'The channel is offline', + }, { + 'url': 'https://www.flextv.co.kr/channels/746/live', + 'only_matching': True, + }] + + def _real_extract(self, url): + channel_id = self._match_id(url) + + try: + stream_data = self._download_json( + f'https://api.flextv.co.kr/api/channels/{channel_id}/stream', + channel_id, query={'option': 'all'}) + except ExtractorError as e: + if isinstance(e.cause, HTTPError) and e.cause.status == 400: + raise UserNotLive(video_id=channel_id) + raise + + playlist_url = stream_data['sources'][0]['url'] + formats, subtitles = self._extract_m3u8_formats_and_subtitles( + playlist_url, channel_id, 'mp4') + + return { + 'id': channel_id, + 'formats': formats, + 'subtitles': subtitles, + 'is_live': True, + **traverse_obj(stream_data, { + 'title': ('stream', 'title', {str}), + 'timestamp': ('stream', 'createdAt', {parse_iso8601}), + 'thumbnail': ('thumbUrl', {url_or_none}), + 'channel': ('owner', 'name', {str}), + 'channel_id': ('owner', 'id', {str_or_none}), + }), + } diff --git a/yt_dlp/extractor/francetv.py b/yt_dlp/extractor/francetv.py index 0ceecde74..7b8f7dd04 100644 --- a/yt_dlp/extractor/francetv.py +++ b/yt_dlp/extractor/francetv.py @@ -1,60 +1,49 @@ +import re +import urllib.parse + from .common import InfoExtractor from .dailymotion import DailymotionIE +from ..networking import HEADRequest from ..utils import ( - ExtractorError, determine_ext, + filter_dict, format_field, int_or_none, join_nonempty, parse_iso8601, - parse_qs, + smuggle_url, + unsmuggle_url, + url_or_none, ) +from ..utils.traversal import traverse_obj class FranceTVBaseInfoExtractor(InfoExtractor): - def _make_url_result(self, video_or_full_id, catalog=None): - full_id = 'francetv:%s' % video_or_full_id - if '@' not in video_or_full_id and catalog: - full_id += '@%s' % catalog - return self.url_result( - full_id, ie=FranceTVIE.ie_key(), - video_id=video_or_full_id.split('@')[0]) + def _make_url_result(self, video_id, url=None): + video_id = video_id.split('@')[0] # for compat with old @catalog IDs + full_id = f'francetv:{video_id}' + if url: + full_id = smuggle_url(full_id, {'hostname': urllib.parse.urlparse(url).hostname}) + return self.url_result(full_id, FranceTVIE, video_id) class FranceTVIE(InfoExtractor): - _VALID_URL = r'''(?x) - (?: - https?:// - sivideo\.webservices\.francetelevisions\.fr/tools/getInfosOeuvre/v2/\? - .*?\bidDiffusion=[^&]+| - (?: - https?://videos\.francetv\.fr/video/| - francetv: - ) - (?P[^@]+)(?:@(?P.+))? - ) - ''' - _EMBED_REGEX = [r']+?src=(["\'])(?P(?:https?://)?embed\.francetv\.fr/\?ue=.+?)\1'] + _VALID_URL = r'francetv:(?P[^@#]+)' + _GEO_COUNTRIES = ['FR'] + _GEO_BYPASS = False _TESTS = [{ - # without catalog - 'url': 'https://sivideo.webservices.francetelevisions.fr/tools/getInfosOeuvre/v2/?idDiffusion=162311093&callback=_jsonp_loader_callback_request_0', - 'md5': 'c2248a8de38c4e65ea8fae7b5df2d84f', + 'url': 'francetv:ec217ecc-0733-48cf-ac06-af1347b849d1', 'info_dict': { - 'id': '162311093', + 'id': 'ec217ecc-0733-48cf-ac06-af1347b849d1', 'ext': 'mp4', 'title': '13h15, le dimanche... - Les mystères de Jésus', - 'description': 'md5:75efe8d4c0a8205e5904498ffe1e1a42', 'timestamp': 1502623500, + 'duration': 2580, + 'thumbnail': r're:^https?://.*\.jpg$', 'upload_date': '20170813', }, - }, { - # with catalog - 'url': 'https://sivideo.webservices.francetelevisions.fr/tools/getInfosOeuvre/v2/?idDiffusion=NI_1004933&catalogue=Zouzous&callback=_jsonp_loader_callback_request_4', - 'only_matching': True, - }, { - 'url': 'http://videos.francetv.fr/video/NI_657393@Regions', - 'only_matching': True, + 'params': {'skip_download': 'm3u8'}, }, { 'url': 'francetv:162311093', 'only_matching': True, @@ -76,10 +65,7 @@ class FranceTVIE(InfoExtractor): 'only_matching': True, }] - def _extract_video(self, video_id, catalogue=None): - # Videos are identified by idDiffusion so catalogue part is optional. - # However when provided, some extra formats may be returned so we pass - # it if available. + def _extract_video(self, video_id, hostname=None): is_live = None videos = [] title = None @@ -91,18 +77,20 @@ def _extract_video(self, video_id, catalogue=None): timestamp = None spritesheets = None - for device_type in ('desktop', 'mobile'): + # desktop+chrome returns dash; mobile+safari returns hls + for device_type, browser in [('desktop', 'chrome'), ('mobile', 'safari')]: dinfo = self._download_json( - 'https://player.webservices.francetelevisions.fr/v1/videos/%s' % video_id, - video_id, 'Downloading %s video JSON' % device_type, query={ + f'https://k7.ftven.fr/videos/{video_id}', video_id, + f'Downloading {device_type} {browser} video JSON', query=filter_dict({ 'device_type': device_type, - 'browser': 'chrome', - }, fatal=False) + 'browser': browser, + 'domain': hostname, + }), fatal=False) if not dinfo: continue - video = dinfo.get('video') + video = traverse_obj(dinfo, ('video', {dict})) if video: videos.append(video) if duration is None: @@ -112,7 +100,7 @@ def _extract_video(self, video_id, catalogue=None): if spritesheets is None: spritesheets = video.get('spritesheets') - meta = dinfo.get('meta') + meta = traverse_obj(dinfo, ('meta', {dict})) if meta: if title is None: title = meta.get('title') @@ -126,43 +114,46 @@ def _extract_video(self, video_id, catalogue=None): if timestamp is None: timestamp = parse_iso8601(meta.get('broadcasted_at')) - formats = [] - subtitles = {} - for video in videos: + formats, subtitles, video_url = [], {}, None + for video in traverse_obj(videos, lambda _, v: url_or_none(v['url'])): + video_url = video['url'] format_id = video.get('format') - video_url = None - if video.get('workflow') == 'token-akamai': - token_url = video.get('token') - if token_url: - token_json = self._download_json( - token_url, video_id, - 'Downloading signed %s manifest URL' % format_id) - if token_json: - video_url = token_json.get('url') - if not video_url: - video_url = video.get('url') + if token_url := url_or_none(video.get('token')): + tokenized_url = traverse_obj(self._download_json( + token_url, video_id, f'Downloading signed {format_id} manifest URL', + fatal=False, query={ + 'format': 'json', + 'url': video_url, + }), ('url', {url_or_none})) + if tokenized_url: + video_url = tokenized_url ext = determine_ext(video_url) if ext == 'f4m': formats.extend(self._extract_f4m_formats( - video_url, video_id, f4m_id=format_id, fatal=False)) + video_url, video_id, f4m_id=format_id or ext, fatal=False)) elif ext == 'm3u8': + format_id = format_id or 'hls' fmts, subs = self._extract_m3u8_formats_and_subtitles( - video_url, video_id, 'mp4', - entry_protocol='m3u8_native', m3u8_id=format_id, - fatal=False) + video_url, video_id, 'mp4', m3u8_id=format_id, fatal=False) + for f in traverse_obj(fmts, lambda _, v: v['vcodec'] == 'none' and v.get('tbr') is None): + if mobj := re.match(rf'{format_id}-[Aa]udio-\w+-(?P\d+)', f['format_id']): + f.update({ + 'tbr': int_or_none(mobj.group('bitrate')), + 'acodec': 'mp4a', + }) formats.extend(fmts) self._merge_subtitles(subs, target=subtitles) elif ext == 'mpd': fmts, subs = self._extract_mpd_formats_and_subtitles( - video_url, video_id, mpd_id=format_id, fatal=False) + video_url, video_id, mpd_id=format_id or 'dash', fatal=False) formats.extend(fmts) self._merge_subtitles(subs, target=subtitles) elif video_url.startswith('rtmp'): formats.append({ 'url': video_url, - 'format_id': 'rtmp-%s' % format_id, + 'format_id': join_nonempty('rtmp', format_id), 'ext': 'flv', }) else: @@ -174,6 +165,13 @@ def _extract_video(self, video_id, catalogue=None): # XXX: what is video['captions']? + if not formats and video_url: + urlh = self._request_webpage( + HEADRequest(video_url), video_id, 'Checking for geo-restriction', + fatal=False, expected_status=403) + if urlh and urlh.headers.get('x-errortype') == 'geo': + self.raise_geo_restricted(countries=self._GEO_COUNTRIES, metadata_available=True) + for f in formats: if f.get('acodec') != 'none' and f.get('language') in ('qtz', 'qad'): f['language_preference'] = -10 @@ -194,7 +192,7 @@ def _extract_video(self, video_id, catalogue=None): # a 10×10 grid of thumbnails corresponding to approximately # 2 seconds of the video; the last spritesheet may be shorter 'duration': 200, - } for sheet in spritesheets] + } for sheet in traverse_obj(spritesheets, (..., {url_or_none}))] }) return { @@ -210,21 +208,15 @@ def _extract_video(self, video_id, catalogue=None): 'series': title if episode_number else None, 'episode_number': int_or_none(episode_number), 'season_number': int_or_none(season_number), + '_format_sort_fields': ('res', 'tbr', 'proto'), # prioritize m3u8 over dash } def _real_extract(self, url): - mobj = self._match_valid_url(url) - video_id = mobj.group('id') - catalog = mobj.group('catalog') + url, smuggled_data = unsmuggle_url(url, {}) + video_id = self._match_id(url) + hostname = smuggled_data.get('hostname') or 'www.france.tv' - if not video_id: - qs = parse_qs(url) - video_id = qs.get('idDiffusion', [None])[0] - catalog = qs.get('catalogue', [None])[0] - if not video_id: - raise ExtractorError('Invalid URL', expected=True) - - return self._extract_video(video_id, catalog) + return self._extract_video(video_id, hostname=hostname) class FranceTVSiteIE(FranceTVBaseInfoExtractor): @@ -246,6 +238,7 @@ class FranceTVSiteIE(FranceTVBaseInfoExtractor): }, 'add_ie': [FranceTVIE.ie_key()], }, { + # geo-restricted 'url': 'https://www.france.tv/enfants/six-huit-ans/foot2rue/saison-1/3066387-duel-au-vieux-port.html', 'info_dict': { 'id': 'a9050959-eedd-4b4a-9b0d-de6eeaa73e44', @@ -261,6 +254,26 @@ class FranceTVSiteIE(FranceTVBaseInfoExtractor): 'thumbnail': r're:^https?://.*\.jpg$', 'duration': 1441, }, + }, { + # geo-restricted livestream (workflow == 'token-akamai') + 'url': 'https://www.france.tv/france-4/direct.html', + 'info_dict': { + 'id': '9a6a7670-dde9-4264-adbc-55b89558594b', + 'ext': 'mp4', + 'title': r're:France 4 en direct .+', + 'live_status': 'is_live', + }, + 'skip': 'geo-restricted livestream', + }, { + # livestream (workflow == 'dai') + 'url': 'https://www.france.tv/france-2/direct.html', + 'info_dict': { + 'id': '006194ea-117d-4bcf-94a9-153d999c59ae', + 'ext': 'mp4', + 'title': r're:France 2 en direct .+', + 'live_status': 'is_live', + }, + 'params': {'skip_download': 'livestream'}, }, { # france3 'url': 'https://www.france.tv/france-3/des-chiffres-et-des-lettres/139063-emission-du-mardi-9-mai-2017.html', @@ -277,10 +290,6 @@ class FranceTVSiteIE(FranceTVBaseInfoExtractor): # franceo 'url': 'https://www.france.tv/france-o/archipels/132249-mon-ancetre-l-esclave.html', 'only_matching': True, - }, { - # france2 live - 'url': 'https://www.france.tv/france-2/direct.html', - 'only_matching': True, }, { 'url': 'https://www.france.tv/documentaires/histoire/136517-argentine-les-500-bebes-voles-de-la-dictature.html', 'only_matching': True, @@ -304,17 +313,16 @@ def _real_extract(self, url): webpage = self._download_webpage(url, display_id) - catalogue = None video_id = self._search_regex( r'(?:data-main-video\s*=|videoId["\']?\s*[:=])\s*(["\'])(?P(?:(?!\1).)+)\1', webpage, 'video id', default=None, group='id') if not video_id: - video_id, catalogue = self._html_search_regex( - r'(?:href=|player\.setVideo\(\s*)"http://videos?\.francetv\.fr/video/([^@]+@[^"]+)"', - webpage, 'video ID').split('@') + video_id = self._html_search_regex( + r'(?:href=|player\.setVideo\(\s*)"http://videos?\.francetv\.fr/video/([^@"]+@[^"]+)"', + webpage, 'video ID') - return self._make_url_result(video_id, catalogue) + return self._make_url_result(video_id, url=url) class FranceTVInfoIE(FranceTVBaseInfoExtractor): @@ -328,8 +336,9 @@ class FranceTVInfoIE(FranceTVBaseInfoExtractor): 'ext': 'mp4', 'title': 'Soir 3', 'upload_date': '20190822', - 'timestamp': 1566510900, - 'description': 'md5:72d167097237701d6e8452ff03b83c00', + 'timestamp': 1566510730, + 'thumbnail': r're:^https?://.*\.jpe?g$', + 'duration': 1637, 'subtitles': { 'fr': 'mincount:2', }, @@ -344,8 +353,8 @@ class FranceTVInfoIE(FranceTVBaseInfoExtractor): 'info_dict': { 'id': '7d204c9e-a2d3-11eb-9e4c-000d3a23d482', 'ext': 'mp4', - 'title': 'Covid-19 : une situation catastrophique à New Dehli', - 'thumbnail': str, + 'title': 'Covid-19 : une situation catastrophique à New Dehli - Édition du mercredi 21 avril 2021', + 'thumbnail': r're:^https?://.*\.jpe?g$', 'duration': 76, 'timestamp': 1619028518, 'upload_date': '20210421', @@ -371,11 +380,17 @@ class FranceTVInfoIE(FranceTVBaseInfoExtractor): 'id': 'x4iiko0', 'ext': 'mp4', 'title': 'NDDL, référendum, Brexit : Cécile Duflot répond à Patrick Cohen', - 'description': 'Au lendemain de la victoire du "oui" au référendum sur l\'aéroport de Notre-Dame-des-Landes, l\'ancienne ministre écologiste est l\'invitée de Patrick Cohen. Plus d\'info : https://www.franceinter.fr/emissions/le-7-9/le-7-9-27-juin-2016', + 'description': 'md5:fdcb582c370756293a65cdfbc6ecd90e', 'timestamp': 1467011958, - 'upload_date': '20160627', 'uploader': 'France Inter', 'uploader_id': 'x2q2ez', + 'upload_date': '20160627', + 'view_count': int, + 'tags': ['Politique', 'France Inter', '27 juin 2016', 'Linvité de 8h20', 'Cécile Duflot', 'Patrick Cohen'], + 'age_limit': 0, + 'duration': 640, + 'like_count': int, + 'thumbnail': r're:https://[^/?#]+/v/[^/?#]+/x1080', }, 'add_ie': ['Dailymotion'], }, { @@ -405,4 +420,4 @@ def _real_extract(self, url): r'(?:data-id|[^.?&#]+)' - _TESTS = [{ - # normal Brightcove embed code extracted with BrightcoveNewIE._extract_url - 'url': 'http://www.gameinformer.com/b/features/archive/2015/09/26/replay-animal-crossing.aspx', - 'md5': '292f26da1ab4beb4c9099f1304d2b071', - 'info_dict': { - 'id': '4515472681001', - 'ext': 'mp4', - 'title': 'Replay - Animal Crossing', - 'description': 'md5:2e211891b215c85d061adc7a4dd2d930', - 'timestamp': 1443457610, - 'upload_date': '20150928', - 'uploader_id': '694940074001', - }, - }, { - # Brightcove id inside unique element with field--name-field-brightcove-video-id class - 'url': 'https://www.gameinformer.com/video-feature/new-gameplay-today/2019/07/09/new-gameplay-today-streets-of-rogue', - 'info_dict': { - 'id': '6057111913001', - 'ext': 'mp4', - 'title': 'New Gameplay Today – Streets Of Rogue', - 'timestamp': 1562699001, - 'upload_date': '20190709', - 'uploader_id': '694940074001', - - }, - }] - BRIGHTCOVE_URL_TEMPLATE = 'http://players.brightcove.net/694940074001/default_default/index.html?videoId=%s' - - def _real_extract(self, url): - display_id = self._match_id(url) - webpage = self._download_webpage( - url, display_id, headers=self.geo_verification_headers()) - brightcove_id = clean_html(get_element_by_class('field--name-field-brightcove-video-id', webpage) or get_element_by_id('video-source-content', webpage)) - brightcove_url = self.BRIGHTCOVE_URL_TEMPLATE % brightcove_id if brightcove_id else BrightcoveNewIE._extract_url(self, webpage) - return self.url_result(brightcove_url, 'BrightcoveNew', brightcove_id) diff --git a/yt_dlp/extractor/gamejolt.py b/yt_dlp/extractor/gamejolt.py index 8ec046bb3..1d3c0b110 100644 --- a/yt_dlp/extractor/gamejolt.py +++ b/yt_dlp/extractor/gamejolt.py @@ -88,7 +88,7 @@ def _parse_post(self, post_data): 'uploader_id': user_data.get('username'), 'uploader_url': format_field(user_data, 'url', 'https://gamejolt.com%s'), 'categories': [try_get(category, lambda x: '%s - %s' % (x['community']['name'], x['channel'].get('display_title') or x['channel']['title'])) - for category in post_data.get('communities' or [])], + for category in post_data.get('communities') or []], 'tags': traverse_obj( lead_content, ('content', ..., 'content', ..., 'marks', ..., 'attrs', 'tag'), expected_type=str_or_none), 'like_count': int_or_none(post_data.get('like_count')), @@ -267,9 +267,9 @@ class GameJoltIE(GameJoltBaseIE): 'id': 'dszyjnwi', 'ext': 'webm', 'title': 'gif-presentacion-mejorado-dszyjnwi', - 'n_entries': 1, } - }] + }], + 'playlist_count': 1, }, { # Multiple GIFs 'url': 'https://gamejolt.com/p/gif-yhsqkumq', @@ -374,7 +374,6 @@ class GameJoltGameSoundtrackIE(GameJoltBaseIE): 'info_dict': { 'id': '657899', 'title': 'Friday Night Funkin\': Vs Oswald', - 'n_entries': None, }, 'playlist': [{ 'info_dict': { @@ -384,7 +383,6 @@ class GameJoltGameSoundtrackIE(GameJoltBaseIE): 'url': r're:^https://.+vs-oswald-menu-music\.mp3$', 'release_timestamp': 1635190816, 'release_date': '20211025', - 'n_entries': 3, } }, { 'info_dict': { @@ -394,7 +392,6 @@ class GameJoltGameSoundtrackIE(GameJoltBaseIE): 'url': r're:^https://.+rabbit-s-luck--full-version-\.mp3$', 'release_timestamp': 1635190841, 'release_date': '20211025', - 'n_entries': 3, } }, { 'info_dict': { @@ -404,9 +401,9 @@ class GameJoltGameSoundtrackIE(GameJoltBaseIE): 'url': r're:^https://.+last-straw\.mp3$', 'release_timestamp': 1635881104, 'release_date': '20211102', - 'n_entries': 3, } - }] + }], + 'playlist_count': 3, }] def _real_extract(self, url): diff --git a/yt_dlp/extractor/gaskrank.py b/yt_dlp/extractor/gaskrank.py index e0bbdae0a..bc56b03e3 100644 --- a/yt_dlp/extractor/gaskrank.py +++ b/yt_dlp/extractor/gaskrank.py @@ -21,7 +21,6 @@ class GaskrankIE(InfoExtractor): 'display_id': 'strike-einparken-durch-anfaenger-crash-mit-groesserem-flurschaden', 'uploader_id': 'Bikefun', 'upload_date': '20170110', - 'uploader_url': None, } }, { 'url': 'http://www.gaskrank.tv/tv/racing/isle-of-man-tt-2011-michael-du-15920.htm', diff --git a/yt_dlp/extractor/gazeta.py b/yt_dlp/extractor/gazeta.py index c6868a672..8925b69fd 100644 --- a/yt_dlp/extractor/gazeta.py +++ b/yt_dlp/extractor/gazeta.py @@ -2,6 +2,7 @@ class GazetaIE(InfoExtractor): + _WORKING = False _VALID_URL = r'(?Phttps?://(?:www\.)?gazeta\.ru/(?:[^/]+/)?video/(?:main/)*(?:\d{4}/\d{2}/\d{2}/)?(?P[A-Za-z0-9-_.]+)\.s?html)' _TESTS = [{ 'url': 'http://www.gazeta.ru/video/main/zadaite_vopros_vladislavu_yurevichu.shtml', diff --git a/yt_dlp/extractor/gdcvault.py b/yt_dlp/extractor/gdcvault.py index 4265feb61..b4d81b2e8 100644 --- a/yt_dlp/extractor/gdcvault.py +++ b/yt_dlp/extractor/gdcvault.py @@ -7,6 +7,7 @@ class GDCVaultIE(InfoExtractor): + _WORKING = False _VALID_URL = r'https?://(?:www\.)?gdcvault\.com/play/(?P\d+)(?:/(?P[\w-]+))?' _NETRC_MACHINE = 'gdcvault' _TESTS = [ diff --git a/yt_dlp/extractor/generic.py b/yt_dlp/extractor/generic.py index 1f0011c09..9d8251582 100644 --- a/yt_dlp/extractor/generic.py +++ b/yt_dlp/extractor/generic.py @@ -2394,7 +2394,6 @@ def _real_extract(self, url): 'Referer': smuggled_data.get('referer'), })) new_url = full_response.url - url = urllib.parse.urlparse(url)._replace(scheme=urllib.parse.urlparse(new_url).scheme).geturl() if new_url != extract_basic_auth(url)[0]: self.report_following_redirect(new_url) if force_videoid: diff --git a/yt_dlp/extractor/getcourseru.py b/yt_dlp/extractor/getcourseru.py index 6fdbcd736..144321ad6 100644 --- a/yt_dlp/extractor/getcourseru.py +++ b/yt_dlp/extractor/getcourseru.py @@ -160,9 +160,8 @@ def _real_extract(self, url): self._login(hostname, username, password) display_id = self._match_id(url) - # NB: 404 is returned due to yt-dlp not properly following redirects #9020 - webpage, urlh = self._download_webpage_handle(url, display_id, expected_status=404) - if self._LOGIN_URL_PATH in urlh.url or urlh.status == 404: + webpage, urlh = self._download_webpage_handle(url, display_id) + if self._LOGIN_URL_PATH in urlh.url: raise ExtractorError( f'This video is only available for registered users. {self._login_hint("any", netrc=hostname)}', expected=True) diff --git a/yt_dlp/extractor/giga.py b/yt_dlp/extractor/giga.py deleted file mode 100644 index b59c129ab..000000000 --- a/yt_dlp/extractor/giga.py +++ /dev/null @@ -1,93 +0,0 @@ -import itertools - -from .common import InfoExtractor -from ..compat import compat_str -from ..utils import parse_duration, parse_iso8601, qualities, str_to_int - - -class GigaIE(InfoExtractor): - _VALID_URL = r'https?://(?:www\.)?giga\.de/(?:[^/]+/)*(?P[^/]+)' - _TESTS = [{ - 'url': 'http://www.giga.de/filme/anime-awesome/trailer/anime-awesome-chihiros-reise-ins-zauberland-das-beste-kommt-zum-schluss/', - 'md5': '6bc5535e945e724640664632055a584f', - 'info_dict': { - 'id': '2622086', - 'display_id': 'anime-awesome-chihiros-reise-ins-zauberland-das-beste-kommt-zum-schluss', - 'ext': 'mp4', - 'title': 'Anime Awesome: Chihiros Reise ins Zauberland – Das Beste kommt zum Schluss', - 'description': 'md5:afdf5862241aded4718a30dff6a57baf', - 'thumbnail': r're:^https?://.*\.jpg$', - 'duration': 578, - 'timestamp': 1414749706, - 'upload_date': '20141031', - 'uploader': 'Robin Schweiger', - 'view_count': int, - }, - }, { - 'url': 'http://www.giga.de/games/channel/giga-top-montag/giga-topmontag-die-besten-serien-2014/', - 'only_matching': True, - }, { - 'url': 'http://www.giga.de/extra/netzkultur/videos/giga-games-tom-mats-robin-werden-eigene-wege-gehen-eine-ankuendigung/', - 'only_matching': True, - }, { - 'url': 'http://www.giga.de/tv/jonas-liest-spieletitel-eingedeutscht-episode-2/', - 'only_matching': True, - }] - - def _real_extract(self, url): - display_id = self._match_id(url) - - webpage = self._download_webpage(url, display_id) - - video_id = self._search_regex( - [r'data-video-id="(\d+)"', r'/api/video/jwplayer/#v=(\d+)'], - webpage, 'video id') - - playlist = self._download_json( - 'http://www.giga.de/api/syndication/video/video_id/%s/playlist.json?content=syndication/key/368b5f151da4ae05ced7fa296bdff65a/' - % video_id, video_id)[0] - - quality = qualities(['normal', 'hd720']) - - formats = [] - for format_id in itertools.count(0): - fmt = playlist.get(compat_str(format_id)) - if not fmt: - break - formats.append({ - 'url': fmt['src'], - 'format_id': '%s-%s' % (fmt['quality'], fmt['type'].split('/')[-1]), - 'quality': quality(fmt['quality']), - }) - - title = self._html_search_meta( - 'title', webpage, 'title', fatal=True) - description = self._html_search_meta( - 'description', webpage, 'description') - thumbnail = self._og_search_thumbnail(webpage) - - duration = parse_duration(self._search_regex( - r'(?s)(?:data-video-id="{0}"|data-video="[^"]*/api/video/jwplayer/#v={0}[^"]*")[^>]*>.+?([^<]+)'.format(video_id), - webpage, 'duration', fatal=False)) - - timestamp = parse_iso8601(self._search_regex( - r'datetime="([^"]+)"', webpage, 'upload date', fatal=False)) - uploader = self._search_regex( - r'class="author">([^<]+)', webpage, 'uploader', fatal=False) - - view_count = str_to_int(self._search_regex( - r'([\d.,]+)', - webpage, 'view count', fatal=False)) - - return { - 'id': video_id, - 'display_id': display_id, - 'title': title, - 'description': description, - 'thumbnail': thumbnail, - 'duration': duration, - 'timestamp': timestamp, - 'uploader': uploader, - 'view_count': view_count, - 'formats': formats, - } diff --git a/yt_dlp/extractor/godtube.py b/yt_dlp/extractor/godtube.py index 697540155..35fb7a9c9 100644 --- a/yt_dlp/extractor/godtube.py +++ b/yt_dlp/extractor/godtube.py @@ -6,6 +6,7 @@ class GodTubeIE(InfoExtractor): + _WORKING = False _VALID_URL = r'https?://(?:www\.)?godtube\.com/watch/\?v=(?P[\da-zA-Z]+)' _TESTS = [ { diff --git a/yt_dlp/extractor/gofile.py b/yt_dlp/extractor/gofile.py index eb1dcf85f..c6eca0c4d 100644 --- a/yt_dlp/extractor/gofile.py +++ b/yt_dlp/extractor/gofile.py @@ -58,21 +58,18 @@ def _real_initialize(self): return account_data = self._download_json( - 'https://api.gofile.io/createAccount', None, note='Getting a new guest account') + 'https://api.gofile.io/accounts', None, 'Getting a new guest account', data=b'{}') self._TOKEN = account_data['data']['token'] self._set_cookie('.gofile.io', 'accountToken', self._TOKEN) def _entries(self, file_id): - query_params = { - 'contentId': file_id, - 'token': self._TOKEN, - 'wt': '4fd6sg89d7s6', # From https://gofile.io/dist/js/alljs.js - } + query_params = {'wt': '4fd6sg89d7s6'} # From https://gofile.io/dist/js/alljs.js password = self.get_param('videopassword') if password: query_params['password'] = hashlib.sha256(password.encode('utf-8')).hexdigest() files = self._download_json( - 'https://api.gofile.io/getContent', file_id, note='Getting filelist', query=query_params) + f'https://api.gofile.io/contents/{file_id}', file_id, 'Getting filelist', + query=query_params, headers={'Authorization': f'Bearer {self._TOKEN}'}) status = files['status'] if status == 'error-passwordRequired': @@ -82,7 +79,7 @@ def _entries(self, file_id): raise ExtractorError(f'{self.IE_NAME} said: status {status}', expected=True) found_files = False - for file in (try_get(files, lambda x: x['data']['contents'], dict) or {}).values(): + for file in (try_get(files, lambda x: x['data']['children'], dict) or {}).values(): file_type, file_format = file.get('mimetype').split('/', 1) if file_type not in ('video', 'audio') and file_format != 'vnd.mts': continue diff --git a/yt_dlp/extractor/goplay.py b/yt_dlp/extractor/goplay.py index 0a3c8340f..74aad1192 100644 --- a/yt_dlp/extractor/goplay.py +++ b/yt_dlp/extractor/goplay.py @@ -40,6 +40,22 @@ class GoPlayIE(InfoExtractor): 'title': 'A Family for the Holidays', }, 'skip': 'This video is only available for registered users' + }, { + 'url': 'https://www.goplay.be/video/de-mol/de-mol-s11/de-mol-s11-aflevering-1#autoplay', + 'info_dict': { + 'id': '03eb8f2f-153e-41cb-9805-0d3a29dab656', + 'ext': 'mp4', + 'title': 'S11 - Aflevering 1', + 'episode': 'Episode 1', + 'series': 'De Mol', + 'season_number': 11, + 'episode_number': 1, + 'season': 'Season 11' + }, + 'params': { + 'skip_download': True + }, + 'skip': 'This video is only available for registered users' }] _id_token = None @@ -77,16 +93,39 @@ def _real_extract(self, url): api = self._download_json( f'https://api.goplay.be/web/v1/videos/long-form/{video_id}', - video_id, headers={'Authorization': 'Bearer %s' % self._id_token}) + video_id, headers={ + 'Authorization': 'Bearer %s' % self._id_token, + **self.geo_verification_headers(), + }) - formats, subs = self._extract_m3u8_formats_and_subtitles( - api['manifestUrls']['hls'], video_id, ext='mp4', m3u8_id='HLS') + if 'manifestUrls' in api: + formats, subtitles = self._extract_m3u8_formats_and_subtitles( + api['manifestUrls']['hls'], video_id, ext='mp4', m3u8_id='HLS') + + else: + if 'ssai' not in api: + raise ExtractorError('expecting Google SSAI stream') + + ssai_content_source_id = api['ssai']['contentSourceID'] + ssai_video_id = api['ssai']['videoID'] + + dai = self._download_json( + f'https://dai.google.com/ondemand/dash/content/{ssai_content_source_id}/vid/{ssai_video_id}/streams', + video_id, data=b'{"api-key":"null"}', + headers={'content-type': 'application/json'}) + + periods = self._extract_mpd_periods(dai['stream_manifest'], video_id) + + # skip pre-roll and mid-roll ads + periods = [p for p in periods if '-ad-' not in p['id']] + + formats, subtitles = self._merge_mpd_periods(periods) info_dict.update({ 'id': video_id, 'formats': formats, + 'subtitles': subtitles, }) - return info_dict diff --git a/yt_dlp/extractor/hotnewhiphop.py b/yt_dlp/extractor/hotnewhiphop.py index 3007fbb53..4f506cde7 100644 --- a/yt_dlp/extractor/hotnewhiphop.py +++ b/yt_dlp/extractor/hotnewhiphop.py @@ -5,6 +5,7 @@ class HotNewHipHopIE(InfoExtractor): + _WORKING = False _VALID_URL = r'https?://(?:www\.)?hotnewhiphop\.com/.*\.(?P.*)\.html' _TEST = { 'url': 'http://www.hotnewhiphop.com/freddie-gibbs-lay-it-down-song.1435540.html', diff --git a/yt_dlp/extractor/hotstar.py b/yt_dlp/extractor/hotstar.py index 541792b90..a3a3c20c9 100644 --- a/yt_dlp/extractor/hotstar.py +++ b/yt_dlp/extractor/hotstar.py @@ -115,11 +115,11 @@ class HotStarIE(HotStarBaseIE): 'upload_date': '20190501', 'duration': 1219, 'channel': 'StarPlus', - 'channel_id': 3, + 'channel_id': '3', 'series': 'Ek Bhram - Sarvagun Sampanna', 'season': 'Chapter 1', 'season_number': 1, - 'season_id': 6771, + 'season_id': '6771', 'episode': 'Janhvi Targets Suman', 'episode_number': 8, } @@ -135,12 +135,12 @@ class HotStarIE(HotStarBaseIE): 'channel': 'StarPlus', 'series': 'Anupama', 'season_number': 1, - 'season_id': 7399, + 'season_id': '7399', 'upload_date': '20230307', 'episode': 'Anupama, Anuj Share a Moment', 'episode_number': 853, 'duration': 1272, - 'channel_id': 3, + 'channel_id': '3', }, 'skip': 'HTTP Error 504: Gateway Time-out', # XXX: Investigate 504 errors on some episodes }, { @@ -155,12 +155,12 @@ class HotStarIE(HotStarBaseIE): 'channel': 'Hotstar Specials', 'series': 'Kana Kaanum Kaalangal', 'season_number': 1, - 'season_id': 9441, + 'season_id': '9441', 'upload_date': '20220421', 'episode': 'Back To School', 'episode_number': 1, 'duration': 1810, - 'channel_id': 54, + 'channel_id': '54', }, }, { 'url': 'https://www.hotstar.com/in/clips/e3-sairat-kahani-pyaar-ki/1000262286', @@ -325,11 +325,11 @@ def _real_extract(self, url): 'formats': formats, 'subtitles': subs, 'channel': video_data.get('channelName'), - 'channel_id': video_data.get('channelId'), + 'channel_id': str_or_none(video_data.get('channelId')), 'series': video_data.get('showName'), 'season': video_data.get('seasonName'), 'season_number': int_or_none(video_data.get('seasonNo')), - 'season_id': video_data.get('seasonId'), + 'season_id': str_or_none(video_data.get('seasonId')), 'episode': video_data.get('title'), 'episode_number': int_or_none(video_data.get('episodeNo')), } diff --git a/yt_dlp/extractor/hungama.py b/yt_dlp/extractor/hungama.py index cdec36838..7da8aad7a 100644 --- a/yt_dlp/extractor/hungama.py +++ b/yt_dlp/extractor/hungama.py @@ -114,7 +114,6 @@ class HungamaSongIE(InfoExtractor): 'title': 'Lucky Ali - Kitni Haseen Zindagi', 'track': 'Kitni Haseen Zindagi', 'artist': 'Lucky Ali', - 'album': None, 'release_year': 2000, 'thumbnail': 'https://stat2.hungama.ind.in/assets/images/default_images/da-200x200.png', }, diff --git a/yt_dlp/extractor/hypergryph.py b/yt_dlp/extractor/hypergryph.py index 9ca6caebc..96e452a51 100644 --- a/yt_dlp/extractor/hypergryph.py +++ b/yt_dlp/extractor/hypergryph.py @@ -9,7 +9,7 @@ class MonsterSirenHypergryphMusicIE(InfoExtractor): 'info_dict': { 'id': '514562', 'ext': 'wav', - 'artist': ['塞壬唱片-MSR'], + 'artists': ['塞壬唱片-MSR'], 'album': 'Flame Shadow', 'title': 'Flame Shadow', } @@ -27,6 +27,6 @@ def _real_extract(self, url): 'url': traverse_obj(json_data, ('player', 'songDetail', 'sourceUrl')), 'ext': 'wav', 'vcodec': 'none', - 'artist': traverse_obj(json_data, ('player', 'songDetail', 'artists')), + 'artists': traverse_obj(json_data, ('player', 'songDetail', 'artists', ...)), 'album': traverse_obj(json_data, ('musicPlay', 'albumDetail', 'name')) } diff --git a/yt_dlp/extractor/instagram.py b/yt_dlp/extractor/instagram.py index dbaa332c2..f7f21505e 100644 --- a/yt_dlp/extractor/instagram.py +++ b/yt_dlp/extractor/instagram.py @@ -617,6 +617,7 @@ def _real_extract(self, url): class InstagramUserIE(InstagramPlaylistBaseIE): + _WORKING = False _VALID_URL = r'https?://(?:www\.)?instagram\.com/(?P[^/]{2,})/?(?:$|[?#])' IE_DESC = 'Instagram user profile' IE_NAME = 'instagram:user' diff --git a/yt_dlp/extractor/itprotv.py b/yt_dlp/extractor/itprotv.py index b9d5c196d..713fd4ec5 100644 --- a/yt_dlp/extractor/itprotv.py +++ b/yt_dlp/extractor/itprotv.py @@ -31,7 +31,7 @@ def _check_if_logged_in(self, webpage): class ITProTVIE(ITProTVBaseIE): - _VALID_URL = r'https://app\.itpro\.tv/course/(?P[\w-]+)/(?P[\w-]+)' + _VALID_URL = r'https?://app\.itpro\.tv/course/(?P[\w-]+)/(?P[\w-]+)' _TESTS = [{ 'url': 'https://app.itpro.tv/course/guided-tour/introductionitprotv', 'md5': 'bca4a28c2667fd1a63052e71a94bb88c', diff --git a/yt_dlp/extractor/jeuxvideo.py b/yt_dlp/extractor/jeuxvideo.py index 56ea15cf9..793820600 100644 --- a/yt_dlp/extractor/jeuxvideo.py +++ b/yt_dlp/extractor/jeuxvideo.py @@ -2,6 +2,8 @@ class JeuxVideoIE(InfoExtractor): + _WORKING = False + _ENABLED = None # XXX: pass through to GenericIE _VALID_URL = r'https?://.*?\.jeuxvideo\.com/.*/(.*?)\.htm' _TESTS = [{ diff --git a/yt_dlp/extractor/kanal2.py b/yt_dlp/extractor/kanal2.py deleted file mode 100644 index 3c0efe598..000000000 --- a/yt_dlp/extractor/kanal2.py +++ /dev/null @@ -1,66 +0,0 @@ -from .common import InfoExtractor -from ..utils import ( - ExtractorError, - join_nonempty, - traverse_obj, - unified_timestamp, - update_url_query, -) - - -class Kanal2IE(InfoExtractor): - _VALID_URL = r'https?://kanal2\.postimees\.ee/[^?#]+\?([^#]+&)?id=(?P\d+)' - _TESTS = [{ - 'note': 'Test standard url (#5575)', - 'url': 'https://kanal2.postimees.ee/pluss/video/?id=40792', - 'md5': '7ea7b16266ec1798743777df241883dd', - 'info_dict': { - 'id': '40792', - 'ext': 'mp4', - 'title': 'Aedniku aabits / Osa 53 (05.08.2016 20:00)', - 'thumbnail': r're:https?://.*\.jpg$', - 'description': 'md5:53cabf3c5d73150d594747f727431248', - 'upload_date': '20160805', - 'timestamp': 1470420000, - }, - }] - - def _real_extract(self, url): - video_id = self._match_id(url) - playlist = self._download_json( - f'https://kanal2.postimees.ee/player/playlist/{video_id}', - video_id, query={'type': 'episodes'}, - headers={'X-Requested-With': 'XMLHttpRequest'}) - - return { - 'id': video_id, - 'title': join_nonempty(*traverse_obj(playlist, ('info', ('title', 'subtitle'))), delim=' / '), - 'description': traverse_obj(playlist, ('info', 'description')), - 'thumbnail': traverse_obj(playlist, ('data', 'image')), - 'formats': self.get_formats(playlist, video_id), - 'timestamp': unified_timestamp(self._search_regex( - r'\((\d{2}\.\d{2}\.\d{4}\s\d{2}:\d{2})\)$', - traverse_obj(playlist, ('info', 'subtitle')), 'timestamp', default='') + ' +0200'), - } - - def get_formats(self, playlist, video_id): - path = traverse_obj(playlist, ('data', 'path')) - if not path: - raise ExtractorError('Path value not found in playlist JSON response') - session = self._download_json( - 'https://sts.postimees.ee/session/register', - video_id, note='Creating session', errnote='Error creating session', - headers={ - 'X-Original-URI': path, - 'Accept': 'application/json', - }) - if session.get('reason') != 'OK' or not session.get('session'): - reason = session.get('reason', 'unknown error') - raise ExtractorError(f'Unable to obtain session: {reason}') - - formats = [] - for stream in traverse_obj(playlist, ('data', 'streams', ..., 'file')): - formats.extend(self._extract_m3u8_formats( - update_url_query(stream, {'s': session['session']}), video_id, 'mp4')) - - return formats diff --git a/yt_dlp/extractor/kankanews.py b/yt_dlp/extractor/kankanews.py index 46e239bd6..8f247b305 100644 --- a/yt_dlp/extractor/kankanews.py +++ b/yt_dlp/extractor/kankanews.py @@ -8,6 +8,7 @@ class KankaNewsIE(InfoExtractor): + _WORKING = False _VALID_URL = r'https?://(?:www\.)?kankanews\.com/a/\d+\-\d+\-\d+/(?P\d+)\.shtml' _TESTS = [{ 'url': 'https://www.kankanews.com/a/2022-11-08/00310276054.shtml?appid=1088227', diff --git a/yt_dlp/extractor/karrierevideos.py b/yt_dlp/extractor/karrierevideos.py deleted file mode 100644 index 28d4841aa..000000000 --- a/yt_dlp/extractor/karrierevideos.py +++ /dev/null @@ -1,96 +0,0 @@ -from .common import InfoExtractor -from ..compat import compat_urlparse -from ..utils import ( - fix_xml_ampersands, - float_or_none, - xpath_with_ns, - xpath_text, -) - - -class KarriereVideosIE(InfoExtractor): - _VALID_URL = r'https?://(?:www\.)?karrierevideos\.at(?:/[^/]+)+/(?P[^/]+)' - _TESTS = [{ - 'url': 'http://www.karrierevideos.at/berufsvideos/mittlere-hoehere-schulen/altenpflegerin', - 'info_dict': { - 'id': '32c91', - 'ext': 'flv', - 'title': 'AltenpflegerIn', - 'description': 'md5:dbadd1259fde2159a9b28667cb664ae2', - 'thumbnail': r're:^http://.*\.png', - }, - 'params': { - # rtmp download - 'skip_download': True, - } - }, { - # broken ampersands - 'url': 'http://www.karrierevideos.at/orientierung/vaeterkarenz-und-neue-chancen-fuer-muetter-baby-was-nun', - 'info_dict': { - 'id': '5sniu', - 'ext': 'flv', - 'title': 'Väterkarenz und neue Chancen für Mütter - "Baby - was nun?"', - 'description': 'md5:97092c6ad1fd7d38e9d6a5fdeb2bcc33', - 'thumbnail': r're:^http://.*\.png', - }, - 'params': { - # rtmp download - 'skip_download': True, - } - }] - - def _real_extract(self, url): - video_id = self._match_id(url) - - webpage = self._download_webpage(url, video_id) - - title = (self._html_search_meta('title', webpage, default=None) - or self._search_regex(r'

([^<]+)

', webpage, 'video title')) - - video_id = self._search_regex( - r'/config/video/(.+?)\.xml', webpage, 'video id') - # Server returns malformed headers - # Force Accept-Encoding: * to prevent gzipped results - playlist = self._download_xml( - 'http://www.karrierevideos.at/player-playlist.xml.php?p=%s' % video_id, - video_id, transform_source=fix_xml_ampersands, - headers={'Accept-Encoding': '*'}) - - NS_MAP = { - 'jwplayer': 'http://developer.longtailvideo.com/trac/wiki/FlashFormats' - } - - def ns(path): - return xpath_with_ns(path, NS_MAP) - - item = playlist.find('./tracklist/item') - video_file = xpath_text( - item, ns('./jwplayer:file'), 'video url', fatal=True) - streamer = xpath_text( - item, ns('./jwplayer:streamer'), 'streamer', fatal=True) - - uploader = xpath_text( - item, ns('./jwplayer:author'), 'uploader') - duration = float_or_none( - xpath_text(item, ns('./jwplayer:duration'), 'duration')) - - description = self._html_search_regex( - r'(?s)
(.+?)
', - webpage, 'description') - - thumbnail = self._html_search_meta( - 'thumbnail', webpage, 'thumbnail') - if thumbnail: - thumbnail = compat_urlparse.urljoin(url, thumbnail) - - return { - 'id': video_id, - 'url': streamer.replace('rtmpt', 'rtmp'), - 'play_path': 'mp4:%s' % video_file, - 'ext': 'flv', - 'title': title, - 'description': description, - 'thumbnail': thumbnail, - 'uploader': uploader, - 'duration': duration, - } diff --git a/yt_dlp/extractor/kelbyone.py b/yt_dlp/extractor/kelbyone.py index 2ca9ad426..bba527e29 100644 --- a/yt_dlp/extractor/kelbyone.py +++ b/yt_dlp/extractor/kelbyone.py @@ -3,6 +3,7 @@ class KelbyOneIE(InfoExtractor): + _WORKING = False _VALID_URL = r'https?://members\.kelbyone\.com/course/(?P[^$&?#/]+)' _TESTS = [{ diff --git a/yt_dlp/extractor/kommunetv.py b/yt_dlp/extractor/kommunetv.py index a30905b57..432816cd8 100644 --- a/yt_dlp/extractor/kommunetv.py +++ b/yt_dlp/extractor/kommunetv.py @@ -3,7 +3,7 @@ class KommunetvIE(InfoExtractor): - _VALID_URL = r'https://\w+\.kommunetv\.no/archive/(?P\w+)' + _VALID_URL = r'https?://\w+\.kommunetv\.no/archive/(?P\w+)' _TEST = { 'url': 'https://oslo.kommunetv.no/archive/921', 'md5': '5f102be308ee759be1e12b63d5da4bbc', diff --git a/yt_dlp/extractor/konserthusetplay.py b/yt_dlp/extractor/konserthusetplay.py deleted file mode 100644 index 10767f1b6..000000000 --- a/yt_dlp/extractor/konserthusetplay.py +++ /dev/null @@ -1,119 +0,0 @@ -from .common import InfoExtractor -from ..utils import ( - determine_ext, - float_or_none, - int_or_none, - url_or_none, -) - - -class KonserthusetPlayIE(InfoExtractor): - _VALID_URL = r'https?://(?:www\.)?(?:konserthusetplay|rspoplay)\.se/\?.*\bm=(?P[^&]+)' - _TESTS = [{ - 'url': 'http://www.konserthusetplay.se/?m=CKDDnlCY-dhWAAqiMERd-A', - 'md5': 'e3fd47bf44e864bd23c08e487abe1967', - 'info_dict': { - 'id': 'CKDDnlCY-dhWAAqiMERd-A', - 'ext': 'mp4', - 'title': 'Orkesterns instrument: Valthornen', - 'description': 'md5:f10e1f0030202020396a4d712d2fa827', - 'thumbnail': 're:^https?://.*$', - 'duration': 398.76, - }, - }, { - 'url': 'http://rspoplay.se/?m=elWuEH34SMKvaO4wO_cHBw', - 'only_matching': True, - }] - - def _real_extract(self, url): - video_id = self._match_id(url) - - webpage = self._download_webpage(url, video_id) - - e = self._search_regex( - r'https?://csp\.picsearch\.com/rest\?.*\be=(.+?)[&"\']', webpage, 'e') - - rest = self._download_json( - 'http://csp.picsearch.com/rest?e=%s&containerId=mediaplayer&i=object' % e, - video_id, transform_source=lambda s: s[s.index('{'):s.rindex('}') + 1]) - - media = rest['media'] - player_config = media['playerconfig'] - playlist = player_config['playlist'] - - source = next(f for f in playlist if f.get('bitrates') or f.get('provider')) - - FORMAT_ID_REGEX = r'_([^_]+)_h264m\.mp4' - - formats = [] - - m3u8_url = source.get('url') - if m3u8_url and determine_ext(m3u8_url) == 'm3u8': - formats.extend(self._extract_m3u8_formats( - m3u8_url, video_id, 'mp4', entry_protocol='m3u8_native', - m3u8_id='hls', fatal=False)) - - fallback_url = source.get('fallbackUrl') - fallback_format_id = None - if fallback_url: - fallback_format_id = self._search_regex( - FORMAT_ID_REGEX, fallback_url, 'format id', default=None) - - connection_url = (player_config.get('rtmp', {}).get( - 'netConnectionUrl') or player_config.get( - 'plugins', {}).get('bwcheck', {}).get('netConnectionUrl')) - if connection_url: - for f in source['bitrates']: - video_url = f.get('url') - if not video_url: - continue - format_id = self._search_regex( - FORMAT_ID_REGEX, video_url, 'format id', default=None) - f_common = { - 'vbr': int_or_none(f.get('bitrate')), - 'width': int_or_none(f.get('width')), - 'height': int_or_none(f.get('height')), - } - f = f_common.copy() - f.update({ - 'url': connection_url, - 'play_path': video_url, - 'format_id': 'rtmp-%s' % format_id if format_id else 'rtmp', - 'ext': 'flv', - }) - formats.append(f) - if format_id and format_id == fallback_format_id: - f = f_common.copy() - f.update({ - 'url': fallback_url, - 'format_id': 'http-%s' % format_id if format_id else 'http', - }) - formats.append(f) - - if not formats and fallback_url: - formats.append({ - 'url': fallback_url, - }) - - title = player_config.get('title') or media['title'] - description = player_config.get('mediaInfo', {}).get('description') - thumbnail = media.get('image') - duration = float_or_none(media.get('duration'), 1000) - - subtitles = {} - captions = source.get('captionsAvailableLanguages') - if isinstance(captions, dict): - for lang, subtitle_url in captions.items(): - subtitle_url = url_or_none(subtitle_url) - if lang != 'none' and subtitle_url: - subtitles.setdefault(lang, []).append({'url': subtitle_url}) - - return { - 'id': video_id, - 'title': title, - 'description': description, - 'thumbnail': thumbnail, - 'duration': duration, - 'formats': formats, - 'subtitles': subtitles, - } diff --git a/yt_dlp/extractor/koo.py b/yt_dlp/extractor/koo.py index 9cfec5eb9..c78a7b9ca 100644 --- a/yt_dlp/extractor/koo.py +++ b/yt_dlp/extractor/koo.py @@ -6,6 +6,7 @@ class KooIE(InfoExtractor): + _WORKING = False _VALID_URL = r'https?://(?:www\.)?kooapp\.com/koo/[^/]+/(?P[^/&#$?]+)' _TESTS = [{ # Test for video in the comments 'url': 'https://www.kooapp.com/koo/ytdlpTestAccount/946c4189-bc2d-4524-b95b-43f641e2adde', diff --git a/yt_dlp/extractor/krasview.py b/yt_dlp/extractor/krasview.py index 4323aa429..0febf759b 100644 --- a/yt_dlp/extractor/krasview.py +++ b/yt_dlp/extractor/krasview.py @@ -8,6 +8,7 @@ class KrasViewIE(InfoExtractor): + _WORKING = False IE_DESC = 'Красвью' _VALID_URL = r'https?://krasview\.ru/(?:video|embed)/(?P\d+)' diff --git a/yt_dlp/extractor/kusi.py b/yt_dlp/extractor/kusi.py deleted file mode 100644 index a23ad8945..000000000 --- a/yt_dlp/extractor/kusi.py +++ /dev/null @@ -1,83 +0,0 @@ -import random -import urllib.parse - -from .common import InfoExtractor -from ..utils import ( - float_or_none, - int_or_none, - timeconvert, - update_url_query, - xpath_text, -) - - -class KUSIIE(InfoExtractor): - _VALID_URL = r'https?://(?:www\.)?kusi\.com/(?Pstory/.+|video\?clipId=(?P\d+))' - _TESTS = [{ - 'url': 'http://www.kusi.com/story/32849881/turko-files-refused-to-help-it-aint-right', - 'md5': '4e76ce8e53660ce9697d06c0ba6fc47d', - 'info_dict': { - 'id': '12689020', - 'ext': 'mp4', - 'title': "Turko Files: Refused to Help, It Ain't Right!", - 'duration': 223.586, - 'upload_date': '20160826', - 'timestamp': 1472233118, - 'thumbnail': r're:^https?://.*\.jpg$' - }, - }, { - 'url': 'http://kusi.com/video?clipId=12203019', - 'only_matching': True, - }] - - def _real_extract(self, url): - mobj = self._match_valid_url(url) - clip_id = mobj.group('clipId') - video_id = clip_id or mobj.group('path') - - webpage = self._download_webpage(url, video_id) - - if clip_id is None: - video_id = clip_id = self._html_search_regex( - r'"clipId"\s*,\s*"(\d+)"', webpage, 'clip id') - - affiliate_id = self._search_regex( - r'affiliateId\s*:\s*\'([^\']+)\'', webpage, 'affiliate id') - - # See __Packages/worldnow/model/GalleryModel.as of WNGallery.swf - xml_url = update_url_query('http://www.kusi.com/build.asp', { - 'buildtype': 'buildfeaturexmlrequest', - 'featureType': 'Clip', - 'featureid': clip_id, - 'affiliateno': affiliate_id, - 'clientgroupid': '1', - 'rnd': int(round(random.random() * 1000000)), - }) - - doc = self._download_xml(xml_url, video_id) - - video_title = xpath_text(doc, 'HEADLINE', fatal=True) - duration = float_or_none(xpath_text(doc, 'DURATION'), scale=1000) - description = xpath_text(doc, 'ABSTRACT') - thumbnail = xpath_text(doc, './THUMBNAILIMAGE/FILENAME') - creation_time = timeconvert(xpath_text(doc, 'rfc822creationdate')) - - quality_options = doc.find('{http://search.yahoo.com/mrss/}group').findall('{http://search.yahoo.com/mrss/}content') - formats = [] - for quality in quality_options: - formats.append({ - 'url': urllib.parse.unquote_plus(quality.attrib['url']), - 'height': int_or_none(quality.attrib.get('height')), - 'width': int_or_none(quality.attrib.get('width')), - 'vbr': float_or_none(quality.attrib.get('bitratebits'), scale=1000), - }) - - return { - 'id': video_id, - 'title': video_title, - 'description': description, - 'duration': duration, - 'formats': formats, - 'thumbnail': thumbnail, - 'timestamp': creation_time, - } diff --git a/yt_dlp/extractor/kuwo.py b/yt_dlp/extractor/kuwo.py index e8a061a10..3c93dedac 100644 --- a/yt_dlp/extractor/kuwo.py +++ b/yt_dlp/extractor/kuwo.py @@ -54,6 +54,7 @@ def _get_formats(self, song_id, tolerate_ip_deny=False): class KuwoIE(KuwoBaseIE): + _WORKING = False IE_NAME = 'kuwo:song' IE_DESC = '酷我音乐' _VALID_URL = r'https?://(?:www\.)?kuwo\.cn/yinyue/(?P\d+)' @@ -133,6 +134,7 @@ def _real_extract(self, url): class KuwoAlbumIE(InfoExtractor): + _WORKING = False IE_NAME = 'kuwo:album' IE_DESC = '酷我音乐 - 专辑' _VALID_URL = r'https?://(?:www\.)?kuwo\.cn/album/(?P\d+?)/' @@ -169,6 +171,7 @@ def _real_extract(self, url): class KuwoChartIE(InfoExtractor): + _WORKING = False IE_NAME = 'kuwo:chart' IE_DESC = '酷我音乐 - 排行榜' _VALID_URL = r'https?://yinyue\.kuwo\.cn/billboard_(?P[^.]+).htm' @@ -194,6 +197,7 @@ def _real_extract(self, url): class KuwoSingerIE(InfoExtractor): + _WORKING = False IE_NAME = 'kuwo:singer' IE_DESC = '酷我音乐 - 歌手' _VALID_URL = r'https?://(?:www\.)?kuwo\.cn/mingxing/(?P[^/]+)' @@ -251,6 +255,7 @@ def page_func(page_num): class KuwoCategoryIE(InfoExtractor): + _WORKING = False IE_NAME = 'kuwo:category' IE_DESC = '酷我音乐 - 分类' _VALID_URL = r'https?://yinyue\.kuwo\.cn/yy/cinfo_(?P\d+?).htm' @@ -290,6 +295,7 @@ def _real_extract(self, url): class KuwoMvIE(KuwoBaseIE): + _WORKING = False IE_NAME = 'kuwo:mv' IE_DESC = '酷我音乐 - MV' _VALID_URL = r'https?://(?:www\.)?kuwo\.cn/mv/(?P\d+?)/' diff --git a/yt_dlp/extractor/lbry.py b/yt_dlp/extractor/lbry.py index cc37c41e8..dcb44d07f 100644 --- a/yt_dlp/extractor/lbry.py +++ b/yt_dlp/extractor/lbry.py @@ -231,7 +231,6 @@ class LBRYIE(LBRYBaseIE): 'release_timestamp': int, 'release_date': str, 'tags': list, - 'duration': None, 'channel': 'RT', 'channel_id': 'fdd11cb3ab75f95efb7b3bc2d726aa13ac915b66', 'channel_url': 'https://odysee.com/@RT:fdd11cb3ab75f95efb7b3bc2d726aa13ac915b66', diff --git a/yt_dlp/extractor/lecture2go.py b/yt_dlp/extractor/lecture2go.py index 3a9b30a3c..10fb5d479 100644 --- a/yt_dlp/extractor/lecture2go.py +++ b/yt_dlp/extractor/lecture2go.py @@ -10,6 +10,7 @@ class Lecture2GoIE(InfoExtractor): + _WORKING = False _VALID_URL = r'https?://lecture2go\.uni-hamburg\.de/veranstaltungen/-/v/(?P\d+)' _TEST = { 'url': 'https://lecture2go.uni-hamburg.de/veranstaltungen/-/v/17473', diff --git a/yt_dlp/extractor/lecturio.py b/yt_dlp/extractor/lecturio.py index 795012541..629d208fc 100644 --- a/yt_dlp/extractor/lecturio.py +++ b/yt_dlp/extractor/lecturio.py @@ -172,7 +172,7 @@ def _real_extract(self, url): class LecturioCourseIE(LecturioBaseIE): - _VALID_URL = r'https://app\.lecturio\.com/(?:[^/]+/(?P[^/?#&]+)\.course|(?:#/)?course/c/(?P\d+))' + _VALID_URL = r'https?://app\.lecturio\.com/(?:[^/]+/(?P[^/?#&]+)\.course|(?:#/)?course/c/(?P\d+))' _TESTS = [{ 'url': 'https://app.lecturio.com/medical-courses/microbiology-introduction.course#/', 'info_dict': { @@ -209,7 +209,7 @@ def _real_extract(self, url): class LecturioDeCourseIE(LecturioBaseIE): - _VALID_URL = r'https://(?:www\.)?lecturio\.de/[^/]+/(?P[^/?#&]+)\.kurs' + _VALID_URL = r'https?://(?:www\.)?lecturio\.de/[^/]+/(?P[^/?#&]+)\.kurs' _TEST = { 'url': 'https://www.lecturio.de/jura/grundrechte.kurs', 'only_matching': True, diff --git a/yt_dlp/extractor/lenta.py b/yt_dlp/extractor/lenta.py index 10aac984e..fe01bda1c 100644 --- a/yt_dlp/extractor/lenta.py +++ b/yt_dlp/extractor/lenta.py @@ -2,6 +2,7 @@ class LentaIE(InfoExtractor): + _WORKING = False _VALID_URL = r'https?://(?:www\.)?lenta\.ru/[^/]+/\d+/\d+/\d+/(?P[^/?#&]+)' _TESTS = [{ 'url': 'https://lenta.ru/news/2018/03/22/savshenko_go/', diff --git a/yt_dlp/extractor/likee.py b/yt_dlp/extractor/likee.py index 74ee2bea9..324463136 100644 --- a/yt_dlp/extractor/likee.py +++ b/yt_dlp/extractor/likee.py @@ -22,8 +22,6 @@ class LikeeIE(InfoExtractor): 'description': 'md5:9a7ebe816f0e78722ee5ed76f75983b4', 'thumbnail': r're:^https?://.+\.jpg', 'uploader': 'Huỳnh Hồng Quân ', - 'play_count': int, - 'download_count': int, 'artist': 'Huỳnh Hồng Quân ', 'timestamp': 1651571320, 'upload_date': '20220503', @@ -44,11 +42,9 @@ class LikeeIE(InfoExtractor): 'comment_count': int, 'like_count': int, 'uploader': 'Vương Phước Nhi', - 'download_count': int, 'timestamp': 1651506835, 'upload_date': '20220502', 'duration': 60024, - 'play_count': int, 'artist': 'Vương Phước Nhi', 'uploader_id': '649222262', 'view_count': int, @@ -65,9 +61,7 @@ class LikeeIE(InfoExtractor): 'duration': 9684, 'uploader_id': 'fernanda_rivasg', 'view_count': int, - 'play_count': int, 'artist': 'La Cami La✨', - 'download_count': int, 'like_count': int, 'uploader': 'Fernanda Rivas🎶', 'timestamp': 1614034308, @@ -83,13 +77,11 @@ class LikeeIE(InfoExtractor): 'thumbnail': r're:^https?://.+\.jpg', 'comment_count': int, 'duration': 18014, - 'play_count': int, 'view_count': int, 'timestamp': 1611694774, 'like_count': int, 'uploader': 'Fernanda Rivas🎶', 'uploader_id': 'fernanda_rivasg', - 'download_count': int, 'artist': 'ʟᴇʀɪᴋ_ᴜɴɪᴄᴏʀɴ♡︎', 'upload_date': '20210126', }, @@ -128,8 +120,6 @@ def _real_extract(self, url): 'description': info.get('share_desc'), 'view_count': int_or_none(info.get('video_count')), 'like_count': int_or_none(info.get('likeCount')), - 'play_count': int_or_none(info.get('play_count')), - 'download_count': int_or_none(info.get('download_count')), 'comment_count': int_or_none(info.get('comment_count')), 'uploader': str_or_none(info.get('nick_name')), 'uploader_id': str_or_none(info.get('likeeId')), diff --git a/yt_dlp/extractor/linkedin.py b/yt_dlp/extractor/linkedin.py index 2bf2e9a11..ad41c0e20 100644 --- a/yt_dlp/extractor/linkedin.py +++ b/yt_dlp/extractor/linkedin.py @@ -3,16 +3,15 @@ from .common import InfoExtractor from ..utils import ( - clean_html, - extract_attributes, ExtractorError, + extract_attributes, float_or_none, - get_element_by_class, int_or_none, srt_subtitles_timecode, - strip_or_none, mimetype2ext, + traverse_obj, try_get, + url_or_none, urlencode_postdata, urljoin, ) @@ -83,15 +82,29 @@ def _get_video_id(self, video_data, course_slug, video_slug): class LinkedInIE(LinkedInBaseIE): - _VALID_URL = r'https?://(?:www\.)?linkedin\.com/posts/.+?(?P\d+)' + _VALID_URL = r'https?://(?:www\.)?linkedin\.com/posts/[^/?#]+-(?P\d+)-\w{4}/?(?:[?#]|$)' _TESTS = [{ 'url': 'https://www.linkedin.com/posts/mishalkhawaja_sendinblueviews-toronto-digitalmarketing-ugcPost-6850898786781339649-mM20', 'info_dict': { 'id': '6850898786781339649', 'ext': 'mp4', - 'title': 'Mishal K. on LinkedIn: #sendinblueviews #toronto #digitalmarketing', - 'description': 'md5:be125430bab1c574f16aeb186a4d5b19', - 'creator': 'Mishal K.' + 'title': 'Mishal K. on LinkedIn: #sendinblueviews #toronto #digitalmarketing #nowhiring #sendinblue…', + 'description': 'md5:2998a31f6f479376dd62831f53a80f71', + 'uploader': 'Mishal K.', + 'thumbnail': 're:^https?://media.licdn.com/dms/image/.*$', + 'like_count': int + }, + }, { + 'url': 'https://www.linkedin.com/posts/the-mathworks_2_what-is-mathworks-cloud-center-activity-7151241570371948544-4Gu7', + 'info_dict': { + 'id': '7151241570371948544', + 'ext': 'mp4', + 'title': 'MathWorks on LinkedIn: What Is MathWorks Cloud Center?', + 'description': 'md5:95f9d4eeb6337882fb47eefe13d7a40c', + 'uploader': 'MathWorks', + 'thumbnail': 're:^https?://media.licdn.com/dms/image/.*$', + 'like_count': int, + 'subtitles': 'mincount:1' }, }] @@ -99,26 +112,30 @@ def _real_extract(self, url): video_id = self._match_id(url) webpage = self._download_webpage(url, video_id) - title = self._html_extract_title(webpage) - description = clean_html(get_element_by_class('share-update-card__update-text', webpage)) - like_count = int_or_none(get_element_by_class('social-counts-reactions__social-counts-numRections', webpage)) - creator = strip_or_none(clean_html(get_element_by_class('comment__actor-name', webpage))) - - sources = self._parse_json(extract_attributes(self._search_regex(r'(]+>)', webpage, 'video'))['data-sources'], video_id) + video_attrs = extract_attributes(self._search_regex(r'(]+>)', webpage, 'video')) + sources = self._parse_json(video_attrs['data-sources'], video_id) formats = [{ 'url': source['src'], 'ext': mimetype2ext(source.get('type')), 'tbr': float_or_none(source.get('data-bitrate'), scale=1000), } for source in sources] + subtitles = {'en': [{ + 'url': video_attrs['data-captions-url'], + 'ext': 'vtt', + }]} if url_or_none(video_attrs.get('data-captions-url')) else {} return { 'id': video_id, 'formats': formats, - 'title': title, - 'like_count': like_count, - 'creator': creator, + 'title': self._og_search_title(webpage, default=None) or self._html_extract_title(webpage), + 'like_count': int_or_none(self._search_regex( + r'\bdata-num-reactions="(\d+)"', webpage, 'reactions', default=None)), + 'uploader': traverse_obj( + self._yield_json_ld(webpage, video_id), + (lambda _, v: v['@type'] == 'SocialMediaPosting', 'author', 'name', {str}), get_all=False), 'thumbnail': self._og_search_thumbnail(webpage), - 'description': description, + 'description': self._og_search_description(webpage, default=None), + 'subtitles': subtitles, } diff --git a/yt_dlp/extractor/localnews8.py b/yt_dlp/extractor/localnews8.py deleted file mode 100644 index 6f3f02c70..000000000 --- a/yt_dlp/extractor/localnews8.py +++ /dev/null @@ -1,42 +0,0 @@ -from .common import InfoExtractor - - -class LocalNews8IE(InfoExtractor): - _VALID_URL = r'https?://(?:www\.)?localnews8\.com/(?:[^/]+/)*(?P[^/]+)/(?P[0-9]+)' - _TEST = { - 'url': 'http://www.localnews8.com/news/rexburg-business-turns-carbon-fiber-scraps-into-wedding-rings/35183304', - 'md5': 'be4d48aea61aa2bde7be2ee47691ad20', - 'info_dict': { - 'id': '35183304', - 'display_id': 'rexburg-business-turns-carbon-fiber-scraps-into-wedding-rings', - 'ext': 'mp4', - 'title': 'Rexburg business turns carbon fiber scraps into wedding ring', - 'description': 'The process was first invented by Lamborghini and less than a dozen companies around the world use it.', - 'duration': 153, - 'timestamp': 1441844822, - 'upload_date': '20150910', - 'uploader_id': 'api', - } - } - - def _real_extract(self, url): - mobj = self._match_valid_url(url) - video_id = mobj.group('id') - display_id = mobj.group('display_id') - - webpage = self._download_webpage(url, display_id) - - partner_id = self._search_regex( - r'partnerId\s*[:=]\s*(["\'])(?P\d+)\1', - webpage, 'partner id', group='id') - kaltura_id = self._search_regex( - r'videoIdString\s*[:=]\s*(["\'])kaltura:(?P[0-9a-z_]+)\1', - webpage, 'videl id', group='id') - - return { - '_type': 'url_transparent', - 'url': 'kaltura:%s:%s' % (partner_id, kaltura_id), - 'ie_key': 'Kaltura', - 'id': video_id, - 'display_id': display_id, - } diff --git a/yt_dlp/extractor/lumni.py b/yt_dlp/extractor/lumni.py index 5810da0c8..5a9538336 100644 --- a/yt_dlp/extractor/lumni.py +++ b/yt_dlp/extractor/lumni.py @@ -1,8 +1,7 @@ -from .common import InfoExtractor -from .francetv import FranceTVIE +from .francetv import FranceTVBaseInfoExtractor -class LumniIE(InfoExtractor): +class LumniIE(FranceTVBaseInfoExtractor): _VALID_URL = r'https?://(?:www\.)?lumni\.fr/video/(?P[\w-]+)' _TESTS = [{ 'url': 'https://www.lumni.fr/video/l-homme-et-son-environnement-dans-la-revolution-industrielle', @@ -21,4 +20,4 @@ def _real_extract(self, url): webpage = self._download_webpage(url, display_id) video_id = self._html_search_regex( r']+data-factoryid\s*=\s*["\']([^"\']+)', webpage, 'video id') - return self.url_result(f'francetv:{video_id}', FranceTVIE, video_id) + return self._make_url_result(video_id, url=url) diff --git a/yt_dlp/extractor/malltv.py b/yt_dlp/extractor/malltv.py deleted file mode 100644 index e1031d8da..000000000 --- a/yt_dlp/extractor/malltv.py +++ /dev/null @@ -1,107 +0,0 @@ -from .common import InfoExtractor -from ..utils import ( - clean_html, - dict_get, - float_or_none, - int_or_none, - merge_dicts, - parse_duration, - try_get, -) - - -class MallTVIE(InfoExtractor): - _VALID_URL = r'https?://(?:(?:www|sk)\.)?mall\.tv/(?:[^/]+/)*(?P[^/?#&]+)' - _TESTS = [{ - 'url': 'https://www.mall.tv/18-miliard-pro-neziskovky-opravdu-jsou-sportovci-nebo-clovek-v-tisni-pijavice', - 'md5': 'cd69ce29176f6533b65bff69ed9a5f2a', - 'info_dict': { - 'id': 't0zzt0', - 'display_id': '18-miliard-pro-neziskovky-opravdu-jsou-sportovci-nebo-clovek-v-tisni-pijavice', - 'ext': 'mp4', - 'title': '18 miliard pro neziskovky. Opravdu jsou sportovci nebo Člověk v tísni pijavice?', - 'description': 'md5:db7d5744a4bd4043d9d98324aa72ab35', - 'duration': 216, - 'timestamp': 1538870400, - 'upload_date': '20181007', - 'view_count': int, - 'comment_count': int, - 'thumbnail': 'https://cdn.vpplayer.tech/agmipnzv/encode/vjsnigfq/thumbnails/retina.jpg', - 'average_rating': 9.060869565217391, - 'dislike_count': int, - 'like_count': int, - } - }, { - 'url': 'https://www.mall.tv/kdo-to-plati/18-miliard-pro-neziskovky-opravdu-jsou-sportovci-nebo-clovek-v-tisni-pijavice', - 'only_matching': True, - }, { - 'url': 'https://sk.mall.tv/gejmhaus/reklamacia-nehreje-vyrobnik-tepla-alebo-spekacka', - 'only_matching': True, - }, { - 'url': 'https://www.mall.tv/zivoty-slavnych/nadeje-vychodu-i-zapadu-jak-michail-gorbacov-zmenil-politickou-mapu-sveta-a-ziskal-za-to-nobelovu-cenu-miru', - 'info_dict': { - 'id': 'yx010y', - 'ext': 'mp4', - 'dislike_count': int, - 'description': 'md5:aee02bee5a8d072c6a8207b91d1905a9', - 'thumbnail': 'https://cdn.vpplayer.tech/agmipnzv/encode/vjsnjdeu/thumbnails/retina.jpg', - 'comment_count': int, - 'display_id': 'md5:0ec2afa94d2e2b7091c019cef2a43a9b', - 'like_count': int, - 'duration': 752, - 'timestamp': 1646956800, - 'title': 'md5:fe79385daaf16d74c12c1ec4a26687af', - 'view_count': int, - 'upload_date': '20220311', - 'average_rating': 9.685714285714285, - } - }] - - def _real_extract(self, url): - display_id = self._match_id(url) - - webpage = self._download_webpage( - url, display_id, headers=self.geo_verification_headers()) - - video = self._parse_json(self._search_regex( - r'videoObject\s*=\s*JSON\.parse\(JSON\.stringify\(({.+?})\)\);', - webpage, 'video object'), display_id) - - video_id = self._search_regex( - r']+value\s*=\s*(\w+)', webpage, 'video id') - - formats = self._extract_m3u8_formats( - video['VideoSource'], video_id, 'mp4', 'm3u8_native') - - subtitles = {} - for s in (video.get('Subtitles') or {}): - s_url = s.get('Url') - if not s_url: - continue - subtitles.setdefault(s.get('Language') or 'cz', []).append({ - 'url': s_url, - }) - - entity_counts = video.get('EntityCounts') or {} - - def get_count(k): - v = entity_counts.get(k + 's') or {} - return int_or_none(dict_get(v, ('Count', 'StrCount'))) - - info = self._search_json_ld(webpage, video_id, default={}) - - return merge_dicts({ - 'id': str(video_id), - 'display_id': display_id, - 'title': video.get('Title'), - 'description': clean_html(video.get('Description')), - 'thumbnail': video.get('ThumbnailUrl'), - 'formats': formats, - 'subtitles': subtitles, - 'duration': int_or_none(video.get('DurationSeconds')) or parse_duration(video.get('Duration')), - 'view_count': get_count('View'), - 'like_count': get_count('Like'), - 'dislike_count': get_count('Dislike'), - 'average_rating': float_or_none(try_get(video, lambda x: x['EntityRating']['AvarageRate'])), - 'comment_count': get_count('Comment'), - }, info) diff --git a/yt_dlp/extractor/manyvids.py b/yt_dlp/extractor/manyvids.py index 741745378..2aa3a3c93 100644 --- a/yt_dlp/extractor/manyvids.py +++ b/yt_dlp/extractor/manyvids.py @@ -12,6 +12,7 @@ class ManyVidsIE(InfoExtractor): + _WORKING = False _VALID_URL = r'(?i)https?://(?:www\.)?manyvids\.com/video/(?P\d+)' _TESTS = [{ # preview video diff --git a/yt_dlp/extractor/markiza.py b/yt_dlp/extractor/markiza.py index 53ed79158..ca465eae9 100644 --- a/yt_dlp/extractor/markiza.py +++ b/yt_dlp/extractor/markiza.py @@ -10,6 +10,7 @@ class MarkizaIE(InfoExtractor): + _WORKING = False _VALID_URL = r'https?://(?:www\.)?videoarchiv\.markiza\.sk/(?:video/(?:[^/]+/)*|embed/)(?P\d+)(?:[_/]|$)' _TESTS = [{ 'url': 'http://videoarchiv.markiza.sk/video/oteckovia/84723_oteckovia-109', @@ -68,6 +69,7 @@ def _real_extract(self, url): class MarkizaPageIE(InfoExtractor): + _WORKING = False _VALID_URL = r'https?://(?:www\.)?(?:(?:[^/]+\.)?markiza|tvnoviny)\.sk/(?:[^/]+/)*(?P\d+)_' _TESTS = [{ 'url': 'http://www.markiza.sk/soubiz/zahranicny/1923705_oteckovia-maju-svoj-den-ti-slavni-nie-su-o-nic-menej-rozkosni', diff --git a/yt_dlp/extractor/medaltv.py b/yt_dlp/extractor/medaltv.py index eeb5b85f3..675ad8ccc 100644 --- a/yt_dlp/extractor/medaltv.py +++ b/yt_dlp/extractor/medaltv.py @@ -9,7 +9,6 @@ int_or_none, str_or_none, traverse_obj, - update_url_query, ) @@ -82,7 +81,7 @@ class MedalTVIE(InfoExtractor): def _real_extract(self, url): video_id = self._match_id(url) - webpage = self._download_webpage(update_url_query(url, {'mobilebypass': 'true'}), video_id) + webpage = self._download_webpage(url, video_id, query={'mobilebypass': 'true'}) hydration_data = self._search_json( r']*>[^<]*\bhydrationData\s*=', webpage, diff --git a/yt_dlp/extractor/megaphone.py b/yt_dlp/extractor/megaphone.py index af80523e3..d249a8492 100644 --- a/yt_dlp/extractor/megaphone.py +++ b/yt_dlp/extractor/megaphone.py @@ -5,18 +5,18 @@ class MegaphoneIE(InfoExtractor): IE_NAME = 'megaphone.fm' IE_DESC = 'megaphone.fm embedded players' - _VALID_URL = r'https://player\.megaphone\.fm/(?P[A-Z0-9]+)' + _VALID_URL = r'https?://player\.megaphone\.fm/(?P[A-Z0-9]+)' _EMBED_REGEX = [rf']*?\ssrc=["\'](?P{_VALID_URL})'] _TEST = { - 'url': 'https://player.megaphone.fm/GLT9749789991?"', + 'url': 'https://player.megaphone.fm/GLT9749789991', 'md5': '4816a0de523eb3e972dc0dda2c191f96', 'info_dict': { 'id': 'GLT9749789991', 'ext': 'mp3', 'title': '#97 What Kind Of Idiot Gets Phished?', 'thumbnail': r're:^https://.*\.png.*$', - 'duration': 1776.26375, - 'author': 'Reply All', + 'duration': 1998.36, + 'creators': ['Reply All'], }, } @@ -40,7 +40,7 @@ def _real_extract(self, url): 'id': video_id, 'thumbnail': thumbnail, 'title': title, - 'author': author, + 'creators': [author] if author else None, 'duration': episode_data['duration'], 'formats': formats, } diff --git a/yt_dlp/extractor/miaopai.py b/yt_dlp/extractor/miaopai.py deleted file mode 100644 index 329ce3658..000000000 --- a/yt_dlp/extractor/miaopai.py +++ /dev/null @@ -1,36 +0,0 @@ -from .common import InfoExtractor - - -class MiaoPaiIE(InfoExtractor): - _VALID_URL = r'https?://(?:www\.)?miaopai\.com/show/(?P[-A-Za-z0-9~_]+)' - _TEST = { - 'url': 'http://www.miaopai.com/show/n~0hO7sfV1nBEw4Y29-Hqg__.htm', - 'md5': '095ed3f1cd96b821add957bdc29f845b', - 'info_dict': { - 'id': 'n~0hO7sfV1nBEw4Y29-Hqg__', - 'ext': 'mp4', - 'title': '西游记音乐会的秒拍视频', - 'thumbnail': 're:^https?://.*/n~0hO7sfV1nBEw4Y29-Hqg___m.jpg', - } - } - - _USER_AGENT_IPAD = 'Mozilla/5.0 (iPad; CPU OS 9_1 like Mac OS X) AppleWebKit/601.1.46 (KHTML, like Gecko) Version/9.0 Mobile/13B143 Safari/601.1' - - def _real_extract(self, url): - video_id = self._match_id(url) - webpage = self._download_webpage( - url, video_id, headers={'User-Agent': self._USER_AGENT_IPAD}) - - title = self._html_extract_title(webpage) - thumbnail = self._html_search_regex( - r']+class=(?P[\'"]).*\bvideo_img\b.*(?P=q1)[^>]+data-url=(?P[\'"])(?P[^\'"]+)(?P=q2)', - webpage, 'thumbnail', fatal=False, group='url') - videos = self._parse_html5_media_entries(url, webpage, video_id) - info = videos[0] - - info.update({ - 'id': video_id, - 'title': title, - 'thumbnail': thumbnail, - }) - return info diff --git a/yt_dlp/extractor/ministrygrid.py b/yt_dlp/extractor/ministrygrid.py deleted file mode 100644 index 053c6726c..000000000 --- a/yt_dlp/extractor/ministrygrid.py +++ /dev/null @@ -1,55 +0,0 @@ -from .common import InfoExtractor -from ..utils import ( - ExtractorError, - smuggle_url, -) - - -class MinistryGridIE(InfoExtractor): - _VALID_URL = r'https?://(?:www\.)?ministrygrid\.com/([^/?#]*/)*(?P[^/#?]+)/?(?:$|[?#])' - - _TEST = { - 'url': 'http://www.ministrygrid.com/training-viewer/-/training/t4g-2014-conference/the-gospel-by-numbers-4/the-gospel-by-numbers', - 'md5': '844be0d2a1340422759c2a9101bab017', - 'info_dict': { - 'id': '3453494717001', - 'ext': 'mp4', - 'title': 'The Gospel by Numbers', - 'thumbnail': r're:^https?://.*\.jpg', - 'upload_date': '20140410', - 'description': 'Coming soon from T4G 2014!', - 'uploader_id': '2034960640001', - 'timestamp': 1397145591, - }, - 'params': { - # m3u8 download - 'skip_download': True, - }, - 'add_ie': ['TDSLifeway'], - } - - def _real_extract(self, url): - video_id = self._match_id(url) - - webpage = self._download_webpage(url, video_id) - portlets = self._parse_json(self._search_regex( - r'Liferay\.Portlet\.list=(\[.+?\])', webpage, 'portlet list'), - video_id) - pl_id = self._search_regex( - r'getPlid:function\(\){return"(\d+)"}', webpage, 'p_l_id') - - for i, portlet in enumerate(portlets): - portlet_url = 'http://www.ministrygrid.com/c/portal/render_portlet?p_l_id=%s&p_p_id=%s' % (pl_id, portlet) - portlet_code = self._download_webpage( - portlet_url, video_id, - note='Looking in portlet %s (%d/%d)' % (portlet, i + 1, len(portlets)), - fatal=False) - video_iframe_url = self._search_regex( - r'\d+)' + _VALID_URL = r'https?://www\.monstercat\.com/release/(?P\d+)' _TESTS = [{ 'url': 'https://www.monstercat.com/release/742779548009', 'playlist_count': 20, diff --git a/yt_dlp/extractor/morningstar.py b/yt_dlp/extractor/morningstar.py deleted file mode 100644 index e9fcfe3e2..000000000 --- a/yt_dlp/extractor/morningstar.py +++ /dev/null @@ -1,45 +0,0 @@ -from .common import InfoExtractor - - -class MorningstarIE(InfoExtractor): - IE_DESC = 'morningstar.com' - _VALID_URL = r'https?://(?:(?:www|news)\.)morningstar\.com/[cC]over/video[cC]enter\.aspx\?id=(?P[0-9]+)' - _TESTS = [{ - 'url': 'http://www.morningstar.com/cover/videocenter.aspx?id=615869', - 'md5': '6c0acface7a787aadc8391e4bbf7b0f5', - 'info_dict': { - 'id': '615869', - 'ext': 'mp4', - 'title': 'Get Ahead of the Curve on 2013 Taxes', - 'description': "Vanguard's Joel Dickson on managing higher tax rates for high-income earners and fund capital-gain distributions in 2013.", - 'thumbnail': r're:^https?://.*m(?:orning)?star\.com/.+thumb\.jpg$' - } - }, { - 'url': 'http://news.morningstar.com/cover/videocenter.aspx?id=825556', - 'only_matching': True, - }] - - def _real_extract(self, url): - mobj = self._match_valid_url(url) - video_id = mobj.group('id') - - webpage = self._download_webpage(url, video_id) - title = self._html_search_regex( - r'

(.*?)

', webpage, 'title') - video_url = self._html_search_regex( - r'(.*?)', - webpage, 'description', fatal=False) - - return { - 'id': video_id, - 'title': title, - 'url': video_url, - 'thumbnail': thumbnail, - 'description': description, - } diff --git a/yt_dlp/extractor/motorsport.py b/yt_dlp/extractor/motorsport.py index efb087d03..167d85fa9 100644 --- a/yt_dlp/extractor/motorsport.py +++ b/yt_dlp/extractor/motorsport.py @@ -5,6 +5,7 @@ class MotorsportIE(InfoExtractor): + _WORKING = False IE_DESC = 'motorsport.com' _VALID_URL = r'https?://(?:www\.)?motorsport\.com/[^/?#]+/video/(?:[^/?#]+/)(?P[^/]+)/?(?:$|[?#])' _TEST = { diff --git a/yt_dlp/extractor/mtv.py b/yt_dlp/extractor/mtv.py index e192453c7..404e431bc 100644 --- a/yt_dlp/extractor/mtv.py +++ b/yt_dlp/extractor/mtv.py @@ -451,6 +451,7 @@ def _real_extract(self, url): class MTVDEIE(MTVServicesInfoExtractor): + _WORKING = False IE_NAME = 'mtv.de' _VALID_URL = r'https?://(?:www\.)?mtv\.de/(?:musik/videoclips|folgen|news)/(?P[0-9a-z]+)' _TESTS = [{ diff --git a/yt_dlp/extractor/muenchentv.py b/yt_dlp/extractor/muenchentv.py index 36a2d4688..934cd4fbc 100644 --- a/yt_dlp/extractor/muenchentv.py +++ b/yt_dlp/extractor/muenchentv.py @@ -9,6 +9,7 @@ class MuenchenTVIE(InfoExtractor): + _WORKING = False _VALID_URL = r'https?://(?:www\.)?muenchen\.tv/livestream' IE_DESC = 'münchen.tv' _TEST = { diff --git a/yt_dlp/extractor/murrtube.py b/yt_dlp/extractor/murrtube.py index 6cdbbda16..74365c0c0 100644 --- a/yt_dlp/extractor/murrtube.py +++ b/yt_dlp/extractor/murrtube.py @@ -12,6 +12,7 @@ class MurrtubeIE(InfoExtractor): + _WORKING = False _VALID_URL = r'''(?x) (?: murrtube:| @@ -100,6 +101,7 @@ def _real_extract(self, url): class MurrtubeUserIE(MurrtubeIE): # XXX: Do not subclass from concrete IE + _WORKING = False IE_DESC = 'Murrtube user profile' _VALID_URL = r'https?://murrtube\.net/(?P[^/]+)$' _TEST = { diff --git a/yt_dlp/extractor/musicdex.py b/yt_dlp/extractor/musicdex.py index 48f29702c..a86351458 100644 --- a/yt_dlp/extractor/musicdex.py +++ b/yt_dlp/extractor/musicdex.py @@ -17,11 +17,11 @@ def _return_info(self, track_json, album_json, id): 'track_number': track_json.get('number'), 'url': format_field(track_json, 'url', 'https://www.musicdex.org/%s'), 'duration': track_json.get('duration'), - 'genre': [genre.get('name') for genre in track_json.get('genres') or []], + 'genres': [genre.get('name') for genre in track_json.get('genres') or []], 'like_count': track_json.get('likes_count'), 'view_count': track_json.get('plays'), - 'artist': [artist.get('name') for artist in track_json.get('artists') or []], - 'album_artist': [artist.get('name') for artist in album_json.get('artists') or []], + 'artists': [artist.get('name') for artist in track_json.get('artists') or []], + 'album_artists': [artist.get('name') for artist in album_json.get('artists') or []], 'thumbnail': format_field(album_json, 'image', 'https://www.musicdex.org/%s'), 'album': album_json.get('name'), 'release_year': try_get(album_json, lambda x: date_from_str(unified_strdate(x['release_date'])).year), @@ -43,11 +43,11 @@ class MusicdexSongIE(MusicdexBaseIE): 'track': 'dual existence', 'track_number': 1, 'duration': 266000, - 'genre': ['Anime'], + 'genres': ['Anime'], 'like_count': int, 'view_count': int, - 'artist': ['fripSide'], - 'album_artist': ['fripSide'], + 'artists': ['fripSide'], + 'album_artists': ['fripSide'], 'thumbnail': 'https://www.musicdex.org/storage/album/9iDIam1DHTVqUG4UclFIEq1WAFGXfPW4y0TtZa91.png', 'album': 'To Aru Kagaku no Railgun T OP2 Single - dual existence', 'release_year': 2020 @@ -69,9 +69,9 @@ class MusicdexAlbumIE(MusicdexBaseIE): 'playlist_mincount': 28, 'info_dict': { 'id': '56', - 'genre': ['OST'], + 'genres': ['OST'], 'view_count': int, - 'artist': ['TENMON & Eiichiro Yanagi / minori'], + 'artists': ['TENMON & Eiichiro Yanagi / minori'], 'title': 'ef - a tale of memories Original Soundtrack 2 ~fortissimo~', 'release_year': 2008, 'thumbnail': 'https://www.musicdex.org/storage/album/2rSHkyYBYfB7sbvElpEyTMcUn6toY7AohOgJuDlE.jpg', @@ -88,9 +88,9 @@ def _real_extract(self, url): 'id': id, 'title': data_json.get('name'), 'description': data_json.get('description'), - 'genre': [genre.get('name') for genre in data_json.get('genres') or []], + 'genres': [genre.get('name') for genre in data_json.get('genres') or []], 'view_count': data_json.get('plays'), - 'artist': [artist.get('name') for artist in data_json.get('artists') or []], + 'artists': [artist.get('name') for artist in data_json.get('artists') or []], 'thumbnail': format_field(data_json, 'image', 'https://www.musicdex.org/%s'), 'release_year': try_get(data_json, lambda x: date_from_str(unified_strdate(x['release_date'])).year), 'entries': entries, diff --git a/yt_dlp/extractor/myvideoge.py b/yt_dlp/extractor/myvideoge.py index 64cee48e7..3e0bb2499 100644 --- a/yt_dlp/extractor/myvideoge.py +++ b/yt_dlp/extractor/myvideoge.py @@ -64,7 +64,7 @@ def _real_extract(self, url): # translate any ka month to an en one re.sub('|'.join(self._MONTH_NAMES_KA), lambda m: MONTH_NAMES['en'][self._MONTH_NAMES_KA.index(m.group(0))], - upload_date, re.I)) + upload_date, flags=re.I)) if upload_date else None) return { diff --git a/yt_dlp/extractor/myvidster.py b/yt_dlp/extractor/myvidster.py index c91f294bf..e3b700dbb 100644 --- a/yt_dlp/extractor/myvidster.py +++ b/yt_dlp/extractor/myvidster.py @@ -2,7 +2,7 @@ class MyVidsterIE(InfoExtractor): - _VALID_URL = r'https?://(?:www\.)?myvidster\.com/video/(?P\d+)/' + _VALID_URL = r'https?://(?:www\.)?myvidster\.com/video/(?P\d+)' _TEST = { 'url': 'http://www.myvidster.com/video/32059805/Hot_chemistry_with_raw_love_making', diff --git a/yt_dlp/extractor/ndtv.py b/yt_dlp/extractor/ndtv.py index bfe52f77d..d099db37b 100644 --- a/yt_dlp/extractor/ndtv.py +++ b/yt_dlp/extractor/ndtv.py @@ -5,6 +5,7 @@ class NDTVIE(InfoExtractor): + _WORKING = False _VALID_URL = r'https?://(?:[^/]+\.)?ndtv\.com/(?:[^/]+/)*videos?/?(?:[^/]+/)*[^/?^&]+-(?P\d+)' _TESTS = [ diff --git a/yt_dlp/extractor/nebula.py b/yt_dlp/extractor/nebula.py index 136b0e10a..cb8f6a67d 100644 --- a/yt_dlp/extractor/nebula.py +++ b/yt_dlp/extractor/nebula.py @@ -1,6 +1,7 @@ import itertools import json +from .art19 import Art19IE from .common import InfoExtractor from ..networking.exceptions import HTTPError from ..utils import ( @@ -112,7 +113,8 @@ def _extract_video_metadata(self, episode): class NebulaIE(NebulaBaseIE): - _VALID_URL = rf'{_BASE_URL_RE}/videos/(?P[-\w]+)' + IE_NAME = 'nebula:video' + _VALID_URL = rf'{_BASE_URL_RE}/videos/(?P[\w-]+)' _TESTS = [{ 'url': 'https://nebula.tv/videos/that-time-disney-remade-beauty-and-the-beast', 'info_dict': { @@ -236,8 +238,8 @@ def _real_extract(self, url): class NebulaClassIE(NebulaBaseIE): - IE_NAME = 'nebula:class' - _VALID_URL = rf'{_BASE_URL_RE}/(?P[-\w]+)/(?P\d+)' + IE_NAME = 'nebula:media' + _VALID_URL = rf'{_BASE_URL_RE}/(?!(?:myshows|library|videos)/)(?P[\w-]+)/(?P[\w-]+)/?(?:$|[?#])' _TESTS = [{ 'url': 'https://nebula.tv/copyright-for-fun-and-profit/14', 'info_dict': { @@ -253,6 +255,46 @@ class NebulaClassIE(NebulaBaseIE): 'title': 'Photos, Sculpture, and Video', }, 'params': {'skip_download': 'm3u8'}, + }, { + 'url': 'https://nebula.tv/extremitiespodcast/pyramiden-the-high-arctic-soviet-ghost-town', + 'info_dict': { + 'ext': 'mp3', + 'id': '018f65f0-0033-4021-8f87-2d132beb19aa', + 'description': 'md5:05d2b23ab780c955e2511a2b9127acff', + 'series_id': '335e8159-d663-491a-888f-1732285706ac', + 'modified_timestamp': 1599091504, + 'episode_id': '018f65f0-0033-4021-8f87-2d132beb19aa', + 'series': 'Extremities', + 'modified_date': '20200903', + 'upload_date': '20200902', + 'title': 'Pyramiden: The High-Arctic Soviet Ghost Town', + 'release_timestamp': 1571237958, + 'thumbnail': r're:^https?://content\.production\.cdn\.art19\.com.*\.jpeg$', + 'duration': 1546.05714, + 'timestamp': 1599085608, + 'release_date': '20191016', + }, + }, { + 'url': 'https://nebula.tv/thelayover/the-layover-episode-1', + 'info_dict': { + 'ext': 'mp3', + 'id': '9d74a762-00bb-45a8-9e8d-9ed47c04a1d0', + 'episode_number': 1, + 'thumbnail': r're:^https?://content\.production\.cdn\.art19\.com.*\.jpeg$', + 'release_date': '20230304', + 'modified_date': '20230403', + 'series': 'The Layover', + 'episode_id': '9d74a762-00bb-45a8-9e8d-9ed47c04a1d0', + 'modified_timestamp': 1680554566, + 'duration': 3130.46401, + 'release_timestamp': 1677943800, + 'title': 'The Layover — Episode 1', + 'series_id': '874303a5-4900-4626-a4b6-2aacac34466a', + 'upload_date': '20230303', + 'episode': 'Episode 1', + 'timestamp': 1677883672, + 'description': 'md5:002cca89258e3bc7c268d5b8c24ba482', + }, }] def _real_extract(self, url): @@ -268,16 +310,38 @@ def _real_extract(self, url): metadata = self._call_api( f'https://content.api.nebula.app/content/{slug}/{episode}/?include=lessons', - slug, note='Fetching video metadata') - return { - **self._extract_video_metadata(metadata), - **self._extract_formats(metadata['id'], slug), - } + slug, note='Fetching class/podcast metadata') + content_type = metadata.get('type') + if content_type == 'lesson': + return { + **self._extract_video_metadata(metadata), + **self._extract_formats(metadata['id'], slug), + } + elif content_type == 'podcast_episode': + episode_url = metadata['episode_url'] + if not episode_url and metadata.get('premium'): + self.raise_login_required() + + if Art19IE.suitable(episode_url): + return self.url_result(episode_url, Art19IE) + return traverse_obj(metadata, { + 'id': ('id', {str}), + 'url': ('episode_url', {url_or_none}), + 'title': ('title', {str}), + 'description': ('description', {str}), + 'timestamp': ('published_at', {parse_iso8601}), + 'duration': ('duration', {int_or_none}), + 'channel_id': ('channel_id', {str}), + 'chnanel': ('channel_title', {str}), + 'thumbnail': ('assets', 'regular', {url_or_none}), + }) + + raise ExtractorError(f'Unexpected content type {content_type!r}') class NebulaSubscriptionsIE(NebulaBaseIE): IE_NAME = 'nebula:subscriptions' - _VALID_URL = rf'{_BASE_URL_RE}/(?Pmyshows|library/latest-videos)' + _VALID_URL = rf'{_BASE_URL_RE}/(?Pmyshows|library/latest-videos)/?(?:$|[?#])' _TESTS = [{ 'url': 'https://nebula.tv/myshows', 'playlist_mincount': 1, @@ -310,7 +374,7 @@ def _real_extract(self, url): class NebulaChannelIE(NebulaBaseIE): IE_NAME = 'nebula:channel' - _VALID_URL = rf'{_BASE_URL_RE}/(?!myshows|library|videos/)(?P[-\w]+)/?(?:$|[?#])' + _VALID_URL = rf'{_BASE_URL_RE}/(?!myshows|library|videos)(?P[\w-]+)/?(?:$|[?#])' _TESTS = [{ 'url': 'https://nebula.tv/tom-scott-presents-money', 'info_dict': { @@ -343,6 +407,14 @@ class NebulaChannelIE(NebulaBaseIE): 'description': 'md5:6690248223eed044a9f11cd5a24f9742', }, 'playlist_count': 23, + }, { + 'url': 'https://nebula.tv/trussissuespodcast', + 'info_dict': { + 'id': 'trussissuespodcast', + 'title': 'The TLDR News Podcast', + 'description': 'md5:a08c4483bc0b705881d3e0199e721385', + }, + 'playlist_mincount': 80, }] def _generate_playlist_entries(self, collection_id, collection_slug): @@ -365,6 +437,17 @@ def _generate_class_entries(self, channel): lesson.get('share_url') or f'https://nebula.tv/{metadata["class_slug"]}/{metadata["slug"]}', {'id': lesson['id']}), NebulaClassIE, url_transparent=True, **metadata) + def _generate_podcast_entries(self, collection_id, collection_slug): + next_url = f'https://content.api.nebula.app/podcast_channels/{collection_id}/podcast_episodes/?ordering=-published_at&premium=true' + for page_num in itertools.count(1): + episodes = self._call_api(next_url, collection_slug, note=f'Retrieving podcast page {page_num}') + + for episode in traverse_obj(episodes, ('results', lambda _, v: url_or_none(v['share_url']))): + yield self.url_result(episode['share_url'], NebulaClassIE) + next_url = episodes.get('next') + if not next_url: + break + def _real_extract(self, url): collection_slug = self._match_id(url) channel = self._call_api( @@ -373,6 +456,8 @@ def _real_extract(self, url): if channel.get('type') == 'class': entries = self._generate_class_entries(channel) + elif channel.get('type') == 'podcast_channel': + entries = self._generate_podcast_entries(channel['id'], collection_slug) else: entries = self._generate_playlist_entries(channel['id'], collection_slug) diff --git a/yt_dlp/extractor/nekohacker.py b/yt_dlp/extractor/nekohacker.py index e10ffe925..24b66570e 100644 --- a/yt_dlp/extractor/nekohacker.py +++ b/yt_dlp/extractor/nekohacker.py @@ -118,7 +118,6 @@ class NekoHackerIE(InfoExtractor): 'artist': 'Neko Hacker', 'track': 'md5:1a5fcbc96ca3c3265b1c6f9f79f30fd0', 'track_number': 1, - 'duration': None } }, { @@ -136,7 +135,6 @@ class NekoHackerIE(InfoExtractor): 'artist': 'Neko Hacker', 'track': 'むじな de なじむ feat. 六科なじむ (CV: 日高里菜 )', 'track_number': 2, - 'duration': None } }, { @@ -154,7 +152,6 @@ class NekoHackerIE(InfoExtractor): 'artist': 'Neko Hacker', 'track': '進め！むじなカンパニー (instrumental)', 'track_number': 3, - 'duration': None } }, { @@ -172,7 +169,6 @@ class NekoHackerIE(InfoExtractor): 'artist': 'Neko Hacker', 'track': 'むじな de なじむ (instrumental)', 'track_number': 4, - 'duration': None } } ] diff --git a/yt_dlp/extractor/nerdcubed.py b/yt_dlp/extractor/nerdcubed.py index 7c801b5d3..5f5607a20 100644 --- a/yt_dlp/extractor/nerdcubed.py +++ b/yt_dlp/extractor/nerdcubed.py @@ -1,33 +1,38 @@ -import datetime - from .common import InfoExtractor +from .youtube import YoutubeIE +from ..utils import parse_iso8601, url_or_none +from ..utils.traversal import traverse_obj class NerdCubedFeedIE(InfoExtractor): - _VALID_URL = r'https?://(?:www\.)?nerdcubed\.co\.uk/feed\.json' + _VALID_URL = r'https?://(?:www\.)?nerdcubed\.co\.uk/?(?:$|[#?])' _TEST = { - 'url': 'http://www.nerdcubed.co.uk/feed.json', + 'url': 'http://www.nerdcubed.co.uk/', 'info_dict': { 'id': 'nerdcubed-feed', 'title': 'nerdcubed.co.uk feed', }, - 'playlist_mincount': 1300, + 'playlist_mincount': 5500, } + def _extract_video(self, feed_entry): + return self.url_result( + f'https://www.youtube.com/watch?v={feed_entry["id"]}', YoutubeIE, + **traverse_obj(feed_entry, { + 'id': ('id', {str}), + 'title': ('title', {str}), + 'description': ('description', {str}), + 'timestamp': ('publishedAt', {parse_iso8601}), + 'channel': ('source', 'name', {str}), + 'channel_id': ('source', 'id', {str}), + 'channel_url': ('source', 'url', {str}), + 'thumbnail': ('thumbnail', 'source', {url_or_none}), + }), url_transparent=True) + def _real_extract(self, url): - feed = self._download_json(url, url, 'Downloading NerdCubed JSON feed') + video_id = 'nerdcubed-feed' + feed = self._download_json('https://www.nerdcubed.co.uk/_/cdn/videos.json', video_id) - entries = [{ - '_type': 'url', - 'title': feed_entry['title'], - 'uploader': feed_entry['source']['name'] if feed_entry['source'] else None, - 'upload_date': datetime.datetime.strptime(feed_entry['date'], '%Y-%m-%d').strftime('%Y%m%d'), - 'url': 'http://www.youtube.com/watch?v=' + feed_entry['youtube_id'], - } for feed_entry in feed] - - return { - '_type': 'playlist', - 'title': 'nerdcubed.co.uk feed', - 'id': 'nerdcubed-feed', - 'entries': entries, - } + return self.playlist_result( + map(self._extract_video, traverse_obj(feed, ('videos', lambda _, v: v['id']))), + video_id, 'nerdcubed.co.uk feed') diff --git a/yt_dlp/extractor/netzkino.py b/yt_dlp/extractor/netzkino.py index 9c314e223..e9422eebf 100644 --- a/yt_dlp/extractor/netzkino.py +++ b/yt_dlp/extractor/netzkino.py @@ -8,6 +8,7 @@ class NetzkinoIE(InfoExtractor): + _WORKING = False _VALID_URL = r'https?://(?:www\.)?netzkino\.de/\#!/[^/]+/(?P[^/]+)' _TESTS = [{ diff --git a/yt_dlp/extractor/newgrounds.py b/yt_dlp/extractor/newgrounds.py index 9601cd10e..67e52efd6 100644 --- a/yt_dlp/extractor/newgrounds.py +++ b/yt_dlp/extractor/newgrounds.py @@ -2,7 +2,9 @@ import re from .common import InfoExtractor +from ..networking.exceptions import HTTPError from ..utils import ( + ExtractorError, OnDemandPagedList, clean_html, extract_attributes, @@ -10,12 +12,16 @@ int_or_none, parse_count, parse_duration, - traverse_obj, unified_timestamp, + url_or_none, + urlencode_postdata, + urljoin, ) +from ..utils.traversal import traverse_obj class NewgroundsIE(InfoExtractor): + _NETRC_MACHINE = 'newgrounds' _VALID_URL = r'https?://(?:www\.)?newgrounds\.com/(?:audio/listen|portal/view)/(?P\d+)(?:/format/flash)?' _TESTS = [{ 'url': 'https://www.newgrounds.com/audio/listen/549479', @@ -25,11 +31,13 @@ class NewgroundsIE(InfoExtractor): 'ext': 'mp3', 'title': 'B7 - BusMode', 'uploader': 'Burn7', - 'timestamp': 1378878540, + 'timestamp': 1378892945, 'upload_date': '20130911', 'duration': 143, 'view_count': int, 'description': 'md5:b8b3c2958875189f07d8e313462e8c4f', + 'age_limit': 0, + 'thumbnail': r're:^https://aicon\.ngfiles\.com/549/549479\.png', }, }, { 'url': 'https://www.newgrounds.com/portal/view/1', @@ -39,11 +47,12 @@ class NewgroundsIE(InfoExtractor): 'ext': 'mp4', 'title': 'Scrotum 1', 'uploader': 'Brian-Beaton', - 'timestamp': 955064100, - 'upload_date': '20000406', + 'timestamp': 955078533, + 'upload_date': '20000407', 'view_count': int, 'description': 'Scrotum plays "catch."', 'age_limit': 17, + 'thumbnail': r're:^https://picon\.ngfiles\.com/0/flash_1_card\.png', }, }, { # source format unavailable, additional mp4 formats @@ -53,11 +62,12 @@ class NewgroundsIE(InfoExtractor): 'ext': 'mp4', 'title': 'ZTV News Episode 8', 'uploader': 'ZONE-SAMA', - 'timestamp': 1487965140, - 'upload_date': '20170224', + 'timestamp': 1487983183, + 'upload_date': '20170225', 'view_count': int, 'description': 'md5:aff9b330ec2e78ed93b1ad6d017accc6', 'age_limit': 17, + 'thumbnail': r're:^https://picon\.ngfiles\.com/689000/flash_689400_card\.png', }, 'params': { 'skip_download': True, @@ -70,11 +80,12 @@ class NewgroundsIE(InfoExtractor): 'ext': 'mp4', 'title': 'Metal Gear Awesome', 'uploader': 'Egoraptor', - 'timestamp': 1140663240, + 'timestamp': 1140681292, 'upload_date': '20060223', 'view_count': int, 'description': 'md5:9246c181614e23754571995104da92e0', 'age_limit': 13, + 'thumbnail': r're:^https://picon\.ngfiles\.com/297000/flash_297383_card\.png', } }, { 'url': 'https://www.newgrounds.com/portal/view/297383/format/flash', @@ -86,8 +97,24 @@ class NewgroundsIE(InfoExtractor): 'description': 'Metal Gear Awesome', 'uploader': 'Egoraptor', 'upload_date': '20060223', - 'timestamp': 1140663240, + 'timestamp': 1140681292, + 'view_count': int, 'age_limit': 13, + 'thumbnail': r're:^https://picon\.ngfiles\.com/297000/flash_297383_card\.png', + } + }, { + 'url': 'https://www.newgrounds.com/portal/view/823109', + 'info_dict': { + 'id': '823109', + 'ext': 'mp4', + 'title': 'Rouge Futa Fleshlight Fuck', + 'description': 'I made a fleshlight model and I wanted to use it in an animation. Based on a video by CDNaturally.', + 'uploader': 'DefaultUser12', + 'upload_date': '20211122', + 'timestamp': 1637611540, + 'view_count': int, + 'age_limit': 18, + 'thumbnail': r're:^https://picon\.ngfiles\.com/823000/flash_823109_card\.png', } }] _AGE_LIMIT = { @@ -96,42 +123,59 @@ class NewgroundsIE(InfoExtractor): 'm': 17, 'a': 18, } + _LOGIN_URL = 'https://www.newgrounds.com/passport' + + def _perform_login(self, username, password): + login_webpage = self._download_webpage(self._LOGIN_URL, None, 'Downloading login page') + login_url = urljoin(self._LOGIN_URL, self._search_regex( + r'
]+>([^<]+)'), webpage, 'uploader', fatal=False) - age_limit = self._html_search_regex( - r']+>', webpage, 'age_limit', default='e') - age_limit = self._AGE_LIMIT.get(age_limit) - - timestamp = unified_timestamp(self._html_search_regex( - (r'
\s*Uploaded\s*
\s*
([^<]+
\s*
[^<]+)', - r'
\s*Uploaded\s*
\s*
([^<]+)'), webpage, 'timestamp', - default=None)) - - duration = parse_duration(self._html_search_regex( - r'"duration"\s*:\s*["\']?(\d+)["\']?', webpage, - 'duration', default=None)) - - description = clean_html(get_element_by_id('author_comments', webpage)) or self._og_search_description(webpage) - - view_count = parse_count(self._html_search_regex( - r'(?s)
\s*(?:Views|Listens)\s*
\s*
([\d\.,]+)
', webpage, - 'view count', default=None)) - - filesize = int_or_none(self._html_search_regex( - r'"filesize"\s*:\s*["\']?([\d]+)["\']?,', webpage, 'filesize', - default=None)) - - video_type_description = self._html_search_regex( - r'"description"\s*:\s*["\']?([^"\']+)["\']?,', webpage, 'filesize', - default=None) - if len(formats) == 1: - formats[0]['filesize'] = filesize + formats[0]['filesize'] = int_or_none(self._html_search_regex( + r'"filesize"\s*:\s*["\']?([\d]+)["\']?,', webpage, 'filesize', default=None)) + + video_type_description = self._html_search_regex( + r'"description"\s*:\s*["\']?([^"\']+)["\']?,', webpage, 'media type', default=None) + if video_type_description == 'Audio File': + formats[0]['vcodec'] = 'none' - if video_type_description == 'Audio File': - formats[0]['vcodec'] = 'none' self._check_formats(formats, media_id) - return { 'id': media_id, - 'title': title, + 'title': self._html_extract_title(webpage), 'uploader': uploader, - 'timestamp': timestamp, - 'duration': duration, + 'timestamp': unified_timestamp(self._search_regex( + r'itemprop="(?:uploadDate|datePublished)"\s+content="([^"]+)"', + webpage, 'timestamp', default=None)), + 'duration': parse_duration(self._html_search_regex( + r'"duration"\s*:\s*["\']?(\d+)["\']?', webpage, 'duration', default=None)), 'formats': formats, 'thumbnail': self._og_search_thumbnail(webpage), - 'description': description, - 'age_limit': age_limit, - 'view_count': view_count, + 'description': ( + clean_html(get_element_by_id('author_comments', webpage)) + or self._og_search_description(webpage)), + 'age_limit': self._AGE_LIMIT.get(self._html_search_regex( + r'\s*(?:Views|Listens)\s*\s*
([\d\.,]+)
', + webpage, 'view count', default=None)), } diff --git a/yt_dlp/extractor/newspicks.py b/yt_dlp/extractor/newspicks.py index b6334dcba..4a1cb0a73 100644 --- a/yt_dlp/extractor/newspicks.py +++ b/yt_dlp/extractor/newspicks.py @@ -5,7 +5,7 @@ class NewsPicksIE(InfoExtractor): - _VALID_URL = r'https://newspicks\.com/movie-series/(?P\d+)\?movieId=(?P\d+)' + _VALID_URL = r'https?://newspicks\.com/movie-series/(?P\d+)\?movieId=(?P\d+)' _TESTS = [{ 'url': 'https://newspicks.com/movie-series/11?movieId=1813', diff --git a/yt_dlp/extractor/nextmedia.py b/yt_dlp/extractor/nextmedia.py index 0e47a4d45..871d3e669 100644 --- a/yt_dlp/extractor/nextmedia.py +++ b/yt_dlp/extractor/nextmedia.py @@ -191,6 +191,8 @@ def _fetch_description(self, page): class NextTVIE(InfoExtractor): + _WORKING = False + _ENABLED = None # XXX: pass through to GenericIE IE_DESC = '壹電視' _VALID_URL = r'https?://(?:www\.)?nexttv\.com\.tw/(?:[^/]+/)+(?P\d+)' diff --git a/yt_dlp/extractor/nhk.py b/yt_dlp/extractor/nhk.py index 4b3d185a3..7cf5b246b 100644 --- a/yt_dlp/extractor/nhk.py +++ b/yt_dlp/extractor/nhk.py @@ -9,6 +9,7 @@ join_nonempty, parse_duration, traverse_obj, + try_call, unescapeHTML, unified_timestamp, url_or_none, @@ -473,22 +474,21 @@ class NhkRadiruIE(InfoExtractor): IE_DESC = 'NHK らじる (Radiru/Rajiru)' _VALID_URL = r'https?://www\.nhk\.or\.jp/radio/(?:player/ondemand|ondemand/detail)\.html\?p=(?P[\da-zA-Z]+)_(?P[\da-zA-Z]+)(?:_(?P[\da-zA-Z]+))?' _TESTS = [{ - 'url': 'https://www.nhk.or.jp/radio/player/ondemand.html?p=0449_01_3853544', - 'skip': 'Episode expired on 2023-04-16', + 'url': 'https://www.nhk.or.jp/radio/player/ondemand.html?p=0449_01_3926210', + 'skip': 'Episode expired on 2024-02-24', 'info_dict': { - 'channel': 'NHK-FM', - 'uploader': 'NHK-FM', - 'description': 'md5:94b08bdeadde81a97df4ec882acce3e9', + 'title': 'ジャズ・トゥナイト　シリーズＪＡＺＺジャイアンツ　５６　ジョニー・ホッジス', + 'id': '0449_01_3926210', 'ext': 'm4a', - 'id': '0449_01_3853544', 'series': 'ジャズ・トゥナイト', + 'uploader': 'NHK-FM', + 'channel': 'NHK-FM', 'thumbnail': 'https://www.nhk.or.jp/prog/img/449/g449.jpg', - 'timestamp': 1680969600, - 'title': 'ジャズ・トゥナイト　ＮＥＷジャズ特集', - 'upload_date': '20230408', - 'release_timestamp': 1680962400, - 'release_date': '20230408', - 'was_live': True, + 'release_date': '20240217', + 'description': 'md5:a456ee8e5e59e6dd2a7d32e62386e811', + 'timestamp': 1708185600, + 'release_timestamp': 1708178400, + 'upload_date': '20240217', }, }, { # playlist, airs every weekday so it should _hopefully_ be okay forever @@ -519,7 +519,8 @@ class NhkRadiruIE(InfoExtractor): 'series': 'らじる文庫 by ラジオ深夜便 ', 'release_timestamp': 1481126700, 'upload_date': '20211101', - } + }, + 'expected_warnings': ['Unable to download JSON metadata', 'Failed to get extended description'], }, { # news 'url': 'https://www.nhk.or.jp/radio/player/ondemand.html?p=F261_01_3855109', @@ -539,9 +540,28 @@ class NhkRadiruIE(InfoExtractor): }, }] + _API_URL_TMPL = None + + def _extract_extended_description(self, episode_id, episode): + service, _, area = traverse_obj(episode, ('aa_vinfo2', {str}, {lambda x: (x or '').partition(',')})) + aa_vinfo3 = traverse_obj(episode, ('aa_vinfo3', {str})) + detail_url = try_call( + lambda: self._API_URL_TMPL.format(service=service, area=area, dateid=aa_vinfo3)) + if not detail_url: + return + + full_meta = traverse_obj( + self._download_json(detail_url, episode_id, 'Downloading extended metadata', fatal=False), + ('list', service, 0, {dict})) or {} + return join_nonempty('subtitle', 'content', 'act', 'music', delim='\n\n', from_dict=full_meta) + def _extract_episode_info(self, headline, programme_id, series_meta): episode_id = f'{programme_id}_{headline["headline_id"]}' episode = traverse_obj(headline, ('file_list', 0, {dict})) + description = self._extract_extended_description(episode_id, episode) + if not description: + self.report_warning('Failed to get extended description, falling back to summary') + description = traverse_obj(episode, ('file_title_sub', {str})) return { **series_meta, @@ -551,14 +571,21 @@ def _extract_episode_info(self, headline, programme_id, series_meta): 'was_live': True, 'series': series_meta.get('title'), 'thumbnail': url_or_none(headline.get('headline_image')) or series_meta.get('thumbnail'), + 'description': description, **traverse_obj(episode, { 'title': 'file_title', - 'description': 'file_title_sub', 'timestamp': ('open_time', {unified_timestamp}), 'release_timestamp': ('aa_vinfo4', {lambda x: x.split('_')[0]}, {unified_timestamp}), }), } + def _real_initialize(self): + if self._API_URL_TMPL: + return + api_config = self._download_xml( + 'https://www.nhk.or.jp/radio/config/config_web.xml', None, 'Downloading API config', fatal=False) + NhkRadiruIE._API_URL_TMPL = try_call(lambda: f'https:{api_config.find(".//url_program_detail").text}') + def _real_extract(self, url): site_id, corner_id, headline_id = self._match_valid_url(url).group('site', 'corner', 'headline') programme_id = f'{site_id}_{corner_id}' diff --git a/yt_dlp/extractor/niconico.py b/yt_dlp/extractor/niconico.py index 797b5268a..6a4624602 100644 --- a/yt_dlp/extractor/niconico.py +++ b/yt_dlp/extractor/niconico.py @@ -13,13 +13,11 @@ from ..utils import ( ExtractorError, OnDemandPagedList, - bug_reports_message, clean_html, float_or_none, int_or_none, join_nonempty, parse_duration, - parse_filesize, parse_iso8601, parse_resolution, qualities, @@ -38,6 +36,8 @@ class NiconicoIE(InfoExtractor): IE_NAME = 'niconico' IE_DESC = 'ニコニコ動画' + _GEO_COUNTRIES = ['JP'] + _GEO_BYPASS = False _TESTS = [{ 'url': 'http://www.nicovideo.jp/watch/sm22312215', @@ -55,25 +55,31 @@ class NiconicoIE(InfoExtractor): 'duration': 33, 'view_count': int, 'comment_count': int, + 'genres': ['未設定'], + 'tags': [], + 'expected_protocol': str, }, - 'skip': 'Requires an account', }, { # File downloaded with and without credentials are different, so omit # the md5 field 'url': 'http://www.nicovideo.jp/watch/nm14296458', 'info_dict': { 'id': 'nm14296458', - 'ext': 'swf', - 'title': '【鏡音リン】Dance on media【オリジナル】take2!', - 'description': 'md5:689f066d74610b3b22e0f1739add0f58', + 'ext': 'mp4', + 'title': '【Kagamine Rin】Dance on media【Original】take2!', + 'description': 'md5:9368f2b1f4178de64f2602c2f3d6cbf5', 'thumbnail': r're:https?://.*', 'uploader': 'りょうた', 'uploader_id': '18822557', 'upload_date': '20110429', 'timestamp': 1304065916, - 'duration': 209, + 'duration': 208.0, + 'comment_count': int, + 'view_count': int, + 'genres': ['音楽・サウンド'], + 'tags': ['Translation_Request', 'Kagamine_Rin', 'Rin_Original'], + 'expected_protocol': str, }, - 'skip': 'Requires an account', }, { # 'video exists but is marked as "deleted" # md5 is unstable @@ -107,22 +113,24 @@ class NiconicoIE(InfoExtractor): }, { # video not available via `getflv`; "old" HTML5 video 'url': 'http://www.nicovideo.jp/watch/sm1151009', - 'md5': '8fa81c364eb619d4085354eab075598a', + 'md5': 'f95a3d259172667b293530cc2e41ebda', 'info_dict': { 'id': 'sm1151009', 'ext': 'mp4', 'title': 'マスターシステム本体内蔵のスペハリのメインテーマ（ＰＳＧ版）', - 'description': 'md5:6ee077e0581ff5019773e2e714cdd0b7', + 'description': 'md5:f95a3d259172667b293530cc2e41ebda', 'thumbnail': r're:https?://.*', 'duration': 184, - 'timestamp': 1190868283, - 'upload_date': '20070927', + 'timestamp': 1190835883, + 'upload_date': '20070926', 'uploader': 'denden2', 'uploader_id': '1392194', 'view_count': int, 'comment_count': int, + 'genres': ['ゲーム'], + 'tags': [], + 'expected_protocol': str, }, - 'skip': 'Requires an account', }, { # "New" HTML5 video # md5 is unstable @@ -132,16 +140,18 @@ class NiconicoIE(InfoExtractor): 'ext': 'mp4', 'title': '新作TVアニメ「戦姫絶唱シンフォギアAXZ」PV 最高画質', 'description': 'md5:e52974af9a96e739196b2c1ca72b5feb', - 'timestamp': 1498514060, + 'timestamp': 1498481660, 'upload_date': '20170626', - 'uploader': 'ゲスト', + 'uploader': 'no-namamae', 'uploader_id': '40826363', 'thumbnail': r're:https?://.*', 'duration': 198, 'view_count': int, 'comment_count': int, + 'genres': ['アニメ'], + 'tags': [], + 'expected_protocol': str, }, - 'skip': 'Requires an account', }, { # Video without owner 'url': 'http://www.nicovideo.jp/watch/sm18238488', @@ -151,16 +161,16 @@ class NiconicoIE(InfoExtractor): 'ext': 'mp4', 'title': '【実写版】ミュータントタートルズ', 'description': 'md5:15df8988e47a86f9e978af2064bf6d8e', - 'timestamp': 1341160408, + 'timestamp': 1341128008, 'upload_date': '20120701', - 'uploader': None, - 'uploader_id': None, 'thumbnail': r're:https?://.*', 'duration': 5271, 'view_count': int, 'comment_count': int, + 'genres': ['エンターテイメント'], + 'tags': [], + 'expected_protocol': str, }, - 'skip': 'Requires an account', }, { 'url': 'http://sp.nicovideo.jp/watch/sm28964488?ss_pos=1&cp_in=wt_tg', 'only_matching': True, @@ -172,9 +182,6 @@ class NiconicoIE(InfoExtractor): _VALID_URL = r'https?://(?:(?:www\.|secure\.|sp\.)?nicovideo\.jp/watch|nico\.ms)/(?P(?:[a-z]{2})?[0-9]+)' _NETRC_MACHINE = 'niconico' - _COMMENT_API_ENDPOINTS = ( - 'https://nvcomment.nicovideo.jp/legacy/api.json', - 'https://nmsg.nicovideo.jp/api.json',) _API_HEADERS = { 'X-Frontend-ID': '6', 'X-Frontend-Version': '0', @@ -356,15 +363,10 @@ def _extract_format_for_quality(self, video_id, audio_quality, video_quality, dm if not audio_quality.get('isAvailable') or not video_quality.get('isAvailable'): return None - def extract_video_quality(video_quality): - return parse_filesize('%sB' % self._search_regex( - r'\| ([0-9]*\.?[0-9]*[MK])', video_quality, 'vbr', default='')) - format_id = '-'.join( [remove_start(s['id'], 'archive_') for s in (video_quality, audio_quality)] + [dmc_protocol]) vid_qual_label = traverse_obj(video_quality, ('metadata', 'label')) - vid_quality = traverse_obj(video_quality, ('metadata', 'bitrate')) return { 'url': 'niconico_dmc:%s/%s/%s' % (video_id, video_quality['id'], audio_quality['id']), @@ -373,10 +375,15 @@ def extract_video_quality(video_quality): 'ext': 'mp4', # Session API are used in HTML5, which always serves mp4 'acodec': 'aac', 'vcodec': 'h264', - 'abr': float_or_none(traverse_obj(audio_quality, ('metadata', 'bitrate')), 1000), - 'vbr': float_or_none(vid_quality if vid_quality > 0 else extract_video_quality(vid_qual_label), 1000), - 'height': traverse_obj(video_quality, ('metadata', 'resolution', 'height')), - 'width': traverse_obj(video_quality, ('metadata', 'resolution', 'width')), + **traverse_obj(audio_quality, ('metadata', { + 'abr': ('bitrate', {functools.partial(float_or_none, scale=1000)}), + 'asr': ('samplingRate', {int_or_none}), + })), + **traverse_obj(video_quality, ('metadata', { + 'vbr': ('bitrate', {functools.partial(float_or_none, scale=1000)}), + 'height': ('resolution', 'height', {int_or_none}), + 'width': ('resolution', 'width', {int_or_none}), + })), 'quality': -2 if 'low' in video_quality['id'] else None, 'protocol': 'niconico_dmc', 'expected_protocol': dmc_protocol, # XXX: This is not a documented field @@ -386,6 +393,63 @@ def extract_video_quality(video_quality): } } + def _yield_dmc_formats(self, api_data, video_id): + dmc_data = traverse_obj(api_data, ('media', 'delivery', 'movie')) + audios = traverse_obj(dmc_data, ('audios', ..., {dict})) + videos = traverse_obj(dmc_data, ('videos', ..., {dict})) + protocols = traverse_obj(dmc_data, ('session', 'protocols', ..., {str})) + if not all((audios, videos, protocols)): + return + + for audio_quality, video_quality, protocol in itertools.product(audios, videos, protocols): + if fmt := self._extract_format_for_quality(video_id, audio_quality, video_quality, protocol): + yield fmt + + def _yield_dms_formats(self, api_data, video_id): + fmt_filter = lambda _, v: v['isAvailable'] and v['id'] + videos = traverse_obj(api_data, ('media', 'domand', 'videos', fmt_filter)) + audios = traverse_obj(api_data, ('media', 'domand', 'audios', fmt_filter)) + access_key = traverse_obj(api_data, ('media', 'domand', 'accessRightKey', {str})) + track_id = traverse_obj(api_data, ('client', 'watchTrackId', {str})) + if not all((videos, audios, access_key, track_id)): + return + + dms_m3u8_url = self._download_json( + f'https://nvapi.nicovideo.jp/v1/watch/{video_id}/access-rights/hls', video_id, + data=json.dumps({ + 'outputs': list(itertools.product((v['id'] for v in videos), (a['id'] for a in audios))) + }).encode(), query={'actionTrackId': track_id}, headers={ + 'x-access-right-key': access_key, + 'x-frontend-id': 6, + 'x-frontend-version': 0, + 'x-request-with': 'https://www.nicovideo.jp', + })['data']['contentUrl'] + # Getting all audio formats results in duplicate video formats which we filter out later + dms_fmts = self._extract_m3u8_formats(dms_m3u8_url, video_id) + + # m3u8 extraction does not provide audio bitrates, so extract from the API data and fix + for audio_fmt in traverse_obj(dms_fmts, lambda _, v: v['vcodec'] == 'none'): + yield { + **audio_fmt, + **traverse_obj(audios, (lambda _, v: audio_fmt['format_id'].startswith(v['id']), { + 'format_id': ('id', {str}), + 'abr': ('bitRate', {functools.partial(float_or_none, scale=1000)}), + 'asr': ('samplingRate', {int_or_none}), + }), get_all=False), + 'acodec': 'aac', + 'ext': 'm4a', + } + + # Sort before removing dupes to keep the format dicts with the lowest tbr + video_fmts = sorted((fmt for fmt in dms_fmts if fmt['vcodec'] != 'none'), key=lambda f: f['tbr']) + self._remove_duplicate_formats(video_fmts) + # Calculate the true vbr/tbr by subtracting the lowest abr + min_abr = min(traverse_obj(audios, (..., 'bitRate', {float_or_none})), default=0) / 1000 + for video_fmt in video_fmts: + video_fmt['tbr'] -= min_abr + video_fmt['format_id'] = f'video-{video_fmt["tbr"]:.0f}' + yield video_fmt + def _real_extract(self, url): video_id = self._match_id(url) @@ -412,19 +476,29 @@ def _real_extract(self, url): webpage, 'error reason', default=None) if not error_msg: raise - raise ExtractorError(re.sub(r'\s+', ' ', error_msg), expected=True) + raise ExtractorError(clean_html(error_msg), expected=True) - formats = [] - - def get_video_info(*items, get_first=True, **kwargs): - return traverse_obj(api_data, ('video', *items), get_all=not get_first, **kwargs) - - quality_info = api_data['media']['delivery']['movie'] - session_api_data = quality_info['session'] - for (audio_quality, video_quality, protocol) in itertools.product(quality_info['audios'], quality_info['videos'], session_api_data['protocols']): - fmt = self._extract_format_for_quality(video_id, audio_quality, video_quality, protocol) - if fmt: - formats.append(fmt) + availability = self._availability(**(traverse_obj(api_data, ('payment', 'video', { + 'needs_premium': ('isPremium', {bool}), + 'needs_subscription': ('isAdmission', {bool}), + })) or {'needs_auth': True})) + formats = [*self._yield_dmc_formats(api_data, video_id), + *self._yield_dms_formats(api_data, video_id)] + if not formats: + fail_msg = clean_html(self._html_search_regex( + r']+\bclass="fail-message"[^>]*>(?P.+?)

', + webpage, 'fail message', default=None, group='msg')) + if fail_msg: + self.to_screen(f'Niconico said: {fail_msg}') + if fail_msg and 'された地域と同じ地域からのみ視聴できます。' in fail_msg: + availability = None + self.raise_geo_restricted(countries=self._GEO_COUNTRIES, metadata_available=True) + elif availability == 'premium_only': + self.raise_login_required('This video requires premium', metadata_available=True) + elif availability == 'subscriber_only': + self.raise_login_required('This video is for members only', metadata_available=True) + elif availability == 'needs_auth': + self.raise_login_required(metadata_available=False) # Start extracting information tags = None @@ -443,11 +517,15 @@ def get_video_info(*items, get_first=True, **kwargs): thumb_prefs = qualities(['url', 'middleUrl', 'largeUrl', 'player', 'ogp']) + def get_video_info(*items, get_first=True, **kwargs): + return traverse_obj(api_data, ('video', *items), get_all=not get_first, **kwargs) + return { 'id': video_id, '_api_data': api_data, 'title': get_video_info(('originalTitle', 'title')) or self._og_search_title(webpage, default=None), 'formats': formats, + 'availability': availability, 'thumbnails': [{ 'id': key, 'url': url, @@ -470,93 +548,19 @@ def get_video_info(*items, get_first=True, **kwargs): parse_duration(self._html_search_meta('video:duration', webpage, 'video duration', default=None)) or get_video_info('duration')), 'webpage_url': url_or_none(url) or f'https://www.nicovideo.jp/watch/{video_id}', - 'subtitles': self.extract_subtitles(video_id, api_data, session_api_data), + 'subtitles': self.extract_subtitles(video_id, api_data), } - def _get_subtitles(self, video_id, api_data, session_api_data): - comment_user_key = traverse_obj(api_data, ('comment', 'keys', 'userKey')) - user_id_str = session_api_data.get('serviceUserId') - - thread_ids = traverse_obj(api_data, ('comment', 'threads', lambda _, v: v['isActive'])) - legacy_danmaku = self._extract_legacy_comments(video_id, thread_ids, user_id_str, comment_user_key) or [] - - new_comments = traverse_obj(api_data, ('comment', 'nvComment')) - new_danmaku = self._extract_new_comments( - new_comments.get('server'), video_id, - new_comments.get('params'), new_comments.get('threadKey')) - - if not legacy_danmaku and not new_danmaku: - self.report_warning(f'Failed to get comments. {bug_reports_message()}') + def _get_subtitles(self, video_id, api_data): + comments_info = traverse_obj(api_data, ('comment', 'nvComment', {dict})) or {} + if not comments_info.get('server'): return - return { - 'comments': [{ - 'ext': 'json', - 'data': json.dumps(legacy_danmaku + new_danmaku), - }], - } - - def _extract_legacy_comments(self, video_id, threads, user_id, user_key): - auth_data = { - 'user_id': user_id, - 'userkey': user_key, - } if user_id and user_key else {'user_id': ''} - - api_url = traverse_obj(threads, (..., 'server'), get_all=False) - - # Request Start - post_data = [{'ping': {'content': 'rs:0'}}] - for i, thread in enumerate(threads): - thread_id = thread['id'] - thread_fork = thread['fork'] - # Post Start (2N) - post_data.append({'ping': {'content': f'ps:{i * 2}'}}) - post_data.append({'thread': { - 'fork': thread_fork, - 'language': 0, - 'nicoru': 3, - 'scores': 1, - 'thread': thread_id, - 'version': '20090904', - 'with_global': 1, - **auth_data, - }}) - # Post Final (2N) - post_data.append({'ping': {'content': f'pf:{i * 2}'}}) - - # Post Start (2N+1) - post_data.append({'ping': {'content': f'ps:{i * 2 + 1}'}}) - post_data.append({'thread_leaves': { - # format is '-:,\d+)' _TEST = { 'url': 'http://www.nobelprize.org/mediaplayer/?id=2636', diff --git a/yt_dlp/extractor/nova.py b/yt_dlp/extractor/nova.py index 8a7dfceeb..72884aaaa 100644 --- a/yt_dlp/extractor/nova.py +++ b/yt_dlp/extractor/nova.py @@ -135,14 +135,15 @@ class NovaIE(InfoExtractor): _VALID_URL = r'https?://(?:[^.]+\.)?(?Ptv(?:noviny)?|tn|novaplus|vymena|fanda|krasna|doma|prask)\.nova\.cz/(?:[^/]+/)+(?P[^/]+?)(?:\.html|/|$)' _TESTS = [{ 'url': 'http://tn.nova.cz/clanek/tajemstvi-ukryte-v-podzemi-specialni-nemocnice-v-prazske-krci.html#player_13260', - 'md5': '249baab7d0104e186e78b0899c7d5f28', + 'md5': 'da8f3f1fcdaf9fb0f112a32a165760a3', 'info_dict': { - 'id': '1757139', - 'display_id': 'tajemstvi-ukryte-v-podzemi-specialni-nemocnice-v-prazske-krci', + 'id': '8OvQqEvV3MW', + 'display_id': '8OvQqEvV3MW', 'ext': 'mp4', 'title': 'Podzemní nemocnice v pražské Krči', 'description': 'md5:f0a42dd239c26f61c28f19e62d20ef53', 'thumbnail': r're:^https?://.*\.(?:jpg)', + 'duration': 151, } }, { 'url': 'http://fanda.nova.cz/clanek/fun-and-games/krvavy-epos-zaklinac-3-divoky-hon-vychazi-vyhrajte-ho-pro-sebe.html', @@ -210,7 +211,7 @@ def _real_extract(self, url): # novaplus embed_id = self._search_regex( - r']+\bsrc=["\'](?:https?:)?//media\.cms\.nova\.cz/embed/([^/?#&]+)', + r']+\bsrc=["\'](?:https?:)?//media(?:tn)?\.cms\.nova\.cz/embed/([^/?#&"\']+)', webpage, 'embed url', default=None) if embed_id: return { diff --git a/yt_dlp/extractor/novaplay.py b/yt_dlp/extractor/novaplay.py index d8849cd88..adab33f59 100644 --- a/yt_dlp/extractor/novaplay.py +++ b/yt_dlp/extractor/novaplay.py @@ -3,7 +3,7 @@ class NovaPlayIE(InfoExtractor): - _VALID_URL = r'https://play\.nova\.bg/video/[^?#]+/(?P\d+)' + _VALID_URL = r'https?://play\.nova\.bg/video/[^?#]+/(?P\d+)' _TESTS = [ { 'url': 'https://play.nova.bg/video/ochakvaite/season-0/ochakvaite-2022-07-22-sybudi-se-sat/606627', @@ -18,7 +18,6 @@ class NovaPlayIE(InfoExtractor): 'upload_date': '20220722', 'thumbnail': 'https://nbg-img.fite.tv/img/606627_460x260.jpg', 'description': '29 сек', - 'view_count': False }, }, { @@ -34,7 +33,6 @@ class NovaPlayIE(InfoExtractor): 'upload_date': '20220722', 'thumbnail': 'https://nbg-img.fite.tv/img/606609_460x260.jpg', 'description': '29 сек', - 'view_count': False }, } ] diff --git a/yt_dlp/extractor/noz.py b/yt_dlp/extractor/noz.py index 59d259f9d..c7b803803 100644 --- a/yt_dlp/extractor/noz.py +++ b/yt_dlp/extractor/noz.py @@ -9,6 +9,7 @@ class NozIE(InfoExtractor): + _WORKING = False _VALID_URL = r'https?://(?:www\.)?noz\.de/video/(?P[0-9]+)/' _TESTS = [{ 'url': 'http://www.noz.de/video/25151/32-Deutschland-gewinnt-Badminton-Lnderspiel-in-Melle', diff --git a/yt_dlp/extractor/ntvru.py b/yt_dlp/extractor/ntvru.py index 91b7724eb..fe3965729 100644 --- a/yt_dlp/extractor/ntvru.py +++ b/yt_dlp/extractor/ntvru.py @@ -35,6 +35,7 @@ class NTVRuIE(InfoExtractor): 'duration': 172, 'view_count': int, }, + 'skip': '404 Not Found', }, { 'url': 'http://www.ntv.ru/peredacha/segodnya/m23700/o232416', 'md5': '82dbd49b38e3af1d00df16acbeab260c', @@ -78,7 +79,8 @@ class NTVRuIE(InfoExtractor): }] _VIDEO_ID_REGEXES = [ - r'http.+?)\1', webpage, 'video url', - default=None if no_video else NO_DEFAULT, group='url') - - if no_video: - raise ExtractorError('Video %s does not exist' % video_id, expected=True) - - return { - 'id': video_id, - 'url': video_url, - 'title': remove_start(self._og_search_title(webpage), 'Video: '), - 'thumbnail': self._og_search_thumbnail(webpage), - } diff --git a/yt_dlp/extractor/ondemandkorea.py b/yt_dlp/extractor/ondemandkorea.py index 94fcac720..591b4147e 100644 --- a/yt_dlp/extractor/ondemandkorea.py +++ b/yt_dlp/extractor/ondemandkorea.py @@ -11,6 +11,7 @@ join_nonempty, parse_age_limit, parse_qs, + str_or_none, unified_strdate, url_or_none, ) @@ -32,7 +33,7 @@ class OnDemandKoreaIE(InfoExtractor): 'duration': 5486.955, 'release_date': '20220924', 'series': 'Ask Us Anything', - 'series_id': 11790, + 'series_id': '11790', 'episode_number': 351, 'episode': 'Jung Sung-ho, Park Seul-gi, Kim Bo-min, Yang Seung-won', }, @@ -47,7 +48,7 @@ class OnDemandKoreaIE(InfoExtractor): 'duration': 1586.0, 'release_date': '20231001', 'series': 'Breakup Probation, A Week', - 'series_id': 22912, + 'series_id': '22912', 'episode_number': 8, 'episode': 'E08', }, @@ -117,7 +118,7 @@ def if_series(key=None): 'duration': ('duration', {functools.partial(float_or_none, scale=1000)}), 'age_limit': ('age_rating', 'name', {lambda x: x.replace('R', '')}, {parse_age_limit}), 'series': ('episode', {if_series(key='program')}, 'title'), - 'series_id': ('episode', {if_series(key='program')}, 'id'), + 'series_id': ('episode', {if_series(key='program')}, 'id', {str_or_none}), 'episode': ('episode', {if_series(key='title')}), 'episode_number': ('episode', {if_series(key='number')}, {int_or_none}), }, get_all=False), diff --git a/yt_dlp/extractor/onefootball.py b/yt_dlp/extractor/onefootball.py index 591d15732..e1b726830 100644 --- a/yt_dlp/extractor/onefootball.py +++ b/yt_dlp/extractor/onefootball.py @@ -1,4 +1,6 @@ from .common import InfoExtractor +from .jwplatform import JWPlatformIE +from ..utils import make_archive_id class OneFootballIE(InfoExtractor): @@ -7,41 +9,43 @@ class OneFootballIE(InfoExtractor): _TESTS = [{ 'url': 'https://onefootball.com/en/video/highlights-fc-zuerich-3-3-fc-basel-34012334', 'info_dict': { - 'id': '34012334', + 'id': 'Y2VtcWAT', 'ext': 'mp4', 'title': 'Highlights: FC Zürich 3-3 FC Basel', 'description': 'md5:33d9855cb790702c4fe42a513700aba8', - 'thumbnail': 'https://photobooth-api.onefootball.com/api/screenshot/https:%2F%2Fperegrine-api.onefootball.com%2Fv2%2Fphotobooth%2Fcms%2Fen%2F34012334', - 'timestamp': 1635874604, - 'upload_date': '20211102' + 'thumbnail': 'https://cdn.jwplayer.com/v2/media/Y2VtcWAT/poster.jpg?width=720', + 'timestamp': 1635874895, + 'upload_date': '20211102', + 'duration': 375.0, + 'tags': ['Football', 'Soccer', 'OneFootball'], + '_old_archive_ids': ['onefootball 34012334'], }, - 'params': {'skip_download': True} + 'params': {'skip_download': True}, + 'expected_warnings': ['Failed to download m3u8 information'], }, { 'url': 'https://onefootball.com/en/video/klopp-fumes-at-var-decisions-in-west-ham-defeat-34041020', 'info_dict': { - 'id': '34041020', + 'id': 'leVJrMho', 'ext': 'mp4', 'title': 'Klopp fumes at VAR decisions in West Ham defeat', 'description': 'md5:9c50371095a01ad3f63311c73d8f51a5', - 'thumbnail': 'https://photobooth-api.onefootball.com/api/screenshot/https:%2F%2Fperegrine-api.onefootball.com%2Fv2%2Fphotobooth%2Fcms%2Fen%2F34041020', - 'timestamp': 1636314103, - 'upload_date': '20211107' + 'thumbnail': 'https://cdn.jwplayer.com/v2/media/leVJrMho/poster.jpg?width=720', + 'timestamp': 1636315232, + 'upload_date': '20211107', + 'duration': 93.0, + 'tags': ['Football', 'Soccer', 'OneFootball'], + '_old_archive_ids': ['onefootball 34041020'], }, 'params': {'skip_download': True} }] def _real_extract(self, url): - id = self._match_id(url) - webpage = self._download_webpage(url, id) - data_json = self._search_json_ld(webpage, id) - m3u8_url = self._html_search_regex(r'(https://cdn\.jwplayer\.com/manifests/.+\.m3u8)', webpage, 'm3u8_url') - formats, subtitles = self._extract_m3u8_formats_and_subtitles(m3u8_url, id) - return { - 'id': id, - 'title': data_json.get('title'), - 'description': data_json.get('description'), - 'thumbnail': data_json.get('thumbnail'), - 'timestamp': data_json.get('timestamp'), - 'formats': formats, - 'subtitles': subtitles, - } + video_id = self._match_id(url) + webpage = self._download_webpage(url, video_id) + data_json = self._search_json_ld(webpage, video_id, fatal=False) + data_json.pop('url', None) + m3u8_url = self._html_search_regex(r'(https://cdn\.jwplayer\.com/manifests/\w+\.m3u8)', webpage, 'm3u8_url') + + return self.url_result( + m3u8_url, JWPlatformIE, video_id, _old_archive_ids=[make_archive_id(self, video_id)], + **data_json, url_transparent=True) diff --git a/yt_dlp/extractor/openrec.py b/yt_dlp/extractor/openrec.py index 86dc9bb89..82a81c6c2 100644 --- a/yt_dlp/extractor/openrec.py +++ b/yt_dlp/extractor/openrec.py @@ -12,6 +12,8 @@ class OpenRecBaseIE(InfoExtractor): + _M3U8_HEADERS = {'Referer': 'https://www.openrec.tv/'} + def _extract_pagestore(self, webpage, video_id): return self._parse_json( self._search_regex(r'(?m)window\.pageStore\s*=\s*(\{.+?\});$', webpage, 'window.pageStore'), video_id) @@ -21,7 +23,7 @@ def _expand_media(self, video_id, media): if not m3u8_url: continue yield from self._extract_m3u8_formats( - m3u8_url, video_id, ext='mp4', m3u8_id=name) + m3u8_url, video_id, ext='mp4', m3u8_id=name, headers=self._M3U8_HEADERS) def _extract_movie(self, webpage, video_id, name, is_live): window_stores = self._extract_pagestore(webpage, video_id) @@ -60,6 +62,7 @@ def _extract_movie(self, webpage, video_id, name, is_live): 'uploader_id': get_first(movie_stores, ('channel', 'user', 'id')), 'timestamp': int_or_none(get_first(movie_stores, ['publishedAt', 'time']), scale=1000) or unified_timestamp(get_first(movie_stores, 'publishedAt')), 'is_live': is_live, + 'http_headers': self._M3U8_HEADERS, } @@ -110,7 +113,7 @@ def _real_extract(self, url): raise ExtractorError('Cannot extract title') formats = self._extract_m3u8_formats( - capture_data.get('source'), video_id, ext='mp4') + capture_data.get('source'), video_id, ext='mp4', headers=self._M3U8_HEADERS) return { 'id': video_id, @@ -121,6 +124,7 @@ def _real_extract(self, url): 'uploader': traverse_obj(movie_store, ('channel', 'name'), expected_type=compat_str), 'uploader_id': traverse_obj(movie_store, ('channel', 'id'), expected_type=compat_str), 'upload_date': unified_strdate(capture_data.get('createdAt')), + 'http_headers': self._M3U8_HEADERS, } diff --git a/yt_dlp/extractor/orf.py b/yt_dlp/extractor/orf.py index 1b2a79a62..526e9acaf 100644 --- a/yt_dlp/extractor/orf.py +++ b/yt_dlp/extractor/orf.py @@ -49,7 +49,6 @@ class ORFTVthekIE(InfoExtractor): 'url': 'https://tvthek.orf.at/profile/ZIB-2/1211/ZIB-2/14121079/Umfrage-Welches-Tier-ist-Sebastian-Kurz/15083150', 'info_dict': { 'id': '14121079', - 'playlist_count': 1 }, 'playlist': [{ 'info_dict': { diff --git a/yt_dlp/extractor/parler.py b/yt_dlp/extractor/parler.py index 2af805e7f..563012f35 100644 --- a/yt_dlp/extractor/parler.py +++ b/yt_dlp/extractor/parler.py @@ -14,7 +14,7 @@ class ParlerIE(InfoExtractor): IE_DESC = 'Posts on parler.com' - _VALID_URL = r'https://parler\.com/feed/(?P[0-9a-f]{8}-(?:[0-9a-f]{4}-){3}[0-9a-f]{12})' + _VALID_URL = r'https?://parler\.com/feed/(?P[0-9a-f]{8}-(?:[0-9a-f]{4}-){3}[0-9a-f]{12})' _TESTS = [ { 'url': 'https://parler.com/feed/df79fdba-07cc-48fe-b085-3293897520d7', diff --git a/yt_dlp/extractor/parlview.py b/yt_dlp/extractor/parlview.py index 0b547917c..777b00889 100644 --- a/yt_dlp/extractor/parlview.py +++ b/yt_dlp/extractor/parlview.py @@ -8,7 +8,7 @@ class ParlviewIE(InfoExtractor): - + _WORKING = False _VALID_URL = r'https?://(?:www\.)?parlview\.aph\.gov\.au/(?:[^/]+)?\bvideoID=(?P\d{6})' _TESTS = [{ 'url': 'https://parlview.aph.gov.au/mediaPlayer.php?videoID=542661', diff --git a/yt_dlp/extractor/peekvids.py b/yt_dlp/extractor/peekvids.py index 41f591b09..939c26dc7 100644 --- a/yt_dlp/extractor/peekvids.py +++ b/yt_dlp/extractor/peekvids.py @@ -157,7 +157,6 @@ class PlayVidsIE(PeekVidsBaseIE): 'display_id': '47iUho33toY', 'ext': 'mp4', 'title': 'KATEE OWEN STRIPTIASE IN SEXY RED LINGERIE', - 'description': None, 'timestamp': 1507052209, 'upload_date': '20171003', 'thumbnail': r're:^https?://.*\.jpg$', @@ -176,7 +175,6 @@ class PlayVidsIE(PeekVidsBaseIE): 'display_id': 'z3_7iwWCmqt', 'ext': 'mp4', 'title': 'SEXY TEEN FILIPINA STRIPTEASE - Beautiful Pinay Bargirl Strips and Dances', - 'description': None, 'timestamp': 1607470323, 'upload_date': '20201208', 'thumbnail': r're:^https?://.*\.jpg$', diff --git a/yt_dlp/extractor/pladform.py b/yt_dlp/extractor/pladform.py index 00500686f..d67f6005c 100644 --- a/yt_dlp/extractor/pladform.py +++ b/yt_dlp/extractor/pladform.py @@ -35,7 +35,6 @@ class PladformIE(InfoExtractor): 'thumbnail': str, 'view_count': int, 'description': str, - 'category': list, 'uploader_id': '12082', 'uploader': 'Comedy Club', 'duration': 367, diff --git a/yt_dlp/extractor/planetmarathi.py b/yt_dlp/extractor/planetmarathi.py index 25753fe7e..a4b612a6e 100644 --- a/yt_dlp/extractor/planetmarathi.py +++ b/yt_dlp/extractor/planetmarathi.py @@ -20,7 +20,6 @@ class PlanetMarathiIE(InfoExtractor): 'title': 'ek unad divas', 'alt_title': 'चित्रपट', 'description': 'md5:41c7ed6b041c2fea9820a3f3125bd881', - 'season_number': None, 'episode_number': 1, 'duration': 5539, 'upload_date': '20210829', diff --git a/yt_dlp/extractor/playstuff.py b/yt_dlp/extractor/playstuff.py deleted file mode 100644 index b424ba187..000000000 --- a/yt_dlp/extractor/playstuff.py +++ /dev/null @@ -1,63 +0,0 @@ -from .common import InfoExtractor -from ..compat import compat_str -from ..utils import ( - smuggle_url, - try_get, -) - - -class PlayStuffIE(InfoExtractor): - _VALID_URL = r'https?://(?:www\.)?play\.stuff\.co\.nz/details/(?P[^/?#&]+)' - _TESTS = [{ - 'url': 'https://play.stuff.co.nz/details/608778ac1de1c4001a3fa09a', - 'md5': 'c82d3669e5247c64bc382577843e5bd0', - 'info_dict': { - 'id': '6250584958001', - 'ext': 'mp4', - 'title': 'Episode 1: Rotorua/Mt Maunganui/Tauranga', - 'description': 'md5:c154bafb9f0dd02d01fd4100fb1c1913', - 'uploader_id': '6005208634001', - 'timestamp': 1619491027, - 'upload_date': '20210427', - }, - 'add_ie': ['BrightcoveNew'], - }, { - # geo restricted, bypassable - 'url': 'https://play.stuff.co.nz/details/_6155660351001', - 'only_matching': True, - }] - BRIGHTCOVE_URL_TEMPLATE = 'http://players.brightcove.net/%s/%s_default/index.html?videoId=%s' - - def _real_extract(self, url): - video_id = self._match_id(url) - - webpage = self._download_webpage(url, video_id) - - state = self._parse_json( - self._search_regex( - r'__INITIAL_STATE__\s*=\s*({.+?})\s*;', webpage, 'state'), - video_id) - - account_id = try_get( - state, lambda x: x['configurations']['accountId'], - compat_str) or '6005208634001' - player_id = try_get( - state, lambda x: x['configurations']['playerId'], - compat_str) or 'default' - - entries = [] - for item_id, video in state['items'].items(): - if not isinstance(video, dict): - continue - asset_id = try_get( - video, lambda x: x['content']['attributes']['assetId'], - compat_str) - if not asset_id: - continue - entries.append(self.url_result( - smuggle_url( - self.BRIGHTCOVE_URL_TEMPLATE % (account_id, player_id, asset_id), - {'geo_countries': ['NZ']}), - 'BrightcoveNew', video_id)) - - return self.playlist_result(entries, video_id) diff --git a/yt_dlp/extractor/plutotv.py b/yt_dlp/extractor/plutotv.py index caffeb21d..5898d927c 100644 --- a/yt_dlp/extractor/plutotv.py +++ b/yt_dlp/extractor/plutotv.py @@ -16,6 +16,7 @@ class PlutoTVIE(InfoExtractor): + _WORKING = False _VALID_URL = r'''(?x) https?://(?:www\.)?pluto\.tv(?:/[^/]+)?/on-demand /(?Pmovies|series) diff --git a/yt_dlp/extractor/podchaser.py b/yt_dlp/extractor/podchaser.py index 290c48817..fc2d407b1 100644 --- a/yt_dlp/extractor/podchaser.py +++ b/yt_dlp/extractor/podchaser.py @@ -29,7 +29,7 @@ class PodchaserIE(InfoExtractor): 'duration': 3708, 'timestamp': 1636531259, 'upload_date': '20211110', - 'rating': 4.0 + 'average_rating': 4.0 } }, { 'url': 'https://www.podchaser.com/podcasts/the-bone-zone-28853', @@ -59,7 +59,7 @@ def _parse_episode(episode, podcast): 'thumbnail': episode.get('image_url'), 'duration': str_to_int(episode.get('length')), 'timestamp': unified_timestamp(episode.get('air_date')), - 'rating': float_or_none(episode.get('rating')), + 'average_rating': float_or_none(episode.get('rating')), 'categories': list(set(traverse_obj(podcast, (('summary', None), 'categories', ..., 'text')))), 'tags': traverse_obj(podcast, ('tags', ..., 'text')), 'series': podcast.get('title'), diff --git a/yt_dlp/extractor/podomatic.py b/yt_dlp/extractor/podomatic.py index 985bfae9d..37b68694b 100644 --- a/yt_dlp/extractor/podomatic.py +++ b/yt_dlp/extractor/podomatic.py @@ -5,6 +5,7 @@ class PodomaticIE(InfoExtractor): + _WORKING = False IE_NAME = 'podomatic' _VALID_URL = r'''(?x) (?Phttps?):// diff --git a/yt_dlp/extractor/pornhub.py b/yt_dlp/extractor/pornhub.py index 999d038d4..29a3e43cc 100644 --- a/yt_dlp/extractor/pornhub.py +++ b/yt_dlp/extractor/pornhub.py @@ -87,8 +87,8 @@ def _login(self, host): def is_logged(webpage): return any(re.search(p, webpage) for p in ( - r'class=["\']signOut', - r'>Sign\s+[Oo]ut\s*<')) + r'id="profileMenuDropdown"', + r'class="ph-icon-logout"')) if is_logged(login_page): self._logged_in = True diff --git a/yt_dlp/extractor/pornovoisines.py b/yt_dlp/extractor/pornovoisines.py index aa48da06b..2e51b4f6b 100644 --- a/yt_dlp/extractor/pornovoisines.py +++ b/yt_dlp/extractor/pornovoisines.py @@ -7,6 +7,7 @@ class PornoVoisinesIE(InfoExtractor): + _WORKING = False _VALID_URL = r'https?://(?:www\.)?pornovoisines\.com/videos/show/(?P\d+)/(?P[^/.]+)' _TEST = { diff --git a/yt_dlp/extractor/pornoxo.py b/yt_dlp/extractor/pornoxo.py index 5104d8a49..049feb4ec 100644 --- a/yt_dlp/extractor/pornoxo.py +++ b/yt_dlp/extractor/pornoxo.py @@ -5,6 +5,7 @@ class PornoXOIE(InfoExtractor): + _WORKING = False _VALID_URL = r'https?://(?:www\.)?pornoxo\.com/videos/(?P\d+)/(?P[^/]+)\.html' _TEST = { 'url': 'http://www.pornoxo.com/videos/7564/striptease-from-sexy-secretary.html', diff --git a/yt_dlp/extractor/pr0gramm.py b/yt_dlp/extractor/pr0gramm.py index 36e415f4a..66f8a5f44 100644 --- a/yt_dlp/extractor/pr0gramm.py +++ b/yt_dlp/extractor/pr0gramm.py @@ -1,5 +1,4 @@ import json -from datetime import date from urllib.parse import unquote from .common import InfoExtractor @@ -10,6 +9,7 @@ int_or_none, make_archive_id, mimetype2ext, + str_or_none, urljoin, ) from ..utils.traversal import traverse_obj @@ -25,8 +25,8 @@ class Pr0grammIE(InfoExtractor): 'title': 'pr0gramm-5466437 by g11st', 'tags': ['Neon Genesis Evangelion', 'Touhou Project', 'Fly me to the Moon', 'Marisad', 'Marisa Kirisame', 'video', 'sound', 'Marisa', 'Anime'], 'uploader': 'g11st', - 'uploader_id': 394718, - 'upload_timestamp': 1671590240, + 'uploader_id': '394718', + 'timestamp': 1671590240, 'upload_date': '20221221', 'like_count': int, 'dislike_count': int, @@ -42,8 +42,8 @@ class Pr0grammIE(InfoExtractor): 'title': 'pr0gramm-3052805 by Hansking1', 'tags': 'count:15', 'uploader': 'Hansking1', - 'uploader_id': 385563, - 'upload_timestamp': 1552930408, + 'uploader_id': '385563', + 'timestamp': 1552930408, 'upload_date': '20190318', 'like_count': int, 'dislike_count': int, @@ -60,8 +60,8 @@ class Pr0grammIE(InfoExtractor): 'title': 'pr0gramm-5848332 by erd0pfel', 'tags': 'count:18', 'uploader': 'erd0pfel', - 'uploader_id': 349094, - 'upload_timestamp': 1694489652, + 'uploader_id': '349094', + 'timestamp': 1694489652, 'upload_date': '20230912', 'like_count': int, 'dislike_count': int, @@ -77,8 +77,8 @@ class Pr0grammIE(InfoExtractor): 'title': 'pr0gramm-5895149 by algoholigSeeManThrower', 'tags': 'count:19', 'uploader': 'algoholigSeeManThrower', - 'uploader_id': 457556, - 'upload_timestamp': 1697580902, + 'uploader_id': '457556', + 'timestamp': 1697580902, 'upload_date': '20231018', 'like_count': int, 'dislike_count': int, @@ -192,11 +192,10 @@ def _real_extract(self, url): '_old_archive_ids': [make_archive_id('Pr0grammStatic', video_id)], **traverse_obj(video_info, { 'uploader': ('user', {str}), - 'uploader_id': ('userId', {int}), + 'uploader_id': ('userId', {str_or_none}), 'like_count': ('up', {int}), 'dislike_count': ('down', {int}), - 'upload_timestamp': ('created', {int}), - 'upload_date': ('created', {int}, {date.fromtimestamp}, {lambda x: x.strftime('%Y%m%d')}), + 'timestamp': ('created', {int}), 'thumbnail': ('thumb', {lambda x: urljoin('https://thumb.pr0gramm.com', x)}) }), } diff --git a/yt_dlp/extractor/prankcast.py b/yt_dlp/extractor/prankcast.py index 562aca0ff..56cd40d8a 100644 --- a/yt_dlp/extractor/prankcast.py +++ b/yt_dlp/extractor/prankcast.py @@ -16,7 +16,7 @@ class PrankCastIE(InfoExtractor): 'display_id': 'Beverly-is-back-like-a-heart-attack-', 'timestamp': 1661391575, 'uploader': 'Devonanustart', - 'channel_id': 4, + 'channel_id': '4', 'duration': 7918, 'cast': ['Devonanustart', 'Phonelosers'], 'description': '', @@ -33,7 +33,7 @@ class PrankCastIE(InfoExtractor): 'display_id': 'NOT-COOL', 'timestamp': 1665028364, 'uploader': 'phonelosers', - 'channel_id': 6, + 'channel_id': '6', 'duration': 4044, 'cast': ['phonelosers'], 'description': '', @@ -60,7 +60,7 @@ def _real_extract(self, url): 'url': f'{json_info["broadcast_url"]}{json_info["recording_hash"]}.mp3', 'timestamp': start_date, 'uploader': uploader, - 'channel_id': json_info.get('user_id'), + 'channel_id': str_or_none(json_info.get('user_id')), 'duration': try_call(lambda: parse_iso8601(json_info['end_date']) - start_date), 'cast': list(filter(None, [uploader] + traverse_obj(guests_json, (..., 'name')))), 'description': json_info.get('broadcast_description'), diff --git a/yt_dlp/extractor/projectveritas.py b/yt_dlp/extractor/projectveritas.py index 0e029ce8c..daf14054c 100644 --- a/yt_dlp/extractor/projectveritas.py +++ b/yt_dlp/extractor/projectveritas.py @@ -7,6 +7,7 @@ class ProjectVeritasIE(InfoExtractor): + _WORKING = False _VALID_URL = r'https?://(?:www\.)?projectveritas\.com/(?Pnews|video)/(?P[^/?#]+)' _TESTS = [{ 'url': 'https://www.projectveritas.com/news/exclusive-inside-the-new-york-and-new-jersey-hospitals-battling-coronavirus/', diff --git a/yt_dlp/extractor/r7.py b/yt_dlp/extractor/r7.py index f067a0571..36f0b52bd 100644 --- a/yt_dlp/extractor/r7.py +++ b/yt_dlp/extractor/r7.py @@ -3,6 +3,8 @@ class R7IE(InfoExtractor): + _WORKING = False + _ENABLED = None # XXX: pass through to GenericIE _VALID_URL = r'''(?x) https?:// (?: @@ -86,6 +88,8 @@ def _real_extract(self, url): class R7ArticleIE(InfoExtractor): + _WORKING = False + _ENABLED = None # XXX: pass through to GenericIE _VALID_URL = r'https?://(?:[a-zA-Z]+)\.r7\.com/(?:[^/]+/)+[^/?#&]+-(?P\d+)' _TEST = { 'url': 'http://tv.r7.com/record-play/balanco-geral/videos/policiais-humilham-suspeito-a-beira-da-morte-morre-com-dignidade-16102015', diff --git a/yt_dlp/extractor/radiko.py b/yt_dlp/extractor/radiko.py index 2b6405999..f0135827b 100644 --- a/yt_dlp/extractor/radiko.py +++ b/yt_dlp/extractor/radiko.py @@ -162,10 +162,8 @@ def _extract_formats(self, video_id, station, is_onair, ft, cursor, auth_token, return formats def _extract_performers(self, prog): - performers = traverse_obj(prog, ( - 'pfm/text()', ..., {lambda x: re.split(r'[/／、　,，]', x)}, ..., {str.strip})) - # TODO: change 'artist' fields to 'artists' and return traversal list instead of str - return ', '.join(performers) or None + return traverse_obj(prog, ( + 'pfm/text()', ..., {lambda x: re.split(r'[/／、　,，]', x)}, ..., {str.strip})) or None class RadikoIE(RadikoBaseIE): @@ -194,7 +192,7 @@ def _real_extract(self, url): return { 'id': video_id, 'title': try_call(lambda: prog.find('title').text), - 'artist': self._extract_performers(prog), + 'cast': self._extract_performers(prog), 'description': clean_html(try_call(lambda: prog.find('info').text)), 'uploader': try_call(lambda: station_program.find('.//name').text), 'uploader_id': station, @@ -253,7 +251,7 @@ def _real_extract(self, url): return { 'id': station, 'title': title, - 'artist': self._extract_performers(prog), + 'cast': self._extract_performers(prog), 'description': description, 'uploader': station_name, 'uploader_id': station, diff --git a/yt_dlp/extractor/radiocomercial.py b/yt_dlp/extractor/radiocomercial.py index 07891fe41..38f8cf786 100644 --- a/yt_dlp/extractor/radiocomercial.py +++ b/yt_dlp/extractor/radiocomercial.py @@ -30,7 +30,8 @@ class RadioComercialIE(InfoExtractor): 'title': 'Taylor Swift entranhando-se que nem uma espada no ventre dos fãs.', 'release_date': '20231025', 'thumbnail': r're:https://radiocomercial.pt/upload/[^.]+.jpg', - 'season': 6 + 'season': 'Season 6', + 'season_number': 6, } }, { 'url': 'https://radiocomercial.pt/podcasts/convenca-me-num-minuto/t3/convenca-me-num-minuto-que-os-lobisomens-existem', @@ -41,7 +42,8 @@ class RadioComercialIE(InfoExtractor): 'title': 'Convença-me num minuto que os lobisomens existem', 'release_date': '20231026', 'thumbnail': r're:https://radiocomercial.pt/upload/[^.]+.jpg', - 'season': 3 + 'season': 'Season 3', + 'season_number': 3, } }, { 'url': 'https://radiocomercial.pt/podcasts/inacreditavel-by-ines-castel-branco/t2/o-desastre-de-aviao', @@ -53,7 +55,8 @@ class RadioComercialIE(InfoExtractor): 'description': 'md5:8a82beeb372641614772baab7246245f', 'release_date': '20231101', 'thumbnail': r're:https://radiocomercial.pt/upload/[^.]+.jpg', - 'season': 2 + 'season': 'Season 2', + 'season_number': 2, }, 'params': { # inconsistant md5 @@ -68,7 +71,8 @@ class RadioComercialIE(InfoExtractor): 'title': 'T.N.T 29 de outubro', 'release_date': '20231029', 'thumbnail': r're:https://radiocomercial.pt/upload/[^.]+.jpg', - 'season': 2023 + 'season': 'Season 2023', + 'season_number': 2023, } }] @@ -82,7 +86,7 @@ def _real_extract(self, url): 'release_date': unified_strdate(get_element_by_class( 'date', get_element_html_by_class('descriptions', webpage) or '')), 'thumbnail': self._og_search_thumbnail(webpage), - 'season': int_or_none(season), + 'season_number': int_or_none(season), 'url': extract_attributes(get_element_html_by_class('audiofile', webpage) or '').get('href'), } diff --git a/yt_dlp/extractor/radiode.py b/yt_dlp/extractor/radiode.py index 32c36d557..726207825 100644 --- a/yt_dlp/extractor/radiode.py +++ b/yt_dlp/extractor/radiode.py @@ -2,6 +2,7 @@ class RadioDeIE(InfoExtractor): + _WORKING = False IE_NAME = 'radio.de' _VALID_URL = r'https?://(?P.+?)\.(?:radio\.(?:de|at|fr|pt|es|pl|it)|rad\.io)' _TEST = { diff --git a/yt_dlp/extractor/radiojavan.py b/yt_dlp/extractor/radiojavan.py index 6a9139466..b3befaef9 100644 --- a/yt_dlp/extractor/radiojavan.py +++ b/yt_dlp/extractor/radiojavan.py @@ -11,6 +11,7 @@ class RadioJavanIE(InfoExtractor): + _WORKING = False _VALID_URL = r'https?://(?:www\.)?radiojavan\.com/videos/video/(?P[^/]+)/?' _TEST = { 'url': 'http://www.radiojavan.com/videos/video/chaartaar-ashoobam', diff --git a/yt_dlp/extractor/radlive.py b/yt_dlp/extractor/radlive.py index 9bcbb11d5..3c00183be 100644 --- a/yt_dlp/extractor/radlive.py +++ b/yt_dlp/extractor/radlive.py @@ -38,10 +38,6 @@ class RadLiveIE(InfoExtractor): 'language': 'en', 'thumbnail': 'https://lsp.littlstar.com/channels/WHISTLE/BAD_JOKES/SEASON_1/BAD_JOKES_101/poster.jpg', 'description': 'Bad Jokes - Champions, Adam Pally, Super Troopers, Team Edge and 2Hype', - 'release_timestamp': None, - 'channel': None, - 'channel_id': None, - 'channel_url': None, 'episode': 'E01: Bad Jokes 1', 'episode_number': 1, 'episode_id': '336', diff --git a/yt_dlp/extractor/rai.py b/yt_dlp/extractor/rai.py index df4102a40..c1fc65c81 100644 --- a/yt_dlp/extractor/rai.py +++ b/yt_dlp/extractor/rai.py @@ -1,6 +1,7 @@ import re from .common import InfoExtractor +from ..networking import HEADRequest from ..utils import ( clean_html, determine_ext, @@ -27,6 +28,29 @@ class RaiBaseIE(InfoExtractor): _GEO_COUNTRIES = ['IT'] _GEO_BYPASS = False + def _fix_m3u8_formats(self, media_url, video_id): + fmts = self._extract_m3u8_formats( + media_url, video_id, 'mp4', m3u8_id='hls', fatal=False) + + # Fix malformed m3u8 manifests by setting audio-only/video-only formats + for f in fmts: + if not f.get('acodec'): + f['acodec'] = 'mp4a' + if not f.get('vcodec'): + f['vcodec'] = 'avc1' + man_url = f['url'] + if re.search(r'chunklist(?:_b\d+)*_ao[_.]', man_url): # audio only + f['vcodec'] = 'none' + elif re.search(r'chunklist(?:_b\d+)*_vo[_.]', man_url): # video only + f['acodec'] = 'none' + else: # video+audio + if f['acodec'] == 'none': + f['acodec'] = 'mp4a' + if f['vcodec'] == 'none': + f['vcodec'] = 'avc1' + + return fmts + def _extract_relinker_info(self, relinker_url, video_id, audio_only=False): def fix_cdata(s): # remove \r\n\t before and after to avoid @@ -68,8 +92,7 @@ def fix_cdata(s): 'format_id': 'https-mp3', }) elif ext == 'm3u8' or 'format=m3u8' in media_url: - formats.extend(self._extract_m3u8_formats( - media_url, video_id, 'mp4', m3u8_id='hls', fatal=False)) + formats.extend(self._fix_m3u8_formats(media_url, video_id)) elif ext == 'f4m': # very likely no longer needed. Cannot find any url that uses it. manifest_url = update_url_query( @@ -91,7 +114,7 @@ def fix_cdata(s): self.raise_geo_restricted(countries=self._GEO_COUNTRIES, metadata_available=True) if not audio_only and not is_live: - formats.extend(self._create_http_urls(media_url, relinker_url, formats)) + formats.extend(self._create_http_urls(media_url, relinker_url, formats, video_id)) return filter_dict({ 'is_live': is_live, @@ -99,7 +122,7 @@ def fix_cdata(s): 'formats': formats, }) - def _create_http_urls(self, manifest_url, relinker_url, fmts): + def _create_http_urls(self, manifest_url, relinker_url, fmts, video_id): _MANIFEST_REG = r'/(?P\w+)(?:_(?P[\d\,]+))?(?:\.mp4)?(?:\.csmil)?/playlist\.m3u8' _MP4_TMPL = '%s&overrideUserAgentRule=mp4-%s' _QUALITY = { @@ -152,10 +175,10 @@ def get_format_info(tbr): 'format_id': f'https-{tbr}', 'width': format_copy.get('width'), 'height': format_copy.get('height'), - 'tbr': format_copy.get('tbr'), - 'vcodec': format_copy.get('vcodec'), - 'acodec': format_copy.get('acodec'), - 'fps': format_copy.get('fps'), + 'tbr': format_copy.get('tbr') or tbr, + 'vcodec': format_copy.get('vcodec') or 'avc1', + 'acodec': format_copy.get('acodec') or 'mp4a', + 'fps': format_copy.get('fps') or 25, } if format_copy else { 'format_id': f'https-{tbr}', 'width': _QUALITY[tbr][0], @@ -166,6 +189,14 @@ def get_format_info(tbr): 'fps': 25, } + # Check if MP4 download is available + try: + self._request_webpage( + HEADRequest(_MP4_TMPL % (relinker_url, '*')), video_id, 'Checking MP4 availability') + except ExtractorError as e: + self.to_screen(f'{video_id}: MP4 direct download is not available: {e.cause}') + return [] + # filter out single-stream formats fmts = [f for f in fmts if not f.get('vcodec') == 'none' and not f.get('acodec') == 'none'] @@ -236,7 +267,7 @@ class RaiPlayIE(RaiBaseIE): 'series': 'Report', 'season': '2013/14', 'subtitles': {'it': 'count:4'}, - 'release_year': 2022, + 'release_year': 2024, 'episode': 'Espresso nel caffè - 07/04/2014', 'timestamp': 1396919880, 'upload_date': '20140408', @@ -244,7 +275,7 @@ class RaiPlayIE(RaiBaseIE): }, 'params': {'skip_download': True}, }, { - # 1080p direct mp4 url + # 1080p 'url': 'https://www.raiplay.it/video/2021/11/Blanca-S1E1-Senza-occhi-b1255a4a-8e72-4a2f-b9f3-fc1308e00736.html', 'md5': 'aeda7243115380b2dd5e881fd42d949a', 'info_dict': { @@ -265,7 +296,7 @@ class RaiPlayIE(RaiBaseIE): 'episode': 'Senza occhi', 'timestamp': 1637318940, 'upload_date': '20211119', - 'formats': 'count:12', + 'formats': 'count:7', }, 'params': {'skip_download': True}, 'expected_warnings': ['Video not available. Likely due to geo-restriction.'] @@ -518,7 +549,7 @@ class RaiPlaySoundPlaylistIE(InfoExtractor): 'info_dict': { 'id': 'ilruggitodelconiglio', 'title': 'Il Ruggito del Coniglio', - 'description': 'md5:48cff6972435964284614d70474132e6', + 'description': 'md5:62a627b3a2d0635d08fa8b6e0a04f27e', }, 'playlist_mincount': 65, }, { @@ -625,19 +656,20 @@ def _real_extract(self, url): } -class RaiNewsIE(RaiIE): # XXX: Do not subclass from concrete IE +class RaiNewsIE(RaiBaseIE): _VALID_URL = rf'https?://(www\.)?rainews\.it/(?!articoli)[^?#]+-(?P{RaiBaseIE._UUID_RE})(?:-[^/?#]+)?\.html' _EMBED_REGEX = [rf']+data-src="(?P/iframe/[^?#]+?{RaiBaseIE._UUID_RE}\.html)'] _TESTS = [{ # new rainews player (#3911) - 'url': 'https://www.rainews.it/rubriche/24mm/video/2022/05/24mm-del-29052022-12cf645d-1ffd-4220-b27c-07c226dbdecf.html', + 'url': 'https://www.rainews.it/video/2024/02/membri-della-croce-rossa-evacuano-gli-abitanti-di-un-villaggio-nella-regione-ucraina-di-kharkiv-il-filmato-dallucraina--31e8017c-845c-43f5-9c48-245b43c3a079.html', 'info_dict': { - 'id': '12cf645d-1ffd-4220-b27c-07c226dbdecf', + 'id': '31e8017c-845c-43f5-9c48-245b43c3a079', 'ext': 'mp4', - 'title': 'Puntata del 29/05/2022', - 'duration': 1589, - 'upload_date': '20220529', + 'title': 'md5:1e81364b09de4a149042bac3c7d36f0b', + 'duration': 196, + 'upload_date': '20240225', 'uploader': 'rainews', + 'formats': 'count:2', }, 'params': {'skip_download': True}, }, { @@ -650,7 +682,8 @@ class RaiNewsIE(RaiIE): # XXX: Do not subclass from concrete IE 'description': 'I film in uscita questa settimana.', 'thumbnail': r're:^https?://.*\.png$', 'duration': 833, - 'upload_date': '20161103' + 'upload_date': '20161103', + 'formats': 'count:8', }, 'params': {'skip_download': True}, 'expected_warnings': ['unable to extract player_data'], @@ -675,7 +708,7 @@ def _real_extract(self, url): if not relinker_url: # fallback on old implementation for some old content try: - return self._extract_from_content_id(video_id, url) + return RaiIE._real_extract(self, url) except GeoRestrictedError: raise except ExtractorError as e: diff --git a/yt_dlp/extractor/rbgtum.py b/yt_dlp/extractor/rbgtum.py index c8a331f3e..54f194cbd 100644 --- a/yt_dlp/extractor/rbgtum.py +++ b/yt_dlp/extractor/rbgtum.py @@ -5,7 +5,7 @@ class RbgTumIE(InfoExtractor): - _VALID_URL = r'https://(?:live\.rbg\.tum\.de|tum\.live)/w/(?P[^?#]+)' + _VALID_URL = r'https?://(?:live\.rbg\.tum\.de|tum\.live)/w/(?P[^?#]+)' _TESTS = [{ # Combined view 'url': 'https://live.rbg.tum.de/w/cpp/22128', @@ -60,7 +60,7 @@ def _real_extract(self, url): class RbgTumCourseIE(InfoExtractor): - _VALID_URL = r'https://(?P(?:live\.rbg\.tum\.de|tum\.live))/old/course/(?P(?P\d+)/(?P\w+)/(?P[^/?#]+))' + _VALID_URL = r'https?://(?P(?:live\.rbg\.tum\.de|tum\.live))/old/course/(?P(?P\d+)/(?P\w+)/(?P[^/?#]+))' _TESTS = [{ 'url': 'https://live.rbg.tum.de/old/course/2022/S/fpv', 'info_dict': { @@ -105,7 +105,7 @@ def _real_extract(self, url): class RbgTumNewCourseIE(InfoExtractor): - _VALID_URL = r'https://(?P(?:live\.rbg\.tum\.de|tum\.live))/\?' + _VALID_URL = r'https?://(?P(?:live\.rbg\.tum\.de|tum\.live))/\?' _TESTS = [{ 'url': 'https://live.rbg.tum.de/?year=2022&term=S&slug=fpv&view=3', 'info_dict': { diff --git a/yt_dlp/extractor/rbmaradio.py b/yt_dlp/extractor/rbmaradio.py deleted file mode 100644 index 86c63dbb7..000000000 --- a/yt_dlp/extractor/rbmaradio.py +++ /dev/null @@ -1,68 +0,0 @@ -from .common import InfoExtractor -from ..compat import compat_str -from ..utils import ( - clean_html, - int_or_none, - unified_timestamp, - update_url_query, -) - - -class RBMARadioIE(InfoExtractor): - _VALID_URL = r'https?://(?:www\.)?(?:rbmaradio|redbullradio)\.com/shows/(?P[^/]+)/episodes/(?P[^/?#&]+)' - _TEST = { - 'url': 'https://www.rbmaradio.com/shows/main-stage/episodes/ford-lopatin-live-at-primavera-sound-2011', - 'md5': '6bc6f9bcb18994b4c983bc3bf4384d95', - 'info_dict': { - 'id': 'ford-lopatin-live-at-primavera-sound-2011', - 'ext': 'mp3', - 'title': 'Main Stage - Ford & Lopatin at Primavera Sound', - 'description': 'md5:d41d8cd98f00b204e9800998ecf8427e', - 'thumbnail': r're:^https?://.*\.jpg', - 'duration': 2452, - 'timestamp': 1307103164, - 'upload_date': '20110603', - }, - } - - def _real_extract(self, url): - mobj = self._match_valid_url(url) - show_id = mobj.group('show_id') - episode_id = mobj.group('id') - - webpage = self._download_webpage(url, episode_id) - - episode = self._parse_json( - self._search_regex( - r'__INITIAL_STATE__\s*=\s*({.+?})\s*', - webpage, 'json data'), - episode_id)['episodes'][show_id][episode_id] - - title = episode['title'] - - show_title = episode.get('showTitle') - if show_title: - title = '%s - %s' % (show_title, title) - - formats = [{ - 'url': update_url_query(episode['audioURL'], query={'cbr': abr}), - 'format_id': compat_str(abr), - 'abr': abr, - 'vcodec': 'none', - } for abr in (96, 128, 192, 256)] - self._check_formats(formats, episode_id) - - description = clean_html(episode.get('longTeaser')) - thumbnail = self._proto_relative_url(episode.get('imageURL', {}).get('landscape')) - duration = int_or_none(episode.get('duration')) - timestamp = unified_timestamp(episode.get('publishedAt')) - - return { - 'id': episode_id, - 'title': title, - 'description': description, - 'thumbnail': thumbnail, - 'duration': duration, - 'timestamp': timestamp, - 'formats': formats, - } diff --git a/yt_dlp/extractor/rcti.py b/yt_dlp/extractor/rcti.py index 79d9c8e31..6a7c7f399 100644 --- a/yt_dlp/extractor/rcti.py +++ b/yt_dlp/extractor/rcti.py @@ -28,7 +28,7 @@ def _call_api(self, url, video_id, note=None): class RCTIPlusIE(RCTIPlusBaseIE): - _VALID_URL = r'https://www\.rctiplus\.com/(?:programs/\d+?/.*?/)?(?Pepisode|clip|extra|live-event|missed-event)/(?P\d+)/(?P[^/?#&]+)' + _VALID_URL = r'https?://www\.rctiplus\.com/(?:programs/\d+?/.*?/)?(?Pepisode|clip|extra|live-event|missed-event)/(?P\d+)/(?P[^/?#&]+)' _TESTS = [{ 'url': 'https://www.rctiplus.com/programs/1259/kiko-untuk-lola/episode/22124/untuk-lola', 'md5': '56ed45affad45fa18d5592a1bc199997', @@ -218,7 +218,7 @@ def _real_extract(self, url): class RCTIPlusSeriesIE(RCTIPlusBaseIE): - _VALID_URL = r'https://www\.rctiplus\.com/programs/(?P\d+)/(?P[^/?#&]+)(?:/(?Pepisodes|extras|clips))?' + _VALID_URL = r'https?://www\.rctiplus\.com/programs/(?P\d+)/(?P[^/?#&]+)(?:/(?Pepisodes|extras|clips))?' _TESTS = [{ 'url': 'https://www.rctiplus.com/programs/829/putri-untuk-pangeran', 'playlist_mincount': 1019, @@ -229,7 +229,7 @@ class RCTIPlusSeriesIE(RCTIPlusBaseIE): 'age_limit': 2, 'cast': ['Verrel Bramasta', 'Ranty Maria', 'Riza Syah', 'Ivan Fadilla', 'Nicole Parham', 'Dll', 'Aviv Elham'], 'display_id': 'putri-untuk-pangeran', - 'tag': 'count:18', + 'tags': 'count:18', }, }, { # No episodes 'url': 'https://www.rctiplus.com/programs/615/inews-pagi', @@ -239,7 +239,7 @@ class RCTIPlusSeriesIE(RCTIPlusBaseIE): 'title': 'iNews Pagi', 'description': 'md5:f18ee3d4643cfb41c358e5a9b693ee04', 'age_limit': 2, - 'tag': 'count:11', + 'tags': 'count:11', 'display_id': 'inews-pagi', } }] @@ -327,8 +327,8 @@ def _real_extract(self, url): 'age_limit': try_get(series_meta, lambda x: self._AGE_RATINGS[x['age_restriction'][0]['code']]), 'cast': traverse_obj(series_meta, (('starring', 'creator', 'writer'), ..., 'name'), expected_type=lambda x: strip_or_none(x) or None), - 'tag': traverse_obj(series_meta, ('tag', ..., 'name'), - expected_type=lambda x: strip_or_none(x) or None), + 'tags': traverse_obj(series_meta, ('tag', ..., 'name'), + expected_type=lambda x: strip_or_none(x) or None), } return self.playlist_result( self._series_entries(series_id, display_id, video_type, metadata), series_id, @@ -336,7 +336,7 @@ def _real_extract(self, url): class RCTIPlusTVIE(RCTIPlusBaseIE): - _VALID_URL = r'https://www\.rctiplus\.com/((tv/(?P\w+))|(?Plive-event|missed-event))' + _VALID_URL = r'https?://www\.rctiplus\.com/((tv/(?P\w+))|(?Plive-event|missed-event))' _TESTS = [{ 'url': 'https://www.rctiplus.com/tv/rcti', 'info_dict': { diff --git a/yt_dlp/extractor/rds.py b/yt_dlp/extractor/rds.py index 9a2e0d985..1a1c6634e 100644 --- a/yt_dlp/extractor/rds.py +++ b/yt_dlp/extractor/rds.py @@ -8,6 +8,7 @@ class RDSIE(InfoExtractor): + _WORKING = False IE_DESC = 'RDS.ca' _VALID_URL = r'https?://(?:www\.)?rds\.ca/vid(?:[eé]|%C3%A9)os/(?:[^/]+/)*(?P[^/]+)-\d+\.\d+' diff --git a/yt_dlp/extractor/redbee.py b/yt_dlp/extractor/redbee.py index b59b518b1..4d71133b3 100644 --- a/yt_dlp/extractor/redbee.py +++ b/yt_dlp/extractor/redbee.py @@ -134,6 +134,7 @@ def _real_extract(self, url): class RTBFIE(RedBeeBaseIE): + _WORKING = False _VALID_URL = r'''(?x) https?://(?:www\.)?rtbf\.be/ (?: diff --git a/yt_dlp/extractor/regiotv.py b/yt_dlp/extractor/regiotv.py deleted file mode 100644 index edb6ae5bc..000000000 --- a/yt_dlp/extractor/regiotv.py +++ /dev/null @@ -1,55 +0,0 @@ -from .common import InfoExtractor -from ..networking import Request -from ..utils import xpath_text, xpath_with_ns - - -class RegioTVIE(InfoExtractor): - _VALID_URL = r'https?://(?:www\.)?regio-tv\.de/video/(?P[0-9]+)' - _TESTS = [{ - 'url': 'http://www.regio-tv.de/video/395808.html', - 'info_dict': { - 'id': '395808', - 'ext': 'mp4', - 'title': 'Wir in Ludwigsburg', - 'description': 'Mit unseren zuckersüßen Adventskindern, außerdem besuchen wir die Abendsterne!', - } - }, { - 'url': 'http://www.regio-tv.de/video/395808', - 'only_matching': True, - }] - - def _real_extract(self, url): - video_id = self._match_id(url) - - webpage = self._download_webpage(url, video_id) - - key = self._search_regex( - r'key\s*:\s*(["\'])(?P.+?)\1', webpage, 'key', group='key') - title = self._og_search_title(webpage) - - SOAP_TEMPLATE = '<{0} xmlns="http://v.telvi.de/">{1}' - - request = Request( - 'http://v.telvi.de/', - SOAP_TEMPLATE.format('GetHTML5VideoData', key).encode('utf-8')) - video_data = self._download_xml(request, video_id, 'Downloading video XML') - - NS_MAP = { - 'xsi': 'http://www.w3.org/2001/XMLSchema-instance', - 'soap': 'http://schemas.xmlsoap.org/soap/envelope/', - } - - video_url = xpath_text( - video_data, xpath_with_ns('.//video', NS_MAP), 'video url', fatal=True) - thumbnail = xpath_text( - video_data, xpath_with_ns('.//image', NS_MAP), 'thumbnail') - description = self._og_search_description( - webpage) or self._html_search_meta('description', webpage) - - return { - 'id': video_id, - 'url': video_url, - 'title': title, - 'description': description, - 'thumbnail': thumbnail, - } diff --git a/yt_dlp/extractor/rentv.py b/yt_dlp/extractor/rentv.py index fdde31704..abb537cf3 100644 --- a/yt_dlp/extractor/rentv.py +++ b/yt_dlp/extractor/rentv.py @@ -8,6 +8,7 @@ class RENTVIE(InfoExtractor): + _WORKING = False _VALID_URL = r'(?:rentv:|https?://(?:www\.)?ren\.tv/(?:player|video/epizod)/)(?P\d+)' _TESTS = [{ 'url': 'http://ren.tv/video/epizod/118577', @@ -59,6 +60,7 @@ def _real_extract(self, url): class RENTVArticleIE(InfoExtractor): + _WORKING = False _VALID_URL = r'https?://(?:www\.)?ren\.tv/novosti/\d{4}-\d{2}-\d{2}/(?P[^/?#]+)' _TESTS = [{ 'url': 'http://ren.tv/novosti/2016-10-26/video-mikroavtobus-popavshiy-v-dtp-s-gruzovikami-v-podmoskove-prevratilsya-v', diff --git a/yt_dlp/extractor/restudy.py b/yt_dlp/extractor/restudy.py index 6d032564d..f49262a65 100644 --- a/yt_dlp/extractor/restudy.py +++ b/yt_dlp/extractor/restudy.py @@ -2,6 +2,7 @@ class RestudyIE(InfoExtractor): + _WORKING = False _VALID_URL = r'https?://(?:(?:www|portal)\.)?restudy\.dk/video/[^/]+/id/(?P[0-9]+)' _TESTS = [{ 'url': 'https://www.restudy.dk/video/play/id/1637', diff --git a/yt_dlp/extractor/reuters.py b/yt_dlp/extractor/reuters.py index 6919425f3..0a8f13b9f 100644 --- a/yt_dlp/extractor/reuters.py +++ b/yt_dlp/extractor/reuters.py @@ -9,6 +9,7 @@ class ReutersIE(InfoExtractor): + _WORKING = False _VALID_URL = r'https?://(?:www\.)?reuters\.com/.*?\?.*?videoId=(?P[0-9]+)' _TEST = { 'url': 'http://www.reuters.com/video/2016/05/20/san-francisco-police-chief-resigns?videoId=368575562', diff --git a/yt_dlp/extractor/ridehome.py b/yt_dlp/extractor/ridehome.py new file mode 100644 index 000000000..78f838ac1 --- /dev/null +++ b/yt_dlp/extractor/ridehome.py @@ -0,0 +1,96 @@ +from .art19 import Art19IE +from .common import InfoExtractor +from ..utils import extract_attributes, get_elements_html_by_class +from ..utils.traversal import traverse_obj + + +class RideHomeIE(InfoExtractor): + _VALID_URL = r'https?://(?:www\.)?ridehome\.info/show/[\w-]+/(?P[\w-]+)/?(?:$|[?#])' + _TESTS = [{ + 'url': 'https://www.ridehome.info/show/techmeme-ride-home/thu-1228-will-2024-be-the-year-apple-gets-serious-about-gaming-on-macs/', + 'info_dict': { + 'id': 'thu-1228-will-2024-be-the-year-apple-gets-serious-about-gaming-on-macs', + }, + 'playlist_count': 1, + 'playlist': [{ + 'md5': 'c84ea3cc96950a9ab86fe540f3edc588', + 'info_dict': { + 'id': '540e5493-9fe6-4c14-a488-dc508d8794b2', + 'ext': 'mp3', + 'title': 'Thu. 12/28 – Will 2024 Be The Year Apple Gets Serious About Gaming On Macs?', + 'description': 'md5:9dba86ae9b5047a8150eceddeeb629c2', + 'series': 'Techmeme Ride Home', + 'series_id': '3c30e8f4-ab48-415b-9421-1ae06cd4058b', + 'upload_date': '20231228', + 'timestamp': 1703780995, + 'modified_date': '20231230', + 'episode_id': '540e5493-9fe6-4c14-a488-dc508d8794b2', + 'modified_timestamp': 1703912404, + 'release_date': '20231228', + 'release_timestamp': 1703782800, + 'duration': 1000.1502, + 'thumbnail': r're:^https?://content\.production\.cdn\.art19\.com/images/.*\.jpeg$', + }, + }], + }, { + 'url': 'https://www.ridehome.info/show/techmeme-ride-home/portfolio-profile-sensel-with-ilyarosenberg/', + 'info_dict': { + 'id': 'portfolio-profile-sensel-with-ilyarosenberg', + }, + 'playlist_count': 1, + 'playlist': [{ + 'md5': 'bf9d6efad221008ce71aea09d5533cf6', + 'info_dict': { + 'id': '6beed803-b1ef-4536-9fef-c23cf6b4dcac', + 'ext': 'mp3', + 'title': '(Portfolio Profile) Sensel - With @IlyaRosenberg', + 'description': 'md5:e1e4a970bce04290e0ba6f030b0125db', + 'series': 'Techmeme Ride Home', + 'series_id': '3c30e8f4-ab48-415b-9421-1ae06cd4058b', + 'upload_date': '20220108', + 'timestamp': 1641656064, + 'modified_date': '20230418', + 'episode_id': '6beed803-b1ef-4536-9fef-c23cf6b4dcac', + 'modified_timestamp': 1681843318, + 'release_date': '20220108', + 'release_timestamp': 1641672000, + 'duration': 2789.38122, + 'thumbnail': r're:^https?://content\.production\.cdn\.art19\.com/images/.*\.jpeg$' + }, + }], + }, { + 'url': 'https://www.ridehome.info/show/spacecasts/big-tech-news-apples-macbook-pro-event/', + 'info_dict': { + 'id': 'big-tech-news-apples-macbook-pro-event', + }, + 'playlist_count': 1, + 'playlist': [{ + 'md5': 'b1428530c6e03904a8271e978007fc05', + 'info_dict': { + 'id': 'f4780044-6c4b-4ce0-8215-8a86cc66bff7', + 'ext': 'mp3', + 'title': 'md5:e6c05d44d59b6577a4145ac339de5040', + 'description': 'md5:14152f7228c8a301a77e3d6bc891b145', + 'series': 'SpaceCasts', + 'series_id': '8e3e837d-7fe0-4a23-8e11-894917e07e17', + 'upload_date': '20211026', + 'timestamp': 1635271450, + 'modified_date': '20230502', + 'episode_id': 'f4780044-6c4b-4ce0-8215-8a86cc66bff7', + 'modified_timestamp': 1683057500, + 'release_date': '20211026', + 'release_timestamp': 1635272124, + 'duration': 2266.30531, + 'thumbnail': r're:^https?://content\.production\.cdn\.art19\.com/images/.*\.jpeg$' + }, + }], + }] + + def _real_extract(self, url): + article_id = self._match_id(url) + webpage = self._download_webpage(url, article_id) + + urls = traverse_obj( + get_elements_html_by_class('iframeContainer', webpage), + (..., {extract_attributes}, lambda k, v: k == 'data-src' and Art19IE.suitable(v))) + return self.playlist_from_matches(urls, article_id, ie=Art19IE) diff --git a/yt_dlp/extractor/rockstargames.py b/yt_dlp/extractor/rockstargames.py index c491aaf53..16622430c 100644 --- a/yt_dlp/extractor/rockstargames.py +++ b/yt_dlp/extractor/rockstargames.py @@ -6,6 +6,7 @@ class RockstarGamesIE(InfoExtractor): + _WORKING = False _VALID_URL = r'https?://(?:www\.)?rockstargames\.com/videos(?:/video/|#?/?\?.*\bvideo=)(?P\d+)' _TESTS = [{ 'url': 'https://www.rockstargames.com/videos/video/11544/', @@ -37,14 +38,14 @@ def _real_extract(self, url): title = video['title'] formats = [] - for video in video['files_processed']['video/mp4']: - if not video.get('src'): + for v in video['files_processed']['video/mp4']: + if not v.get('src'): continue - resolution = video.get('resolution') + resolution = v.get('resolution') height = int_or_none(self._search_regex( r'^(\d+)[pP]$', resolution or '', 'height', default=None)) formats.append({ - 'url': self._proto_relative_url(video['src']), + 'url': self._proto_relative_url(v['src']), 'format_id': resolution, 'height': height, }) diff --git a/yt_dlp/extractor/rokfin.py b/yt_dlp/extractor/rokfin.py index cad76f0c9..5099f3ae4 100644 --- a/yt_dlp/extractor/rokfin.py +++ b/yt_dlp/extractor/rokfin.py @@ -38,7 +38,7 @@ class RokfinIE(InfoExtractor): 'upload_date': '20211023', 'timestamp': 1634998029, 'channel': 'Jimmy Dore', - 'channel_id': 65429, + 'channel_id': '65429', 'channel_url': 'https://rokfin.com/TheJimmyDoreShow', 'availability': 'public', 'live_status': 'not_live', @@ -56,7 +56,7 @@ class RokfinIE(InfoExtractor): 'upload_date': '20190412', 'timestamp': 1555052644, 'channel': 'Ron Placone', - 'channel_id': 10, + 'channel_id': '10', 'channel_url': 'https://rokfin.com/RonPlacone', 'availability': 'public', 'live_status': 'not_live', @@ -73,7 +73,7 @@ class RokfinIE(InfoExtractor): 'thumbnail': r're:https://img\.production\.rokfin\.com/.+', 'description': 'md5:324ce2d3e3b62e659506409e458b9d8e', 'channel': 'TLAVagabond', - 'channel_id': 53856, + 'channel_id': '53856', 'channel_url': 'https://rokfin.com/TLAVagabond', 'availability': 'public', 'is_live': False, @@ -86,7 +86,6 @@ class RokfinIE(InfoExtractor): 'dislike_count': int, 'like_count': int, 'tags': ['FreeThinkingMedia^'], - 'duration': None, } }, { 'url': 'https://rokfin.com/post/126703/Brave-New-World--Aldous-Huxley-DEEPDIVE--Chpts-13--Quite-Frankly--Jay-Dyer', @@ -96,7 +95,7 @@ class RokfinIE(InfoExtractor): 'title': 'Brave New World - Aldous Huxley DEEPDIVE! (Chpts 1-3) - Quite Frankly & Jay Dyer', 'thumbnail': r're:https://img\.production\.rokfin\.com/.+', 'channel': 'Jay Dyer', - 'channel_id': 186881, + 'channel_id': '186881', 'channel_url': 'https://rokfin.com/jaydyer', 'availability': 'premium_only', 'live_status': 'not_live', @@ -116,7 +115,7 @@ class RokfinIE(InfoExtractor): 'title': 'The Grayzone live on Nordstream blame game', 'thumbnail': r're:https://image\.v\.rokfin\.com/.+', 'channel': 'Max Blumenthal', - 'channel_id': 248902, + 'channel_id': '248902', 'channel_url': 'https://rokfin.com/MaxBlumenthal', 'availability': 'premium_only', 'live_status': 'was_live', @@ -174,7 +173,7 @@ def _real_extract(self, url): 'like_count': int_or_none(metadata.get('likeCount')), 'dislike_count': int_or_none(metadata.get('dislikeCount')), 'channel': str_or_none(traverse_obj(metadata, ('createdBy', 'name'), ('creator', 'name'))), - 'channel_id': traverse_obj(metadata, ('createdBy', 'id'), ('creator', 'id')), + 'channel_id': str_or_none(traverse_obj(metadata, ('createdBy', 'id'), ('creator', 'id'))), 'channel_url': url_or_none(f'https://rokfin.com/{uploader}') if uploader else None, 'timestamp': timestamp, 'release_timestamp': timestamp if live_status != 'not_live' else None, diff --git a/yt_dlp/extractor/roosterteeth.py b/yt_dlp/extractor/roosterteeth.py index 94e673b13..5c622399d 100644 --- a/yt_dlp/extractor/roosterteeth.py +++ b/yt_dlp/extractor/roosterteeth.py @@ -2,17 +2,18 @@ from ..networking.exceptions import HTTPError from ..utils import ( ExtractorError, + LazyList, int_or_none, join_nonempty, - LazyList, + parse_iso8601, parse_qs, + smuggle_url, str_or_none, - traverse_obj, url_or_none, urlencode_postdata, urljoin, - update_url_query, ) +from ..utils.traversal import traverse_obj class RoosterTeethBaseIE(InfoExtractor): @@ -57,19 +58,27 @@ def _extract_video_info(self, data): title = traverse_obj(attributes, 'title', 'display_title') sub_only = attributes.get('is_sponsors_only') + episode_id = str_or_none(data.get('uuid')) + video_id = str_or_none(data.get('id')) + if video_id and 'parent_content_id' in attributes: # parent_content_id is a bonus-only key + video_id += '-bonus' # there are collisions with bonus ids and regular ids + elif not video_id: + video_id = episode_id + return { - 'id': str(data.get('id')), + 'id': video_id, 'display_id': attributes.get('slug'), 'title': title, 'description': traverse_obj(attributes, 'description', 'caption'), - 'series': attributes.get('show_title'), + 'series': traverse_obj(attributes, 'show_title', 'parent_content_title'), 'season_number': int_or_none(attributes.get('season_number')), - 'season_id': attributes.get('season_id'), + 'season_id': str_or_none(attributes.get('season_id')), 'episode': title, 'episode_number': int_or_none(attributes.get('number')), - 'episode_id': str_or_none(data.get('uuid')), + 'episode_id': episode_id, 'channel_id': attributes.get('channel_id'), 'duration': int_or_none(attributes.get('length')), + 'release_timestamp': parse_iso8601(attributes.get('original_air_date')), 'thumbnails': thumbnails, 'availability': self._availability( needs_premium=sub_only, needs_subscription=sub_only, needs_auth=sub_only, @@ -79,7 +88,7 @@ def _extract_video_info(self, data): class RoosterTeethIE(RoosterTeethBaseIE): - _VALID_URL = r'https?://(?:.+?\.)?roosterteeth\.com/(?:episode|watch)/(?P[^/?#&]+)' + _VALID_URL = r'https?://(?:.+?\.)?roosterteeth\.com/(?:bonus-feature|episode|watch)/(?P[^/?#&]+)' _TESTS = [{ 'url': 'http://roosterteeth.com/episode/million-dollars-but-season-2-million-dollars-but-the-game-announcement', 'info_dict': { @@ -91,6 +100,17 @@ class RoosterTeethIE(RoosterTeethBaseIE): 'thumbnail': r're:^https?://.*\.png$', 'series': 'Million Dollars, But...', 'episode': 'Million Dollars, But... The Game Announcement', + 'tags': ['Game Show', 'Sketch'], + 'season_number': 2, + 'availability': 'public', + 'episode_number': 10, + 'episode_id': '00374575-464e-11e7-a302-065410f210c4', + 'season': 'Season 2', + 'season_id': 'ffa27d48-464d-11e7-a302-065410f210c4', + 'channel_id': '92b6bb21-91d2-4b1b-bf95-3268fa0d9939', + 'duration': 145, + 'release_timestamp': 1462982400, + 'release_date': '20160511', }, 'params': {'skip_download': True}, }, { @@ -104,8 +124,90 @@ class RoosterTeethIE(RoosterTeethBaseIE): 'channel_id': '92f780eb-ebfe-4bf5-a3b5-c6ad5460a5f1', 'thumbnail': r're:^https?://.*\.(png|jpe?g)$', 'ext': 'mp4', + 'availability': 'public', + 'episode_id': 'f8117b13-f068-499e-803e-eec9ea2dec8c', + 'episode_number': 3, + 'tags': ['Animation'], + 'season_id': '4b8f0a9e-12c4-41ed-8caa-fed15a85bab8', + 'season': 'Season 1', + 'series': 'RWBY: World of Remnant', + 'season_number': 1, + 'duration': 216, + 'release_timestamp': 1413489600, + 'release_date': '20141016', }, 'params': {'skip_download': True}, + }, { + # bonus feature with /watch/ url + 'url': 'https://roosterteeth.com/watch/rwby-bonus-21', + 'info_dict': { + 'id': '33-bonus', + 'display_id': 'rwby-bonus-21', + 'title': 'Volume 5 Yang Character Short', + 'description': 'md5:8c2440bc763ea90c52cfe0a68093e1f7', + 'episode': 'Volume 5 Yang Character Short', + 'channel_id': '92f780eb-ebfe-4bf5-a3b5-c6ad5460a5f1', + 'thumbnail': r're:^https?://.*\.(png|jpe?g)$', + 'ext': 'mp4', + 'availability': 'public', + 'episode_id': 'f2a9f132-1fe2-44ad-8956-63d7c0267720', + 'episode_number': 55, + 'series': 'RWBY', + 'duration': 255, + 'release_timestamp': 1507993200, + 'release_date': '20171014', + }, + 'params': {'skip_download': True}, + }, { + # only works with video_data['attributes']['url'] m3u8 url + 'url': 'https://www.roosterteeth.com/watch/achievement-hunter-achievement-hunter-fatality-walkthrough-deathstroke-lex-luthor-captain-marvel-green-lantern-and-wonder-woman', + 'info_dict': { + 'id': '25394', + 'ext': 'mp4', + 'title': 'Fatality Walkthrough: Deathstroke, Lex Luthor, Captain Marvel, Green Lantern, and Wonder Woman', + 'description': 'md5:91bb934698344fb9647b1c7351f16964', + 'availability': 'public', + 'thumbnail': r're:^https?://.*\.(png|jpe?g)$', + 'episode': 'Fatality Walkthrough: Deathstroke, Lex Luthor, Captain Marvel, Green Lantern, and Wonder Woman', + 'episode_number': 71, + 'episode_id': 'ffaec998-464d-11e7-a302-065410f210c4', + 'season': 'Season 2008', + 'tags': ['Gaming'], + 'series': 'Achievement Hunter', + 'display_id': 'md5:4465ce4f001735f9d7a2ae529a543d31', + 'season_id': 'ffa13340-464d-11e7-a302-065410f210c4', + 'season_number': 2008, + 'channel_id': '2cb2a70c-be50-46f5-93d7-84a1baabb4f7', + 'duration': 189, + 'release_timestamp': 1228317300, + 'release_date': '20081203', + }, + 'params': {'skip_download': True}, + }, { + # brightcove fallback extraction needed + 'url': 'https://roosterteeth.com/watch/lets-play-2013-126', + 'info_dict': { + 'id': '17845', + 'ext': 'mp4', + 'title': 'WWE \'13', + 'availability': 'public', + 'series': 'Let\'s Play', + 'episode_number': 10, + 'season_id': 'ffa23d9c-464d-11e7-a302-065410f210c4', + 'channel_id': '75ba87e8-06fd-4482-bad9-52a4da2c6181', + 'episode': 'WWE \'13', + 'episode_id': 'ffdbe55e-464d-11e7-a302-065410f210c4', + 'thumbnail': r're:^https?://.*\.(png|jpe?g)$', + 'tags': ['Gaming', 'Our Favorites'], + 'description': 'md5:b4a5226d2bbcf0dafbde11a2ba27262d', + 'display_id': 'lets-play-2013-126', + 'season_number': 3, + 'season': 'Season 3', + 'release_timestamp': 1359999840, + 'release_date': '20130204', + }, + 'expected_warnings': ['Direct m3u8 URL returned HTTP Error 403'], + 'params': {'skip_download': True}, }, { 'url': 'http://achievementhunter.roosterteeth.com/episode/off-topic-the-achievement-hunter-podcast-2016-i-didn-t-think-it-would-pass-31', 'only_matching': True, @@ -125,18 +227,29 @@ class RoosterTeethIE(RoosterTeethBaseIE): }, { 'url': 'https://roosterteeth.com/watch/million-dollars-but-season-2-million-dollars-but-the-game-announcement', 'only_matching': True, + }, { + 'url': 'https://roosterteeth.com/bonus-feature/camp-camp-soundtrack-another-rap-song-about-foreign-cars-richie-branson', + 'only_matching': True, }] + _BRIGHTCOVE_ACCOUNT_ID = '6203312018001' + + def _extract_brightcove_formats_and_subtitles(self, bc_id, url, m3u8_url): + account_id = self._search_regex( + r'/accounts/(\d+)/videos/', m3u8_url, 'account id', default=self._BRIGHTCOVE_ACCOUNT_ID) + info = self._downloader.get_info_extractor('BrightcoveNew').extract(smuggle_url( + f'https://players.brightcove.net/{account_id}/default_default/index.html?videoId={bc_id}', + {'referrer': url})) + return info['formats'], info['subtitles'] + def _real_extract(self, url): display_id = self._match_id(url) api_episode_url = f'{self._API_BASE_URL}/watch/{display_id}' try: video_data = self._download_json( - api_episode_url + '/videos', display_id, - 'Downloading video JSON metadata')['data'][0] - m3u8_url = video_data['attributes']['url'] - # XXX: additional URL at video_data['links']['download'] + api_episode_url + '/videos', display_id, 'Downloading video JSON metadata', + headers={'Client-Type': 'web'})['data'][0] # web client-type yields ad-free streams except ExtractorError as e: if isinstance(e.cause, HTTPError) and e.cause.status == 403: if self._parse_json(e.cause.response.read().decode(), display_id).get('access') is False: @@ -144,8 +257,21 @@ def _real_extract(self, url): '%s is only available for FIRST members' % display_id) raise - formats, subtitles = self._extract_m3u8_formats_and_subtitles( - m3u8_url, display_id, 'mp4', 'm3u8_native', m3u8_id='hls') + # XXX: additional ad-free URL at video_data['links']['download'] but often gives 403 errors + m3u8_url = video_data['attributes']['url'] + is_brightcove = traverse_obj(video_data, ('attributes', 'encoding_pipeline')) == 'brightcove' + bc_id = traverse_obj(video_data, ('attributes', 'uid', {str})) + + try: + formats, subtitles = self._extract_m3u8_formats_and_subtitles( + m3u8_url, display_id, 'mp4', 'm3u8_native', m3u8_id='hls') + except ExtractorError as e: + if is_brightcove and bc_id and isinstance(e.cause, HTTPError) and e.cause.status == 403: + self.report_warning( + 'Direct m3u8 URL returned HTTP Error 403; retrying with Brightcove extraction') + formats, subtitles = self._extract_brightcove_formats_and_subtitles(bc_id, url, m3u8_url) + else: + raise episode = self._download_json( api_episode_url, display_id, @@ -167,38 +293,53 @@ class RoosterTeethSeriesIE(RoosterTeethBaseIE): 'info_dict': { 'id': 'rwby-7', 'title': 'RWBY - Season 7', - } + }, + }, { + 'url': 'https://roosterteeth.com/series/the-weird-place', + 'playlist_count': 7, + 'info_dict': { + 'id': 'the-weird-place', + 'title': 'The Weird Place', + }, }, { 'url': 'https://roosterteeth.com/series/role-initiative', 'playlist_mincount': 16, 'info_dict': { 'id': 'role-initiative', 'title': 'Role Initiative', - } + }, }, { 'url': 'https://roosterteeth.com/series/let-s-play-minecraft?season=9', 'playlist_mincount': 50, 'info_dict': { 'id': 'let-s-play-minecraft-9', 'title': 'Let\'s Play Minecraft - Season 9', - } + }, }] def _entries(self, series_id, season_number): display_id = join_nonempty(series_id, season_number) - # TODO: extract bonus material - for data in self._download_json( - f'{self._API_BASE_URL}/shows/{series_id}/seasons?order=asc&order_by', display_id)['data']: - idx = traverse_obj(data, ('attributes', 'number')) - if season_number and idx != season_number: - continue - season_url = update_url_query(urljoin(self._API_BASE, data['links']['episodes']), {'per_page': 1000}) - season = self._download_json(season_url, display_id, f'Downloading season {idx} JSON metadata')['data'] - for episode in season: + + def yield_episodes(data): + for episode in traverse_obj(data, ('data', lambda _, v: v['canonical_links']['self'])): yield self.url_result( - f'https://www.roosterteeth.com{episode["canonical_links"]["self"]}', - RoosterTeethIE.ie_key(), - **self._extract_video_info(episode)) + urljoin('https://www.roosterteeth.com', episode['canonical_links']['self']), + RoosterTeethIE, **self._extract_video_info(episode)) + + series_data = self._download_json( + f'{self._API_BASE_URL}/shows/{series_id}/seasons?order=asc&order_by', display_id) + for season_data in traverse_obj(series_data, ('data', lambda _, v: v['links']['episodes'])): + idx = traverse_obj(season_data, ('attributes', 'number')) + if season_number is not None and idx != season_number: + continue + yield from yield_episodes(self._download_json( + urljoin(self._API_BASE, season_data['links']['episodes']), display_id, + f'Downloading season {idx} JSON metadata', query={'per_page': 1000})) + + if season_number is None: # extract series-level bonus features + yield from yield_episodes(self._download_json( + f'{self._API_BASE_URL}/shows/{series_id}/bonus_features?order=asc&order_by&per_page=1000', + display_id, 'Downloading bonus features JSON metadata', fatal=False)) def _real_extract(self, url): series_id = self._match_id(url) diff --git a/yt_dlp/extractor/rozhlas.py b/yt_dlp/extractor/rozhlas.py index 63134322d..411a62519 100644 --- a/yt_dlp/extractor/rozhlas.py +++ b/yt_dlp/extractor/rozhlas.py @@ -247,17 +247,17 @@ class MujRozhlasIE(RozhlasBaseIE): 'url': 'https://www.mujrozhlas.cz/vykopavky/ach-jo-zase-teleci-rizek-je-mnohem-min-cesky-nez-jsme-si-mysleli', 'md5': '6f8fd68663e64936623e67c152a669e0', 'info_dict': { - 'id': '10739193', + 'id': '10787730', 'ext': 'mp3', 'title': 'Ach jo, zase to telecí! Řízek je mnohem míň český, než jsme si mysleli', 'description': 'md5:db7141e9caaedc9041ec7cefb9a62908', 'timestamp': 1684915200, - 'modified_timestamp': 1684922446, + 'modified_timestamp': 1687550432, 'series': 'Vykopávky', 'thumbnail': 'https://portal.rozhlas.cz/sites/default/files/images/84377046610af6ddc54d910b1dd7a22b.jpg', 'channel_id': 'radio-wave', 'upload_date': '20230524', - 'modified_date': '20230524', + 'modified_date': '20230623', }, }, { # serial extraction @@ -277,6 +277,26 @@ class MujRozhlasIE(RozhlasBaseIE): 'title': 'Nespavci', 'description': 'md5:c430adcbf9e2b9eac88b745881e814dc', }, + }, { + # serialPart + 'url': 'https://www.mujrozhlas.cz/povidka/gustavo-adolfo-becquer-hora-duchu', + 'info_dict': { + 'id': '8889035', + 'ext': 'm4a', + 'title': 'Gustavo Adolfo Bécquer: Hora duchů', + 'description': 'md5:343a15257b376c276e210b78e900ffea', + 'chapter': 'Hora duchů a Polibek – dva tajemné příběhy Gustava Adolfa Bécquera', + 'thumbnail': 'https://portal.rozhlas.cz/sites/default/files/images/2adfe1387fb140634be725c1ccf26214.jpg', + 'timestamp': 1708173000, + 'episode': 'Episode 1', + 'episode_number': 1, + 'series': 'Povídka', + 'modified_date': '20240217', + 'upload_date': '20240217', + 'modified_timestamp': 1708173198, + 'channel_id': 'vltava', + }, + 'params': {'skip_download': 'dash'}, }] def _call_api(self, path, item_id, msg='API JSON'): @@ -322,7 +342,7 @@ def _real_extract(self, url): entity = info['siteEntityBundle'] - if entity == 'episode': + if entity in ('episode', 'serialPart'): return self._extract_audio_entry(self._call_api( 'episodes', info['contentId'], 'episode info API JSON')) diff --git a/yt_dlp/extractor/rts.py b/yt_dlp/extractor/rts.py index 9f73d1811..bce5cba82 100644 --- a/yt_dlp/extractor/rts.py +++ b/yt_dlp/extractor/rts.py @@ -13,6 +13,7 @@ class RTSIE(SRGSSRIE): # XXX: Do not subclass from concrete IE + _WORKING = False IE_DESC = 'RTS.ch' _VALID_URL = r'rts:(?P\d+)|https?://(?:.+?\.)?rts\.ch/(?:[^/]+/){2,}(?P[0-9]+)-(?P.+?)\.html' diff --git a/yt_dlp/extractor/rule34video.py b/yt_dlp/extractor/rule34video.py index 85ad7e2ff..11095b262 100644 --- a/yt_dlp/extractor/rule34video.py +++ b/yt_dlp/extractor/rule34video.py @@ -9,7 +9,6 @@ get_element_html_by_class, get_elements_by_class, int_or_none, - join_nonempty, parse_count, parse_duration, unescapeHTML, @@ -57,7 +56,7 @@ class Rule34VideoIE(InfoExtractor): 'comment_count': int, 'timestamp': 1640131200, 'description': '', - 'creator': 'WildeerStudio', + 'creators': ['WildeerStudio'], 'upload_date': '20211222', 'uploader': 'CerZule', 'uploader_url': 'https://rule34video.com/members/36281/', @@ -81,13 +80,13 @@ def _real_extract(self, url): 'quality': quality, }) - categories, creator, uploader, uploader_url = [None] * 4 + categories, creators, uploader, uploader_url = [None] * 4 for col in get_elements_by_class('col', webpage): label = clean_html(get_element_by_class('label', col)) if label == 'Categories:': categories = list(map(clean_html, get_elements_by_class('item', col))) elif label == 'Artist:': - creator = join_nonempty(*map(clean_html, get_elements_by_class('item', col)), delim=', ') + creators = list(map(clean_html, get_elements_by_class('item', col))) elif label == 'Uploaded By:': uploader = clean_html(get_element_by_class('name', col)) uploader_url = extract_attributes(get_element_html_by_class('name', col) or '').get('href') @@ -115,7 +114,7 @@ def _real_extract(self, url): 'comment_count': int_or_none(self._search_regex( r'[^(]+\((\d+)\)', get_element_by_attribute('href', '#tab_comments', webpage), 'comment count', fatal=False)), 'age_limit': 18, - 'creator': creator, + 'creators': creators, 'uploader': uploader, 'uploader_url': uploader_url, 'categories': categories, diff --git a/yt_dlp/extractor/rumble.py b/yt_dlp/extractor/rumble.py index 1dc049ac8..837a324e6 100644 --- a/yt_dlp/extractor/rumble.py +++ b/yt_dlp/extractor/rumble.py @@ -90,7 +90,6 @@ class RumbleEmbedIE(InfoExtractor): 'channel_url': 'https://rumble.com/c/LofiGirl', 'channel': 'Lofi Girl', 'thumbnail': r're:https://.+\.jpg', - 'duration': None, 'uploader': 'Lofi Girl', 'live_status': 'is_live', }, diff --git a/yt_dlp/extractor/rutube.py b/yt_dlp/extractor/rutube.py index 08d9b9257..287824d08 100644 --- a/yt_dlp/extractor/rutube.py +++ b/yt_dlp/extractor/rutube.py @@ -46,7 +46,7 @@ def _extract_info(self, video, video_id=None, require_title=True): 'uploader': try_get(video, lambda x: x['author']['name']), 'uploader_id': compat_str(uploader_id) if uploader_id else None, 'timestamp': unified_timestamp(video.get('created_ts')), - 'category': [category] if category else None, + 'categories': [category] if category else None, 'age_limit': age_limit, 'view_count': int_or_none(video.get('hits')), 'comment_count': int_or_none(video.get('comments_count')), @@ -112,7 +112,7 @@ class RutubeIE(RutubeBaseIE): 'age_limit': 0, 'view_count': int, 'thumbnail': 'http://pic.rutubelist.ru/video/d2/a0/d2a0aec998494a396deafc7ba2c82add.jpg', - 'category': ['Новости и СМИ'], + 'categories': ['Новости и СМИ'], 'chapters': [], }, 'expected_warnings': ['Unable to download f4m'], @@ -144,7 +144,7 @@ class RutubeIE(RutubeBaseIE): 'age_limit': 0, 'view_count': int, 'thumbnail': 'http://pic.rutubelist.ru/video/f2/d4/f2d42b54be0a6e69c1c22539e3152156.jpg', - 'category': ['Видеоигры'], + 'categories': ['Видеоигры'], 'chapters': [], }, 'expected_warnings': ['Unable to download f4m'], @@ -154,7 +154,7 @@ class RutubeIE(RutubeBaseIE): 'id': 'c65b465ad0c98c89f3b25cb03dcc87c6', 'ext': 'mp4', 'chapters': 'count:4', - 'category': ['Бизнес и предпринимательство'], + 'categories': ['Бизнес и предпринимательство'], 'description': 'md5:252feac1305257d8c1bab215cedde75d', 'thumbnail': 'http://pic.rutubelist.ru/video/71/8f/718f27425ea9706073eb80883dd3787b.png', 'duration': 782, diff --git a/yt_dlp/extractor/saitosan.py b/yt_dlp/extractor/saitosan.py index d2f60e92f..a5f05e1d0 100644 --- a/yt_dlp/extractor/saitosan.py +++ b/yt_dlp/extractor/saitosan.py @@ -3,6 +3,7 @@ class SaitosanIE(InfoExtractor): + _WORKING = False IE_NAME = 'Saitosan' _VALID_URL = r'https?://(?:www\.)?saitosan\.net/bview.html\?id=(?P[0-9]+)' _TESTS = [{ diff --git a/yt_dlp/extractor/savefrom.py b/yt_dlp/extractor/savefrom.py deleted file mode 100644 index 9c9e74b6d..000000000 --- a/yt_dlp/extractor/savefrom.py +++ /dev/null @@ -1,30 +0,0 @@ -import os.path - -from .common import InfoExtractor - - -class SaveFromIE(InfoExtractor): - IE_NAME = 'savefrom.net' - _VALID_URL = r'https?://[^.]+\.savefrom\.net/\#url=(?P.*)$' - - _TEST = { - 'url': 'http://en.savefrom.net/#url=http://youtube.com/watch?v=UlVRAPW2WJY&utm_source=youtube.com&utm_medium=short_domains&utm_campaign=ssyoutube.com', - 'info_dict': { - 'id': 'UlVRAPW2WJY', - 'ext': 'mp4', - 'title': 'About Team Radical MMA | MMA Fighting', - 'upload_date': '20120816', - 'uploader': 'Howcast', - 'uploader_id': 'Howcast', - 'description': r're:(?s).* Hi, my name is Rene Dreifuss\. And I\'m here to show you some MMA.*', - }, - 'params': { - 'skip_download': True - } - } - - def _real_extract(self, url): - mobj = self._match_valid_url(url) - video_id = os.path.splitext(url.split('/')[-1])[0] - - return self.url_result(mobj.group('url'), video_id=video_id) diff --git a/yt_dlp/extractor/sbs.py b/yt_dlp/extractor/sbs.py index 7a9115047..8d61e22fc 100644 --- a/yt_dlp/extractor/sbs.py +++ b/yt_dlp/extractor/sbs.py @@ -44,8 +44,6 @@ class SBSIE(InfoExtractor): 'timestamp': 1408613220, 'upload_date': '20140821', 'uploader': 'SBSC', - 'tags': None, - 'categories': None, }, 'expected_warnings': ['Unable to download JSON metadata'], }, { diff --git a/yt_dlp/extractor/screencastify.py b/yt_dlp/extractor/screencastify.py index 136b8479b..3c43043de 100644 --- a/yt_dlp/extractor/screencastify.py +++ b/yt_dlp/extractor/screencastify.py @@ -5,7 +5,10 @@ class ScreencastifyIE(InfoExtractor): - _VALID_URL = r'https?://watch\.screencastify\.com/v/(?P[^/?#]+)' + _VALID_URL = [ + r'https?://watch\.screencastify\.com/v/(?P[^/?#]+)', + r'https?://app\.screencastify\.com/v[23]/watch/(?P[^/?#]+)', + ] _TESTS = [{ 'url': 'https://watch.screencastify.com/v/sYVkZip3quLKhHw4Ybk8', 'info_dict': { @@ -19,6 +22,21 @@ class ScreencastifyIE(InfoExtractor): 'params': { 'skip_download': 'm3u8', }, + }, { + 'url': 'https://app.screencastify.com/v3/watch/J5N7H11wofDN1jZUCr3t', + 'info_dict': { + 'id': 'J5N7H11wofDN1jZUCr3t', + 'ext': 'mp4', + 'uploader': 'Scott Piesen', + 'description': '', + 'title': 'Lesson Recording 1-17 Burrr...', + }, + 'params': { + 'skip_download': 'm3u8', + }, + }, { + 'url': 'https://app.screencastify.com/v2/watch/BQ26VbUdfbQLhKzkktOk', + 'only_matching': True, }] def _real_extract(self, url): diff --git a/yt_dlp/extractor/seeker.py b/yt_dlp/extractor/seeker.py deleted file mode 100644 index 65eb16a09..000000000 --- a/yt_dlp/extractor/seeker.py +++ /dev/null @@ -1,55 +0,0 @@ -import re - -from .common import InfoExtractor -from ..utils import ( - get_element_by_class, - strip_or_none, -) - - -class SeekerIE(InfoExtractor): - _VALID_URL = r'https?://(?:www\.)?seeker\.com/(?P.*)-(?P\d+)\.html' - _TESTS = [{ - 'url': 'http://www.seeker.com/should-trump-be-required-to-release-his-tax-returns-1833805621.html', - 'md5': '897d44bbe0d8986a2ead96de565a92db', - 'info_dict': { - 'id': 'Elrn3gnY', - 'ext': 'mp4', - 'title': 'Should Trump Be Required To Release His Tax Returns?', - 'description': 'md5:41efa8cfa8d627841045eec7b018eb45', - 'timestamp': 1490090165, - 'upload_date': '20170321', - } - }, { - 'url': 'http://www.seeker.com/changes-expected-at-zoos-following-recent-gorilla-lion-shootings-1834116536.html', - 'playlist': [ - { - 'md5': '0497b9f20495174be73ae136949707d2', - 'info_dict': { - 'id': 'FihYQ8AE', - 'ext': 'mp4', - 'title': 'The Pros & Cons Of Zoos', - 'description': 'md5:d88f99a8ea8e7d25e6ff77f271b1271c', - 'timestamp': 1490039133, - 'upload_date': '20170320', - }, - } - ], - 'info_dict': { - 'id': '1834116536', - 'title': 'After Gorilla Killing, Changes Ahead for Zoos', - 'description': 'The largest association of zoos and others are hoping to learn from recent incidents that led to the shooting deaths of a gorilla and two lions.', - }, - }] - - def _real_extract(self, url): - display_id, article_id = self._match_valid_url(url).groups() - webpage = self._download_webpage(url, display_id) - entries = [] - for jwp_id in re.findall(r'data-video-id="([a-zA-Z0-9]{8})"', webpage): - entries.append(self.url_result( - 'jwplatform:' + jwp_id, 'JWPlatform', jwp_id)) - return self.playlist_result( - entries, article_id, - self._og_search_title(webpage), - strip_or_none(get_element_by_class('subtitle__text', webpage)) or self._og_search_description(webpage)) diff --git a/yt_dlp/extractor/senalcolombia.py b/yt_dlp/extractor/senalcolombia.py index f3c066da7..b2f354fae 100644 --- a/yt_dlp/extractor/senalcolombia.py +++ b/yt_dlp/extractor/senalcolombia.py @@ -3,6 +3,7 @@ class SenalColombiaLiveIE(InfoExtractor): + _WORKING = False _VALID_URL = r'https?://(?:www\.)?senalcolombia\.tv/(?Psenal-en-vivo)' _TESTS = [{ diff --git a/yt_dlp/extractor/sendtonews.py b/yt_dlp/extractor/sendtonews.py index 3600e2e74..1ecea71fc 100644 --- a/yt_dlp/extractor/sendtonews.py +++ b/yt_dlp/extractor/sendtonews.py @@ -12,6 +12,7 @@ class SendtoNewsIE(InfoExtractor): + _WORKING = False _VALID_URL = r'https?://embed\.sendtonews\.com/player2/embedplayer\.php\?.*\bSC=(?P[0-9A-Za-z-]+)' _TEST = { diff --git a/yt_dlp/extractor/sexu.py b/yt_dlp/extractor/sexu.py index 3117f81e3..989b63c72 100644 --- a/yt_dlp/extractor/sexu.py +++ b/yt_dlp/extractor/sexu.py @@ -2,6 +2,7 @@ class SexuIE(InfoExtractor): + _WORKING = False _VALID_URL = r'https?://(?:www\.)?sexu\.com/(?P\d+)' _TEST = { 'url': 'http://sexu.com/961791/', diff --git a/yt_dlp/extractor/skeb.py b/yt_dlp/extractor/skeb.py index e02f8cef0..54dfdc441 100644 --- a/yt_dlp/extractor/skeb.py +++ b/yt_dlp/extractor/skeb.py @@ -10,7 +10,7 @@ class SkebIE(InfoExtractor): 'info_dict': { 'id': '466853', 'title': '内容はおまかせします！ by 姫ノ森りぃる@一周年', - 'descripion': 'md5:1ec50901efc3437cfbfe3790468d532d', + 'description': 'md5:1ec50901efc3437cfbfe3790468d532d', 'uploader': '姫ノ森りぃる@一周年', 'uploader_id': 'riiru_wm', 'age_limit': 0, @@ -34,7 +34,7 @@ class SkebIE(InfoExtractor): 'info_dict': { 'id': '489408', 'title': 'いつもお世話になってお... by 古川ノブ@音楽とVlo...', - 'descripion': 'md5:5adc2e41d06d33b558bf7b1faeb7b9c2', + 'description': 'md5:5adc2e41d06d33b558bf7b1faeb7b9c2', 'uploader': '古川ノブ@音楽とVlogのVtuber', 'uploader_id': 'furukawa_nob', 'age_limit': 0, @@ -61,12 +61,12 @@ class SkebIE(InfoExtractor): 'info_dict': { 'id': '6', 'title': 'ヒロ。\n\n私のキャラク... by 諸々', - 'descripion': 'md5:aa6cbf2ba320b50bce219632de195f07', + 'description': 'md5:aa6cbf2ba320b50bce219632de195f07', '_type': 'playlist', 'entries': [{ 'id': '486430', 'title': 'ヒロ。\n\n私のキャラク... by 諸々', - 'descripion': 'md5:aa6cbf2ba320b50bce219632de195f07', + 'description': 'md5:aa6cbf2ba320b50bce219632de195f07', }, { 'id': '486431', 'title': 'ヒロ。\n\n私のキャラク... by 諸々', @@ -81,7 +81,7 @@ def _real_extract(self, url): parent = { 'id': video_id, 'title': nuxt_data.get('title'), - 'descripion': nuxt_data.get('description'), + 'description': nuxt_data.get('description'), 'uploader': traverse_obj(nuxt_data, ('creator', 'name')), 'uploader_id': traverse_obj(nuxt_data, ('creator', 'screen_name')), 'age_limit': 18 if nuxt_data.get('nsfw') else 0, diff --git a/yt_dlp/extractor/skylinewebcams.py b/yt_dlp/extractor/skylinewebcams.py index 4292bb2ae..197407c18 100644 --- a/yt_dlp/extractor/skylinewebcams.py +++ b/yt_dlp/extractor/skylinewebcams.py @@ -2,6 +2,7 @@ class SkylineWebcamsIE(InfoExtractor): + _WORKING = False _VALID_URL = r'https?://(?:www\.)?skylinewebcams\.com/[^/]+/webcam/(?:[^/]+/)+(?P[^/]+)\.html' _TEST = { 'url': 'https://www.skylinewebcams.com/it/webcam/italia/lazio/roma/scalinata-piazza-di-spagna-barcaccia.html', diff --git a/yt_dlp/extractor/skynewsarabia.py b/yt_dlp/extractor/skynewsarabia.py index 6264b04bb..867782778 100644 --- a/yt_dlp/extractor/skynewsarabia.py +++ b/yt_dlp/extractor/skynewsarabia.py @@ -38,6 +38,7 @@ def _extract_video_info(self, video_data): class SkyNewsArabiaIE(SkyNewsArabiaBaseIE): + _WORKING = False IE_NAME = 'skynewsarabia:video' _VALID_URL = r'https?://(?:www\.)?skynewsarabia\.com/web/video/(?P[0-9]+)' _TEST = { @@ -64,6 +65,7 @@ def _real_extract(self, url): class SkyNewsArabiaArticleIE(SkyNewsArabiaBaseIE): + _WORKING = False IE_NAME = 'skynewsarabia:article' _VALID_URL = r'https?://(?:www\.)?skynewsarabia\.com/web/article/(?P[0-9]+)' _TESTS = [{ diff --git a/yt_dlp/extractor/slideslive.py b/yt_dlp/extractor/slideslive.py index df2af3b35..a1328dee2 100644 --- a/yt_dlp/extractor/slideslive.py +++ b/yt_dlp/extractor/slideslive.py @@ -25,8 +25,8 @@ class SlidesLiveIE(InfoExtractor): 'id': '38902413', 'ext': 'mp4', 'title': 'GCC IA16 backend', - 'timestamp': 1648189972, - 'upload_date': '20220325', + 'timestamp': 1697793372, + 'upload_date': '20231020', 'thumbnail': r're:^https?://.*\.jpg', 'thumbnails': 'count:42', 'chapters': 'count:41', @@ -42,8 +42,8 @@ class SlidesLiveIE(InfoExtractor): 'id': '38935785', 'ext': 'mp4', 'title': 'Offline Reinforcement Learning: From Algorithms to Practical Challenges', - 'upload_date': '20211115', - 'timestamp': 1636996003, + 'upload_date': '20231020', + 'timestamp': 1697807002, 'thumbnail': r're:^https?://.*\.(?:jpg|png)', 'thumbnails': 'count:640', 'chapters': 'count:639', @@ -59,9 +59,9 @@ class SlidesLiveIE(InfoExtractor): 'id': '38973182', 'ext': 'mp4', 'title': 'How Should a Machine Learning Researcher Think About AI Ethics?', - 'upload_date': '20220201', + 'upload_date': '20231020', 'thumbnail': r're:^https?://.*\.jpg', - 'timestamp': 1643728135, + 'timestamp': 1697822521, 'thumbnails': 'count:3', 'chapters': 'count:2', 'duration': 5889, @@ -70,37 +70,22 @@ class SlidesLiveIE(InfoExtractor): 'skip_download': 'm3u8', }, }, { - # service_name = youtube, only XML slides info + # formerly youtube, converted to native 'url': 'https://slideslive.com/38897546/special-metaprednaska-petra-ludwiga-hodnoty-pro-lepsi-spolecnost', 'md5': '8a79b5e3d700837f40bd2afca3c8fa01', 'info_dict': { - 'id': 'jmg02wCJD5M', - 'display_id': '38897546', + 'id': '38897546', 'ext': 'mp4', 'title': 'SPECIÁL: Meta-přednáška Petra Ludwiga - Hodnoty pro lepší společnost', - 'description': 'Watch full version of this video at https://slideslive.com/38897546.', - 'channel_url': 'https://www.youtube.com/channel/UCZWdAkNYFncuX0khyvhqnxw', - 'channel': 'SlidesLive Videos - G1', - 'channel_id': 'UCZWdAkNYFncuX0khyvhqnxw', - 'uploader_id': 'UCZWdAkNYFncuX0khyvhqnxw', - 'uploader': 'SlidesLive Videos - G1', - 'uploader_url': 'http://www.youtube.com/channel/UCZWdAkNYFncuX0khyvhqnxw', - 'live_status': 'not_live', - 'upload_date': '20160710', - 'timestamp': 1618786715, - 'duration': 6827, - 'like_count': int, - 'view_count': int, - 'comment_count': int, - 'channel_follower_count': int, - 'age_limit': 0, - 'thumbnail': r're:^https?://.*\.(?:jpg|webp)', + 'thumbnail': r're:^https?://.*\.jpg', + 'upload_date': '20231029', + 'timestamp': 1698588144, 'thumbnails': 'count:169', - 'playable_in_embed': True, - 'availability': 'unlisted', - 'tags': [], - 'categories': ['People & Blogs'], 'chapters': 'count:168', + 'duration': 6827, + }, + 'params': { + 'skip_download': 'm3u8', }, }, { # embed-only presentation, only XML slides info @@ -111,8 +96,8 @@ class SlidesLiveIE(InfoExtractor): 'title': 'Towards a Deep Network Architecture for Structured Smoothness', 'thumbnail': r're:^https?://.*\.jpg', 'thumbnails': 'count:8', - 'timestamp': 1629671508, - 'upload_date': '20210822', + 'timestamp': 1697803109, + 'upload_date': '20231020', 'chapters': 'count:7', 'duration': 326, }, @@ -128,8 +113,8 @@ class SlidesLiveIE(InfoExtractor): 'title': 'MoReL: Multi-omics Relational Learning', 'thumbnail': r're:^https?://.*\.(?:jpg|png)', 'thumbnails': 'count:7', - 'timestamp': 1654714970, - 'upload_date': '20220608', + 'timestamp': 1697824939, + 'upload_date': '20231020', 'chapters': 'count:6', 'duration': 171, }, @@ -145,8 +130,8 @@ class SlidesLiveIE(InfoExtractor): 'title': 'Decentralized Attribution of Generative Models', 'thumbnail': r're:^https?://.*\.jpg', 'thumbnails': 'count:16', - 'timestamp': 1622806321, - 'upload_date': '20210604', + 'timestamp': 1697814901, + 'upload_date': '20231020', 'chapters': 'count:15', 'duration': 306, }, @@ -162,8 +147,8 @@ class SlidesLiveIE(InfoExtractor): 'title': 'Efficient Active Search for Combinatorial Optimization Problems', 'thumbnail': r're:^https?://.*\.(?:jpg|png)', 'thumbnails': 'count:9', - 'timestamp': 1654714896, - 'upload_date': '20220608', + 'timestamp': 1697824757, + 'upload_date': '20231020', 'chapters': 'count:8', 'duration': 295, }, @@ -177,10 +162,10 @@ class SlidesLiveIE(InfoExtractor): 'id': '38979880', 'ext': 'mp4', 'title': 'The Representation Power of Neural Networks', - 'timestamp': 1654714962, + 'timestamp': 1697824919, 'thumbnail': r're:^https?://.*\.(?:jpg|png)', 'thumbnails': 'count:22', - 'upload_date': '20220608', + 'upload_date': '20231020', 'chapters': 'count:21', 'duration': 294, }, @@ -200,10 +185,10 @@ class SlidesLiveIE(InfoExtractor): 'id': '38979682', 'ext': 'mp4', 'title': 'LoRA: Low-Rank Adaptation of Large Language Models', - 'timestamp': 1654714920, + 'timestamp': 1697824815, 'thumbnail': r're:^https?://.*\.(?:jpg|png)', 'thumbnails': 'count:30', - 'upload_date': '20220608', + 'upload_date': '20231020', 'chapters': 'count:31', 'duration': 272, }, @@ -213,8 +198,8 @@ class SlidesLiveIE(InfoExtractor): 'ext': 'mp4', 'title': 'LoRA: Low-Rank Adaptation of Large Language Models - Slide 021', 'duration': 3, - 'timestamp': 1654714920, - 'upload_date': '20220608', + 'timestamp': 1697824815, + 'upload_date': '20231020', }, }, { 'info_dict': { @@ -222,8 +207,8 @@ class SlidesLiveIE(InfoExtractor): 'ext': 'mp4', 'title': 'LoRA: Low-Rank Adaptation of Large Language Models - Slide 024', 'duration': 4, - 'timestamp': 1654714920, - 'upload_date': '20220608', + 'timestamp': 1697824815, + 'upload_date': '20231020', }, }], 'params': { @@ -242,10 +227,10 @@ class SlidesLiveIE(InfoExtractor): 'id': '38979481', 'ext': 'mp4', 'title': 'How to Train Your MAML to Excel in Few-Shot Classification', - 'timestamp': 1654714877, + 'timestamp': 1697824716, 'thumbnail': r're:^https?://.*\.(?:jpg|png)', 'thumbnails': 'count:43', - 'upload_date': '20220608', + 'upload_date': '20231020', 'chapters': 'count:43', 'duration': 315, }, @@ -255,8 +240,8 @@ class SlidesLiveIE(InfoExtractor): 'ext': 'mp4', 'title': 'How to Train Your MAML to Excel in Few-Shot Classification - Slide 013', 'duration': 3, - 'timestamp': 1654714877, - 'upload_date': '20220608', + 'timestamp': 1697824716, + 'upload_date': '20231020', }, }], 'params': { @@ -275,10 +260,10 @@ class SlidesLiveIE(InfoExtractor): 'channel_id': 'UC62SdArr41t_-_fX40QCLRw', 'channel_url': 'https://www.youtube.com/channel/UC62SdArr41t_-_fX40QCLRw', 'uploader': 'SlidesLive Videos - A', - 'uploader_id': 'UC62SdArr41t_-_fX40QCLRw', - 'uploader_url': 'http://www.youtube.com/channel/UC62SdArr41t_-_fX40QCLRw', + 'uploader_id': '@slideslivevideos-a6075', + 'uploader_url': 'https://www.youtube.com/@slideslivevideos-a6075', 'upload_date': '20200903', - 'timestamp': 1602599092, + 'timestamp': 1697805922, 'duration': 942, 'age_limit': 0, 'live_status': 'not_live', @@ -303,8 +288,8 @@ class SlidesLiveIE(InfoExtractor): 'id': '38983994', 'ext': 'mp4', 'title': 'Zero-Shot AutoML with Pretrained Models', - 'timestamp': 1662384834, - 'upload_date': '20220905', + 'timestamp': 1697826708, + 'upload_date': '20231020', 'thumbnail': r're:^https?://.*\.(?:jpg|png)', 'thumbnails': 'count:23', 'chapters': 'count:22', @@ -336,8 +321,8 @@ class SlidesLiveIE(InfoExtractor): 'title': 'Towards a Deep Network Architecture for Structured Smoothness', 'thumbnail': r're:^https?://.*\.jpg', 'thumbnails': 'count:8', - 'timestamp': 1629671508, - 'upload_date': '20210822', + 'timestamp': 1697803109, + 'upload_date': '20231020', 'chapters': 'count:7', 'duration': 326, }, @@ -386,7 +371,7 @@ def _extract_custom_m3u8_info(self, m3u8_data): if not line.startswith('#EXT-SL-'): continue tag, _, value = line.partition(':') - key = lookup.get(tag.lstrip('#EXT-SL-')) + key = lookup.get(tag[8:]) if not key: continue m3u8_dict[key] = value diff --git a/yt_dlp/extractor/stageplus.py b/yt_dlp/extractor/stageplus.py index 4bed4d646..77e4362fc 100644 --- a/yt_dlp/extractor/stageplus.py +++ b/yt_dlp/extractor/stageplus.py @@ -21,7 +21,7 @@ class StagePlusVODConcertIE(InfoExtractor): 'id': 'vod_concert_APNM8GRFDPHMASJKBSPJACG', 'title': 'Yuja Wang plays Rachmaninoff\'s Piano Concerto No. 2 – from Odeonsplatz', 'description': 'md5:50f78ec180518c9bdb876bac550996fc', - 'artist': ['Yuja Wang', 'Lorenzo Viotti'], + 'artists': ['Yuja Wang', 'Lorenzo Viotti'], 'upload_date': '20230331', 'timestamp': 1680249600, 'release_date': '20210709', @@ -40,10 +40,10 @@ class StagePlusVODConcertIE(InfoExtractor): 'release_timestamp': 1625788800, 'duration': 2207, 'chapters': 'count:5', - 'artist': ['Yuja Wang'], - 'composer': ['Sergei Rachmaninoff'], + 'artists': ['Yuja Wang'], + 'composers': ['Sergei Rachmaninoff'], 'album': 'Yuja Wang plays Rachmaninoff\'s Piano Concerto No. 2 – from Odeonsplatz', - 'album_artist': ['Yuja Wang', 'Lorenzo Viotti'], + 'album_artists': ['Yuja Wang', 'Lorenzo Viotti'], 'track': 'Piano Concerto No. 2 in C Minor, Op. 18', 'track_number': 1, 'genre': 'Instrumental Concerto', @@ -474,7 +474,7 @@ def _real_extract(self, url): metadata = traverse_obj(data, { 'title': 'title', 'description': ('shortDescription', {str}), - 'artist': ('artists', 'edges', ..., 'node', 'name'), + 'artists': ('artists', 'edges', ..., 'node', 'name'), 'timestamp': ('archiveReleaseDate', {unified_timestamp}), 'release_timestamp': ('productionDate', {unified_timestamp}), }) @@ -494,7 +494,7 @@ def _real_extract(self, url): 'formats': formats, 'subtitles': subtitles, 'album': metadata.get('title'), - 'album_artist': metadata.get('artist'), + 'album_artists': metadata.get('artist'), 'track_number': idx, **metadata, **traverse_obj(video, { @@ -506,8 +506,8 @@ def _real_extract(self, url): 'title': 'title', 'start_time': ('mark', {float_or_none}), }), - 'artist': ('artists', 'edges', ..., 'node', 'name'), - 'composer': ('work', 'composers', ..., 'name'), + 'artists': ('artists', 'edges', ..., 'node', 'name'), + 'composers': ('work', 'composers', ..., 'name'), 'genre': ('work', 'genre', 'title'), }), }) diff --git a/yt_dlp/extractor/startrek.py b/yt_dlp/extractor/startrek.py index e92122f9b..94efb589c 100644 --- a/yt_dlp/extractor/startrek.py +++ b/yt_dlp/extractor/startrek.py @@ -3,6 +3,7 @@ class StarTrekIE(InfoExtractor): + _WORKING = False _VALID_URL = r'(?Phttps?://(?:intl|www)\.startrek\.com)/videos/(?P[^/]+)' _TESTS = [{ 'url': 'https://intl.startrek.com/videos/watch-welcoming-jess-bush-to-the-ready-room', diff --git a/yt_dlp/extractor/steam.py b/yt_dlp/extractor/steam.py index 7daee2fe0..63da9662a 100644 --- a/yt_dlp/extractor/steam.py +++ b/yt_dlp/extractor/steam.py @@ -2,9 +2,10 @@ from .common import InfoExtractor from ..utils import ( - extract_attributes, ExtractorError, + extract_attributes, get_element_by_class, + str_or_none, ) @@ -30,7 +31,6 @@ class SteamIE(InfoExtractor): 'ext': 'mp4', 'title': 'Terraria video 256785003', 'thumbnail': r're:^https://cdn\.[^\.]+\.steamstatic\.com', - 'n_entries': 2, } }, { @@ -39,9 +39,7 @@ class SteamIE(InfoExtractor): 'id': '2040428', 'ext': 'mp4', 'title': 'Terraria video 2040428', - 'playlist_index': 2, 'thumbnail': r're:^https://cdn\.[^\.]+\.steamstatic\.com', - 'n_entries': 2, } } ], @@ -55,12 +53,10 @@ class SteamIE(InfoExtractor): }, { 'url': 'https://store.steampowered.com/app/271590/Grand_Theft_Auto_V/', 'info_dict': { - 'id': '256757115', - 'title': 'Grand Theft Auto V video 256757115', - 'ext': 'mp4', - 'thumbnail': r're:^https://cdn\.[^\.]+\.steamstatic\.com', - 'n_entries': 20, + 'id': '271590', + 'title': 'Grand Theft Auto V', }, + 'playlist_count': 23, }] def _real_extract(self, url): @@ -136,7 +132,7 @@ class SteamCommunityBroadcastIE(InfoExtractor): 'id': '76561199073851486', 'title': r're:Steam Community :: pepperm!nt :: Broadcast 2022-06-26 \d{2}:\d{2}', 'ext': 'mp4', - 'uploader_id': 1113585758, + 'uploader_id': '1113585758', 'uploader': 'pepperm!nt', 'live_status': 'is_live', }, @@ -169,6 +165,6 @@ def _real_extract(self, url): 'live_status': 'is_live', 'view_count': json_data.get('num_view'), 'uploader': uploader_json.get('persona_name'), - 'uploader_id': uploader_json.get('accountid'), + 'uploader_id': str_or_none(uploader_json.get('accountid')), 'subtitles': subs, } diff --git a/yt_dlp/extractor/streamff.py b/yt_dlp/extractor/streamff.py deleted file mode 100644 index 93c42942c..000000000 --- a/yt_dlp/extractor/streamff.py +++ /dev/null @@ -1,30 +0,0 @@ -from .common import InfoExtractor -from ..utils import int_or_none, parse_iso8601 - - -class StreamFFIE(InfoExtractor): - _VALID_URL = r'https?://(?:www\.)?streamff\.com/v/(?P[a-zA-Z0-9]+)' - - _TESTS = [{ - 'url': 'https://streamff.com/v/55cc94', - 'md5': '8745a67bb5e5c570738efe7983826370', - 'info_dict': { - 'id': '55cc94', - 'ext': 'mp4', - 'title': '55cc94', - 'timestamp': 1634764643, - 'upload_date': '20211020', - 'view_count': int, - } - }] - - def _real_extract(self, url): - video_id = self._match_id(url) - json_data = self._download_json(f'https://streamff.com/api/videos/{video_id}', video_id) - return { - 'id': video_id, - 'title': json_data.get('name') or video_id, - 'url': 'https://streamff.com/%s' % json_data['videoLink'], - 'view_count': int_or_none(json_data.get('views')), - 'timestamp': parse_iso8601(json_data.get('date')), - } diff --git a/yt_dlp/extractor/svt.py b/yt_dlp/extractor/svt.py index 18da87534..573147a45 100644 --- a/yt_dlp/extractor/svt.py +++ b/yt_dlp/extractor/svt.py @@ -7,8 +7,6 @@ determine_ext, dict_get, int_or_none, - str_or_none, - strip_or_none, traverse_obj, try_get, unified_timestamp, @@ -388,15 +386,55 @@ def _real_extract(self, url): dict_get(series, ('longDescription', 'shortDescription'))) -class SVTPageIE(InfoExtractor): - _VALID_URL = r'https?://(?:www\.)?svt\.se/(?P(?:[^/]+/)*(?P[^/?&#]+))' +class SVTPageIE(SVTBaseIE): + _VALID_URL = r'https?://(?:www\.)?svt\.se/(?:[^/?#]+/)*(?P[^/?&#]+)' _TESTS = [{ + 'url': 'https://www.svt.se/nyheter/lokalt/skane/viktor-18-forlorade-armar-och-ben-i-sepsis-vill-ateruppta-karaten-och-bli-svetsare', + 'info_dict': { + 'title': 'Viktor, 18, förlorade armar och ben i sepsis – vill återuppta karaten och bli svetsare', + 'id': 'viktor-18-forlorade-armar-och-ben-i-sepsis-vill-ateruppta-karaten-och-bli-svetsare', + }, + 'playlist_count': 2, + }, { + 'url': 'https://www.svt.se/nyheter/lokalt/skane/forsvarsmakten-om-trafikkaoset-pa-e22-kunde-inte-varit-dar-snabbare', + 'info_dict': { + 'id': 'jXvk42E', + 'title': 'Försvarsmakten om trafikkaoset på E22: Kunde inte varit där snabbare', + 'ext': 'mp4', + "duration": 80, + 'age_limit': 0, + 'timestamp': 1704370009, + 'episode': 'Försvarsmakten om trafikkaoset på E22: Kunde inte varit där snabbare', + 'series': 'Lokala Nyheter Skåne', + 'upload_date': '20240104' + }, + 'params': { + 'skip_download': True, + } + }, { + 'url': 'https://www.svt.se/nyheter/svtforum/2023-tungt-ar-for-svensk-media', + 'info_dict': { + 'title': '2023 tungt år för svensk media', + 'id': 'ewqAZv4', + 'ext': 'mp4', + "duration": 3074, + 'age_limit': 0, + 'series': '', + 'timestamp': 1702980479, + 'upload_date': '20231219', + 'episode': 'Mediestudier' + }, + 'params': { + 'skip_download': True, + } + }, { 'url': 'https://www.svt.se/sport/ishockey/bakom-masken-lehners-kamp-mot-mental-ohalsa', 'info_dict': { 'id': '25298267', 'title': 'Bakom masken – Lehners kamp mot mental ohälsa', }, 'playlist_count': 4, + 'skip': 'Video is gone' }, { 'url': 'https://www.svt.se/nyheter/utrikes/svenska-andrea-ar-en-mil-fran-branderna-i-kalifornien', 'info_dict': { @@ -404,6 +442,7 @@ class SVTPageIE(InfoExtractor): 'title': 'Svenska Andrea redo att fly sitt hem i Kalifornien', }, 'playlist_count': 2, + 'skip': 'Video is gone' }, { # only programTitle 'url': 'http://www.svt.se/sport/ishockey/jagr-tacklar-giroux-under-intervjun', @@ -414,6 +453,7 @@ class SVTPageIE(InfoExtractor): 'duration': 27, 'age_limit': 0, }, + 'skip': 'Video is gone' }, { 'url': 'https://www.svt.se/nyheter/lokalt/vast/svt-testar-tar-nagon-upp-skrapet-1', 'only_matching': True, @@ -427,26 +467,23 @@ def suitable(cls, url): return False if SVTIE.suitable(url) or SVTPlayIE.suitable(url) else super(SVTPageIE, cls).suitable(url) def _real_extract(self, url): - path, display_id = self._match_valid_url(url).groups() + display_id = self._match_id(url) - article = self._download_json( - 'https://api.svt.se/nss-api/page/' + path, display_id, - query={'q': 'articles'})['articles']['content'][0] + webpage = self._download_webpage(url, display_id) + title = self._og_search_title(webpage) - entries = [] + urql_state = self._search_json( + r'window\.svt\.nyh\.urqlState\s*=', webpage, 'json data', display_id) - def _process_content(content): - if content.get('_type') in ('VIDEOCLIP', 'VIDEOEPISODE'): - video_id = compat_str(content['image']['svtId']) - entries.append(self.url_result( - 'svt:' + video_id, SVTPlayIE.ie_key(), video_id)) + data = traverse_obj(urql_state, (..., 'data', {str}, {json.loads}), get_all=False) or {} - for media in article.get('media', []): - _process_content(media) + def entries(): + for video_id in set(traverse_obj(data, ( + 'page', (('topMedia', 'svtId'), ('body', ..., 'video', 'svtId')), {str} + ))): + info = self._extract_video( + self._download_json(f'https://api.svt.se/video/{video_id}', video_id), video_id) + info['title'] = title + yield info - for obj in article.get('structuredBody', []): - _process_content(obj.get('content') or {}) - - return self.playlist_result( - entries, str_or_none(article.get('id')), - strip_or_none(article.get('title'))) + return self.playlist_result(entries(), display_id, title) diff --git a/yt_dlp/extractor/swearnet.py b/yt_dlp/extractor/swearnet.py index 6e216a2a5..aeaff28f2 100644 --- a/yt_dlp/extractor/swearnet.py +++ b/yt_dlp/extractor/swearnet.py @@ -1,5 +1,5 @@ from .common import InfoExtractor -from ..utils import int_or_none, traverse_obj +from ..utils import ExtractorError, int_or_none, traverse_obj class SwearnetEpisodeIE(InfoExtractor): @@ -51,7 +51,13 @@ def _real_extract(self, url): display_id, season_number, episode_number = self._match_valid_url(url).group('id', 'season_num', 'episode_num') webpage = self._download_webpage(url, display_id) - external_id = self._search_regex(r'externalid\s*=\s*"([^"]+)', webpage, 'externalid') + try: + external_id = self._search_regex(r'externalid\s*=\s*"([^"]+)', webpage, 'externalid') + except ExtractorError: + if 'Upgrade Now' in webpage: + self.raise_login_required() + raise + json_data = self._download_json( f'https://play.vidyard.com/player/{external_id}.json', display_id)['payload']['chapters'][0] diff --git a/yt_dlp/extractor/syfy.py b/yt_dlp/extractor/syfy.py index afcdbf780..bd2d73842 100644 --- a/yt_dlp/extractor/syfy.py +++ b/yt_dlp/extractor/syfy.py @@ -6,6 +6,7 @@ class SyfyIE(AdobePassIE): + _WORKING = False _VALID_URL = r'https?://(?:www\.)?syfy\.com/(?:[^/]+/)?videos/(?P[^/?#]+)' _TESTS = [{ 'url': 'http://www.syfy.com/theinternetruinedmylife/videos/the-internet-ruined-my-life-season-1-trailer', diff --git a/yt_dlp/extractor/tagesschau.py b/yt_dlp/extractor/tagesschau.py index e23b490b0..c69c13d0b 100644 --- a/yt_dlp/extractor/tagesschau.py +++ b/yt_dlp/extractor/tagesschau.py @@ -12,6 +12,7 @@ class TagesschauIE(InfoExtractor): + _WORKING = False _VALID_URL = r'https?://(?:www\.)?tagesschau\.de/(?P[^/]+/(?:[^/]+/)*?(?P[^/#?]+?(?:-?[0-9]+)?))(?:~_?[^/#?]+?)?\.html' _TESTS = [{ diff --git a/yt_dlp/extractor/tass.py b/yt_dlp/extractor/tass.py index 67e544a6a..d4c5b41a7 100644 --- a/yt_dlp/extractor/tass.py +++ b/yt_dlp/extractor/tass.py @@ -8,6 +8,7 @@ class TassIE(InfoExtractor): + _WORKING = False _VALID_URL = r'https?://(?:tass\.ru|itar-tass\.com)/[^/]+/(?P\d+)' _TESTS = [ { diff --git a/yt_dlp/extractor/tdslifeway.py b/yt_dlp/extractor/tdslifeway.py deleted file mode 100644 index 3623a68c8..000000000 --- a/yt_dlp/extractor/tdslifeway.py +++ /dev/null @@ -1,31 +0,0 @@ -from .common import InfoExtractor - - -class TDSLifewayIE(InfoExtractor): - _VALID_URL = r'https?://tds\.lifeway\.com/v1/trainingdeliverysystem/courses/(?P\d+)/index\.html' - - _TEST = { - # From http://www.ministrygrid.com/training-viewer/-/training/t4g-2014-conference/the-gospel-by-numbers-4/the-gospel-by-numbers - 'url': 'http://tds.lifeway.com/v1/trainingdeliverysystem/courses/3453494717001/index.html?externalRegistration=AssetId%7C34F466F1-78F3-4619-B2AB-A8EFFA55E9E9%21InstanceId%7C0%21UserId%7Caaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa&grouping=http%3A%2F%2Flifeway.com%2Fvideo%2F3453494717001&activity_id=http%3A%2F%2Flifeway.com%2Fvideo%2F3453494717001&content_endpoint=http%3A%2F%2Ftds.lifeway.com%2Fv1%2Ftrainingdeliverysystem%2FScormEngineInterface%2FTCAPI%2Fcontent%2F&actor=%7B%22name%22%3A%5B%22Guest%20Guest%22%5D%2C%22account%22%3A%5B%7B%22accountServiceHomePage%22%3A%22http%3A%2F%2Fscorm.lifeway.com%2F%22%2C%22accountName%22%3A%22aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa%22%7D%5D%2C%22objectType%22%3A%22Agent%22%7D&content_token=462a50b2-b6f9-4970-99b1-930882c499fb®istration=93d6ec8e-7f7b-4ed3-bbc8-a857913c0b2a&externalConfiguration=access%7CFREE%21adLength%7C-1%21assignOrgId%7C4AE36F78-299A-425D-91EF-E14A899B725F%21assignOrgParentId%7C%21courseId%7C%21isAnonymous%7Cfalse%21previewAsset%7Cfalse%21previewLength%7C-1%21previewMode%7Cfalse%21royalty%7CFREE%21sessionId%7C671422F9-8E79-48D4-9C2C-4EE6111EA1CD%21trackId%7C&auth=Basic%20OjhmZjk5MDBmLTBlYTMtNDJhYS04YjFlLWE4MWQ3NGNkOGRjYw%3D%3D&endpoint=http%3A%2F%2Ftds.lifeway.com%2Fv1%2Ftrainingdeliverysystem%2FScormEngineInterface%2FTCAPI%2F', - 'info_dict': { - 'id': '3453494717001', - 'ext': 'mp4', - 'title': 'The Gospel by Numbers', - 'thumbnail': r're:^https?://.*\.jpg', - 'upload_date': '20140410', - 'description': 'Coming soon from T4G 2014!', - 'uploader_id': '2034960640001', - 'timestamp': 1397145591, - }, - 'params': { - # m3u8 download - 'skip_download': True, - }, - 'add_ie': ['BrightcoveNew'], - } - - BRIGHTCOVE_URL_TEMPLATE = 'http://players.brightcove.net/2034960640001/default_default/index.html?videoId=%s' - - def _real_extract(self, url): - brightcove_id = self._match_id(url) - return self.url_result(self.BRIGHTCOVE_URL_TEMPLATE % brightcove_id, 'BrightcoveNew', brightcove_id) diff --git a/yt_dlp/extractor/teachable.py b/yt_dlp/extractor/teachable.py index 01906bda9..5eac9aa3f 100644 --- a/yt_dlp/extractor/teachable.py +++ b/yt_dlp/extractor/teachable.py @@ -99,6 +99,7 @@ def is_logged(webpage): class TeachableIE(TeachableBaseIE): + _WORKING = False _VALID_URL = r'''(?x) (?: %shttps?://(?P[^/]+)| diff --git a/yt_dlp/extractor/teachertube.py b/yt_dlp/extractor/teachertube.py index c3eec2784..90a976297 100644 --- a/yt_dlp/extractor/teachertube.py +++ b/yt_dlp/extractor/teachertube.py @@ -9,6 +9,7 @@ class TeacherTubeIE(InfoExtractor): + _WORKING = False IE_NAME = 'teachertube' IE_DESC = 'teachertube.com videos' @@ -87,6 +88,7 @@ def _real_extract(self, url): class TeacherTubeUserIE(InfoExtractor): + _WORKING = False IE_NAME = 'teachertube:user:collection' IE_DESC = 'teachertube.com user and collection videos' diff --git a/yt_dlp/extractor/teachingchannel.py b/yt_dlp/extractor/teachingchannel.py index 275f6d1f9..5791292a9 100644 --- a/yt_dlp/extractor/teachingchannel.py +++ b/yt_dlp/extractor/teachingchannel.py @@ -2,6 +2,7 @@ class TeachingChannelIE(InfoExtractor): + _WORKING = False _VALID_URL = r'https?://(?:www\.)?teachingchannel\.org/videos?/(?P[^/?&#]+)' _TEST = { diff --git a/yt_dlp/extractor/tele5.py b/yt_dlp/extractor/tele5.py index 9260db2b4..72f67e402 100644 --- a/yt_dlp/extractor/tele5.py +++ b/yt_dlp/extractor/tele5.py @@ -7,6 +7,7 @@ class Tele5IE(DPlayIE): # XXX: Do not subclass from concrete IE + _WORKING = False _VALID_URL = r'https?://(?:www\.)?tele5\.de/(?:[^/]+/)*(?P[^/?#&]+)' _GEO_COUNTRIES = ['DE'] _TESTS = [{ diff --git a/yt_dlp/extractor/telemb.py b/yt_dlp/extractor/telemb.py index 3d29dace3..a71b14c27 100644 --- a/yt_dlp/extractor/telemb.py +++ b/yt_dlp/extractor/telemb.py @@ -5,6 +5,7 @@ class TeleMBIE(InfoExtractor): + _WORKING = False _VALID_URL = r'https?://(?:www\.)?telemb\.be/(?P.+?)_d_(?P\d+)\.html' _TESTS = [ { diff --git a/yt_dlp/extractor/telemundo.py b/yt_dlp/extractor/telemundo.py index 54e74a6c0..84b24dead 100644 --- a/yt_dlp/extractor/telemundo.py +++ b/yt_dlp/extractor/telemundo.py @@ -4,7 +4,7 @@ class TelemundoIE(InfoExtractor): - + _WORKING = False _VALID_URL = r'https?:\/\/(?:www\.)?telemundo\.com\/.+?video\/[^\/]+(?Ptmvo\d{7})' _TESTS = [{ 'url': 'https://www.telemundo.com/noticias/noticias-telemundo-en-la-noche/empleo/video/esta-aplicacion-gratuita-esta-ayudando-los-latinos-encontrar-trabajo-en-estados-unidos-tmvo9829325', diff --git a/yt_dlp/extractor/telequebec.py b/yt_dlp/extractor/telequebec.py index e89137269..08a083714 100644 --- a/yt_dlp/extractor/telequebec.py +++ b/yt_dlp/extractor/telequebec.py @@ -83,7 +83,7 @@ def _real_extract(self, url): class TeleQuebecSquatIE(InfoExtractor): - _VALID_URL = r'https://squat\.telequebec\.tv/videos/(?P\d+)' + _VALID_URL = r'https?://squat\.telequebec\.tv/videos/(?P\d+)' _TESTS = [{ 'url': 'https://squat.telequebec.tv/videos/9314', 'info_dict': { diff --git a/yt_dlp/extractor/teletask.py b/yt_dlp/extractor/teletask.py index a73dd68fb..fd831f580 100644 --- a/yt_dlp/extractor/teletask.py +++ b/yt_dlp/extractor/teletask.py @@ -5,6 +5,7 @@ class TeleTaskIE(InfoExtractor): + _WORKING = False _VALID_URL = r'https?://(?:www\.)?tele-task\.de/archive/video/html5/(?P[0-9]+)' _TEST = { 'url': 'http://www.tele-task.de/archive/video/html5/26168/', diff --git a/yt_dlp/extractor/tenplay.py b/yt_dlp/extractor/tenplay.py index 7ce7cbf84..a98275d86 100644 --- a/yt_dlp/extractor/tenplay.py +++ b/yt_dlp/extractor/tenplay.py @@ -20,7 +20,8 @@ class TenPlayIE(InfoExtractor): 'alt_title': 'Nathan Borg Is The First Aussie Actor With A Cochlear Implant To Join Neighbours', 'description': 'md5:a02d0199c901c2dd4c796f1e7dd0de43', 'duration': 186, - 'season': 39, + 'season': 'Season 39', + 'season_number': 39, 'series': 'Neighbours', 'thumbnail': r're:https://.*\.jpg', 'uploader': 'Channel 10', @@ -108,7 +109,7 @@ def _real_extract(self, url): 'description': data.get('description'), 'age_limit': self._AUS_AGES.get(data.get('classification')), 'series': data.get('tvShow'), - 'season': int_or_none(data.get('season')), + 'season_number': int_or_none(data.get('season')), 'episode_number': int_or_none(data.get('episode')), 'timestamp': data.get('published'), 'thumbnail': data.get('imageUrl'), diff --git a/yt_dlp/extractor/tiktok.py b/yt_dlp/extractor/tiktok.py index f26972cff..02545bc79 100644 --- a/yt_dlp/extractor/tiktok.py +++ b/yt_dlp/extractor/tiktok.py @@ -6,7 +6,7 @@ import time from .common import InfoExtractor -from ..compat import compat_urllib_parse_unquote, compat_urllib_parse_urlparse +from ..compat import compat_urllib_parse_urlparse from ..networking import HEADRequest from ..utils import ( ExtractorError, @@ -15,7 +15,6 @@ UserNotLive, determine_ext, format_field, - get_first, int_or_none, join_nonempty, merge_dicts, @@ -42,7 +41,7 @@ class TikTokBaseIE(InfoExtractor): @property def _API_HOSTNAME(self): return self._configuration_arg( - 'api_hostname', ['api16-normal-c-useast1a.tiktokv.com'], ie_key=TikTokIE)[0] + 'api_hostname', ['api22-normal-c-useast2a.tiktokv.com'], ie_key=TikTokIE)[0] @staticmethod def _create_url(user_id, video_id): @@ -51,7 +50,13 @@ def _create_url(user_id, video_id): def _get_sigi_state(self, webpage, display_id): return self._search_json( r']+\bid="(?:SIGI_STATE|sigi-persisted-data)"[^>]*>', webpage, - 'sigi state', display_id, end_pattern=r'') + 'sigi state', display_id, end_pattern=r'', default={}) + + def _get_universal_data(self, webpage, display_id): + return traverse_obj(self._search_json( + r']+\bid="__UNIVERSAL_DATA_FOR_REHYDRATION__"[^>]*>', webpage, + 'universal data', display_id, end_pattern=r'', default={}), + ('__DEFAULT_SCOPE__', {dict})) or {} def _call_api_impl(self, ep, query, manifest_app_version, video_id, fatal=True, note='Downloading API JSON', errnote='Unable to download API page'): @@ -219,8 +224,8 @@ def audio_meta(url): def extract_addr(addr, add_meta={}): parsed_meta, res = parse_url_key(addr.get('url_key', '')) if res: - known_resolutions.setdefault(res, {}).setdefault('height', add_meta.get('height') or addr.get('height')) - known_resolutions[res].setdefault('width', add_meta.get('width') or addr.get('width')) + known_resolutions.setdefault(res, {}).setdefault('height', int_or_none(addr.get('height'))) + known_resolutions[res].setdefault('width', int_or_none(addr.get('width'))) parsed_meta.update(known_resolutions.get(res, {})) add_meta.setdefault('height', int_or_none(res[:-1])) return [{ @@ -237,22 +242,26 @@ def extract_addr(addr, add_meta={}): # Hack: Add direct video links first to prioritize them when removing duplicate formats formats = [] + width = int_or_none(video_info.get('width')) + height = int_or_none(video_info.get('height')) if video_info.get('play_addr'): formats.extend(extract_addr(video_info['play_addr'], { 'format_id': 'play_addr', 'format_note': 'Direct video', 'vcodec': 'h265' if traverse_obj( video_info, 'is_bytevc1', 'is_h265') else 'h264', # TODO: Check for "direct iOS" videos, like https://www.tiktok.com/@cookierun_dev/video/7039716639834656002 - 'width': video_info.get('width'), - 'height': video_info.get('height'), + 'width': width, + 'height': height, })) if video_info.get('download_addr'): - formats.extend(extract_addr(video_info['download_addr'], { + download_addr = video_info['download_addr'] + dl_width = int_or_none(download_addr.get('width')) + formats.extend(extract_addr(download_addr, { 'format_id': 'download_addr', 'format_note': 'Download video%s' % (', watermarked' if video_info.get('has_watermark') else ''), 'vcodec': 'h264', - 'width': video_info.get('width'), - 'height': video_info.get('height'), + 'width': dl_width or width, + 'height': try_call(lambda: int(dl_width / 0.5625)) or height, # download_addr['height'] is wrong 'preference': -2 if video_info.get('has_watermark') else -1, })) if video_info.get('play_addr_h264'): @@ -311,13 +320,10 @@ def extract_addr(addr, add_meta={}): if is_generic_og_trackname: music_track, music_author = contained_music_track or 'original sound', contained_music_author else: - music_track, music_author = music_info.get('title'), music_info.get('author') + music_track, music_author = music_info.get('title'), traverse_obj(music_info, ('author', {str})) return { 'id': aweme_id, - 'extractor_key': TikTokIE.ie_key(), - 'extractor': TikTokIE.IE_NAME, - 'webpage_url': self._create_url(author_info.get('uid'), aweme_id), **traverse_obj(aweme_detail, { 'title': ('desc', {str}), 'description': ('desc', {str}), @@ -330,15 +336,16 @@ def extract_addr(addr, add_meta={}): 'comment_count': 'comment_count', }, expected_type=int_or_none), **traverse_obj(author_info, { - 'uploader': 'unique_id', - 'uploader_id': 'uid', - 'creator': 'nickname', - 'channel_id': 'sec_uid', - }, expected_type=str_or_none), + 'uploader': ('unique_id', {str}), + 'uploader_id': ('uid', {str_or_none}), + 'creators': ('nickname', {str}, {lambda x: [x] if x else None}), # for compat + 'channel': ('nickname', {str}), + 'channel_id': ('sec_uid', {str}), + }), 'uploader_url': user_url, 'track': music_track, 'album': str_or_none(music_info.get('album')) or None, - 'artist': music_author or None, + 'artists': re.split(r'(?:, | & )', music_author) if music_author else None, 'formats': formats, 'subtitles': self.extract_subtitles(aweme_detail, aweme_id), 'thumbnails': thumbnails, @@ -399,7 +406,8 @@ def _parse_aweme_video_web(self, aweme_detail, webpage_url, video_id): 'timestamp': ('createTime', {int_or_none}), }), **traverse_obj(author_info or aweme_detail, { - 'creator': ('nickname', {str}), + 'creators': ('nickname', {str}, {lambda x: [x] if x else None}), # for compat + 'channel': ('nickname', {str}), 'uploader': (('uniqueId', 'author'), {str}), 'uploader_id': (('authorId', 'uid', 'id'), {str_or_none}), }, get_all=False), @@ -410,10 +418,10 @@ def _parse_aweme_video_web(self, aweme_detail, webpage_url, video_id): 'comment_count': 'commentCount', }, expected_type=int_or_none), **traverse_obj(music_info, { - 'track': 'title', - 'album': ('album', {lambda x: x or None}), - 'artist': 'authorName', - }, expected_type=str), + 'track': ('title', {str}), + 'album': ('album', {str}, {lambda x: x or None}), + 'artists': ('authorName', {str}, {lambda x: [x] if x else None}), + }), 'channel_id': channel_id, 'uploader_url': user_url, 'formats': formats, @@ -470,7 +478,8 @@ class TikTokIE(TikTokBaseIE): 'uploader_id': '18702747', 'uploader_url': 'https://www.tiktok.com/@MS4wLjABAAAAiFnldaILebi5heDoVU6bn4jBWWycX6-9U3xuNPqZ8Ws', 'channel_id': 'MS4wLjABAAAAiFnldaILebi5heDoVU6bn4jBWWycX6-9U3xuNPqZ8Ws', - 'creator': 'patroX', + 'channel': 'patroX', + 'creators': ['patroX'], 'thumbnail': r're:^https?://[\w\/\.\-]+(~[\w\-]+\.image)?', 'upload_date': '20190930', 'timestamp': 1569860870, @@ -478,7 +487,7 @@ class TikTokIE(TikTokBaseIE): 'like_count': int, 'repost_count': int, 'comment_count': int, - 'artist': 'Evan Todd, Jessica Keenan Wynn, Alice Lee, Barrett Wilbert Weed & Jon Eidson', + 'artists': ['Evan Todd', 'Jessica Keenan Wynn', 'Alice Lee', 'Barrett Wilbert Weed', 'Jon Eidson'], 'track': 'Big Fun', }, }, { @@ -490,12 +499,13 @@ class TikTokIE(TikTokBaseIE): 'title': 'Balas @yolaaftwsr hayu yu ? #SquadRandom_ 🔥', 'description': 'Balas @yolaaftwsr hayu yu ? #SquadRandom_ 🔥', 'uploader': 'barudakhb_', - 'creator': 'md5:29f238c49bc0c176cb3cef1a9cea9fa6', + 'channel': 'md5:29f238c49bc0c176cb3cef1a9cea9fa6', + 'creators': ['md5:29f238c49bc0c176cb3cef1a9cea9fa6'], 'uploader_id': '6974687867511718913', 'uploader_url': 'https://www.tiktok.com/@MS4wLjABAAAAbhBwQC-R1iKoix6jDFsF-vBdfx2ABoDjaZrM9fX6arU3w71q3cOWgWuTXn1soZ7d', 'channel_id': 'MS4wLjABAAAAbhBwQC-R1iKoix6jDFsF-vBdfx2ABoDjaZrM9fX6arU3w71q3cOWgWuTXn1soZ7d', 'track': 'Boka Dance', - 'artist': 'md5:29f238c49bc0c176cb3cef1a9cea9fa6', + 'artists': ['md5:29f238c49bc0c176cb3cef1a9cea9fa6'], 'timestamp': 1626121503, 'duration': 18, 'thumbnail': r're:^https?://[\w\/\.\-]+(~[\w\-]+\.image)?', @@ -514,7 +524,8 @@ class TikTokIE(TikTokBaseIE): 'title': 'Slap and Run!', 'description': 'Slap and Run!', 'uploader': 'user440922249', - 'creator': 'Slap And Run', + 'channel': 'Slap And Run', + 'creators': ['Slap And Run'], 'uploader_id': '7036055384943690754', 'uploader_url': 'https://www.tiktok.com/@MS4wLjABAAAATh8Vewkn0LYM7Fo03iec3qKdeCUOcBIouRk1mkiag6h3o_pQu_dUXvZ2EZlGST7_', 'channel_id': 'MS4wLjABAAAATh8Vewkn0LYM7Fo03iec3qKdeCUOcBIouRk1mkiag6h3o_pQu_dUXvZ2EZlGST7_', @@ -538,7 +549,8 @@ class TikTokIE(TikTokBaseIE): 'title': 'TikTok video #7059698374567611694', 'description': '', 'uploader': 'pokemonlife22', - 'creator': 'Pokemon', + 'channel': 'Pokemon', + 'creators': ['Pokemon'], 'uploader_id': '6820838815978423302', 'uploader_url': 'https://www.tiktok.com/@MS4wLjABAAAA0tF1nBwQVVMyrGu3CqttkNgM68Do1OXUFuCY0CRQk8fEtSVDj89HqoqvbSTmUP2W', 'channel_id': 'MS4wLjABAAAA0tF1nBwQVVMyrGu3CqttkNgM68Do1OXUFuCY0CRQk8fEtSVDj89HqoqvbSTmUP2W', @@ -547,7 +559,7 @@ class TikTokIE(TikTokBaseIE): 'duration': 6, 'thumbnail': r're:^https?://[\w\/\.\-]+(~[\w\-]+\.image)?', 'upload_date': '20220201', - 'artist': 'Pokemon', + 'artists': ['Pokemon'], 'view_count': int, 'like_count': int, 'repost_count': int, @@ -584,12 +596,13 @@ class TikTokIE(TikTokBaseIE): 'ext': 'mp3', 'title': 'TikTok video #7139980461132074283', 'description': '', - 'creator': 'Antaura', + 'channel': 'Antaura', + 'creators': ['Antaura'], 'uploader': '_le_cannibale_', 'uploader_id': '6604511138619654149', 'uploader_url': 'https://www.tiktok.com/@MS4wLjABAAAAoShJqaw_5gvy48y3azFeFcT4jeyKWbB0VVYasOCt2tTLwjNFIaDcHAM4D-QGXFOP', 'channel_id': 'MS4wLjABAAAAoShJqaw_5gvy48y3azFeFcT4jeyKWbB0VVYasOCt2tTLwjNFIaDcHAM4D-QGXFOP', - 'artist': 'nathan !', + 'artists': ['nathan !'], 'track': 'grahamscott canon', 'upload_date': '20220905', 'timestamp': 1662406249, @@ -597,23 +610,24 @@ class TikTokIE(TikTokBaseIE): 'like_count': int, 'repost_count': int, 'comment_count': int, - 'thumbnail': r're:^https://.+\.webp', + 'thumbnail': r're:^https://.+\.(?:webp|jpe?g)', }, }, { # only available via web - 'url': 'https://www.tiktok.com/@moxypatch/video/7206382937372134662', + 'url': 'https://www.tiktok.com/@moxypatch/video/7206382937372134662', # FIXME 'md5': '6aba7fad816e8709ff2c149679ace165', 'info_dict': { 'id': '7206382937372134662', 'ext': 'mp4', 'title': 'md5:1d95c0b96560ca0e8a231af4172b2c0a', 'description': 'md5:1d95c0b96560ca0e8a231af4172b2c0a', - 'creator': 'MoxyPatch', + 'channel': 'MoxyPatch', + 'creators': ['MoxyPatch'], 'uploader': 'moxypatch', 'uploader_id': '7039142049363379205', 'uploader_url': 'https://www.tiktok.com/@MS4wLjABAAAAFhqKnngMHJSsifL0w1vFOP5kn3Ndo1ODp0XuIBkNMBCkALTvwILdpu12g3pTtL4V', 'channel_id': 'MS4wLjABAAAAFhqKnngMHJSsifL0w1vFOP5kn3Ndo1ODp0XuIBkNMBCkALTvwILdpu12g3pTtL4V', - 'artist': 'your worst nightmare', + 'artists': ['your worst nightmare'], 'track': 'original sound', 'upload_date': '20230303', 'timestamp': 1677866781, @@ -628,7 +642,7 @@ class TikTokIE(TikTokBaseIE): 'expected_warnings': ['Unable to find video in feed'], }, { # 1080p format - 'url': 'https://www.tiktok.com/@tatemcrae/video/7107337212743830830', + 'url': 'https://www.tiktok.com/@tatemcrae/video/7107337212743830830', # FIXME 'md5': '982512017a8a917124d5a08c8ae79621', 'info_dict': { 'id': '7107337212743830830', @@ -639,8 +653,9 @@ class TikTokIE(TikTokBaseIE): 'uploader_id': '86328792343818240', 'uploader_url': 'https://www.tiktok.com/@MS4wLjABAAAA-0bQT0CqebTRr6I4IkYvMDMKSRSJHLNPBo5HrSklJwyA2psXLSZG5FP-LMNpHnJd', 'channel_id': 'MS4wLjABAAAA-0bQT0CqebTRr6I4IkYvMDMKSRSJHLNPBo5HrSklJwyA2psXLSZG5FP-LMNpHnJd', - 'creator': 'tate mcrae', - 'artist': 'tate mcrae', + 'channel': 'tate mcrae', + 'creators': ['tate mcrae'], + 'artists': ['tate mcrae'], 'track': 'original sound', 'upload_date': '20220609', 'timestamp': 1654805899, @@ -651,7 +666,7 @@ class TikTokIE(TikTokBaseIE): 'comment_count': int, 'thumbnail': r're:^https://.+\.webp', }, - 'params': {'format': 'bytevc1_1080p_808907-0'}, + 'skip': 'Unavailable via feed API, no formats available via web', }, { # Slideshow, audio-only m4a format 'url': 'https://www.tiktok.com/@hara_yoimiya/video/7253412088251534594', @@ -665,8 +680,9 @@ class TikTokIE(TikTokBaseIE): 'uploader_id': '6582536342634676230', 'uploader_url': 'https://www.tiktok.com/@MS4wLjABAAAAIAlDxriiPWLE-p8p1R_0Bx8qWKfi-7zwmGhzU8Mv25W8sNxjfIKrol31qTczzuLB', 'channel_id': 'MS4wLjABAAAAIAlDxriiPWLE-p8p1R_0Bx8qWKfi-7zwmGhzU8Mv25W8sNxjfIKrol31qTczzuLB', - 'creator': 'лампочка', - 'artist': 'Øneheart', + 'channel': 'лампочка', + 'creators': ['лампочка'], + 'artists': ['Øneheart'], 'album': 'watching the stars', 'track': 'watching the stars', 'upload_date': '20230708', @@ -675,7 +691,7 @@ class TikTokIE(TikTokBaseIE): 'like_count': int, 'comment_count': int, 'repost_count': int, - 'thumbnail': r're:^https://.+\.webp', + 'thumbnail': r're:^https://.+\.(?:webp|jpe?g)', }, }, { # Auto-captions available @@ -688,24 +704,35 @@ def _real_extract(self, url): try: return self._extract_aweme_app(video_id) except ExtractorError as e: + e.expected = True self.report_warning(f'{e}; trying with webpage') url = self._create_url(user_id, video_id) webpage = self._download_webpage(url, video_id, headers={'User-Agent': 'Mozilla/5.0'}) - next_data = self._search_nextjs_data(webpage, video_id, default='{}') - if next_data: - status = traverse_obj(next_data, ('props', 'pageProps', 'statusCode'), expected_type=int) or 0 - video_data = traverse_obj(next_data, ('props', 'pageProps', 'itemInfo', 'itemStruct'), expected_type=dict) - else: - sigi_data = self._get_sigi_state(webpage, video_id) - status = traverse_obj(sigi_data, ('VideoPage', 'statusCode'), expected_type=int) or 0 - video_data = traverse_obj(sigi_data, ('ItemModule', video_id), expected_type=dict) - if status == 0: + if universal_data := self._get_universal_data(webpage, video_id): + self.write_debug('Found universal data for rehydration') + status = traverse_obj(universal_data, ('webapp.video-detail', 'statusCode', {int})) or 0 + video_data = traverse_obj(universal_data, ('webapp.video-detail', 'itemInfo', 'itemStruct', {dict})) + + elif sigi_data := self._get_sigi_state(webpage, video_id): + self.write_debug('Found sigi state data') + status = traverse_obj(sigi_data, ('VideoPage', 'statusCode', {int})) or 0 + video_data = traverse_obj(sigi_data, ('ItemModule', video_id, {dict})) + + elif next_data := self._search_nextjs_data(webpage, video_id, default='{}'): + self.write_debug('Found next.js data') + status = traverse_obj(next_data, ('props', 'pageProps', 'statusCode', {int})) or 0 + video_data = traverse_obj(next_data, ('props', 'pageProps', 'itemInfo', 'itemStruct', {dict})) + + else: + raise ExtractorError('Unable to extract webpage video data') + + if video_data and status == 0: return self._parse_aweme_video_web(video_data, url, video_id) elif status == 10216: raise ExtractorError('This video is private', expected=True) - raise ExtractorError('Video not available', video_id=video_id) + raise ExtractorError(f'Video not available, status code {status}', video_id=video_id) class TikTokUserIE(TikTokBaseIE): @@ -921,20 +948,23 @@ class DouyinIE(TikTokBaseIE): _VALID_URL = r'https?://(?:www\.)?douyin\.com/video/(?P[0-9]+)' _TESTS = [{ 'url': 'https://www.douyin.com/video/6961737553342991651', - 'md5': 'a97db7e3e67eb57bf40735c022ffa228', + 'md5': '9ecce7bc5b302601018ecb2871c63a75', 'info_dict': { 'id': '6961737553342991651', 'ext': 'mp4', 'title': '#杨超越 小小水手带你去远航❤️', 'description': '#杨超越 小小水手带你去远航❤️', + 'uploader': '6897520xka', 'uploader_id': '110403406559', 'uploader_url': 'https://www.douyin.com/user/MS4wLjABAAAAEKnfa654JAJ_N5lgZDQluwsxmY0lhfmEYNQBBkwGG98', 'channel_id': 'MS4wLjABAAAAEKnfa654JAJ_N5lgZDQluwsxmY0lhfmEYNQBBkwGG98', - 'creator': '杨超越', - 'duration': 19782, + 'channel': '杨超越', + 'creators': ['杨超越'], + 'duration': 19, 'timestamp': 1620905839, 'upload_date': '20210513', 'track': '@杨超越创作的原声', + 'artists': ['杨超越'], 'view_count': int, 'like_count': int, 'repost_count': int, @@ -943,20 +973,23 @@ class DouyinIE(TikTokBaseIE): }, }, { 'url': 'https://www.douyin.com/video/6982497745948921092', - 'md5': '34a87ebff3833357733da3fe17e37c0e', + 'md5': '15c5e660b7048af3707304e3cc02bbb5', 'info_dict': { 'id': '6982497745948921092', 'ext': 'mp4', 'title': '这个夏日和小羊@杨超越 一起遇见白色幻想', 'description': '这个夏日和小羊@杨超越 一起遇见白色幻想', + 'uploader': '0731chaoyue', 'uploader_id': '408654318141572', 'uploader_url': 'https://www.douyin.com/user/MS4wLjABAAAAZJpnglcjW2f_CMVcnqA_6oVBXKWMpH0F8LIHuUu8-lA', 'channel_id': 'MS4wLjABAAAAZJpnglcjW2f_CMVcnqA_6oVBXKWMpH0F8LIHuUu8-lA', - 'creator': '杨超越工作室', - 'duration': 42479, + 'channel': '杨超越工作室', + 'creators': ['杨超越工作室'], + 'duration': 42, 'timestamp': 1625739481, 'upload_date': '20210708', 'track': '@杨超越工作室创作的原声', + 'artists': ['杨超越工作室'], 'view_count': int, 'like_count': int, 'repost_count': int, @@ -965,20 +998,23 @@ class DouyinIE(TikTokBaseIE): }, }, { 'url': 'https://www.douyin.com/video/6953975910773099811', - 'md5': 'dde3302460f19db59c47060ff013b902', + 'md5': '0e6443758b8355db9a3c34864a4276be', 'info_dict': { 'id': '6953975910773099811', 'ext': 'mp4', 'title': '#一起看海 出现在你的夏日里', 'description': '#一起看海 出现在你的夏日里', + 'uploader': '6897520xka', 'uploader_id': '110403406559', 'uploader_url': 'https://www.douyin.com/user/MS4wLjABAAAAEKnfa654JAJ_N5lgZDQluwsxmY0lhfmEYNQBBkwGG98', 'channel_id': 'MS4wLjABAAAAEKnfa654JAJ_N5lgZDQluwsxmY0lhfmEYNQBBkwGG98', - 'creator': '杨超越', - 'duration': 17343, + 'channel': '杨超越', + 'creators': ['杨超越'], + 'duration': 17, 'timestamp': 1619098692, 'upload_date': '20210422', 'track': '@杨超越创作的原声', + 'artists': ['杨超越'], 'view_count': int, 'like_count': int, 'repost_count': int, @@ -1004,20 +1040,23 @@ class DouyinIE(TikTokBaseIE): 'skip': 'No longer available', }, { 'url': 'https://www.douyin.com/video/6963263655114722595', - 'md5': 'cf9f11f0ec45d131445ec2f06766e122', + 'md5': '1440bcf59d8700f8e014da073a4dfea8', 'info_dict': { 'id': '6963263655114722595', 'ext': 'mp4', 'title': '#哪个爱豆的105度最甜 换个角度看看我哈哈', 'description': '#哪个爱豆的105度最甜 换个角度看看我哈哈', + 'uploader': '6897520xka', 'uploader_id': '110403406559', 'uploader_url': 'https://www.douyin.com/user/MS4wLjABAAAAEKnfa654JAJ_N5lgZDQluwsxmY0lhfmEYNQBBkwGG98', 'channel_id': 'MS4wLjABAAAAEKnfa654JAJ_N5lgZDQluwsxmY0lhfmEYNQBBkwGG98', - 'creator': '杨超越', - 'duration': 15115, + 'channel': '杨超越', + 'creators': ['杨超越'], + 'duration': 15, 'timestamp': 1621261163, 'upload_date': '20210517', 'track': '@杨超越创作的原声', + 'artists': ['杨超越'], 'view_count': int, 'like_count': int, 'repost_count': int, @@ -1025,34 +1064,23 @@ class DouyinIE(TikTokBaseIE): 'thumbnail': r're:https?://.+\.jpe?g', }, }] - _APP_VERSIONS = [('23.3.0', '230300')] - _APP_NAME = 'aweme' - _AID = 1128 - _API_HOSTNAME = 'aweme.snssdk.com' _UPLOADER_URL_FORMAT = 'https://www.douyin.com/user/%s' _WEBPAGE_HOST = 'https://www.douyin.com/' def _real_extract(self, url): video_id = self._match_id(url) - try: - return self._extract_aweme_app(video_id) - except ExtractorError as e: - e.expected = True - self.to_screen(f'{e}; trying with webpage') - - webpage = self._download_webpage(url, video_id) - render_data = self._search_json( - r'