From 2d1d683a541d71f3d3bb999dfe8eeb1976fb91ce Mon Sep 17 00:00:00 2001 From: Simon Sawicki Date: Tue, 26 Dec 2023 18:30:04 +0100 Subject: [PATCH] [devscripts] `run_tests`: Create Python script (#8720) Authored by: Grub4K --- .github/workflows/core.yml | 6 +-- .github/workflows/download.yml | 7 +--- .github/workflows/quick-test.yml | 2 +- CONTRIBUTING.md | 31 +++++++------- devscripts/run_tests.bat | 17 +------- devscripts/run_tests.py | 70 ++++++++++++++++++++++++++++++++ devscripts/run_tests.sh | 14 +------ 7 files changed, 95 insertions(+), 52 deletions(-) create mode 100755 devscripts/run_tests.py diff --git a/.github/workflows/core.yml b/.github/workflows/core.yml index b22adb1b9d..ded7e6d612 100644 --- a/.github/workflows/core.yml +++ b/.github/workflows/core.yml @@ -38,18 +38,14 @@ jobs: os: [ubuntu-latest] # CPython 3.11 is in quick-test python-version: ['3.8', '3.9', '3.10', '3.12', pypy-3.8, pypy-3.10] - run-tests-ext: [sh] include: # atleast one of each CPython/PyPy tests must be in windows - os: windows-latest python-version: '3.8' - run-tests-ext: bat - os: windows-latest python-version: '3.12' - run-tests-ext: bat - os: windows-latest python-version: pypy-3.9 - run-tests-ext: bat steps: - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} @@ -62,4 +58,4 @@ jobs: continue-on-error: False run: | python3 -m yt_dlp -v || true # Print debug head - ./devscripts/run_tests.${{ matrix.run-tests-ext }} core + python3 ./devscripts/run_tests.py core diff --git a/.github/workflows/download.yml b/.github/workflows/download.yml index 73b2f9ca3d..9f47d67187 100644 --- a/.github/workflows/download.yml +++ b/.github/workflows/download.yml @@ -18,7 +18,7 @@ jobs: run: pip install pytest -r requirements.txt - name: Run tests continue-on-error: true - run: ./devscripts/run_tests.sh download + run: python3 ./devscripts/run_tests.py download full: name: Full Download Tests @@ -29,15 +29,12 @@ jobs: matrix: os: [ubuntu-latest] python-version: ['3.10', '3.11', '3.12', pypy-3.8, pypy-3.10] - run-tests-ext: [sh] include: # atleast one of each CPython/PyPy tests must be in windows - os: windows-latest python-version: '3.8' - run-tests-ext: bat - os: windows-latest python-version: pypy-3.9 - run-tests-ext: bat steps: - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} @@ -48,4 +45,4 @@ jobs: run: pip install pytest -r requirements.txt - name: Run tests continue-on-error: true - run: ./devscripts/run_tests.${{ matrix.run-tests-ext }} download + run: python3 ./devscripts/run_tests.py download diff --git a/.github/workflows/quick-test.yml b/.github/workflows/quick-test.yml index edbdaffd74..1ccfbe836f 100644 --- a/.github/workflows/quick-test.yml +++ b/.github/workflows/quick-test.yml @@ -19,7 +19,7 @@ jobs: - name: Run tests run: | python3 -m yt_dlp -v || true - ./devscripts/run_tests.sh core + python3 ./devscripts/run_tests.py core flake8: name: Linter if: "!contains(github.event.head_commit.message, 'ci skip all')" diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c472f32514..248917bf55 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -140,12 +140,9 @@ # DEVELOPER INSTRUCTIONS python -m yt_dlp -To run the test, simply invoke your favorite test runner, or execute a test file directly; any of the following work: +To run all the available core tests, use: - python -m unittest discover - python test/test_download.py - nosetests - pytest + python devscripts/run_tests.py See item 6 of [new extractor tutorial](#adding-support-for-a-new-site) for how to run extractor specific test cases. @@ -187,15 +184,21 @@ ## Adding support for a new site 'url': 'https://yourextractor.com/watch/42', 'md5': 'TODO: md5 sum of the first 10241 bytes of the video file (use --test)', 'info_dict': { + # For videos, only the 'id' and 'ext' fields are required to RUN the test: 'id': '42', 'ext': 'mp4', - 'title': 'Video title goes here', - 'thumbnail': r're:^https?://.*\.jpg$', - # TODO more properties, either as: - # * A value - # * MD5 checksum; start the string with md5: - # * A regular expression; start the string with re: - # * Any Python type, e.g. int or float + # Then if the test run fails, it will output the missing/incorrect fields. + # Properties can be added as: + # * A value, e.g. + # 'title': 'Video title goes here', + # * MD5 checksum; start the string with 'md5:', e.g. + # 'description': 'md5:098f6bcd4621d373cade4e832627b4f6', + # * A regular expression; start the string with 're:', e.g. + # 'thumbnail': r're:^https?://.*\.jpg$', + # * A count of elements in a list; start the string with 'count:', e.g. + # 'tags': 'count:10', + # * Any Python type, e.g. + # 'view_count': int, } }] @@ -215,8 +218,8 @@ ## Adding support for a new site } ``` 1. Add an import in [`yt_dlp/extractor/_extractors.py`](yt_dlp/extractor/_extractors.py). Note that the class name must end with `IE`. -1. Run `python test/test_download.py TestDownload.test_YourExtractor` (note that `YourExtractor` doesn't end with `IE`). This *should fail* at first, but you can continually re-run it until you're done. If you decide to add more than one test, the tests will then be named `TestDownload.test_YourExtractor`, `TestDownload.test_YourExtractor_1`, `TestDownload.test_YourExtractor_2`, etc. Note that tests with `only_matching` key in test's dict are not counted in. You can also run all the tests in one go with `TestDownload.test_YourExtractor_all` -1. Make sure you have atleast one test for your extractor. Even if all videos covered by the extractor are expected to be inaccessible for automated testing, tests should still be added with a `skip` parameter indicating why the particular test is disabled from running. +1. Run `python devscripts/run_tests.py YourExtractor`. This *may fail* at first, but you can continually re-run it until you're done. Upon failure, it will output the missing fields and/or correct values which you can copy. If you decide to add more than one test, the tests will then be named `YourExtractor`, `YourExtractor_1`, `YourExtractor_2`, etc. Note that tests with an `only_matching` key in the test's dict are not included in the count. You can also run all the tests in one go with `YourExtractor_all` +1. Make sure you have at least one test for your extractor. Even if all videos covered by the extractor are expected to be inaccessible for automated testing, tests should still be added with a `skip` parameter indicating why the particular test is disabled from running. 1. Have a look at [`yt_dlp/extractor/common.py`](yt_dlp/extractor/common.py) for possible helper methods and a [detailed description of what your extractor should and may return](yt_dlp/extractor/common.py#L119-L440). Add tests and code for as many as you want. 1. Make sure your code follows [yt-dlp coding conventions](#yt-dlp-coding-conventions) and check the code with [flake8](https://flake8.pycqa.org/en/latest/index.html#quickstart): diff --git a/devscripts/run_tests.bat b/devscripts/run_tests.bat index 190d23918c..57b1f4bf46 100644 --- a/devscripts/run_tests.bat +++ b/devscripts/run_tests.bat @@ -1,17 +1,4 @@ -@setlocal @echo off -cd /d %~dp0.. -if ["%~1"]==[""] ( - set "test_set="test"" -) else if ["%~1"]==["core"] ( - set "test_set="-m not download"" -) else if ["%~1"]==["download"] ( - set "test_set="-m "download"" -) else ( - echo.Invalid test type "%~1". Use "core" ^| "download" - exit /b 1 -) - -set PYTHONWARNINGS=error -pytest %test_set% +>&2 echo run_tests.bat is deprecated. Please use `devscripts/run_tests.py` instead +python %~dp0run_tests.py %~1 diff --git a/devscripts/run_tests.py b/devscripts/run_tests.py new file mode 100755 index 0000000000..b0c6ee67af --- /dev/null +++ b/devscripts/run_tests.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python3 + +import argparse +import functools +import os +import re +import subprocess +import sys +from pathlib import Path + + +fix_test_name = functools.partial(re.compile(r'IE(_all|_\d+)?$').sub, r'\1') + + +def parse_args(): + parser = argparse.ArgumentParser(description='Run selected yt-dlp tests') + parser.add_argument( + 'test', help='a extractor tests, or one of "core" or "download"', nargs='*') + parser.add_argument( + '-k', help='run a test matching EXPRESSION. Same as "pytest -k"', metavar='EXPRESSION') + return parser.parse_args() + + +def run_tests(*tests, pattern=None): + run_core = 'core' in tests or (not pattern and not tests) + run_download = 'download' in tests + tests = list(map(fix_test_name, tests)) + + arguments = ['pytest', '-Werror', '--tb', 'short'] + if run_core: + arguments.extend(['-m', 'not download']) + elif run_download: + arguments.extend(['-m', 'download']) + elif pattern: + arguments.extend(['-k', pattern]) + else: + arguments.extend( + f'test/test_download.py::TestDownload::test_{test}' for test in tests) + + print(f'Running {arguments}') + try: + subprocess.run(arguments) + return + except FileNotFoundError: + pass + + arguments = [sys.executable, '-Werror', '-m', 'unittest'] + if run_core: + print('"pytest" needs to be installed to run core tests', file=sys.stderr) + return + elif run_download: + arguments.append('test.test_download') + elif pattern: + arguments.extend(['-k', pattern]) + else: + arguments.extend( + f'test.test_download.TestDownload.test_{test}' for test in tests) + + print(f'Running {arguments}') + subprocess.run(arguments) + + +if __name__ == '__main__': + try: + args = parse_args() + + os.chdir(Path(__file__).parent.parent) + run_tests(*args.test, pattern=args.k) + except KeyboardInterrupt: + pass diff --git a/devscripts/run_tests.sh b/devscripts/run_tests.sh index faa642e96c..123ceb1ee4 100755 --- a/devscripts/run_tests.sh +++ b/devscripts/run_tests.sh @@ -1,14 +1,4 @@ #!/usr/bin/env sh -if [ -z "$1" ]; then - test_set='test' -elif [ "$1" = 'core' ]; then - test_set="-m not download" -elif [ "$1" = 'download' ]; then - test_set="-m download" -else - echo 'Invalid test type "'"$1"'". Use "core" | "download"' - exit 1 -fi - -python3 -bb -Werror -m pytest "$test_set" +>&2 echo 'run_tests.sh is deprecated. Please use `devscripts/run_tests.py` instead' +python3 devscripts/run_tests.py "$1"