[utils] windows_enable_vt_mode: Proper implementation

Authored by: Grub4K

.github/workflows/core.yml

@@ -12,13 +12,13 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest]
-        # CPython 3.9 is in quick-test
-        python-version: ['3.7', '3.10', 3.11-dev, pypy-3.7, pypy-3.8]
+        # CPython 3.11 is in quick-test
+        python-version: ['3.8', '3.9', '3.10', pypy-3.7, pypy-3.8]
         run-tests-ext: [sh]
         include:
         # atleast one of each CPython/PyPy tests must be in windows
         - os: windows-latest
-          python-version: '3.8'
+          python-version: '3.7'
           run-tests-ext: bat
         - os: windows-latest
           python-version: pypy-3.9
@@ -33,5 +33,6 @@ jobs:
       run: pip install pytest
     - name: Run tests
       continue-on-error: False
-      run: ./devscripts/run_tests.${{ matrix.run-tests-ext }} core
-  # Linter is in quick-test
+      run: |
+        python3 -m yt_dlp -v || true  # Print debug head
+        ./devscripts/run_tests.${{ matrix.run-tests-ext }} core

.github/workflows/quick-test.yml

@@ -10,24 +10,23 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v3
-    - name: Set up Python
+    - name: Set up Python 3.11
       uses: actions/setup-python@v4
       with:
-        python-version: 3.9
+        python-version: '3.11'
     - name: Install test requirements
       run: pip install pytest pycryptodomex
     - name: Run tests
-      run: ./devscripts/run_tests.sh core
+      run: |
+        python3 -m yt_dlp -v || true
+        ./devscripts/run_tests.sh core
   flake8:
     name: Linter
     if: "!contains(github.event.head_commit.message, 'ci skip all')"
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v3
-    - name: Set up Python
-      uses: actions/setup-python@v4
-      with:
-        python-version: 3.9
+    - uses: actions/setup-python@v4
     - name: Install flake8
       run: pip install flake8
     - name: Make lazy extractors

.gitignore

@@ -71,6 +71,7 @@ dist/
 zip/
 tmp/
 venv/
+.venv/
 completions/
 
 # Misc

CONTRIBUTING.md

@@ -351,8 +351,9 @@ #### Example
 ```python
 thumbnail_data = data.get('thumbnails') or []
 thumbnails = [{
-    'url': item['url']
-} for item in thumbnail_data]  # correct
+    'url': item['url'],
+    'height': item.get('h'),
+} for item in thumbnail_data if item.get('url')]  # correct
 ```
 
 and not like:
@@ -360,12 +361,27 @@ #### Example
 ```python
 thumbnail_data = data.get('thumbnails')
 thumbnails = [{
-    'url': item['url']
+    'url': item['url'],
+    'height': item.get('h'),
 } for item in thumbnail_data]  # incorrect
 ```
 
 In this case, `thumbnail_data` will be `None` if the field was not found and this will cause the loop `for item in thumbnail_data` to raise a fatal error. Using `or []` avoids this error and results in setting an empty list in `thumbnails` instead.
 
+Alternately, this can be further simplified by using `traverse_obj`
+
+```python
+thumbnails = [{
+    'url': item['url'],
+    'height': item.get('h'),
+} for item in traverse_obj(data, ('thumbnails', lambda _, v: v['url']))]
+```
+
+or, even better,
+
+```python
+thumbnails = traverse_obj(data, ('thumbnails', ..., {'url': 'url', 'height': 'h'}))
+```
 
 ### Provide fallbacks
 

README.md

@@ -432,19 +432,19 @@ ## Geo-restriction:
                                     explicitly provided IP block in CIDR notation
 
 ## Video Selection:
-    -I, --playlist-items ITEM_SPEC  Comma separated playlist_index of the videos
+    -I, --playlist-items ITEM_SPEC  Comma separated playlist_index of the items
                                     to download. You can specify a range using
                                     "[START]:[STOP][:STEP]". For backward
                                     compatibility, START-STOP is also supported.
                                     Use negative indices to count from the right
                                     and negative STEP to download in reverse
                                     order. E.g. "-I 1:3,7,-5::2" used on a
-                                    playlist of size 15 will download the videos
+                                    playlist of size 15 will download the items
                                     at index 1,2,3,7,11,13,15
-    --min-filesize SIZE             Do not download any videos smaller than
+    --min-filesize SIZE             Abort download if filesize is smaller than
+                                    SIZE, e.g. 50k or 44.6M
+    --max-filesize SIZE             Abort download if filesize is larger than
                                     SIZE, e.g. 50k or 44.6M
-    --max-filesize SIZE             Do not download any videos larger than SIZE,
-                                    e.g. 50k or 44.6M
     --date DATE                     Download only videos uploaded on this date.
                                     The date can be "YYYYMMDD" or in the format 
                                     [now|today|yesterday][-N[day|week|month|year]].
@@ -491,9 +491,9 @@ ## Video Selection:
                                     a file that is in the archive
     --break-on-reject               Stop the download process when encountering
                                     a file that has been filtered out
-    --break-per-input               --break-on-existing, --break-on-reject,
-                                    --max-downloads, and autonumber resets per
-                                    input URL
+    --break-per-input               Alters --max-downloads, --break-on-existing,
+                                    --break-on-reject, and autonumber to reset
+                                    per input URL
     --no-break-per-input            --break-on-existing and similar options
                                     terminates the entire download queue
     --skip-playlist-after-errors N  Number of allowed failures until the rest of
@@ -1046,10 +1046,10 @@ ## SponsorBlock Options:
                                     for, separated by commas. Available
                                     categories are sponsor, intro, outro,
                                     selfpromo, preview, filler, interaction,
-                                    music_offtopic, poi_highlight, chapter, all and
-                                    default (=all). You can prefix the category
-                                    with a "-" to exclude it. See [1] for
-                                    description of the categories. E.g.
+                                    music_offtopic, poi_highlight, chapter, all
+                                    and default (=all). You can prefix the
+                                    category with a "-" to exclude it. See [1]
+                                    for description of the categories. E.g.
                                     --sponsorblock-mark all,-preview
                                     [1] https://wiki.sponsor.ajay.app/w/Segment_Categories
     --sponsorblock-remove CATS      SponsorBlock categories to be removed from
@@ -1058,7 +1058,7 @@ ## SponsorBlock Options:
                                     remove takes precedence. The syntax and
                                     available categories are the same as for
                                     --sponsorblock-mark except that "default"
-                                    refers to "all,-filler" and poi_highlight and
+                                    refers to "all,-filler" and poi_highlight,
                                     chapter are not available
     --sponsorblock-chapter-title TEMPLATE
                                     An output template for the title of the

yt_dlp/YoutubeDL.py

@@ -3123,7 +3123,7 @@ def existing_video_file(*filepaths):
                 fd, success = None, True
                 if info_dict.get('protocol') or info_dict.get('url'):
                     fd = get_suitable_downloader(info_dict, self.params, to_stdout=temp_filename == '-')
-                    if fd is not FFmpegFD and (
+                    if fd is not FFmpegFD and 'no-direct-merge' not in self.params['compat_opts'] and (
                             info_dict.get('section_start') or info_dict.get('section_end')):
                         msg = ('This format cannot be partially downloaded' if FFmpegFD.available()
                                else 'You have requested downloading the video partially, but ffmpeg is not installed')

yt_dlp/__init__.py

@@ -91,12 +91,11 @@ def get_urls(urls, batchfile, verbose):
 
 
 def print_extractor_information(opts, urls):
-    # Importing GenericIE is currently slow since it imports other extractors
-    # TODO: Move this back to module level after generalization of embed detection
-    from .extractor.generic import GenericIE
-
     out = ''
     if opts.list_extractors:
+        # Importing GenericIE is currently slow since it imports YoutubeIE
+        from .extractor.generic import GenericIE
+
         urls = dict.fromkeys(urls, False)
         for ie in list_extractor_classes(opts.age_limit):
             out += ie.IE_NAME + (' (CURRENTLY BROKEN)' if not ie.working() else '') + '\n'

yt_dlp/downloader/common.py

@@ -20,6 +20,7 @@
     RetryManager,
     classproperty,
     decodeArgument,
+    deprecation_warning,
     encodeFilename,
     format_bytes,
     join_nonempty,
@@ -180,7 +181,9 @@ def best_block_size(elapsed_time, bytes):
     @staticmethod
     def parse_bytes(bytestr):
         """Parse a string indicating a byte quantity into an integer."""
-        parse_bytes(bytestr)
+        deprecation_warning('yt_dlp.FileDownloader.parse_bytes is deprecated and '
+                            'may be removed in the future. Use yt_dlp.utils.parse_bytes instead')
+        return parse_bytes(bytestr)
 
     def slow_down(self, start_time, now, byte_counter):
         """Sleep if the download speed is over the rate limit."""

yt_dlp/extractor/common.py

@@ -71,6 +71,7 @@
     str_to_int,
     strip_or_none,
     traverse_obj,
+    truncate_string,
     try_call,
     try_get,
     unescapeHTML,
@@ -674,7 +675,8 @@ def extract(self, url):
             for _ in range(2):
                 try:
                     self.initialize()
-                    self.write_debug('Extracting URL: %s' % url)
+                    self.to_screen('Extracting URL: %s' % (
+                        url if self.get_param('verbose') else truncate_string(url, 100, 20)))
                     ie_result = self._real_extract(url)
                     if ie_result is None:
                         return None
@@ -1906,6 +1908,14 @@ def _extract_m3u8_formats_and_subtitles(
             errnote=None, fatal=True, live=False, data=None, headers={},
             query={}):
 
+        if not m3u8_url:
+            if errnote is not False:
+                errnote = errnote or 'Failed to obtain m3u8 URL'
+                if fatal:
+                    raise ExtractorError(errnote, video_id=video_id)
+                self.report_warning(f'{errnote}{bug_reports_message()}')
+            return [], {}
+
         res = self._download_webpage_handle(
             m3u8_url, video_id,
             note='Downloading m3u8 information' if note is None else note,

yt_dlp/options.py

@@ -535,10 +535,10 @@ def _alias_callback(option, opt_str, value, parser, opts, nargs):
         '-I', '--playlist-items',
         dest='playlist_items', metavar='ITEM_SPEC', default=None,
         help=(
-            'Comma separated playlist_index of the videos to download. '
+            'Comma separated playlist_index of the items to download. '
             'You can specify a range using "[START]:[STOP][:STEP]". For backward compatibility, START-STOP is also supported. '
             'Use negative indices to count from the right and negative STEP to download in reverse order. '
-            'E.g. "-I 1:3,7,-5::2" used on a playlist of size 15 will download the videos at index 1,2,3,7,11,13,15'))
+            'E.g. "-I 1:3,7,-5::2" used on a playlist of size 15 will download the items at index 1,2,3,7,11,13,15'))
     selection.add_option(
         '--match-title',
         dest='matchtitle', metavar='REGEX',
@@ -554,7 +554,7 @@ def _alias_callback(option, opt_str, value, parser, opts, nargs):
     selection.add_option(
         '--max-filesize',
         metavar='SIZE', dest='max_filesize', default=None,
-        help='Abort download if filesize if larger than SIZE, e.g. 50k or 44.6M')
+        help='Abort download if filesize is larger than SIZE, e.g. 50k or 44.6M')
     selection.add_option(
         '--date',
         metavar='DATE', dest='date', default=None,
@@ -635,7 +635,7 @@ def _alias_callback(option, opt_str, value, parser, opts, nargs):
     selection.add_option(
         '--break-per-input',
         action='store_true', dest='break_per_url', default=False,
-        help='--break-on-existing, --break-on-reject, --max-downloads, and autonumber resets per input URL')
+        help='Alters --max-downloads, --break-on-existing, --break-on-reject, and autonumber to reset per input URL')
     selection.add_option(
         '--no-break-per-input',
         action='store_false', dest='break_per_url',

yt_dlp/utils.py

@@ -3872,6 +3872,9 @@ def __eq__(self, other):
         return (isinstance(other, download_range_func)
                 and self.chapters == other.chapters and self.ranges == other.ranges)
 
+    def __repr__(self):
+        return f'{type(self).__name__}({self.chapters}, {self.ranges})'
+
 
 def parse_dfxp_time_expr(time_expr):
     if not time_expr:
@@ -5576,17 +5579,39 @@ def supports_terminal_sequences(stream):
         return False
 
 
-def windows_enable_vt_mode():  # TODO: Do this the proper way https://bugs.python.org/issue30075
+def windows_enable_vt_mode():
+    """Ref: https://bugs.python.org/issue30075 """
     if get_windows_version() < (10, 0, 10586):
         return
-    global WINDOWS_VT_MODE
-    try:
-        Popen.run('', shell=True)
-    except Exception:
-        return
 
-    WINDOWS_VT_MODE = True
-    supports_terminal_sequences.cache_clear()
+    import ctypes
+    import ctypes.wintypes
+    import msvcrt
+
+    ENABLE_VIRTUAL_TERMINAL_PROCESSING = 0x0004
+
+    dll = ctypes.WinDLL('kernel32', use_last_error=False)
+    handle = os.open('CONOUT$', os.O_RDWR)
+
+    try:
+        h_out = ctypes.wintypes.HANDLE(msvcrt.get_osfhandle(handle))
+        dw_original_mode = ctypes.wintypes.DWORD()
+        success = dll.GetConsoleMode(h_out, ctypes.byref(dw_original_mode))
+        if not success:
+            raise Exception('GetConsoleMode failed')
+
+        success = dll.SetConsoleMode(h_out, ctypes.wintypes.DWORD(
+            dw_original_mode.value | ENABLE_VIRTUAL_TERMINAL_PROCESSING))
+        if not success:
+            raise Exception('SetConsoleMode failed')
+    except Exception as e:
+        write_string(f'WARNING: Cannot enable VT mode - {e}')
+    else:
+        global WINDOWS_VT_MODE
+        WINDOWS_VT_MODE = True
+        supports_terminal_sequences.cache_clear()
+    finally:
+        os.close(handle)
 
 
 _terminal_sequences_re = re.compile('\033\\[[^m]+m')
@@ -5976,7 +6001,7 @@ def truncate_string(s, left, right=0):
     assert left > 3 and right >= 0
     if s is None or len(s) <= left + right:
         return s
-    return f'{s[:left-3]}...{s[-right:]}'
+    return f'{s[:left-3]}...{s[-right:] if right else ""}'
 
 
 def orderedSet_from_options(options, alias_dict, *, use_regex=False, start=None):