mirror of
https://github.com/yt-dlp/yt-dlp.git
synced 2024-12-04 12:26:39 +00:00
bcec568ea7
Add ffmpeg progress tracking to FFmpegPostProcessor Apply changes from the code review Fix a bug where the subprocess didn't capture any output thus an empty stdout and stderr were sent back Add missing hooks Revert "Add missing hooks" This reverts commit a359c5ea10bb35b965e80801e736f43cdbcf3294. Add support of -ss=132 timestamp format Infer filename from ffmpeg args instead of info_dic Remove redundant parenthesis and switch from to_stodout to to_screen Add info kwarg with multiple files and ffmpeg to track progress Moved format progress function to util Moved format progress function to util Add progress tracking to postprocessing operations Fix typing error Handle self._downloader is None at __init__ Move format progress functions to utils Move format progress functions to utils Handle case where ydl passed is None Handle case where ydl passed is None Handle case where _multiline isn't initialized Handle case where _multiline isn't initialized Fix streams incorrectly returned Fix case where ydl is nested in the downloader Add progress_hook attribute Fix bug after merge Fix import bugs after merge Catch up with upstream Fix merge errors #1 Adapt tests and implementatation for ffmpeg progress tracking args
326 lines
13 KiB
Python
326 lines
13 KiB
Python
import functools
|
|
import json
|
|
import os
|
|
import sys
|
|
|
|
from ..minicurses import (
|
|
BreaklineStatusPrinter,
|
|
MultilineLogger,
|
|
MultilinePrinter,
|
|
QuietMultilinePrinter,
|
|
)
|
|
from ..networking import Request
|
|
from ..networking.exceptions import HTTPError, network_exceptions
|
|
from ..utils import (
|
|
FormatProgressInfos,
|
|
Namespace,
|
|
PostProcessingError,
|
|
RetryManager,
|
|
_configuration_args,
|
|
deprecation_warning,
|
|
encodeFilename,
|
|
format_bytes,
|
|
join_nonempty,
|
|
try_call,
|
|
)
|
|
|
|
|
|
class PostProcessorMetaClass(type):
|
|
@staticmethod
|
|
def run_wrapper(func):
|
|
@functools.wraps(func)
|
|
def run(self, info, *args, **kwargs):
|
|
info_copy = self._copy_infodict(info)
|
|
self._hook_progress({'status': 'started'}, info_copy)
|
|
ret = func(self, info, *args, **kwargs)
|
|
if ret is not None:
|
|
_, info = ret
|
|
self._hook_progress({'status': 'finished'}, info_copy)
|
|
return ret
|
|
return run
|
|
|
|
def __new__(cls, name, bases, attrs):
|
|
if 'run' in attrs:
|
|
attrs['run'] = cls.run_wrapper(attrs['run'])
|
|
return type.__new__(cls, name, bases, attrs)
|
|
|
|
|
|
class PostProcessor(metaclass=PostProcessorMetaClass):
|
|
"""Post Processor class.
|
|
|
|
PostProcessor objects can be added to downloaders with their
|
|
add_post_processor() method. When the downloader has finished a
|
|
successful download, it will take its internal chain of PostProcessors
|
|
and start calling the run() method on each one of them, first with
|
|
an initial argument and then with the returned value of the previous
|
|
PostProcessor.
|
|
|
|
PostProcessor objects follow a "mutual registration" process similar
|
|
to InfoExtractor objects.
|
|
|
|
Optionally PostProcessor can use a list of additional command-line arguments
|
|
with self._configuration_args.
|
|
"""
|
|
|
|
_downloader = None
|
|
|
|
def __init__(self, downloader=None):
|
|
self._progress_hooks = []
|
|
self.add_progress_hook(self.report_progress)
|
|
self.set_downloader(downloader)
|
|
self._out_files = self.set_out_files()
|
|
self.PP_NAME = self.pp_key()
|
|
self._prepare_multiline_status()
|
|
|
|
@classmethod
|
|
def pp_key(cls):
|
|
name = cls.__name__[:-2]
|
|
return name[6:] if name[:6].lower() == 'ffmpeg' else name
|
|
|
|
def to_screen(self, text, prefix=True, *args, **kwargs):
|
|
if self._downloader:
|
|
tag = '[%s] ' % self.PP_NAME if prefix else ''
|
|
return self._downloader.to_screen(f'{tag}{text}', *args, **kwargs)
|
|
|
|
def report_warning(self, text, *args, **kwargs):
|
|
if self._downloader:
|
|
return self._downloader.report_warning(text, *args, **kwargs)
|
|
|
|
def deprecation_warning(self, msg):
|
|
warn = getattr(self._downloader, 'deprecation_warning', deprecation_warning)
|
|
return warn(msg, stacklevel=1)
|
|
|
|
def deprecated_feature(self, msg):
|
|
if self._downloader:
|
|
return self._downloader.deprecated_feature(msg)
|
|
return deprecation_warning(msg, stacklevel=1)
|
|
|
|
def report_error(self, text, *args, **kwargs):
|
|
self.deprecation_warning('"yt_dlp.postprocessor.PostProcessor.report_error" is deprecated. '
|
|
'raise "yt_dlp.utils.PostProcessingError" instead')
|
|
if self._downloader:
|
|
return self._downloader.report_error(text, *args, **kwargs)
|
|
|
|
def write_debug(self, text, *args, **kwargs):
|
|
if self._downloader:
|
|
return self._downloader.write_debug(text, *args, **kwargs)
|
|
|
|
def _delete_downloaded_files(self, *files_to_delete, **kwargs):
|
|
if self._downloader:
|
|
return self._downloader._delete_downloaded_files(*files_to_delete, **kwargs)
|
|
for filename in set(filter(None, files_to_delete)):
|
|
os.remove(filename)
|
|
|
|
def get_param(self, name, default=None, *args, **kwargs):
|
|
if self._downloader:
|
|
return self._downloader.params.get(name, default, *args, **kwargs)
|
|
return default
|
|
|
|
def set_out_files(self):
|
|
if not self._downloader:
|
|
return None
|
|
return getattr(self._downloader, '_out_files', None) or self._downloader.ydl._out_files
|
|
|
|
def set_downloader(self, downloader):
|
|
"""Sets the downloader for this PP."""
|
|
self._downloader = downloader
|
|
for ph in getattr(downloader, '_postprocessor_hooks', []):
|
|
self.add_progress_hook(ph)
|
|
|
|
def _copy_infodict(self, info_dict):
|
|
return getattr(self._downloader, '_copy_infodict', dict)(info_dict)
|
|
|
|
@staticmethod
|
|
def _restrict_to(*, video=True, audio=True, images=True, simulated=True):
|
|
allowed = {'video': video, 'audio': audio, 'images': images}
|
|
|
|
def decorator(func):
|
|
@functools.wraps(func)
|
|
def wrapper(self, info):
|
|
if not simulated and (self.get_param('simulate') or self.get_param('skip_download')):
|
|
return [], info
|
|
format_type = (
|
|
'video' if info.get('vcodec') != 'none'
|
|
else 'audio' if info.get('acodec') != 'none'
|
|
else 'images')
|
|
if allowed[format_type]:
|
|
return func(self, info)
|
|
else:
|
|
self.to_screen('Skipping %s' % format_type)
|
|
return [], info
|
|
return wrapper
|
|
return decorator
|
|
|
|
def run(self, information):
|
|
"""Run the PostProcessor.
|
|
|
|
The "information" argument is a dictionary like the ones
|
|
composed by InfoExtractors. The only difference is that this
|
|
one has an extra field called "filepath" that points to the
|
|
downloaded file.
|
|
|
|
This method returns a tuple, the first element is a list of the files
|
|
that can be deleted, and the second of which is the updated
|
|
information.
|
|
|
|
In addition, this method may raise a PostProcessingError
|
|
exception if post processing fails.
|
|
"""
|
|
return [], information # by default, keep file and do nothing
|
|
|
|
def try_utime(self, path, atime, mtime, errnote='Cannot update utime of file'):
|
|
try:
|
|
os.utime(encodeFilename(path), (atime, mtime))
|
|
except Exception:
|
|
self.report_warning(errnote)
|
|
|
|
def _configuration_args(self, exe, *args, **kwargs):
|
|
return _configuration_args(
|
|
self.pp_key(), self.get_param('postprocessor_args'), exe, *args, **kwargs)
|
|
|
|
def _hook_progress(self, status, info_dict):
|
|
if not self._progress_hooks:
|
|
return
|
|
status.update({
|
|
'info_dict': info_dict,
|
|
'postprocessor': self.pp_key(),
|
|
})
|
|
for ph in self._progress_hooks:
|
|
ph(status)
|
|
|
|
def add_progress_hook(self, ph):
|
|
# See YoutubeDl.py (search for postprocessor_hooks) for a description of this interface
|
|
self._progress_hooks.append(ph)
|
|
|
|
def report_destination(self, filename):
|
|
"""Report destination filename."""
|
|
self.to_screen('[processing] Destination: ' + filename)
|
|
|
|
def _prepare_multiline_status(self, lines=1):
|
|
if self._downloader:
|
|
if self._downloader.params.get('noprogress'):
|
|
self._multiline = QuietMultilinePrinter()
|
|
elif self._downloader.params.get('logger'):
|
|
self._multiline = MultilineLogger(self._downloader.params['logger'], lines)
|
|
elif self._downloader.params.get('progress_with_newline'):
|
|
self._multiline = BreaklineStatusPrinter(self._downloader._out_files.out, lines)
|
|
elif hasattr(self._downloader, "_out_files"):
|
|
self._multiline = MultilinePrinter(self._downloader._out_files.out, lines, not self._downloader.params.get('quiet'))
|
|
else:
|
|
self._multiline = MultilinePrinter(sys.stdout, lines, not self._downloader.params.get('quiet'))
|
|
self._multiline.allow_colors = self._multiline._HAVE_FULLCAP and not self._downloader.params.get('no_color')
|
|
else:
|
|
self._multiline = MultilinePrinter(sys.stdout, lines, True)
|
|
self._multiline.allow_colors = self._multiline._HAVE_FULLCAP
|
|
|
|
def _finish_multiline_status(self):
|
|
self._multiline.end()
|
|
|
|
ProgressStyles = Namespace(
|
|
processed_bytes='light blue',
|
|
percent='light blue',
|
|
eta='yellow',
|
|
speed='green',
|
|
elapsed='bold white',
|
|
total_bytes='',
|
|
total_bytes_estimate='',
|
|
)
|
|
|
|
def _report_progress_status(self, s, default_template):
|
|
for name, style in self.ProgressStyles.items_:
|
|
name = f'_{name}_str'
|
|
if name not in s:
|
|
continue
|
|
s[name] = self._format_progress(s[name], style)
|
|
s['_default_template'] = default_template % s
|
|
|
|
progress_dict = s.copy()
|
|
progress_dict.pop('info_dict')
|
|
progress_dict = {'info': s['info_dict'], 'progress': progress_dict}
|
|
|
|
progress_template = self._downloader.params.get('progress_template', {})
|
|
self._multiline.print_at_line(self._downloader.evaluate_outtmpl(
|
|
progress_template.get('process') or '[processing] %(progress._default_template)s',
|
|
progress_dict), s.get('progress_idx') or 0)
|
|
self._downloader.to_console_title(self._downloader.evaluate_outtmpl(
|
|
progress_template.get('download-title') or 'yt-dlp %(progress._default_template)s',
|
|
progress_dict))
|
|
|
|
def _format_progress(self, *args, **kwargs):
|
|
return self._downloader._format_text(
|
|
self._multiline.stream, self._multiline.allow_colors, *args, **kwargs)
|
|
|
|
def report_progress(self, s):
|
|
def with_fields(*tups, default=''):
|
|
for *fields, tmpl in tups:
|
|
if all(s.get(f) is not None for f in fields):
|
|
return tmpl
|
|
return default
|
|
|
|
if not self._downloader:
|
|
return
|
|
|
|
if s['status'] == 'finished':
|
|
if self._downloader.params.get('noprogress'):
|
|
self.to_screen('[processing] Download completed')
|
|
speed = try_call(lambda: s['total_bytes'] / s['elapsed'])
|
|
s.update({
|
|
'speed': speed,
|
|
'_speed_str': FormatProgressInfos.format_speed(speed).strip(),
|
|
'_total_bytes_str': format_bytes(s.get('total_bytes')),
|
|
'_elapsed_str': FormatProgressInfos.format_seconds(s.get('elapsed')),
|
|
'_percent_str': FormatProgressInfos.format_percent(100),
|
|
})
|
|
self._report_progress_status(s, join_nonempty(
|
|
'100%%',
|
|
with_fields(('total_bytes', 'of %(_total_bytes_str)s')),
|
|
with_fields(('elapsed', 'in %(_elapsed_str)s')),
|
|
with_fields(('speed', 'at %(_speed_str)s')),
|
|
delim=' '))
|
|
|
|
if s['status'] != 'processing':
|
|
return
|
|
|
|
s.update({
|
|
'_eta_str': FormatProgressInfos.format_eta(s.get('eta')),
|
|
'_speed_str': FormatProgressInfos.format_speed(s.get('speed')),
|
|
'_percent_str': FormatProgressInfos.format_percent(try_call(
|
|
lambda: 100 * s['processed_bytes'] / s['total_bytes'],
|
|
lambda: 100 * s['processed_bytes'] / s['total_bytes_estimate'],
|
|
lambda: s['processed_bytes'] == 0 and 0)),
|
|
'_total_bytes_str': format_bytes(s.get('total_bytes')),
|
|
'_total_bytes_estimate_str': format_bytes(s.get('total_bytes_estimate')),
|
|
'_processed_bytes_str': format_bytes(s.get('processed_bytes')),
|
|
'_elapsed_str': FormatProgressInfos.format_seconds(s.get('elapsed')),
|
|
})
|
|
|
|
msg_template = with_fields(
|
|
('total_bytes', '%(_percent_str)s of %(_total_bytes_str)s at %(_speed_str)s ETA %(_eta_str)s'),
|
|
('processed_bytes', 'elapsed', '%(_processed_bytes_str)s at %(_speed_str)s (%(_elapsed_str)s)'),
|
|
('processed_bytes', '%(_processed_bytes_str)s at %(_speed_str)s'),
|
|
default='%(_percent_str)s at %(_speed_str)s ETA %(_eta_str)s')
|
|
|
|
self._report_progress_status(s, msg_template)
|
|
|
|
def _retry_download(self, err, count, retries):
|
|
# While this is not an extractor, it behaves similar to one and
|
|
# so obey extractor_retries and "--retry-sleep extractor"
|
|
RetryManager.report_retry(err, count, retries, info=self.to_screen, warn=self.report_warning,
|
|
sleep_func=self.get_param('retry_sleep_functions', {}).get('extractor'))
|
|
|
|
def _download_json(self, url, *, expected_http_errors=(404,)):
|
|
self.write_debug(f'{self.PP_NAME} query: {url}')
|
|
for retry in RetryManager(self.get_param('extractor_retries', 3), self._retry_download):
|
|
try:
|
|
rsp = self._downloader.urlopen(Request(url))
|
|
except network_exceptions as e:
|
|
if isinstance(e, HTTPError) and e.status in expected_http_errors:
|
|
return None
|
|
retry.error = PostProcessingError(f'Unable to communicate with {self.PP_NAME} API: {e}')
|
|
continue
|
|
return json.loads(rsp.read().decode(rsp.headers.get_param('charset') or 'utf-8'))
|
|
|
|
|
|
class AudioConversionError(PostProcessingError): # Deprecated
|
|
pass
|