[utils] Move FileDownloader.parse_bytes into utils

This commit is contained in:
pukkandan 2022-11-17 08:40:34 +05:30
parent 4de88a6a36
commit 64c464a144
No known key found for this signature in database
GPG key ID: 7EEE9E1E817D0A39
3 changed files with 25 additions and 21 deletions

View file

@ -16,7 +16,6 @@
from .compat import compat_shlex_quote from .compat import compat_shlex_quote
from .cookies import SUPPORTED_BROWSERS, SUPPORTED_KEYRINGS from .cookies import SUPPORTED_BROWSERS, SUPPORTED_KEYRINGS
from .downloader import FileDownloader
from .downloader.external import get_external_downloader from .downloader.external import get_external_downloader
from .extractor import list_extractor_classes from .extractor import list_extractor_classes
from .extractor.adobepass import MSO_INFO from .extractor.adobepass import MSO_INFO
@ -50,6 +49,7 @@
format_field, format_field,
int_or_none, int_or_none,
match_filter_func, match_filter_func,
parse_bytes,
parse_duration, parse_duration,
preferredencoding, preferredencoding,
read_batch_urls, read_batch_urls,
@ -281,19 +281,19 @@ def parse_sleep_func(expr):
raise ValueError(f'invalid {key} retry sleep expression {expr!r}') raise ValueError(f'invalid {key} retry sleep expression {expr!r}')
# Bytes # Bytes
def parse_bytes(name, value): def validate_bytes(name, value):
if value is None: if value is None:
return None return None
numeric_limit = FileDownloader.parse_bytes(value) numeric_limit = parse_bytes(value)
validate(numeric_limit is not None, 'rate limit', value) validate(numeric_limit is not None, 'rate limit', value)
return numeric_limit return numeric_limit
opts.ratelimit = parse_bytes('rate limit', opts.ratelimit) opts.ratelimit = validate_bytes('rate limit', opts.ratelimit)
opts.throttledratelimit = parse_bytes('throttled rate limit', opts.throttledratelimit) opts.throttledratelimit = validate_bytes('throttled rate limit', opts.throttledratelimit)
opts.min_filesize = parse_bytes('min filesize', opts.min_filesize) opts.min_filesize = validate_bytes('min filesize', opts.min_filesize)
opts.max_filesize = parse_bytes('max filesize', opts.max_filesize) opts.max_filesize = validate_bytes('max filesize', opts.max_filesize)
opts.buffersize = parse_bytes('buffer size', opts.buffersize) opts.buffersize = validate_bytes('buffer size', opts.buffersize)
opts.http_chunk_size = parse_bytes('http chunk size', opts.http_chunk_size) opts.http_chunk_size = validate_bytes('http chunk size', opts.http_chunk_size)
# Output templates # Output templates
def validate_outtmpl(tmpl, msg): def validate_outtmpl(tmpl, msg):

View file

@ -15,7 +15,6 @@
from ..utils import ( from ..utils import (
IDENTITY, IDENTITY,
NO_DEFAULT, NO_DEFAULT,
NUMBER_RE,
LockingUnsupportedError, LockingUnsupportedError,
Namespace, Namespace,
RetryManager, RetryManager,
@ -24,6 +23,7 @@
encodeFilename, encodeFilename,
format_bytes, format_bytes,
join_nonempty, join_nonempty,
parse_bytes,
remove_start, remove_start,
sanitize_open, sanitize_open,
shell_quote, shell_quote,
@ -180,12 +180,7 @@ def best_block_size(elapsed_time, bytes):
@staticmethod @staticmethod
def parse_bytes(bytestr): def parse_bytes(bytestr):
"""Parse a string indicating a byte quantity into an integer.""" """Parse a string indicating a byte quantity into an integer."""
matchobj = re.match(rf'(?i)^({NUMBER_RE})([kMGTPEZY]?)$', bytestr) parse_bytes(bytestr)
if matchobj is None:
return None
number = float(matchobj.group(1))
multiplier = 1024.0 ** 'bkmgtpezy'.index(matchobj.group(2).lower())
return int(round(number * multiplier))
def slow_down(self, start_time, now, byte_counter): def slow_down(self, start_time, now, byte_counter):
"""Sleep if the download speed is over the rate limit.""" """Sleep if the download speed is over the rate limit."""

View file

@ -2289,15 +2289,24 @@ def format_bytes(bytes):
return format_decimal_suffix(bytes, '%.2f%sB', factor=1024) or 'N/A' return format_decimal_suffix(bytes, '%.2f%sB', factor=1024) or 'N/A'
def lookup_unit_table(unit_table, s): def lookup_unit_table(unit_table, s, strict=False):
num_re = NUMBER_RE if strict else NUMBER_RE.replace(R'\.', '[,.]')
units_re = '|'.join(re.escape(u) for u in unit_table) units_re = '|'.join(re.escape(u) for u in unit_table)
m = re.match( m = (re.fullmatch if strict else re.match)(
r'(?P<num>[0-9]+(?:[,.][0-9]*)?)\s*(?P<unit>%s)\b' % units_re, s) rf'(?P<num>{num_re})\s*(?P<unit>{units_re})\b', s)
if not m: if not m:
return None return None
num_str = m.group('num').replace(',', '.')
num = float(m.group('num').replace(',', '.'))
mult = unit_table[m.group('unit')] mult = unit_table[m.group('unit')]
return int(float(num_str) * mult) return round(num * mult)
def parse_bytes(s):
"""Parse a string indicating a byte quantity into an integer"""
return lookup_unit_table(
{u: 1024**i for i, u in enumerate(['', *'KMGTPEZY'])},
s.upper(), strict=True)
def parse_filesize(s): def parse_filesize(s):