Allow multiple and nested configuration files

test/test_options.py

@@ -1,26 +0,0 @@
-# coding: utf-8
-
-from __future__ import unicode_literals
-
-# Allow direct execution
-import os
-import sys
-import unittest
-sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
-
-from yt_dlp.options import _hide_login_info
-
-
-class TestOptions(unittest.TestCase):
-    def test_hide_login_info(self):
-        self.assertEqual(_hide_login_info(['-u', 'foo', '-p', 'bar']),
-                         ['-u', 'PRIVATE', '-p', 'PRIVATE'])
-        self.assertEqual(_hide_login_info(['-u']), ['-u'])
-        self.assertEqual(_hide_login_info(['-u', 'foo', '-u', 'bar']),
-                         ['-u', 'PRIVATE', '-u', 'PRIVATE'])
-        self.assertEqual(_hide_login_info(['--username=foo']),
-                         ['--username=PRIVATE'])
-
-
-if __name__ == '__main__':
-    unittest.main()

test/test_utils.py

@@ -23,6 +23,7 @@
     caesar,
     clean_html,
     clean_podcast_url,
+    Config,
     date_from_str,
     datetime_from_str,
     DateRange,
@@ -1701,6 +1702,15 @@ def test_format_bytes(self):
         self.assertEqual(format_bytes(1024**7), '1.00ZiB')
         self.assertEqual(format_bytes(1024**8), '1.00YiB')
 
+    def test_hide_login_info(self):
+        self.assertEqual(Config.hide_login_info(['-u', 'foo', '-p', 'bar']),
+                         ['-u', 'PRIVATE', '-p', 'PRIVATE'])
+        self.assertEqual(Config.hide_login_info(['-u']), ['-u'])
+        self.assertEqual(Config.hide_login_info(['-u', 'foo', '-u', 'bar']),
+                         ['-u', 'PRIVATE', '-u', 'PRIVATE'])
+        self.assertEqual(Config.hide_login_info(['--username=foo']),
+                         ['--username=PRIVATE'])
+
 
 if __name__ == '__main__':
     unittest.main()

yt_dlp/options.py

@@ -13,11 +13,11 @@
     compat_shlex_split,
 )
 from .utils import (
+    Config,
     expand_path,
     get_executable_path,
     OUTTMPL_TYPES,
     POSTPROCESS_WHEN,
-    preferredencoding,
     remove_end,
     write_string,
 )
@@ -35,39 +35,16 @@
 from .postprocessor.modify_chapters import DEFAULT_SPONSORBLOCK_CHAPTER_TITLE
 
 
-def _hide_login_info(opts):
+def parseOpts(overrideArguments=None, ignore_config_files='if_override'):
-    PRIVATE_OPTS = set(['-p', '--password', '-u', '--username', '--video-password', '--ap-password', '--ap-username'])
+    parser = create_parser()
-    eqre = re.compile('^(?P<key>' + ('|'.join(re.escape(po) for po in PRIVATE_OPTS)) + ')=.+$')
+    root = Config(parser)
 
-    def _scrub_eq(o):
+    if ignore_config_files == 'if_override':
-        m = eqre.match(o)
+        ignore_config_files = overrideArguments is not None
-        if m:
+    if overrideArguments:
-            return m.group('key') + '=PRIVATE'
+        root.append_config(overrideArguments, label='Override')
-        else:
+    else:
-            return o
+        root.append_config(sys.argv[1:], label='Command-line')
-
-    opts = list(map(_scrub_eq, opts))
-    for idx, opt in enumerate(opts):
-        if opt in PRIVATE_OPTS and idx + 1 < len(opts):
-            opts[idx + 1] = 'PRIVATE'
-    return opts
-
-
-def parseOpts(overrideArguments=None):
-    def _readOptions(filename_bytes, default=[]):
-        try:
-            optionf = open(filename_bytes)
-        except IOError:
-            return default  # silently skip if file is not present
-        try:
-            # FIXME: https://github.com/ytdl-org/youtube-dl/commit/dfe5fa49aed02cf36ba9f743b11b0903554b5e56
-            contents = optionf.read()
-            if sys.version_info < (3,):
-                contents = contents.decode(preferredencoding())
-            res = compat_shlex_split(contents, comments=True)
-        finally:
-            optionf.close()
-        return res
 
     def _readUserConf(package_name, default=[]):
         # .config
@@ -75,7 +52,7 @@ def _readUserConf(package_name, default=[]):
         userConfFile = os.path.join(xdg_config_home, package_name, 'config')
         if not os.path.isfile(userConfFile):
             userConfFile = os.path.join(xdg_config_home, '%s.conf' % package_name)
-        userConf = _readOptions(userConfFile, default=None)
+        userConf = Config.read_file(userConfFile, default=None)
         if userConf is not None:
             return userConf, userConfFile
 
@@ -83,24 +60,64 @@ def _readUserConf(package_name, default=[]):
         appdata_dir = compat_getenv('appdata')
         if appdata_dir:
             userConfFile = os.path.join(appdata_dir, package_name, 'config')
-            userConf = _readOptions(userConfFile, default=None)
+            userConf = Config.read_file(userConfFile, default=None)
             if userConf is None:
                 userConfFile += '.txt'
-                userConf = _readOptions(userConfFile, default=None)
+                userConf = Config.read_file(userConfFile, default=None)
         if userConf is not None:
             return userConf, userConfFile
 
         # home
         userConfFile = os.path.join(compat_expanduser('~'), '%s.conf' % package_name)
-        userConf = _readOptions(userConfFile, default=None)
+        userConf = Config.read_file(userConfFile, default=None)
         if userConf is None:
             userConfFile += '.txt'
-            userConf = _readOptions(userConfFile, default=None)
+            userConf = Config.read_file(userConfFile, default=None)
         if userConf is not None:
             return userConf, userConfFile
 
         return default, None
 
+    def add_config(label, path, user=False):
+        """ Adds config and returns whether to continue """
+        if root.parse_args()[0].ignoreconfig:
+            return False
+        # Multiple package names can be given here
+        # Eg: ('yt-dlp', 'youtube-dlc', 'youtube-dl') will look for
+        # the configuration file of any of these three packages
+        for package in ('yt-dlp',):
+            if user:
+                args, current_path = _readUserConf(package, default=None)
+            else:
+                current_path = os.path.join(path, '%s.conf' % package)
+                args = Config.read_file(current_path, default=None)
+            if args is not None:
+                root.append_config(args, current_path, label=label)
+                return True
+        return True
+
+    def load_configs():
+        yield not ignore_config_files
+        yield add_config('Portable', get_executable_path())
+        yield add_config('Home', expand_path(root.parse_args()[0].paths.get('home', '')).strip())
+        yield add_config('User', None, user=True)
+        yield add_config('System', '/etc')
+
+    if all(load_configs()):
+        # If ignoreconfig is found inside the system configuration file,
+        # the user configuration is removed
+        if root.parse_args()[0].ignoreconfig:
+            user_conf = next((i for i, conf in enumerate(root.configs) if conf.label == 'User'), None)
+            if user_conf is not None:
+                root.configs.pop(user_conf)
+
+    opts, args = root.parse_args()
+    if opts.verbose:
+        write_string(f'\n{root}'.replace('\n| ', '\n[debug] ')[1:] + '\n')
+    return parser, opts, args
+
+
+def create_parser():
     def _format_option_string(option):
         ''' ('-o', '--option') -> -o, --format METAVAR'''
 
@@ -244,14 +261,20 @@ def _dict_from_options_callback(
         '--ignore-config', '--no-config',
         action='store_true', dest='ignoreconfig',
         help=(
-            'Disable loading any configuration files except the one provided by --config-location. '
+            'Disable loading any further configuration files except the one provided by --config-locations. '
-            'When given inside a configuration file, no further configuration files are loaded. '
+            'For backward compatibility, if this option is found inside the system configuration file, the user configuration is not loaded'))
-            'Additionally, (for backward compatibility) if this option is found inside the '
-            'system configuration file, the user configuration is not loaded'))
     general.add_option(
-        '--config-location',
+        '--no-config-locations',
-        dest='config_location', metavar='PATH',
+        action='store_const', dest='config_locations', const=[],
-        help='Location of the main configuration file; either the path to the config or its containing directory')
+        help=(
+            'Do not load any custom configuration files (default). When given inside a '
+            'configuration file, ignore all previous --config-locations defined in the current file'))
+    general.add_option(
+        '--config-locations',
+        dest='config_locations', metavar='PATH', action='append',
+        help=(
+            'Location of the main configuration file; either the path to the config or its containing directory. '
+            'Can be used multiple times and inside other configuration files'))
     general.add_option(
         '--flat-playlist',
         action='store_const', dest='extract_flat', const='in_playlist', default=False,
@@ -1634,75 +1657,11 @@ def _dict_from_options_callback(
     parser.add_option_group(sponsorblock)
     parser.add_option_group(extractor)
 
-    if overrideArguments is not None:
+    return parser
-        opts, args = parser.parse_args(overrideArguments)
-        if opts.verbose:
-            write_string('[debug] Override config: ' + repr(overrideArguments) + '\n')
-    else:
-        def compat_conf(conf):
-            if sys.version_info < (3,):
-                return [a.decode(preferredencoding(), 'replace') for a in conf]
-            return conf
 
-        configs = {
-            'command-line': compat_conf(sys.argv[1:]),
-            'custom': [], 'home': [], 'portable': [], 'user': [], 'system': []}
-        paths = {'command-line': False}
 
-        def read_options(name, path, user=False):
+def _hide_login_info(opts):
-            ''' loads config files and returns ignoreconfig '''
+    write_string(
-            # Multiple package names can be given here
+        'DeprecationWarning: "yt_dlp.options._hide_login_info" is deprecated and may be removed in a future version. '
-            # Eg: ('yt-dlp', 'youtube-dlc', 'youtube-dl') will look for
+        'Use "yt_dlp.utils.Config.hide_login_info" instead\n')
-            # the configuration file of any of these three packages
+    return Config.hide_login_info(opts)
-            for package in ('yt-dlp',):
-                if user:
-                    config, current_path = _readUserConf(package, default=None)
-                else:
-                    current_path = os.path.join(path, '%s.conf' % package)
-                    config = _readOptions(current_path, default=None)
-                if config is not None:
-                    current_path = os.path.realpath(current_path)
-                    if current_path in paths.values():
-                        return False
-                    configs[name], paths[name] = config, current_path
-                    return parser.parse_args(config)[0].ignoreconfig
-            return False
-
-        def get_configs():
-            opts, _ = parser.parse_args(configs['command-line'])
-            if opts.config_location is not None:
-                location = compat_expanduser(opts.config_location)
-                if os.path.isdir(location):
-                    location = os.path.join(location, 'yt-dlp.conf')
-                if not os.path.exists(location):
-                    parser.error('config-location %s does not exist.' % location)
-                config = _readOptions(location, default=None)
-                if config:
-                    configs['custom'], paths['custom'] = config, location
-
-            if opts.ignoreconfig:
-                return
-            if parser.parse_args(configs['custom'])[0].ignoreconfig:
-                return
-            if read_options('portable', get_executable_path()):
-                return
-            opts, _ = parser.parse_args(configs['portable'] + configs['custom'] + configs['command-line'])
-            if read_options('home', expand_path(opts.paths.get('home', '')).strip()):
-                return
-            if read_options('system', '/etc'):
-                return
-            if read_options('user', None, user=True):
-                configs['system'], paths['system'] = [], None
-
-        get_configs()
-        argv = configs['system'] + configs['user'] + configs['home'] + configs['portable'] + configs['custom'] + configs['command-line']
-        opts, args = parser.parse_args(argv)
-        if opts.verbose:
-            for label in ('Command-line', 'Custom', 'Portable', 'Home', 'User', 'System'):
-                key = label.lower()
-                if paths.get(key):
-                    write_string(f'[debug] {label} config file: {paths[key]}\n')
-                if paths.get(key) is not None:
-                    write_string(f'[debug] {label} config: {_hide_login_info(configs[key])!r}\n')
-
-    return parser, opts, args

yt_dlp/utils.py

@@ -58,6 +58,7 @@
     compat_kwargs,
     compat_os_name,
     compat_parse_qs,
+    compat_shlex_split,
     compat_shlex_quote,
     compat_str,
     compat_struct_pack,
@@ -5100,3 +5101,90 @@ def join_nonempty(*values, delim='-', from_dict=None):
     if from_dict is not None:
         values = map(from_dict.get, values)
     return delim.join(map(str, filter(None, values)))
+
+
+class Config:
+    own_args = None
+    filename = None
+    __initialized = False
+
+    def __init__(self, parser, label=None):
+        self._parser, self.label = parser, label
+        self._loaded_paths, self.configs = set(), []
+
+    def init(self, args=None, filename=None):
+        assert not self.__initialized
+        if filename:
+            location = os.path.realpath(filename)
+            if location in self._loaded_paths:
+                return False
+            self._loaded_paths.add(location)
+
+        self.__initialized = True
+        self.own_args, self.filename = args, filename
+        for location in self._parser.parse_args(args)[0].config_locations or []:
+            location = compat_expanduser(location)
+            if os.path.isdir(location):
+                location = os.path.join(location, 'yt-dlp.conf')
+            if not os.path.exists(location):
+                self._parser.error(f'config location {location} does not exist')
+            self.append_config(self.read_file(location), location)
+        return True
+
+    def __str__(self):
+        label = join_nonempty(
+            self.label, 'config', f'"{self.filename}"' if self.filename else '',
+            delim=' ')
+        return join_nonempty(
+            self.own_args is not None and f'{label[0].upper()}{label[1:]}: {self.hide_login_info(self.own_args)}',
+            *(f'\n{c}'.replace('\n', '\n| ')[1:] for c in self.configs),
+            delim='\n')
+
+    @staticmethod
+    def read_file(filename, default=[]):
+        try:
+            optionf = open(filename)
+        except IOError:
+            return default  # silently skip if file is not present
+        try:
+            # FIXME: https://github.com/ytdl-org/youtube-dl/commit/dfe5fa49aed02cf36ba9f743b11b0903554b5e56
+            contents = optionf.read()
+            if sys.version_info < (3,):
+                contents = contents.decode(preferredencoding())
+            res = compat_shlex_split(contents, comments=True)
+        finally:
+            optionf.close()
+        return res
+
+    @staticmethod
+    def hide_login_info(opts):
+        PRIVATE_OPTS = set(['-p', '--password', '-u', '--username', '--video-password', '--ap-password', '--ap-username'])
+        eqre = re.compile('^(?P<key>' + ('|'.join(re.escape(po) for po in PRIVATE_OPTS)) + ')=.+$')
+
+        def _scrub_eq(o):
+            m = eqre.match(o)
+            if m:
+                return m.group('key') + '=PRIVATE'
+            else:
+                return o
+
+        opts = list(map(_scrub_eq, opts))
+        for idx, opt in enumerate(opts):
+            if opt in PRIVATE_OPTS and idx + 1 < len(opts):
+                opts[idx + 1] = 'PRIVATE'
+        return opts
+
+    def append_config(self, *args, label=None):
+        config = type(self)(self._parser, label)
+        config._loaded_paths = self._loaded_paths
+        if config.init(*args):
+            self.configs.append(config)
+
+    @property
+    def all_args(self):
+        for config in reversed(self.configs):
+            yield from config.all_args
+        yield from self.own_args or []
+
+    def parse_args(self):
+        return self._parser.parse_args(list(self.all_args))