From e044b7d9f0f0e84bf6309655e18aace50360509d Mon Sep 17 00:00:00 2001 From: riot Date: Sat, 10 Sep 2022 20:49:12 +0300 Subject: [PATCH] Load domains and proxy from files --- main.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/main.py b/main.py index 161db2a..8f597f2 100644 --- a/main.py +++ b/main.py @@ -1,5 +1,5 @@ from loguru import logger -import pretty_errors +# import pretty_errors from colorama import Fore, Back, Style, init import pathlib @@ -45,8 +45,16 @@ def load_config() -> dict: argparser = init_argparser() args = vars(argparser.parse_args()) - with open(args["config"]) as f: - config = yaml.safe_load(f) + with open(args["domains"]) as domains_file: + domains = domains_file.readlines() + args["domains"] = domains + + with open(args["proxy"]) as proxy_file: + proxy = proxy_file.readlines() + args["proxy"] = proxy + + with open(args["config"]) as config_file: + config = yaml.safe_load(config_file) config["settings"].update(args) # Remove config path to pass config values to the Scraper @@ -65,6 +73,7 @@ async def main(): config = load_config() scraper = Scraper(**config["settings"]) + await scraper.scrape() asyncio.run(main())