def/project305
def/project305
Archived
1
1
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Load domains and proxy from files

Browse source
This commit is contained in:
riot 2022-09-10 20:49:12 +03:00
parent 1ffe7a5d7f
commit e044b7d9f0
1 changed files with 12 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

15
main.py
View file

@ -1,5 +1,5 @@
from loguru import logger from loguru import logger
import pretty_errors # import pretty_errors
from colorama import Fore, Back, Style, init from colorama import Fore, Back, Style, init
import pathlib import pathlib
@ -45,8 +45,16 @@ def load_config() -> dict:
argparser = init_argparser() argparser = init_argparser()
args = vars(argparser.parse_args()) args = vars(argparser.parse_args())
with open(args["config"]) as f: with open(args["domains"]) as domains_file:
config = yaml.safe_load(f) domains = domains_file.readlines()
args["domains"] = domains
with open(args["proxy"]) as proxy_file:
proxy = proxy_file.readlines()
args["proxy"] = proxy
with open(args["config"]) as config_file:
config = yaml.safe_load(config_file)
config["settings"].update(args) config["settings"].update(args)
# Remove config path to pass config values to the Scraper # Remove config path to pass config values to the Scraper
@ -65,6 +73,7 @@ async def main():
config = load_config() config = load_config()
scraper = Scraper(**config["settings"]) scraper = Scraper(**config["settings"])
await scraper.scrape()
asyncio.run(main()) asyncio.run(main())