from loguru import logger # import pretty_errors from colorama import Fore, Back, Style, init import pathlib import sys from sys import platform import argparse from argparse import ArgumentParser, Namespace import yaml import asyncio from courts_scraper.scraper import Scraper def init_argparser() -> ArgumentParser: argparser = argparse.ArgumentParser( description="List fish in aquarium.", ) argparser.add_argument( "--config", "-c", help="Path to the config file", type=pathlib.Path, ) argparser.add_argument( "--domains", "-d", help="Path to the domains file", type=pathlib.Path, ) argparser.add_argument( "--proxy", "-p", help="Path to the proxy file", type=pathlib.Path, ) argparser.add_argument("--rps", type=int) return argparser async def main(): logger.add("project.log") logger.info("Starting...") if platform != "linux": logger.critical("""\nNot for windows, run only on GNU/Linux!\n""") input() sys.exit(1) scraper_settings = dict() argparser = init_argparser() args = argparser.parse_args() if args.config is not None: with open(args.config) as config_file: scraper_settings = yaml.safe_load(config_file) else: scraper_settings["rps"] = args.rps with open(args.domains) as domains_file: scraper_settings["domains"] = domains_file.readlines() if args.proxy is not None: # Optional argument with open(args.proxy) as proxy_file: scraper_settings["proxy"] = proxy_file.readlines() scraper = Scraper(**scraper_settings) await scraper.scrape() asyncio.run(main())