72 lines
1.7 KiB
Python
72 lines
1.7 KiB
Python
from loguru import logger
|
|
# import pretty_errors
|
|
from colorama import Fore, Back, Style, init
|
|
|
|
import pathlib
|
|
import sys
|
|
from sys import platform
|
|
|
|
import argparse
|
|
from argparse import ArgumentParser, Namespace
|
|
import yaml
|
|
|
|
import asyncio
|
|
from courts_scraper.scraper import Scraper
|
|
|
|
|
|
def init_argparser() -> ArgumentParser:
|
|
argparser = argparse.ArgumentParser(
|
|
description="List fish in aquarium.",
|
|
)
|
|
argparser.add_argument(
|
|
"--config", "-c",
|
|
help="Path to the config file",
|
|
type=pathlib.Path,
|
|
)
|
|
argparser.add_argument(
|
|
"--domains", "-d",
|
|
help="Path to the domains file",
|
|
type=pathlib.Path,
|
|
)
|
|
argparser.add_argument(
|
|
"--proxy", "-p",
|
|
help="Path to the proxy file",
|
|
type=pathlib.Path,
|
|
)
|
|
argparser.add_argument("--rps", type=int)
|
|
|
|
return argparser
|
|
|
|
|
|
async def main():
|
|
logger.add("project.log")
|
|
logger.info("Starting...")
|
|
|
|
if platform != "linux":
|
|
logger.critical("""\nNot for windows, run only on GNU/Linux!\n""")
|
|
input()
|
|
sys.exit(1)
|
|
|
|
scraper_settings = dict()
|
|
argparser = init_argparser()
|
|
args = argparser.parse_args()
|
|
|
|
if args.config is not None:
|
|
with open(args.config) as config_file:
|
|
scraper_settings = yaml.safe_load(config_file)
|
|
else:
|
|
scraper_settings["rps"] = args.rps
|
|
|
|
with open(args.domains) as domains_file:
|
|
scraper_settings["domains"] = domains_file.readlines()
|
|
|
|
if args.proxy is not None: # Optional argument
|
|
with open(args.proxy) as proxy_file:
|
|
scraper_settings["proxy"] = proxy_file.readlines()
|
|
|
|
scraper = Scraper(**scraper_settings)
|
|
await scraper.scrape()
|
|
|
|
|
|
asyncio.run(main())
|