We are compiling a "chronicle of judicial crimes" / Мы составляем «летопись судейских преступлений»
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
This repo is archived. You can view files and clone it, but cannot push or open issues/pull-requests.
project305/main.py

71 lines
1.7 KiB

from loguru import logger
# import pretty_errors
from colorama import Fore, Back, Style, init
import pathlib
import sys
from sys import platform
import argparse
from argparse import ArgumentParser, Namespace
import yaml
import asyncio
from courts_scraper.scraper import Scraper
def init_argparser() -> ArgumentParser:
argparser = argparse.ArgumentParser(
description="List fish in aquarium.",
)
argparser.add_argument(
"--config", "-c",
help="Path to the config file",
type=pathlib.Path,
)
argparser.add_argument(
"--domains", "-d",
help="Path to the domains file",
type=pathlib.Path,
)
argparser.add_argument(
"--proxy", "-p",
help="Path to the proxy file",
type=pathlib.Path,
)
argparser.add_argument("--rps", type=int)
return argparser
async def main():
logger.add("project.log")
logger.info("Starting...")
if platform != "linux":
logger.critical("""\nNot for windows, run only on GNU/Linux!\n""")
input()
sys.exit(1)
scraper_settings = dict()
argparser = init_argparser()
args = argparser.parse_args()
if args.config is not None:
with open(args.config) as config_file:
scraper_settings = yaml.safe_load(config_file)
else:
scraper_settings["rps"] = args.rps
with open(args.domains) as domains_file:
scraper_settings["domains"] = domains_file.readlines()
if args.proxy is not None: # Optional argument
with open(args.proxy) as proxy_file:
scraper_settings["proxy"] = proxy_file.readlines()
scraper = Scraper(**scraper_settings)
await scraper.scrape()
asyncio.run(main())