diff --git a/main.py b/main.py index c94f329..161db2a 100644 --- a/main.py +++ b/main.py @@ -64,6 +64,7 @@ async def main(): sys.exit(1) config = load_config() + scraper = Scraper(**config["settings"]) asyncio.run(main()) diff --git a/requirements.txt b/requirements.txt index 8627b63..736c648 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ loguru pretty_errors colorama -pyyaml \ No newline at end of file +pyyaml +beautifulsoup4 \ No newline at end of file diff --git a/scraper.py b/scraper.py new file mode 100644 index 0000000..488525e --- /dev/null +++ b/scraper.py @@ -0,0 +1,15 @@ +from bs4 import BeautifulSoup + + +class Scraper: + def __init__( + self, domains: list[str], proxy: list[str], rps_min: int, rps_max: int + ): + self.domains = domains + self.proxy = proxy + + self.rps_min = rps_min + self.rps_max = rps_max + + async def scrape(self): + ...