Create Scraper class

This commit is contained in:
riot 2022-09-10 18:42:43 +03:00
parent 11fe999463
commit 1ffe7a5d7f
3 changed files with 18 additions and 1 deletions

View File

@ -64,6 +64,7 @@ async def main():
sys.exit(1)
config = load_config()
scraper = Scraper(**config["settings"])
asyncio.run(main())

View File

@ -1,4 +1,5 @@
loguru
pretty_errors
colorama
pyyaml
pyyaml
beautifulsoup4

15
scraper.py Normal file
View File

@ -0,0 +1,15 @@
from bs4 import BeautifulSoup
class Scraper:
def __init__(
self, domains: list[str], proxy: list[str], rps_min: int, rps_max: int
):
self.domains = domains
self.proxy = proxy
self.rps_min = rps_min
self.rps_max = rps_max
async def scrape(self):
...