import asyncio

from crawlee.http_clients import HttpxHttpClient
from crawlee.parsel_crawler import ParselCrawler, ParselCrawlingContext
from crawlee.router import Router

parsel_router = Router[ParselCrawlingContext]()


@parsel_router.default_handler
async def request_handler(context: ParselCrawlingContext) -> None:
    context.log.info(f'Parsel processing {context.request.url} ...')

    # Enqueue all links found on the page.
    await context.enqueue_links()


async def main() -> None:
    http_client = HttpxHttpClient(verify=False)
    crawler = ParselCrawler(
        request_handler=parsel_router,
        http_client=http_client,
    )

    # Run the crawler with the initial list of URLs.
    await crawler.run(['https://www.shangbao.com.ph/more/scroll.html'])


if __name__ == '__main__':
    asyncio.run(main())