feat: add scrape_async to ZenRowsScraper for async proxy fetches
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -55,3 +55,22 @@ class ZenRowsScraper:
|
|||||||
f"ZenRows scraped url: {url}. Returned status: {response.status_code} in {elapsed_time}ms"
|
f"ZenRows scraped url: {url}. Returned status: {response.status_code} in {elapsed_time}ms"
|
||||||
)
|
)
|
||||||
return response
|
return response
|
||||||
|
|
||||||
|
async def scrape_async(self, url: str, premium_proxy: bool = False):
|
||||||
|
"""Fetch a URL asynchronously through ZenRows as a plain proxy (no JS rendering).
|
||||||
|
|
||||||
|
Args
|
||||||
|
url: URL to fetch
|
||||||
|
premium_proxy: use residential proxy (ZenRows premium_proxy=true)
|
||||||
|
"""
|
||||||
|
self.logger.debug(f"ZenRows fetching url: {url}...")
|
||||||
|
params: dict = {}
|
||||||
|
if premium_proxy:
|
||||||
|
params["premium_proxy"] = "true"
|
||||||
|
start = time.monotonic()
|
||||||
|
response = await self.client.get_async(url, params=params)
|
||||||
|
elapsed_time = round((time.monotonic() - start) * 1000)
|
||||||
|
self.logger.debug(
|
||||||
|
f"ZenRows fetched url: {url}. Returned status: {response.status_code} in {elapsed_time}ms"
|
||||||
|
)
|
||||||
|
return response
|
||||||
|
|||||||
Reference in New Issue
Block a user