post and post_async on zenrows scraper
This commit is contained in:
@@ -1,6 +1,6 @@
|
||||
[project]
|
||||
name = "vl-apify-shared"
|
||||
version = "0.1.2"
|
||||
version = "0.1.3"
|
||||
description = "Shared utility package for use with Apify actors"
|
||||
authors = [{ name = "Ryan Byrne", email = "ryanjbyrne30@gmail.com" }]
|
||||
requires-python = ">=3.12"
|
||||
|
||||
@@ -109,3 +109,59 @@ class ZenRowsScraper:
|
||||
f"ZenRows fetched url: {url}. Returned status: {response.status_code} in {elapsed_time}ms"
|
||||
)
|
||||
return response
|
||||
|
||||
def post(
|
||||
self,
|
||||
url: str,
|
||||
data: str,
|
||||
js_render: bool = False,
|
||||
wait: Optional[int] = None,
|
||||
wait_for: Optional[str] = None,
|
||||
premium_proxy: bool = False,
|
||||
js_instructions: Optional[list] = None,
|
||||
headers: Optional[dict] = None,
|
||||
):
|
||||
"""POST to a URL through ZenRows."""
|
||||
self.logger.debug(f"ZenRows posting to url: {url}...")
|
||||
params = _build_params(
|
||||
js_render=js_render,
|
||||
wait=wait,
|
||||
wait_for=wait_for,
|
||||
premium_proxy=premium_proxy,
|
||||
js_instructions=js_instructions,
|
||||
)
|
||||
start = time.monotonic()
|
||||
response = self.client.post(url, params=params, data=data, headers=headers)
|
||||
elapsed_time = round((time.monotonic() - start) * 1000)
|
||||
self.logger.debug(
|
||||
f"ZenRows posted to url: {url}. Returned status: {response.status_code} in {elapsed_time}ms"
|
||||
)
|
||||
return response
|
||||
|
||||
async def post_async(
|
||||
self,
|
||||
url: str,
|
||||
data: str,
|
||||
js_render: bool = False,
|
||||
wait: Optional[int] = None,
|
||||
wait_for: Optional[str] = None,
|
||||
premium_proxy: bool = False,
|
||||
js_instructions: Optional[list] = None,
|
||||
headers: Optional[dict] = None,
|
||||
):
|
||||
"""POST to a URL asynchronously through ZenRows."""
|
||||
self.logger.debug(f"ZenRows posting to url: {url}...")
|
||||
params = _build_params(
|
||||
js_render=js_render,
|
||||
wait=wait,
|
||||
wait_for=wait_for,
|
||||
premium_proxy=premium_proxy,
|
||||
js_instructions=js_instructions,
|
||||
)
|
||||
start = time.monotonic()
|
||||
response = await self.client.post_async(url, params=params, data=data, headers=headers)
|
||||
elapsed_time = round((time.monotonic() - start) * 1000)
|
||||
self.logger.debug(
|
||||
f"ZenRows posted to url: {url}. Returned status: {response.status_code} in {elapsed_time}ms"
|
||||
)
|
||||
return response
|
||||
|
||||
Reference in New Issue
Block a user