Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
async def fetch(self) -> Response:
"""Fetch all the information by using aiohttp"""
if self.request_config.get("DELAY", 0) > 0:
await asyncio.sleep(self.request_config["DELAY"])
timeout = self.request_config.get("TIMEOUT", 10)
try:
async with async_timeout.timeout(timeout):
resp = await self._make_request()
try:
resp_data = await resp.text(encoding=self.encoding)
except UnicodeDecodeError:
resp_data = await resp.read()
response = Response(
url=self.url,
method=self.method,
encoding=resp.get_encoding(),
html=resp_data,
metadata=self.metadata,
cookies=resp.cookies,
headers=resp.headers,
history=resp.history,
status=resp.status,
aws_json=resp.json,
aws_text=resp.text,
aws_read=resp.read,
)
# Retry middleware
aws_valid_response = self.request_config.get("VALID")
if aws_valid_response and iscoroutinefunction(aws_valid_response):
#!/usr/bin/env python
from typing import Callable
from ruia.response import Response
class PyppeteerResponse(Response):
def __init__(
self,
url: str,
method: str,
*,
encoding: str = "",
html: str = "",
page,
browser,
metadata: dict,
cookies,
history,
headers: dict = None,
status: int = -1,
aws_json: Callable = None,
aws_read: Callable = None,
page=page,
browser=self.browser,
metadata=self.metadata,
cookies=await page.cookies(),
headers=resp.headers,
history=(),
status=resp.status,
aws_json=resp.json,
aws_text=resp.text,
aws_read=resp.buffer,
)
else:
async with async_timeout.timeout(timeout):
resp = await self._make_request()
resp_data = await resp.text(encoding=self.encoding)
response = Response(
url=self.url,
method=self.method,
encoding=resp.get_encoding(),
html=resp_data,
metadata=self.metadata,
cookies=resp.cookies,
headers=resp.headers,
history=resp.history,
status=resp.status,
aws_json=resp.json,
aws_text=resp.text,
aws_read=resp.read,
)
if not response.ok:
return await self._retry(
error_msg=f"Request url failed with status {response.status}!"
async def _retry(self, error_msg):
"""Manage request"""
if self.retry_times > 0:
retry_times = self.request_config.get("RETRIES", 3) - self.retry_times + 1
self.logger.error(
f", Retry times: {retry_times}, Retry message: {error_msg}>"
)
self.retry_times -= 1
retry_func = self.request_config.get("RETRY_FUNC")
if retry_func and iscoroutinefunction(retry_func):
request_ins = await retry_func(weakref.proxy(self))
if isinstance(request_ins, Request):
return await request_ins.fetch()
return await self.fetch()
else:
response = Response(
url=self.url,
method=self.method,
metadata=self.metadata,
cookies={},
history=(),
headers=None,
)
return response
async def handle_request(self, request: Request) -> typing.Tuple[AsyncGeneratorType, Response]:
"""
Wrap request with middleware.
:param request:
:return:
"""
callback_result, response = None, None
await self._run_request_middleware(request)
try:
callback_result, response = await request.fetch_callback(self.sem)
except NotImplementedParseError as e:
self.logger.error(e)
except NothingMatchedError as e:
self.logger.error(f'')
except Exception as e:
self.logger.error(f'
async def handle_request(
self, request: Request
) -> typing.Tuple[AsyncGeneratorType, Response]:
"""
Wrap request with middleware.
:param request:
:return:
"""
callback_result, response = None, None
await self._run_request_middleware(request)
try:
callback_result, response = await request.fetch_callback(self.sem)
except NotImplementedParseError as e:
self.logger.error(e)
except NothingMatchedError as e:
self.logger.error(f"")
except Exception as e:
async def fetch_callback(
self, sem: Semaphore
) -> Tuple[AsyncGeneratorType, Response]:
"""
Request the target url and then call the callback function
:param sem: Semaphore
:return: Tuple[AsyncGeneratorType, Response]
"""
try:
async with sem:
response = await self.fetch()
except Exception as e:
response = None
self.logger.error(f"")
if self.callback is not None:
if iscoroutinefunction(self.callback):
callback_result = await self.callback(response)
response.callback_result = callback_result