Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
self.request_config = self.request_config or {}
self.headers = self.headers or {}
self.metadata = self.metadata or {}
self.kwargs = self.kwargs or {}
self.request_config = self.request_config or {}
self.is_async_start = is_async_start
self.logger = get_logger(name='Spider')
self.loop = loop
asyncio.set_event_loop(self.loop)
# customize middleware
if isinstance(middleware, list):
self.middleware = reduce(lambda x, y: x + y, middleware)
else:
self.middleware = middleware or Middleware()
# async queue as a producer
self.request_queue = asyncio.Queue()
# semaphore, used for concurrency control
self.sem = asyncio.Semaphore(self.concurrency)
def __add__(self, other):
new_middleware = Middleware()
# asc
new_middleware.request_middleware.extend(self.request_middleware)
new_middleware.request_middleware.extend(other.request_middleware)
# desc
new_middleware.response_middleware.extend(other.response_middleware)
new_middleware.response_middleware.extend(self.response_middleware)
return new_middleware
self.request_config = self.request_config or {}
self.headers = self.headers or {}
self.metadata = self.metadata or {}
self.kwargs = self.kwargs or {}
self.request_config = self.request_config or {}
self.request_session = ClientSession()
self.cancel_tasks = cancel_tasks
self.is_async_start = is_async_start
# customize middleware
if isinstance(middleware, list):
self.middleware = reduce(lambda x, y: x + y, middleware)
else:
self.middleware = middleware or Middleware()
# async queue as a producer
self.request_queue = asyncio.Queue()
# semaphore, used for concurrency control
self.sem = asyncio.Semaphore(self.concurrency)