Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
def get_head_response(url, headers=fake_headers):
try:
req = Request(url, headers=headers)
req.get_method = lambda: 'HEAD'
response = urlopen(req)
except IOError as e:
# if HEAD method is not supported
if 'HTTP Error 405' in str(e):
req = Request(url, headers=headers)
response = urlopen(req)
response.close()
else:
raise
# urllib will follow redirections and it's too much code to tell urllib
# not to do that
return response
def get_head_response(url, headers=fake_headers):
try:
req = Request(url, headers=headers)
req.get_method = lambda: 'HEAD'
response = urlopen(req)
except IOError as e:
# if HEAD method is not supported
if 'HTTP Error 405' in str(e):
req = Request(url, headers=headers)
response = urlopen(req)
response.close()
else:
raise
# urllib will follow redirections and it's too much code to tell urllib
# not to do that
return response
def get_content(url, headers=fake_headers, data=None, charset=None):
"""Gets the content of a URL via sending a HTTP GET request.
Args:
url: A URL.
headers: Request headers used by the client.
decoded: Whether decode the response body using UTF-8 or the charset specified in Content-Type.
Returns:
The content as a string.
"""
logger.debug("get_content> URL: " + url)
req = Request(url, headers=headers, data=data)
#if cookies_txt:
# cookies_txt.add_cookie_header(req)
# req.headers.update(req.unredirected_hdrs)
response = urlopen(req)
data = response.read()
# Handle HTTP compression for gzip and deflate (zlib)
resheader = response.info()
if 'Content-Encoding' in resheader:
content_encoding = resheader['Content-Encoding']
elif hasattr(resheader, 'get_payload'):
payload = resheader.get_payload()
if isinstance(payload, str):
content_encoding = match1(payload, r'Content-Encoding:\s*([\w-]+)')
else:
content_encoding = None