实战代码
import requests
if __name__ == '__main__':
http_headers = {
'Accept': '*/*',
'Connection': 'keep-alive',
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.116 Safari/537.36'
}
try:
resp = requests.get(url=item, headers=http_headers,timeout=10)
except Exception as e:
continue
print("resp", resp.url)
以上代码未有重试机制,下面加上重试机制加以完善
def get_real_url(url,try_count = 1):
if try_count > 3:
return url
try:
rs = requests.get(url,headers=http_headers,timeout=10)
if rs.status_code > 400:
return get_real_url(url,try_count+1)
return rs.url
except:
return get_real_url(url, try_count + 1)