Python请求 - 连接错误 - 超过最大重试次数

Question

我正在使用rauth和请求来调用Beatport API。电话工程，但我很偶然地得到以下错误ConnectionError: HTTPSConnectionPool(host='oauth-api.beatport.com', port=443): Max retries exceeded with url

这是追溯。

Traceback (most recent call last): 
    File "<console>", line 1, in <module> 
    File "scraper/songlist_top100.py", line 88, in <module> 
    'sortBy': 'releaseDate ASC'}) 
    File "C:\Python27\lib\site-packages\requests\sessions.py", line 347, in get 
    return self.request('GET', url, **kwargs) 
    File "C:\Python27\lib\site-packages\rauth\session.py", line 208, in request 
    return super(OAuth1Session, self).request(method, url, **req_kwargs) 
    File "C:\Python27\lib\site-packages\requests\sessions.py", line 335, in reques 
t 
    resp = self.send(prep, **send_kwargs) 
    File "C:\Python27\lib\site-packages\requests\sessions.py", line 438, in send 
    r = adapter.send(request, **kwargs) 
    File "C:\Python27\lib\site-packages\requests\adapters.py", line 327, in send 
    raise ConnectionError(e) 
ConnectionError: HTTPSConnectionPool(host='oauth-api.beatport.com', port=443): 
Max retries exceeded with url: /catalog/3/tracks?perPage=150& 
oauth_nonce=xxxxx&oauth_timestamp=xxxxx&facets=artistName%3A 
Avicii&oauth_signature_method=HMAC-SHA1&oauth_version=1.0& 
oauth_consumer_key=xxxxx&oauth_token=xxxxxx&sortBy=releaseDate+ASC 
&oauth_signature=xxxxx%3D&page=3 (Caused by <class 'httplib.BadStatusLine'>: '') 

这里是我的脚本

from rauth import OAuth1Service 
import requests 
from hunt.models import DJ, Song 

def get_obj_or_none(model, **kwargs): 
    try: 
     return model.objects.get(**kwargs) 
    except model.DoesNotExist: 
     return None 

beatport_login = 'xxx' 
beatport_pass = 'xxx' 

beatport = OAuth1Service(
    name='beatport', 
    consumer_key='xxxxx', 
    consumer_secret='xxxxx', 
    request_token_url= 'https://oauth-api.beatport.com/identity/1/oauth/request-token', 
    access_token_url='https://oauth-api.beatport.com/identity/1/oauth/access-token', 
    authorize_url='https://oauth-api.beatport.com/identity/1/oauth/authorize', 
    base_url='https://oauth-api.beatport.com/json/catalog') 

request_token, request_token_secret = beatport.get_request_token(method='POST', data={ 
    'oauth_callback': 'http://www.edmhunters.com'}) 

authorize_url = beatport.get_authorize_url(request_token) 

values = { 
    'oauth_token': request_token, 
    'username': beatport_login, 
    'password': beatport_pass, 
    'submit' : 'Login', 
} 

r = requests.post('https://oauth-api.beatport.com/identity/1/oauth/authorize-submit', data=values) 

verifier = r.url.split("oauth_verifier=",1)[1] 

tokens = beatport.get_raw_access_token(request_token, request_token_secret, method='POST', data={ 
    'oauth_verifier': verifier}) 

token_string = tokens.content 

access_token = token_string[token_string.find('=')+1:token_string.find('&')] 
access_token_secret = token_string[token_string.find('t=')+2:token_string.rfind('&s')] 

session = beatport.get_session((access_token, access_token_secret)) 

for dj in DJ.objects.all(): 
    r = session.get('https://oauth-api.beatport.com/catalog/3/tracks', params={'facets': "artistName:"+dj.name, 'perPage': 150}) 
    count_response = r.json() 
    results = [] 
    for i in range(1, count_response['metadata']['totalPages']+1): 
     r1 = session.get('https://oauth-api.beatport.com/catalog/3/tracks', params={'facets': "artistName:"+dj.name, 
                        'page': i, 
                        'perPage': 150, 
                        'sortBy': 'releaseDate ASC'}) 
     json_response = r1.json() 
     results += json_response['results'] 

    song_list = []  
    for song in results: 
     artists = [artist['name'] for artist in song['artists'] if str(artist['type'])=='artist'] 
     remixers = [artist['name'] for artist in song['artists'] if str(artist['type'])=='remixer'] 
     if not ((dj.name in artists) and ((dj.name not in remixers) if len(remixers)>0 else False)): 
      song_list.append(song) 

    for song in song_list: 
     artists = [artist['name'] for artist in song['artists'] if str(artist['type'])=='artist'] 
     remixers = [artist['name'] for artist in song['artists'] if str(artist['type'])=='remixer'] 
     artist_list = ', '.join(artists) 
     remixer_list = ', '.join(remixers) 
     song_name = song['name'] 
     if not(song_name.lower().find("feat.") == -1): 
      normal_name=song_name[0:song_name.lower().find("feat.")].rstrip() 
     else: 
      normal_name=song_name 

     genre_list=[] 
     for genre in song['genres']: 
      genre_list.append(genre['name']) 
     genres = ', '.join(genre_list) 

     if not get_obj_or_none(Song, name__iexact=song_name, artist=dj): 
      s = Song(song_id=song['id'], 
        name=song_name, 
        title=song['title'], 
        normalized_name=normal_name, 
        artist=dj, 
        artists=artist_list, 
        remixers=remixer_list, 
        release_date=song['releaseDate'], 
        slug=song['slug'], 
        artwork=song['images']['large']['url'], 
        genres=genres) 
      s.save() 
      print "Added song:", s.song_id, s.artist 

为什么我得到上述错误？

Answer 1

它看起来好像Beatport API过载并且有时会过早地关闭连接。你的第一组请求成功了，第3页抛出错误，因为响应是空的。

你真的应该报告给Beatport，但你也许可以通过指示requests模块重试请求解决此问题：

from requests.adapters import HTTPAdapter 

# .... 

session = beatport.get_session((access_token, access_token_secret)) 
session.mount('https://oauth-api.beatport.com', HTTPAdapter(max_retries=5)) 

会重试您的要求的情况下，发生了错误多做几次。