Improved GoogleTranslate evaluation script: it now avoids rate-limit or server errors.
@@ -182,11 +182,22 @@ class MMTTranslator(Translator):
return text, decoding_time
+class GoogleRateLimitError(TranslateError):
+ def __init__(self, *args, **kwargs):
+ super(GoogleRateLimitError, self).__init__(*args, **kwargs)
+
+
+class GoogleServerError(TranslateError):
+ def __init__(self, *args, **kwargs):
+ super(GoogleServerError, self).__init__(*args, **kwargs)
+
+
class GoogleTranslate(Translator):
def __init__(self, source_lang, target_lang, key=None, nmt=False):
- Translator.__init__(self, source_lang, target_lang, threads=10)
+ Translator.__init__(self, source_lang, target_lang, threads=5)
self._key = key if key is not None else DEFAULT_GOOGLE_KEY
self._nmt = nmt
+ self._delay = 0
self._url = 'https://translation.googleapis.com/language/translate/v2' if self._nmt \
else 'https://www.googleapis.com/language/translate/v2'
@@ -198,20 +209,31 @@ class GoogleTranslate(Translator):
def _pack_error(request):
json = request.json()
- daily_limit = False
-
if request.status_code == 403:
for error in json['error']['errors']:
if error['reason'] == 'dailyLimitExceeded':
- daily_limit = True
- break
-
- if daily_limit:
return TranslateError('Google Translate free quota is over. Please use option --gt-key'
' to specify your GT API key.')
- else:
+ elif error['reason'] == 'userRateLimitExceeded':
+ return GoogleRateLimitError('Google Translate rate limit exceeded')
+ elif 500 <= request.status_code < 600:
+ return GoogleServerError('Google Translate server error (%d): %s' %
+ (request.status_code, json['error']['message']))
+
return TranslateError('Google Translate error (%d): %s' % (request.status_code, json['error']['message']))
+ def _increment_delay(self):
+ if self._delay < 0.002:
+ self._delay = 0.05
+ else:
+ self._delay = min(1, self._delay * 1.05)
+
+ def _decrement_delay(self):
+ self._delay *= 0.95
+
+ if self._delay < 0.002:
+ self._delay = 0
+
def _get_translation(self, line, corpus):
data = {
'source': self.source_lang,
@@ -228,12 +250,37 @@ class GoogleTranslate(Translator):
'X-HTTP-Method-Override': 'GET'
}
+ rate_limit_reached = False
+ server_error_count = 0
+
+ while True:
+ if self._delay > 0:
+ delay = self._delay * random.uniform(0.5, 1)
+ time.sleep(delay)
+
begin = time.time()
r = requests.post(self._url, data=data, headers=headers)
elapsed = time.time() - begin
if r.status_code != requests.codes.ok:
- raise self._pack_error(r)
+ e = self._pack_error(r)
+ if isinstance(e, GoogleRateLimitError):
+ rate_limit_reached = True
+ self._increment_delay()
+ elif isinstance(e, GoogleServerError):
+ server_error_count += 1
+
+ if server_error_count < 10:
+ time.sleep(1.)
+ else:
+ raise e
+ else:
+ raise e
+ else:
+ break
+
+ if not rate_limit_reached and self._delay > 0:
+ self._decrement_delay()
text = r.json()['data']['translations'][0]['translatedText']
