diff options
Diffstat (limited to 'cdsapi/api.py')
| -rw-r--r-- | cdsapi/api.py | 107 |
1 files changed, 56 insertions, 51 deletions
diff --git a/cdsapi/api.py b/cdsapi/api.py index a603716..bb0138f 100644 --- a/cdsapi/api.py +++ b/cdsapi/api.py @@ -9,10 +9,12 @@ from __future__ import absolute_import, division, print_function, unicode_literals import json -import time -import os import logging +import os +import time import uuid + +import pkg_resources import requests try: @@ -35,16 +37,15 @@ def bytes_to_string(n): def read_config(path): config = {} with open(path) as f: - for l in f.readlines(): - if ":" in l: - k, v = l.strip().split(":", 1) + for line in f.readlines(): + if ":" in line: + k, v = line.strip().split(":", 1) if k in ("url", "key", "verify"): config[k] = v.strip() return config def toJSON(obj): - to_json = getattr(obj, "toJSON", None) if callable(to_json): return to_json() @@ -63,7 +64,6 @@ def toJSON(obj): class Result(object): def __init__(self, client, reply): - self.reply = reply self._url = client.url @@ -95,7 +95,6 @@ class Result(object): return r def _download(self, url, size, target): - if target is None: target = url.split("/")[-1] @@ -109,7 +108,6 @@ class Result(object): headers = None while tries < self.retry_max: - r = self.robust(self.session.get)( url, stream=True, @@ -206,12 +204,13 @@ class Result(object): task_url = "%s/tasks/%s" % (self._url, request_id) self.debug("GET %s", task_url) - result = self.robust(self.session.get)(task_url, verify=self.verify) + result = self.robust(self.session.get)( + task_url, verify=self.verify, timeout=self.timeout + ) result.raise_for_status() self.reply = result.json() def delete(self): - if self._deleted: return @@ -221,7 +220,9 @@ class Result(object): task_url = "%s/tasks/%s" % (self._url, rid) self.debug("DELETE %s", task_url) - delete = self.session.delete(task_url, verify=self.verify) + delete = self.session.delete( + task_url, verify=self.verify, timeout=self.timeout + ) self.debug("DELETE returns %s %s", delete.status_code, delete.reason) try: @@ -245,7 +246,6 @@ class Result(object): class Client(object): - logger = logging.getLogger("cdsapi") def __init__( @@ -270,17 +270,20 @@ class Client(object): forget=False, session=requests.Session(), ): - if not quiet: - if debug: level = logging.DEBUG else: level = logging.INFO - logging.basicConfig( - level=level, format="%(asctime)s %(levelname)s %(message)s" - ) + self.logger.setLevel(level) + + # avoid duplicate handlers when creating more than one Client + if not self.logger.handlers: + formatter = logging.Formatter("%(asctime)s %(levelname)s %(message)s") + handler = logging.StreamHandler() + handler.setFormatter(formatter) + self.logger.addHandler(handler) dotrc = os.environ.get("CDSAPI_RC", os.path.expanduser("~/.cdsapirc")) @@ -295,11 +298,15 @@ class Client(object): url = config.get("url") if verify is None: - verify = int(config.get("verify", 1)) + verify = bool(int(config.get("verify", 1))) if url is None or key is None or key is None: raise Exception("Missing/incomplete configuration file: %s" % (dotrc)) + # If verify is still None, then we set to default value of True + if verify is None: + verify = True + self.url = url self.key = key @@ -322,6 +329,15 @@ class Client(object): self.session = session self.session.auth = tuple(self.key.split(":", 2)) + self.session.headers = { + "User-Agent": "cdsapi/%s" + % pkg_resources.get_distribution("cdsapi").version, + } + + assert len(self.session.auth) == 2, ( + "The cdsapi key provided is not the correct format, please ensure it conforms to:\n" + "<UID>:<APIKEY>" + ) self.metadata = metadata self.forget = forget @@ -353,7 +369,7 @@ class Client(object): def service(self, name, *args, **kwargs): self.delete = False # Don't delete results name = "/".join(name.split(".")) - mimic_ui = kwargs.pop('mimic_ui', False) + mimic_ui = kwargs.pop("mimic_ui", False) # To mimic the CDS ui the request should be populated directly with the kwargs if mimic_ui: request = kwargs @@ -377,7 +393,7 @@ class Client(object): def status(self, context=None): url = "%s/status.json" % (self.url,) - r = self.session.get(url, verify=self.verify) + r = self.session.get(url, verify=self.verify, timeout=self.timeout) r.raise_for_status() return r.json() @@ -401,7 +417,6 @@ class Client(object): pass def _api(self, url, request, method): - self._status(url) session = self.session @@ -427,7 +442,6 @@ class Client(object): result.raise_for_status() reply = result.json() except Exception: - if reply is None: try: reply = result.json() @@ -457,7 +471,6 @@ class Client(object): sleep = 1 while True: - self.debug("REPLY %s", reply) if reply["state"] != self.last_state: @@ -535,7 +548,6 @@ class Client(object): self.logger.debug(*args, **kwargs) def _download(self, results, targets=None): - if isinstance(results, Result): if targets: path = targets.pop(0) @@ -547,7 +559,6 @@ class Client(object): return [self._download(x, targets) for x in results] if isinstance(results, dict): - if "location" in results and "contentLength" in results: reply = dict( location=results["location"], @@ -586,7 +597,6 @@ class Client(object): def robust(self, call): def retriable(code, reason): - if code in [ requests.codes.internal_server_error, requests.codes.bad_gateway, @@ -601,40 +611,35 @@ class Client(object): def wrapped(*args, **kwargs): tries = 0 - while tries < self.retry_max: + while True: + txt = "Error" try: - r = call(*args, **kwargs) + resp = call(*args, **kwargs) except ( requests.exceptions.ConnectionError, requests.exceptions.ReadTimeout, ) as e: - r = None - self.warning( - "Recovering from connection error [%s], attemps %s of %s", - e, - tries, - self.retry_max, - ) - - if r is not None: - if not retriable(r.status_code, r.reason): - return r + resp = None + txt = f"Connection error: [{e}]" + + if resp is not None: + if not retriable(resp.status_code, resp.reason): + break try: - self.warning(r.json()["reason"]) + self.warning(resp.json()["reason"]) except Exception: pass - self.warning( - "Recovering from HTTP error [%s %s], attemps %s of %s", - r.status_code, - r.reason, - tries, - self.retry_max, - ) + txt = f"HTTP error: [{resp.status_code} {resp.reason}]" tries += 1 + self.warning(txt + f". Attempt {tries} of {self.retry_max}.") + if tries < self.retry_max: + self.warning(f"Retrying in {self.sleep_max} seconds") + time.sleep(self.sleep_max) + self.info("Retrying now...") + else: + raise Exception("Could not connect") - self.warning("Retrying in %s seconds", self.sleep_max) - time.sleep(self.sleep_max) - self.info("Retrying now...") + return resp return wrapped |