diff options
-rw-r--r-- | .github/workflows/check-and-publish.yml | 37 | ||||
-rw-r--r-- | .gitignore | 1 | ||||
-rw-r--r-- | README.rst | 2 | ||||
-rw-r--r-- | cdsapi/api.py | 107 | ||||
-rw-r--r-- | docker/retrieve.py | 4 | ||||
-rwxr-xr-x | examples/example-era5-update.py | 1 | ||||
-rw-r--r-- | setup.py | 14 | ||||
-rw-r--r-- | tests/test_api.py | 7 | ||||
-rw-r--r-- | tox.ini | 10 |
9 files changed, 102 insertions, 81 deletions
diff --git a/.github/workflows/check-and-publish.yml b/.github/workflows/check-and-publish.yml index 10831bf..60e0319 100644 --- a/.github/workflows/check-and-publish.yml +++ b/.github/workflows/check-and-publish.yml @@ -12,12 +12,26 @@ on: jobs: - checks: + quality-checks: + name: Code QA + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - run: pip install black flake8 isort + - run: black --version + - run: isort --version + - run: flake8 --version + - run: isort --check . + - run: black --check . + - run: flake8 . + + platform-checks: + needs: quality-checks strategy: fail-fast: false matrix: platform: [windows-latest, ubuntu-latest, macos-latest] - python-version: ["2.7", "3.6", "3.7", "3.8", "3.9"] + python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"] name: Python ${{ matrix.python-version }} on ${{ matrix.platform }} runs-on: ${{ matrix.platform }} @@ -31,32 +45,21 @@ jobs: with: python-version: ${{ matrix.python-version }} - - name: Tokens - shell: python - env: - CDSAPIRC: ${{ secrets.CDSAPIRC }} - - run: | - from __future__ import print_function - import os - for n in ('CDSAPIRC',): - m = os.path.expanduser("~/." + n.lower()) - if os.environ[n]: - with open(m, "w") as f: - print(os.environ[n], file=f) - - name: Tests + env: + CDSAPI_URL: https://cds.climate.copernicus.eu/api/v2 + CDSAPI_KEY: ${{ secrets.CDSAPI_KEY }} run: | python setup.py develop pip install pytest pytest deploy: + needs: platform-checks if: ${{ github.event_name == 'release' }} name: Upload to Pypi - needs: checks runs-on: ubuntu-latest @@ -7,3 +7,4 @@ *.grib *.nc cdsapi.egg-info +build/ @@ -10,7 +10,7 @@ Install via `pip` with:: Configure --------- -Get your UID and API key from the CDS portal at the address https://cds.climate.copernicus.eu/user +Get your user ID (UID) and API key from the CDS portal at the address https://cds.climate.copernicus.eu/user and write it into the configuration file, so it looks like:: $ cat ~/.cdsapirc diff --git a/cdsapi/api.py b/cdsapi/api.py index a603716..bb0138f 100644 --- a/cdsapi/api.py +++ b/cdsapi/api.py @@ -9,10 +9,12 @@ from __future__ import absolute_import, division, print_function, unicode_literals import json -import time -import os import logging +import os +import time import uuid + +import pkg_resources import requests try: @@ -35,16 +37,15 @@ def bytes_to_string(n): def read_config(path): config = {} with open(path) as f: - for l in f.readlines(): - if ":" in l: - k, v = l.strip().split(":", 1) + for line in f.readlines(): + if ":" in line: + k, v = line.strip().split(":", 1) if k in ("url", "key", "verify"): config[k] = v.strip() return config def toJSON(obj): - to_json = getattr(obj, "toJSON", None) if callable(to_json): return to_json() @@ -63,7 +64,6 @@ def toJSON(obj): class Result(object): def __init__(self, client, reply): - self.reply = reply self._url = client.url @@ -95,7 +95,6 @@ class Result(object): return r def _download(self, url, size, target): - if target is None: target = url.split("/")[-1] @@ -109,7 +108,6 @@ class Result(object): headers = None while tries < self.retry_max: - r = self.robust(self.session.get)( url, stream=True, @@ -206,12 +204,13 @@ class Result(object): task_url = "%s/tasks/%s" % (self._url, request_id) self.debug("GET %s", task_url) - result = self.robust(self.session.get)(task_url, verify=self.verify) + result = self.robust(self.session.get)( + task_url, verify=self.verify, timeout=self.timeout + ) result.raise_for_status() self.reply = result.json() def delete(self): - if self._deleted: return @@ -221,7 +220,9 @@ class Result(object): task_url = "%s/tasks/%s" % (self._url, rid) self.debug("DELETE %s", task_url) - delete = self.session.delete(task_url, verify=self.verify) + delete = self.session.delete( + task_url, verify=self.verify, timeout=self.timeout + ) self.debug("DELETE returns %s %s", delete.status_code, delete.reason) try: @@ -245,7 +246,6 @@ class Result(object): class Client(object): - logger = logging.getLogger("cdsapi") def __init__( @@ -270,17 +270,20 @@ class Client(object): forget=False, session=requests.Session(), ): - if not quiet: - if debug: level = logging.DEBUG else: level = logging.INFO - logging.basicConfig( - level=level, format="%(asctime)s %(levelname)s %(message)s" - ) + self.logger.setLevel(level) + + # avoid duplicate handlers when creating more than one Client + if not self.logger.handlers: + formatter = logging.Formatter("%(asctime)s %(levelname)s %(message)s") + handler = logging.StreamHandler() + handler.setFormatter(formatter) + self.logger.addHandler(handler) dotrc = os.environ.get("CDSAPI_RC", os.path.expanduser("~/.cdsapirc")) @@ -295,11 +298,15 @@ class Client(object): url = config.get("url") if verify is None: - verify = int(config.get("verify", 1)) + verify = bool(int(config.get("verify", 1))) if url is None or key is None or key is None: raise Exception("Missing/incomplete configuration file: %s" % (dotrc)) + # If verify is still None, then we set to default value of True + if verify is None: + verify = True + self.url = url self.key = key @@ -322,6 +329,15 @@ class Client(object): self.session = session self.session.auth = tuple(self.key.split(":", 2)) + self.session.headers = { + "User-Agent": "cdsapi/%s" + % pkg_resources.get_distribution("cdsapi").version, + } + + assert len(self.session.auth) == 2, ( + "The cdsapi key provided is not the correct format, please ensure it conforms to:\n" + "<UID>:<APIKEY>" + ) self.metadata = metadata self.forget = forget @@ -353,7 +369,7 @@ class Client(object): def service(self, name, *args, **kwargs): self.delete = False # Don't delete results name = "/".join(name.split(".")) - mimic_ui = kwargs.pop('mimic_ui', False) + mimic_ui = kwargs.pop("mimic_ui", False) # To mimic the CDS ui the request should be populated directly with the kwargs if mimic_ui: request = kwargs @@ -377,7 +393,7 @@ class Client(object): def status(self, context=None): url = "%s/status.json" % (self.url,) - r = self.session.get(url, verify=self.verify) + r = self.session.get(url, verify=self.verify, timeout=self.timeout) r.raise_for_status() return r.json() @@ -401,7 +417,6 @@ class Client(object): pass def _api(self, url, request, method): - self._status(url) session = self.session @@ -427,7 +442,6 @@ class Client(object): result.raise_for_status() reply = result.json() except Exception: - if reply is None: try: reply = result.json() @@ -457,7 +471,6 @@ class Client(object): sleep = 1 while True: - self.debug("REPLY %s", reply) if reply["state"] != self.last_state: @@ -535,7 +548,6 @@ class Client(object): self.logger.debug(*args, **kwargs) def _download(self, results, targets=None): - if isinstance(results, Result): if targets: path = targets.pop(0) @@ -547,7 +559,6 @@ class Client(object): return [self._download(x, targets) for x in results] if isinstance(results, dict): - if "location" in results and "contentLength" in results: reply = dict( location=results["location"], @@ -586,7 +597,6 @@ class Client(object): def robust(self, call): def retriable(code, reason): - if code in [ requests.codes.internal_server_error, requests.codes.bad_gateway, @@ -601,40 +611,35 @@ class Client(object): def wrapped(*args, **kwargs): tries = 0 - while tries < self.retry_max: + while True: + txt = "Error" try: - r = call(*args, **kwargs) + resp = call(*args, **kwargs) except ( requests.exceptions.ConnectionError, requests.exceptions.ReadTimeout, ) as e: - r = None - self.warning( - "Recovering from connection error [%s], attemps %s of %s", - e, - tries, - self.retry_max, - ) - - if r is not None: - if not retriable(r.status_code, r.reason): - return r + resp = None + txt = f"Connection error: [{e}]" + + if resp is not None: + if not retriable(resp.status_code, resp.reason): + break try: - self.warning(r.json()["reason"]) + self.warning(resp.json()["reason"]) except Exception: pass - self.warning( - "Recovering from HTTP error [%s %s], attemps %s of %s", - r.status_code, - r.reason, - tries, - self.retry_max, - ) + txt = f"HTTP error: [{resp.status_code} {resp.reason}]" tries += 1 + self.warning(txt + f". Attempt {tries} of {self.retry_max}.") + if tries < self.retry_max: + self.warning(f"Retrying in {self.sleep_max} seconds") + time.sleep(self.sleep_max) + self.info("Retrying now...") + else: + raise Exception("Could not connect") - self.warning("Retrying in %s seconds", self.sleep_max) - time.sleep(self.sleep_max) - self.info("Retrying now...") + return resp return wrapped diff --git a/docker/retrieve.py b/docker/retrieve.py index 01ba3f5..a009fba 100644 --- a/docker/retrieve.py +++ b/docker/retrieve.py @@ -1,4 +1,6 @@ -import json, sys, cdsapi +import json + +import cdsapi with open("/input/request.json") as req: request = json.load(req) diff --git a/examples/example-era5-update.py b/examples/example-era5-update.py index 8b56573..5d10c85 100755 --- a/examples/example-era5-update.py +++ b/examples/example-era5-update.py @@ -9,6 +9,7 @@ # does it submit to any jurisdiction. import time + import cdsapi c = cdsapi.Client(debug=True, wait_until_complete=False) @@ -30,7 +30,7 @@ def read(fname): return io.open(file_path, encoding="utf-8").read() -version = "0.5.1" +version = "0.6.1" setuptools.setup( @@ -50,15 +50,15 @@ setuptools.setup( ], zip_safe=True, classifiers=[ - "Development Status :: 3 - Alpha", + "Development Status :: 4 - Beta", "Intended Audience :: Developers", "Programming Language :: Python", - "Programming Language :: Python :: 2", - "Programming Language :: Python :: 2.7", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.4", - "Programming Language :: Python :: 3.5", - "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Implementation :: PyPy", "Operating System :: OS Independent", diff --git a/tests/test_api.py b/tests/test_api.py index c3f3766..3725f1f 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -1,8 +1,9 @@ -import cdsapi import os -def test_request(): +import cdsapi + +def test_request(): c = cdsapi.Client() r = c.retrieve( @@ -11,7 +12,7 @@ def test_request(): "variable": "2t", "product_type": "reanalysis", "date": "2012-12-01", - "time": "12:00" + "time": "12:00", }, ) @@ -1,5 +1,5 @@ [tox] -envlist = qc, py36, py35, py34, py27, pypy3, pypy, deps +envlist = qc, py311, py310, py39, py38, py37, pypy3, pypy, deps [testenv] setenv = PYTHONPATH = {toxinidir} @@ -14,3 +14,11 @@ commands = pytest -v --pep8 --mccabe --cov=cdsapi --cov-report=html --cache-clea [testenv:deps] deps = commands = python setup.py test + + +[black] +line_length=120 +[isort] +profile=black +[flake8] +max-line-length = 120
\ No newline at end of file |