From 7361974b7665e6564988d16ede6281a7077a5728 Mon Sep 17 00:00:00 2001 From: Eddy Comyn-Platt <53045993+EddyCMWF@users.noreply.github.com> Date: Mon, 27 Feb 2023 11:08:31 +0000 Subject: Assertion error for cdsapi key (#60) --- cdsapi/api.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cdsapi/api.py b/cdsapi/api.py index a603716..79c2ef3 100644 --- a/cdsapi/api.py +++ b/cdsapi/api.py @@ -322,6 +322,10 @@ class Client(object): self.session = session self.session.auth = tuple(self.key.split(":", 2)) + assert len(self.session.auth)==2, ( + "The cdsapi key provided is not the correct format, please ensure it conforms to:\n" + ":" + ) self.metadata = metadata self.forget = forget -- cgit v1.2.3 From 83a1a8a64792008e6fe83e5156369b3ebd26fb97 Mon Sep 17 00:00:00 2001 From: Eddy Comyn-Platt <53045993+EddyCMWF@users.noreply.github.com> Date: Mon, 27 Feb 2023 14:04:25 +0000 Subject: Feature/fix GitHub actions (#62) Use environment variables instead of files containing secrets Remove deprecated python versions and add new python versions 3.10, 3.11 --- .github/workflows/check-and-publish.yml | 19 ++++--------------- 1 file changed, 4 insertions(+), 15 deletions(-) diff --git a/.github/workflows/check-and-publish.yml b/.github/workflows/check-and-publish.yml index 10831bf..9b783de 100644 --- a/.github/workflows/check-and-publish.yml +++ b/.github/workflows/check-and-publish.yml @@ -17,7 +17,7 @@ jobs: fail-fast: false matrix: platform: [windows-latest, ubuntu-latest, macos-latest] - python-version: ["2.7", "3.6", "3.7", "3.8", "3.9"] + python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"] name: Python ${{ matrix.python-version }} on ${{ matrix.platform }} runs-on: ${{ matrix.platform }} @@ -31,21 +31,10 @@ jobs: with: python-version: ${{ matrix.python-version }} - - name: Tokens - shell: python - env: - CDSAPIRC: ${{ secrets.CDSAPIRC }} - - run: | - from __future__ import print_function - import os - for n in ('CDSAPIRC',): - m = os.path.expanduser("~/." + n.lower()) - if os.environ[n]: - with open(m, "w") as f: - print(os.environ[n], file=f) - - name: Tests + env: + CDSAPI_URL: https://cds.climate.copernicus.eu/api/v2 + CDSAPI_KEY: ${{ secrets.CDSAPI_KEY }} run: | python setup.py develop pip install pytest -- cgit v1.2.3 From 503efc0dc7cb261d659cf0d61d979eba5184b330 Mon Sep 17 00:00:00 2001 From: Eddy Comyn-Platt <53045993+EddyCMWF@users.noreply.github.com> Date: Mon, 27 Feb 2023 15:45:32 +0000 Subject: github actions rebase (#64) --- cdsapi/api.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/cdsapi/api.py b/cdsapi/api.py index 79c2ef3..17f8e51 100644 --- a/cdsapi/api.py +++ b/cdsapi/api.py @@ -295,11 +295,15 @@ class Client(object): url = config.get("url") if verify is None: - verify = int(config.get("verify", 1)) + verify = bool(int(config.get("verify", 1))) if url is None or key is None or key is None: raise Exception("Missing/incomplete configuration file: %s" % (dotrc)) + # If verify is still None, then we set to default value of True + if verify is None: + verify = True + self.url = url self.key = key -- cgit v1.2.3 From 85351e4a835a16fff2d64da0f203e0c523432cc1 Mon Sep 17 00:00:00 2001 From: Eddy Comyn-Platt <53045993+EddyCMWF@users.noreply.github.com> Date: Tue, 28 Feb 2023 12:09:54 +0000 Subject: Code quality tests added (#65) * code quality checks to github actions * tox.ini options set --- .github/workflows/check-and-publish.yml | 18 ++++++++++++++++-- README.rst | 2 +- cdsapi/api.py | 29 ++++++++--------------------- docker/retrieve.py | 4 +++- examples/example-era5-update.py | 1 + tests/test_api.py | 7 ++++--- tox.ini | 8 ++++++++ 7 files changed, 41 insertions(+), 28 deletions(-) diff --git a/.github/workflows/check-and-publish.yml b/.github/workflows/check-and-publish.yml index 9b783de..60e0319 100644 --- a/.github/workflows/check-and-publish.yml +++ b/.github/workflows/check-and-publish.yml @@ -12,7 +12,21 @@ on: jobs: - checks: + quality-checks: + name: Code QA + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - run: pip install black flake8 isort + - run: black --version + - run: isort --version + - run: flake8 --version + - run: isort --check . + - run: black --check . + - run: flake8 . + + platform-checks: + needs: quality-checks strategy: fail-fast: false matrix: @@ -41,11 +55,11 @@ jobs: pytest deploy: + needs: platform-checks if: ${{ github.event_name == 'release' }} name: Upload to Pypi - needs: checks runs-on: ubuntu-latest diff --git a/README.rst b/README.rst index 166dac0..608c536 100644 --- a/README.rst +++ b/README.rst @@ -10,7 +10,7 @@ Install via `pip` with:: Configure --------- -Get your UID and API key from the CDS portal at the address https://cds.climate.copernicus.eu/user +Get your user ID (UID) and API key from the CDS portal at the address https://cds.climate.copernicus.eu/user and write it into the configuration file, so it looks like:: $ cat ~/.cdsapirc diff --git a/cdsapi/api.py b/cdsapi/api.py index 17f8e51..27a6b09 100644 --- a/cdsapi/api.py +++ b/cdsapi/api.py @@ -9,10 +9,11 @@ from __future__ import absolute_import, division, print_function, unicode_literals import json -import time -import os import logging +import os +import time import uuid + import requests try: @@ -35,16 +36,15 @@ def bytes_to_string(n): def read_config(path): config = {} with open(path) as f: - for l in f.readlines(): - if ":" in l: - k, v = l.strip().split(":", 1) + for line in f.readlines(): + if ":" in line: + k, v = line.strip().split(":", 1) if k in ("url", "key", "verify"): config[k] = v.strip() return config def toJSON(obj): - to_json = getattr(obj, "toJSON", None) if callable(to_json): return to_json() @@ -63,7 +63,6 @@ def toJSON(obj): class Result(object): def __init__(self, client, reply): - self.reply = reply self._url = client.url @@ -95,7 +94,6 @@ class Result(object): return r def _download(self, url, size, target): - if target is None: target = url.split("/")[-1] @@ -109,7 +107,6 @@ class Result(object): headers = None while tries < self.retry_max: - r = self.robust(self.session.get)( url, stream=True, @@ -211,7 +208,6 @@ class Result(object): self.reply = result.json() def delete(self): - if self._deleted: return @@ -245,7 +241,6 @@ class Result(object): class Client(object): - logger = logging.getLogger("cdsapi") def __init__( @@ -270,9 +265,7 @@ class Client(object): forget=False, session=requests.Session(), ): - if not quiet: - if debug: level = logging.DEBUG else: @@ -326,7 +319,7 @@ class Client(object): self.session = session self.session.auth = tuple(self.key.split(":", 2)) - assert len(self.session.auth)==2, ( + assert len(self.session.auth) == 2, ( "The cdsapi key provided is not the correct format, please ensure it conforms to:\n" ":" ) @@ -361,7 +354,7 @@ class Client(object): def service(self, name, *args, **kwargs): self.delete = False # Don't delete results name = "/".join(name.split(".")) - mimic_ui = kwargs.pop('mimic_ui', False) + mimic_ui = kwargs.pop("mimic_ui", False) # To mimic the CDS ui the request should be populated directly with the kwargs if mimic_ui: request = kwargs @@ -409,7 +402,6 @@ class Client(object): pass def _api(self, url, request, method): - self._status(url) session = self.session @@ -435,7 +427,6 @@ class Client(object): result.raise_for_status() reply = result.json() except Exception: - if reply is None: try: reply = result.json() @@ -465,7 +456,6 @@ class Client(object): sleep = 1 while True: - self.debug("REPLY %s", reply) if reply["state"] != self.last_state: @@ -543,7 +533,6 @@ class Client(object): self.logger.debug(*args, **kwargs) def _download(self, results, targets=None): - if isinstance(results, Result): if targets: path = targets.pop(0) @@ -555,7 +544,6 @@ class Client(object): return [self._download(x, targets) for x in results] if isinstance(results, dict): - if "location" in results and "contentLength" in results: reply = dict( location=results["location"], @@ -594,7 +582,6 @@ class Client(object): def robust(self, call): def retriable(code, reason): - if code in [ requests.codes.internal_server_error, requests.codes.bad_gateway, diff --git a/docker/retrieve.py b/docker/retrieve.py index 01ba3f5..a009fba 100644 --- a/docker/retrieve.py +++ b/docker/retrieve.py @@ -1,4 +1,6 @@ -import json, sys, cdsapi +import json + +import cdsapi with open("/input/request.json") as req: request = json.load(req) diff --git a/examples/example-era5-update.py b/examples/example-era5-update.py index 8b56573..5d10c85 100755 --- a/examples/example-era5-update.py +++ b/examples/example-era5-update.py @@ -9,6 +9,7 @@ # does it submit to any jurisdiction. import time + import cdsapi c = cdsapi.Client(debug=True, wait_until_complete=False) diff --git a/tests/test_api.py b/tests/test_api.py index c3f3766..3725f1f 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -1,8 +1,9 @@ -import cdsapi import os -def test_request(): +import cdsapi + +def test_request(): c = cdsapi.Client() r = c.retrieve( @@ -11,7 +12,7 @@ def test_request(): "variable": "2t", "product_type": "reanalysis", "date": "2012-12-01", - "time": "12:00" + "time": "12:00", }, ) diff --git a/tox.ini b/tox.ini index 1d50849..8553cf4 100644 --- a/tox.ini +++ b/tox.ini @@ -14,3 +14,11 @@ commands = pytest -v --pep8 --mccabe --cov=cdsapi --cov-report=html --cache-clea [testenv:deps] deps = commands = python setup.py test + + +[black] +line_length=120 +[isort] +profile=black +[flake8] +max-line-length = 120 \ No newline at end of file -- cgit v1.2.3 From 5862e1fb09c99a47f5adff0d6ac7625331edf28b Mon Sep 17 00:00:00 2001 From: Kevin Anderson Date: Tue, 28 Feb 2023 09:37:47 -0500 Subject: don't use logging.basicConfig (#47) * replace basicConfig with setLevel and addHandler * avoid duplicate handlers --- cdsapi/api.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/cdsapi/api.py b/cdsapi/api.py index 27a6b09..68befdb 100644 --- a/cdsapi/api.py +++ b/cdsapi/api.py @@ -271,9 +271,14 @@ class Client(object): else: level = logging.INFO - logging.basicConfig( - level=level, format="%(asctime)s %(levelname)s %(message)s" - ) + self.logger.setLevel(level) + + # avoid duplicate handlers when creating more than one Client + if not self.logger.handlers: + formatter = logging.Formatter("%(asctime)s %(levelname)s %(message)s") + handler = logging.StreamHandler() + handler.setFormatter(formatter) + self.logger.addHandler(handler) dotrc = os.environ.get("CDSAPI_RC", os.path.expanduser("~/.cdsapirc")) -- cgit v1.2.3 From c41b0dbc50572d96ecc4cb2298a887f6fcbd5daa Mon Sep 17 00:00:00 2001 From: Luca Fabbri Date: Tue, 28 Feb 2023 16:58:14 +0100 Subject: Replaced default python-requests user-agent with a custom ones (#54) * Replaced default python-requests user-agent with a custom ones This can improve analytics and QOS a bit * isort * made black happy --- cdsapi/api.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/cdsapi/api.py b/cdsapi/api.py index 68befdb..0211856 100644 --- a/cdsapi/api.py +++ b/cdsapi/api.py @@ -14,6 +14,7 @@ import os import time import uuid +import pkg_resources import requests try: @@ -324,6 +325,11 @@ class Client(object): self.session = session self.session.auth = tuple(self.key.split(":", 2)) + self.session.headers = { + "User-Agent": "cdsapi/%s" + % pkg_resources.get_distribution("cdsapi").version, + } + assert len(self.session.auth) == 2, ( "The cdsapi key provided is not the correct format, please ensure it conforms to:\n" ":" -- cgit v1.2.3 From 893dbd6ea7dfb67c85ecb3cc30fc43b764d62a1c Mon Sep 17 00:00:00 2001 From: Eddy Comyn-Platt <53045993+EddyCMWF@users.noreply.github.com> Date: Tue, 28 Feb 2023 16:03:32 +0000 Subject: update metadata (#68) * metadata updates --- setup.py | 12 ++++++------ tox.ini | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/setup.py b/setup.py index 706ac21..8a46a07 100644 --- a/setup.py +++ b/setup.py @@ -50,15 +50,15 @@ setuptools.setup( ], zip_safe=True, classifiers=[ - "Development Status :: 3 - Alpha", + "Development Status :: 4 - Beta", "Intended Audience :: Developers", "Programming Language :: Python", - "Programming Language :: Python :: 2", - "Programming Language :: Python :: 2.7", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.4", - "Programming Language :: Python :: 3.5", - "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Implementation :: PyPy", "Operating System :: OS Independent", diff --git a/tox.ini b/tox.ini index 8553cf4..d97570f 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = qc, py36, py35, py34, py27, pypy3, pypy, deps +envlist = qc, py311, py310, py39, py38, py37, pypy3, pypy, deps [testenv] setenv = PYTHONPATH = {toxinidir} -- cgit v1.2.3 From 30f452163784369571b2d1b478eaae8427461623 Mon Sep 17 00:00:00 2001 From: LukeJones123 <95484041+LukeJones123@users.noreply.github.com> Date: Tue, 28 Feb 2023 16:04:19 +0000 Subject: Bugfix/timeout and sleep (#53) * Client.status uses timeout & no final sleep * Result update and delete use timeout --- cdsapi/api.py | 52 +++++++++++++++++++++++++--------------------------- 1 file changed, 25 insertions(+), 27 deletions(-) diff --git a/cdsapi/api.py b/cdsapi/api.py index 0211856..47dce4f 100644 --- a/cdsapi/api.py +++ b/cdsapi/api.py @@ -204,7 +204,8 @@ class Result(object): task_url = "%s/tasks/%s" % (self._url, request_id) self.debug("GET %s", task_url) - result = self.robust(self.session.get)(task_url, verify=self.verify) + result = self.robust(self.session.get)(task_url, verify=self.verify, + timeout=self.timeout) result.raise_for_status() self.reply = result.json() @@ -218,7 +219,8 @@ class Result(object): task_url = "%s/tasks/%s" % (self._url, rid) self.debug("DELETE %s", task_url) - delete = self.session.delete(task_url, verify=self.verify) + delete = self.session.delete(task_url, verify=self.verify, + timeout=self.timeout) self.debug("DELETE returns %s %s", delete.status_code, delete.reason) try: @@ -389,7 +391,7 @@ class Client(object): def status(self, context=None): url = "%s/status.json" % (self.url,) - r = self.session.get(url, verify=self.verify) + r = self.session.get(url, verify=self.verify, timeout=self.timeout) r.raise_for_status() return r.json() @@ -607,40 +609,36 @@ class Client(object): def wrapped(*args, **kwargs): tries = 0 - while tries < self.retry_max: + while True: + + txt = "Error" try: - r = call(*args, **kwargs) + resp = call(*args, **kwargs) except ( requests.exceptions.ConnectionError, requests.exceptions.ReadTimeout, ) as e: - r = None - self.warning( - "Recovering from connection error [%s], attemps %s of %s", - e, - tries, - self.retry_max, - ) - - if r is not None: - if not retriable(r.status_code, r.reason): - return r + resp = None + txt = f"Connection error: [{e}]" + + if resp is not None: + if not retriable(resp.status_code, resp.reason): + break try: - self.warning(r.json()["reason"]) + self.warning(resp.json()["reason"]) except Exception: pass - self.warning( - "Recovering from HTTP error [%s %s], attemps %s of %s", - r.status_code, - r.reason, - tries, - self.retry_max, - ) + txt = f"HTTP error: [{resp.status_code} {resp.reason}]" tries += 1 + self.warning(txt + f". Attempt {tries} of {self.retry_max}.") + if tries < self.retry_max: + self.warning(f"Retrying in {self.sleep_max} seconds") + time.sleep(self.sleep_max) + self.info("Retrying now...") + else: + raise Exception('Could not connect') - self.warning("Retrying in %s seconds", self.sleep_max) - time.sleep(self.sleep_max) - self.info("Retrying now...") + return resp return wrapped -- cgit v1.2.3 From 1f41f8243233476f98ad339ac26d986883ea016e Mon Sep 17 00:00:00 2001 From: Eddy Comyn-Platt <53045993+EddyCMWF@users.noreply.github.com> Date: Tue, 28 Feb 2023 16:08:35 +0000 Subject: small qa update (#69) * code quality checks --- cdsapi/api.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/cdsapi/api.py b/cdsapi/api.py index 47dce4f..bb0138f 100644 --- a/cdsapi/api.py +++ b/cdsapi/api.py @@ -204,8 +204,9 @@ class Result(object): task_url = "%s/tasks/%s" % (self._url, request_id) self.debug("GET %s", task_url) - result = self.robust(self.session.get)(task_url, verify=self.verify, - timeout=self.timeout) + result = self.robust(self.session.get)( + task_url, verify=self.verify, timeout=self.timeout + ) result.raise_for_status() self.reply = result.json() @@ -219,8 +220,9 @@ class Result(object): task_url = "%s/tasks/%s" % (self._url, rid) self.debug("DELETE %s", task_url) - delete = self.session.delete(task_url, verify=self.verify, - timeout=self.timeout) + delete = self.session.delete( + task_url, verify=self.verify, timeout=self.timeout + ) self.debug("DELETE returns %s %s", delete.status_code, delete.reason) try: @@ -610,7 +612,6 @@ class Client(object): def wrapped(*args, **kwargs): tries = 0 while True: - txt = "Error" try: resp = call(*args, **kwargs) @@ -637,7 +638,7 @@ class Client(object): time.sleep(self.sleep_max) self.info("Retrying now...") else: - raise Exception('Could not connect') + raise Exception("Could not connect") return resp -- cgit v1.2.3 From acb8c791e9a8e552f7c56ccb7ae4f656cf783680 Mon Sep 17 00:00:00 2001 From: Gionata Biavati <4508477+gbiavati@users.noreply.github.com> Date: Wed, 15 Mar 2023 17:17:56 +0100 Subject: Update setup.py (#70) Preparing new release --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 8a46a07..87fae0d 100644 --- a/setup.py +++ b/setup.py @@ -30,7 +30,7 @@ def read(fname): return io.open(file_path, encoding="utf-8").read() -version = "0.5.1" +version = "0.6.0" setuptools.setup( -- cgit v1.2.3 From 9b882a8f14495652c6fcc5d0eb0bd3de60e4bea8 Mon Sep 17 00:00:00 2001 From: Eddy Date: Thu, 16 Mar 2023 14:27:20 +0000 Subject: adding build to .gitingore + release --- .gitignore | 1 + setup.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 2a1348b..46aafc5 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,4 @@ *.grib *.nc cdsapi.egg-info +build/ diff --git a/setup.py b/setup.py index 87fae0d..1692456 100644 --- a/setup.py +++ b/setup.py @@ -30,7 +30,7 @@ def read(fname): return io.open(file_path, encoding="utf-8").read() -version = "0.6.0" +version = "0.6.1" setuptools.setup( -- cgit v1.2.3