Merge tag '0.6.1' into gspr/post-bookworm

9 files changed, 102 insertions, 81 deletions

diff --git a/.github/workflows/check-and-publish.yml b/.github/workflows/check-and-publish.yml
index 10831bf..60e0319 100644
--- a/.github/workflows/check-and-publish.yml
+++ b/.github/workflows/check-and-publish.yml
@@ -12,12 +12,26 @@ on:
 
 
 jobs:
-  checks:
+  quality-checks:
+    name: Code QA
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - run: pip install black flake8 isort
+      - run: black --version
+      - run: isort --version
+      - run: flake8 --version
+      - run: isort --check .
+      - run: black --check .
+      - run: flake8 .
+
+  platform-checks:
+    needs: quality-checks
     strategy:
       fail-fast: false
       matrix:
         platform: [windows-latest, ubuntu-latest, macos-latest]
-        python-version: ["2.7", "3.6", "3.7", "3.8", "3.9"]
+        python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"]
 
     name: Python ${{ matrix.python-version }} on ${{ matrix.platform }}
     runs-on: ${{ matrix.platform }}
@@ -31,32 +45,21 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
 
-      - name: Tokens
-        shell: python
-        env:
-          CDSAPIRC: ${{ secrets.CDSAPIRC }}
-
-        run: |
-          from __future__ import print_function
-          import os
-          for n in ('CDSAPIRC',):
-              m = os.path.expanduser("~/." + n.lower())
-              if os.environ[n]:
-                  with open(m, "w") as f:
-                      print(os.environ[n], file=f)
-
       - name: Tests
+        env:
+          CDSAPI_URL: https://cds.climate.copernicus.eu/api/v2
+          CDSAPI_KEY: ${{ secrets.CDSAPI_KEY }}
         run: |
           python setup.py develop
           pip install pytest
           pytest
 
   deploy:
+    needs: platform-checks
 
     if: ${{ github.event_name == 'release' }}
 
     name: Upload to Pypi
-    needs: checks
 
     runs-on: ubuntu-latest
 
diff --git a/.gitignore b/.gitignore
index 2a1348b..46aafc5 100644
--- a/.gitignore
+++ b/.gitignore
@@ -7,3 +7,4 @@
 *.grib
 *.nc
 cdsapi.egg-info
+build/
diff --git a/README.rst b/README.rst
index 166dac0..608c536 100644
--- a/README.rst
+++ b/README.rst
@@ -10,7 +10,7 @@ Install via `pip` with::
 Configure
 ---------
 
-Get your UID and API key from the CDS portal at the address https://cds.climate.copernicus.eu/user
+Get your user ID (UID) and API key from the CDS portal at the address https://cds.climate.copernicus.eu/user
 and write it into the configuration file, so it looks like::
 
     $ cat ~/.cdsapirc
diff --git a/cdsapi/api.py b/cdsapi/api.py
index a603716..bb0138f 100644
--- a/cdsapi/api.py
+++ b/cdsapi/api.py
@@ -9,10 +9,12 @@
 from __future__ import absolute_import, division, print_function, unicode_literals
 
 import json
-import time
-import os
 import logging
+import os
+import time
 import uuid
+
+import pkg_resources
 import requests
 
 try:
@@ -35,16 +37,15 @@ def bytes_to_string(n):
 def read_config(path):
     config = {}
     with open(path) as f:
-        for l in f.readlines():
-            if ":" in l:
-                k, v = l.strip().split(":", 1)
+        for line in f.readlines():
+            if ":" in line:
+                k, v = line.strip().split(":", 1)
                 if k in ("url", "key", "verify"):
                     config[k] = v.strip()
     return config
 
 
 def toJSON(obj):
-
     to_json = getattr(obj, "toJSON", None)
     if callable(to_json):
         return to_json()
@@ -63,7 +64,6 @@ def toJSON(obj):
 
 class Result(object):
     def __init__(self, client, reply):
-
         self.reply = reply
 
         self._url = client.url
@@ -95,7 +95,6 @@ class Result(object):
         return r
 
     def _download(self, url, size, target):
-
         if target is None:
             target = url.split("/")[-1]
 
@@ -109,7 +108,6 @@ class Result(object):
         headers = None
 
         while tries < self.retry_max:
-
             r = self.robust(self.session.get)(
                 url,
                 stream=True,
@@ -206,12 +204,13 @@ class Result(object):
         task_url = "%s/tasks/%s" % (self._url, request_id)
         self.debug("GET %s", task_url)
 
-        result = self.robust(self.session.get)(task_url, verify=self.verify)
+        result = self.robust(self.session.get)(
+            task_url, verify=self.verify, timeout=self.timeout
+        )
         result.raise_for_status()
         self.reply = result.json()
 
     def delete(self):
-
         if self._deleted:
             return
 
@@ -221,7 +220,9 @@ class Result(object):
             task_url = "%s/tasks/%s" % (self._url, rid)
             self.debug("DELETE %s", task_url)
 
-            delete = self.session.delete(task_url, verify=self.verify)
+            delete = self.session.delete(
+                task_url, verify=self.verify, timeout=self.timeout
+            )
             self.debug("DELETE returns %s %s", delete.status_code, delete.reason)
 
             try:
@@ -245,7 +246,6 @@ class Result(object):
 
 
 class Client(object):
-
     logger = logging.getLogger("cdsapi")
 
     def __init__(
@@ -270,17 +270,20 @@ class Client(object):
         forget=False,
         session=requests.Session(),
     ):
-
         if not quiet:
-
             if debug:
                 level = logging.DEBUG
             else:
                 level = logging.INFO
 
-            logging.basicConfig(
-                level=level, format="%(asctime)s %(levelname)s %(message)s"
-            )
+            self.logger.setLevel(level)
+
+            # avoid duplicate handlers when creating more than one Client
+            if not self.logger.handlers:
+                formatter = logging.Formatter("%(asctime)s %(levelname)s %(message)s")
+                handler = logging.StreamHandler()
+                handler.setFormatter(formatter)
+                self.logger.addHandler(handler)
 
         dotrc = os.environ.get("CDSAPI_RC", os.path.expanduser("~/.cdsapirc"))
 
@@ -295,11 +298,15 @@ class Client(object):
                     url = config.get("url")
 
                 if verify is None:
-                    verify = int(config.get("verify", 1))
+                    verify = bool(int(config.get("verify", 1)))
 
         if url is None or key is None or key is None:
             raise Exception("Missing/incomplete configuration file: %s" % (dotrc))
 
+        # If verify is still None, then we set to default value of True
+        if verify is None:
+            verify = True
+
         self.url = url
         self.key = key
 
@@ -322,6 +329,15 @@ class Client(object):
 
         self.session = session
         self.session.auth = tuple(self.key.split(":", 2))
+        self.session.headers = {
+            "User-Agent": "cdsapi/%s"
+            % pkg_resources.get_distribution("cdsapi").version,
+        }
+
+        assert len(self.session.auth) == 2, (
+            "The cdsapi key provided is not the correct format, please ensure it conforms to:\n"
+            "<UID>:<APIKEY>"
+        )
 
         self.metadata = metadata
         self.forget = forget
@@ -353,7 +369,7 @@ class Client(object):
     def service(self, name, *args, **kwargs):
         self.delete = False  # Don't delete results
         name = "/".join(name.split("."))
-        mimic_ui = kwargs.pop('mimic_ui', False)
+        mimic_ui = kwargs.pop("mimic_ui", False)
         # To mimic the CDS ui the request should be populated directly with the kwargs
         if mimic_ui:
             request = kwargs
@@ -377,7 +393,7 @@ class Client(object):
 
     def status(self, context=None):
         url = "%s/status.json" % (self.url,)
-        r = self.session.get(url, verify=self.verify)
+        r = self.session.get(url, verify=self.verify, timeout=self.timeout)
         r.raise_for_status()
         return r.json()
 
@@ -401,7 +417,6 @@ class Client(object):
             pass
 
     def _api(self, url, request, method):
-
         self._status(url)
 
         session = self.session
@@ -427,7 +442,6 @@ class Client(object):
             result.raise_for_status()
             reply = result.json()
         except Exception:
-
             if reply is None:
                 try:
                     reply = result.json()
@@ -457,7 +471,6 @@ class Client(object):
         sleep = 1
 
         while True:
-
             self.debug("REPLY %s", reply)
 
             if reply["state"] != self.last_state:
@@ -535,7 +548,6 @@ class Client(object):
             self.logger.debug(*args, **kwargs)
 
     def _download(self, results, targets=None):
-
         if isinstance(results, Result):
             if targets:
                 path = targets.pop(0)
@@ -547,7 +559,6 @@ class Client(object):
             return [self._download(x, targets) for x in results]
 
         if isinstance(results, dict):
-
             if "location" in results and "contentLength" in results:
                 reply = dict(
                     location=results["location"],
@@ -586,7 +597,6 @@ class Client(object):
 
     def robust(self, call):
         def retriable(code, reason):
-
             if code in [
                 requests.codes.internal_server_error,
                 requests.codes.bad_gateway,
@@ -601,40 +611,35 @@ class Client(object):
 
         def wrapped(*args, **kwargs):
             tries = 0
-            while tries < self.retry_max:
+            while True:
+                txt = "Error"
                 try:
-                    r = call(*args, **kwargs)
+                    resp = call(*args, **kwargs)
                 except (
                     requests.exceptions.ConnectionError,
                     requests.exceptions.ReadTimeout,
                 ) as e:
-                    r = None
-                    self.warning(
-                        "Recovering from connection error [%s], attemps %s of %s",
-                        e,
-                        tries,
-                        self.retry_max,
-                    )
-
-                if r is not None:
-                    if not retriable(r.status_code, r.reason):
-                        return r
+                    resp = None
+                    txt = f"Connection error: [{e}]"
+
+                if resp is not None:
+                    if not retriable(resp.status_code, resp.reason):
+                        break
                     try:
-                        self.warning(r.json()["reason"])
+                        self.warning(resp.json()["reason"])
                     except Exception:
                         pass
-                    self.warning(
-                        "Recovering from HTTP error [%s %s], attemps %s of %s",
-                        r.status_code,
-                        r.reason,
-                        tries,
-                        self.retry_max,
-                    )
+                    txt = f"HTTP error: [{resp.status_code} {resp.reason}]"
 
                 tries += 1
+                self.warning(txt + f". Attempt {tries} of {self.retry_max}.")
+                if tries < self.retry_max:
+                    self.warning(f"Retrying in {self.sleep_max} seconds")
+                    time.sleep(self.sleep_max)
+                    self.info("Retrying now...")
+                else:
+                    raise Exception("Could not connect")
 
-                self.warning("Retrying in %s seconds", self.sleep_max)
-                time.sleep(self.sleep_max)
-                self.info("Retrying now...")
+            return resp
 
         return wrapped
diff --git a/docker/retrieve.py b/docker/retrieve.py
index 01ba3f5..a009fba 100644
--- a/docker/retrieve.py
+++ b/docker/retrieve.py
@@ -1,4 +1,6 @@
-import json, sys, cdsapi
+import json
+
+import cdsapi
 
 with open("/input/request.json") as req:
     request = json.load(req)
diff --git a/examples/example-era5-update.py b/examples/example-era5-update.py
index 8b56573..5d10c85 100755
--- a/examples/example-era5-update.py
+++ b/examples/example-era5-update.py
@@ -9,6 +9,7 @@
 # does it submit to any jurisdiction.
 
 import time
+
 import cdsapi
 
 c = cdsapi.Client(debug=True, wait_until_complete=False)
diff --git a/setup.py b/setup.py
index 706ac21..1692456 100644
--- a/setup.py
+++ b/setup.py
@@ -30,7 +30,7 @@ def read(fname):
     return io.open(file_path, encoding="utf-8").read()
 
 
-version = "0.5.1"
+version = "0.6.1"
 
 
 setuptools.setup(
@@ -50,15 +50,15 @@ setuptools.setup(
     ],
     zip_safe=True,
     classifiers=[
-        "Development Status :: 3 - Alpha",
+        "Development Status :: 4 - Beta",
         "Intended Audience :: Developers",
         "Programming Language :: Python",
-        "Programming Language :: Python :: 2",
-        "Programming Language :: Python :: 2.7",
         "Programming Language :: Python :: 3",
-        "Programming Language :: Python :: 3.4",
-        "Programming Language :: Python :: 3.5",
-        "Programming Language :: Python :: 3.6",
+        "Programming Language :: Python :: 3.7",
+        "Programming Language :: Python :: 3.8",
+        "Programming Language :: Python :: 3.9",
+        "Programming Language :: Python :: 3.10",
+        "Programming Language :: Python :: 3.11",
         "Programming Language :: Python :: Implementation :: CPython",
         "Programming Language :: Python :: Implementation :: PyPy",
         "Operating System :: OS Independent",
diff --git a/tests/test_api.py b/tests/test_api.py
index c3f3766..3725f1f 100644
--- a/tests/test_api.py
+++ b/tests/test_api.py
@@ -1,8 +1,9 @@
-import cdsapi
 import os
 
-def test_request():
+import cdsapi
 
+
+def test_request():
     c = cdsapi.Client()
 
     r = c.retrieve(
@@ -11,7 +12,7 @@ def test_request():
             "variable": "2t",
             "product_type": "reanalysis",
             "date": "2012-12-01",
-            "time": "12:00"
+            "time": "12:00",
         },
     )
 
diff --git a/tox.ini b/tox.ini
index 1d50849..d97570f 100644
--- a/tox.ini
+++ b/tox.ini
@@ -1,5 +1,5 @@
 [tox]
-envlist = qc, py36, py35, py34, py27, pypy3, pypy, deps
+envlist = qc, py311, py310, py39, py38, py37, pypy3, pypy, deps
 
 [testenv]
 setenv = PYTHONPATH = {toxinidir}
@@ -14,3 +14,11 @@ commands = pytest -v --pep8 --mccabe --cov=cdsapi --cov-report=html --cache-clea
 [testenv:deps]
 deps =
 commands = python setup.py test
+
+
+[black]
+line_length=120
+[isort]
+profile=black
+[flake8]
+max-line-length = 120
\ No newline at end of file