Merge pull request #1493 from tqdm/devel

fix `envwrap` types, update docs

.github/workflows/test.yml

@@ -69,16 +69,13 @@ jobs:
  echo "$HOME/bin" >> $GITHUB_PATH
  - name: tox
  run: |
- TIMEOUT=10m
- if [[ "${{ matrix.python }}" = "2.7" ]]; then
- TIMEOUT=15m
- elif [[ "py${{ matrix.python }}-${{ matrix.os }}" = "py3.8-ubuntu" ]]; then
- export TOXENV="py38-tf,py38-tf-keras" # full
+ if [[ "py${{ matrix.python }}-${{ matrix.os }}" = "py3.11-ubuntu" ]]; then
+ export TOXENV="py311-tf,py311-tf-keras" # full
  fi
  if [[ "${{ matrix.os }}" != "ubuntu" ]]; then
- tox -e py${PYVER/./} # basic
+ tox -e py${PYVER/./}  # basic
  else
- timeout $TIMEOUT tox || timeout $TIMEOUT tox || timeout $TIMEOUT tox
+ timeout 5m tox || timeout 5m tox || timeout 5m tox
  fi
  env:
  PYVER: ${{ matrix.python }}

.meta/.readme.rst

@@ -291,6 +291,12 @@ The most common issues relate to excessive output on multiple lines, instead
 of a neat one-line progress bar.
 
 - Consoles in general: require support for carriage return (``CR``, ``\r``).
+
+ * Some cloud logging consoles which don't support ``\r`` properly
+ (`cloudwatch <https://github.com/tqdm/tqdm/issues/966>`__,
+ `K8s <https://github.com/tqdm/tqdm/issues/1319>`__) may benefit from
+ ``export TQDM_POSITION=-1``.
+
 - Nested progress bars:
 
  * Consoles in general: require support for moving cursors up to the
@@ -327,13 +333,11 @@ of a neat one-line progress bar.
  * The same applies to ``itertools``.
  * Some useful convenience functions can be found under ``tqdm.contrib``.
 
-- `Hanging pipes in python2 <https://github.com/tqdm/tqdm/issues/359>`__:
- when using ``tqdm`` on the CLI, you may need to use Python 3.5+ for correct
- buffering.
 - `No intermediate output in docker-compose <https://github.com/tqdm/tqdm/issues/771>`__:
  use ``docker-compose run`` instead of ``docker-compose up`` and ``tty: true``.
+
 - Overriding defaults via environment variables:
- e.g. in CI jobs, ``export TQDM_MININTERVAL=5`` to avoid log spam.
+ e.g. in CI/cloud jobs, ``export TQDM_MININTERVAL=5`` to avoid log spam.
  This override logic is handled by the ``tqdm.utils.envwrap`` decorator
  (useful independent of ``tqdm``).
 
@@ -349,7 +353,7 @@ Documentation
  class tqdm():
  """{DOC_tqdm}"""
 
- @envwrap("TQDM_", is_method=True) # override defaults via env vars
+ @envwrap("TQDM_") # override defaults via env vars
  def __init__(self, iterable=None, desc=None, total=None, leave=True,
  file=None, ncols=None, mininterval=0.1,
  maxinterval=10.0, miniters=None, ascii=None, disable=False,

CONTRIBUTING.md

@@ -97,9 +97,9 @@ versions of Python.)
 
 Note: to install all versions of the Python interpreter that are specified
 in [tox.ini](https://github.com/tqdm/tqdm/blob/master/tox.ini),
-you can use `MiniConda` to install a minimal setup. You must also make sure
-that each distribution has an alias to call the Python interpreter:
-`python27` for Python 2.7's interpreter, `python32` for Python 3.2's, etc.
+you can use `MiniConda` to install a minimal setup. You must also ensure
+that each distribution has an alias to call the Python interpreter
+(e.g. `python311` for Python 3.11's interpreter).
 
 ### Alternative unit tests with pytest
 

README.rst

@@ -291,6 +291,12 @@ The most common issues relate to excessive output on multiple lines, instead
 of a neat one-line progress bar.
 
 - Consoles in general: require support for carriage return (``CR``, ``\r``).
+
+ * Some cloud logging consoles which don't support ``\r`` properly
+ (`cloudwatch <https://github.com/tqdm/tqdm/issues/966>`__,
+ `K8s <https://github.com/tqdm/tqdm/issues/1319>`__) may benefit from
+ ``export TQDM_POSITION=-1``.
+
 - Nested progress bars:
 
  * Consoles in general: require support for moving cursors up to the
@@ -327,13 +333,11 @@ of a neat one-line progress bar.
  * The same applies to ``itertools``.
  * Some useful convenience functions can be found under ``tqdm.contrib``.
 
-- `Hanging pipes in python2 <https://github.com/tqdm/tqdm/issues/359>`__:
- when using ``tqdm`` on the CLI, you may need to use Python 3.5+ for correct
- buffering.
 - `No intermediate output in docker-compose <https://github.com/tqdm/tqdm/issues/771>`__:
  use ``docker-compose run`` instead of ``docker-compose up`` and ``tty: true``.
+
 - Overriding defaults via environment variables:
- e.g. in CI jobs, ``export TQDM_MININTERVAL=5`` to avoid log spam.
+ e.g. in CI/cloud jobs, ``export TQDM_MININTERVAL=5`` to avoid log spam.
  This override logic is handled by the ``tqdm.utils.envwrap`` decorator
  (useful independent of ``tqdm``).
 
@@ -353,7 +357,7 @@ Documentation
  progressbar every time a value is requested.
  """
 
- @envwrap("TQDM_", is_method=True) # override defaults via env vars
+ @envwrap("TQDM_") # override defaults via env vars
  def __init__(self, iterable=None, desc=None, total=None, leave=True,
  file=None, ncols=None, mininterval=0.1,
  maxinterval=10.0, miniters=None, ascii=None, disable=False,

asv.conf.json

@@ -6,7 +6,7 @@
  "environment_type": "virtualenv",
  "build_command": ["PIP_NO_BUILD_ISOLATION=false python -mpip wheel --no-deps --no-index -w {build_cache_dir} ."],
  "show_commit_url": "https://github.com/tqdm/tqdm/commit/",
- // "pythons": ["2.7", "3.6"],
+ // "pythons": ["3.7", "3.11"],
  // "conda_channels": ["conda-forge", "defaults"],
  "matrix": {
  "alive-progress": [""],

examples/async_coroutines.py

@@ -1,6 +1,4 @@
-"""
-Asynchronous examples using `asyncio`, `async` and `await` on `python>=3.7`.
-"""
+"""Asynchronous examples using `asyncio`, `async` and `await`."""
 import asyncio
 
 from tqdm.asyncio import tqdm, trange

examples/parallel_bars.py

@@ -21,8 +21,7 @@ def progresser(n, auto_position=True, write_safe=False, blocking=True, progress=
  sleep(interval)
  # NB: may not clear instances with higher `position` upon completion
  # since this worker may not know about other bars #796
- if write_safe:
- # we think we know about other bars (currently only py3 threading)
+ if write_safe: # we think we know about other bars
  if n == 6:
  tqdm.write("n == 6 completed")
  return n + 1

tests/tests_asyncio.py

@@ -1,4 +1,4 @@
-"""Tests `tqdm.asyncio` on `python>=3.7`."""
+"""Tests `tqdm.asyncio`."""
 import asyncio
 from functools import partial
 from sys import platform

tests/tests_utils.py

@@ -1,41 +1,51 @@
-from pytest import mark
+from ast import literal_eval
+from collections import defaultdict
+from typing import Union # py<3.10
 
-from tqdm.utils import IS_WIN, envwrap
+from tqdm.utils import envwrap
 
 
 def test_envwrap(monkeypatch):
- """Test envwrap overrides"""
- env_a = 42
- env_c = 1337
- monkeypatch.setenv('FUNC_A', str(env_a))
- monkeypatch.setenv('FUNC_TyPe_HiNt', str(env_c))
+ """Test @envwrap (basic)"""
+ monkeypatch.setenv('FUNC_A', "42")
+ monkeypatch.setenv('FUNC_TyPe_HiNt', "1337")
  monkeypatch.setenv('FUNC_Unused', "x")
 
  @envwrap("FUNC_")
  def func(a=1, b=2, type_hint: int = None):
  return a, b, type_hint
 
- assert (env_a, 2, 1337) == func(), "expected env override"
- assert (99, 2, 1337) == func(a=99), "expected manual override"
+ assert (42, 2, 1337) == func()
+ assert (99, 2, 1337) == func(a=99)
 
- env_literal = 3.14159
- monkeypatch.setenv('FUNC_literal', str(env_literal))
 
- @envwrap("FUNC_", literal_eval=True)
- def another_func(literal="some_string"):
-  return literal
+def test_envwrap_types(monkeypatch):
+ """Test @envwrap(types)"""
+ monkeypatch.setenv('FUNC_notype', "3.14159")
 
- assert env_literal == another_func()
+ @envwrap("FUNC_", types=defaultdict(lambda: literal_eval))
+ def func(notype=None):
+ return notype
 
+ assert 3.14159 == func()
 
-@mark.skipif(IS_WIN, reason="no lowercase environ on Windows")
-def test_envwrap_case(monkeypatch):
- """Test envwrap case-sensitive overrides"""
- env_liTeRaL = 3.14159
- monkeypatch.setenv('FUNC_liTeRaL', str(env_liTeRaL))
+ monkeypatch.setenv('FUNC_number', "1")
+ monkeypatch.setenv('FUNC_string', "1")
 
- @envwrap("FUNC_", literal_eval=True, case_sensitive=True)
- def func(liTeRaL="some_string"):
- return liTeRaL
+ @envwrap("FUNC_", types={'number': int})
+ def nofallback(number=None, string=None):
+ return number, string
 
- assert env_liTeRaL == func()
+ assert 1, "1" == nofallback()
+
+
+def test_envwrap_annotations(monkeypatch):
+ """Test @envwrap with typehints"""
+ monkeypatch.setenv('FUNC_number', "1.1")
+ monkeypatch.setenv('FUNC_string', "1.1")
+
+ @envwrap("FUNC_")
+ def annotated(number: Union[int, float] = None, string: int = None):
+ return number, string
+
+ assert 1.1, "1.1" == annotated()

tox.ini

@@ -45,7 +45,7 @@ deps=
  numpy
  pandas
  rich
- !py311-tf: tensorflow!=2.5.0
+ tf: tensorflow!=2.5.0
  keras: keras
 commands=
  pytest --cov=tqdm --cov-report= -W=ignore tests_notebook.ipynb --nbval --current-env --sanitize-with=.meta/nbval.ini

tqdm/notebook.py

@@ -10,6 +10,7 @@
 # import compatibility functions and utilities
 import re
 import sys
+from html import escape
 from weakref import proxy
 
 # to inherit from the tqdm class
@@ -58,12 +59,6 @@
  except ImportError:
  pass
 
- # HTML encoding
- try: # Py3
- from html import escape
- except ImportError: # Py2
- from cgi import escape
-
 __author__ = {"github.com/": ["lrq3000", "casperdcl", "alexanderkuk"]}
 __all__ = ['tqdm_notebook', 'tnrange', 'tqdm', 'trange']
 WARN_NOIPYW = ("IProgress not found. Please update jupyter and ipywidgets."

tqdm/std.py

@@ -949,13 +949,15 @@ def wrapper(*args, **kwargs):
  elif _Rolling_and_Expanding is not None:
  _Rolling_and_Expanding.progress_apply = inner_generator()
 
- @envwrap("TQDM_", is_method=True) # override defaults via env vars
+ # override defaults via env vars
+ @envwrap("TQDM_", is_method=True, types={'total': float, 'ncols': int, 'miniters': float,
+ 'position': int, 'nrows': int})
  def __init__(self, iterable=None, desc=None, total=None, leave=True, file=None,
  ncols=None, mininterval=0.1, maxinterval=10.0, miniters=None,
  ascii=None, disable=False, unit='it', unit_scale=False,
  dynamic_ncols=False, smoothing=0.3, bar_format=None, initial=0,
  position=None, postfix=None, unit_divisor=1000, write_bytes=False,
- lock_args=None, nrows=None, colour=None, delay=0, gui=False,
+ lock_args=None, nrows=None, colour=None, delay=0.0, gui=False,
  **kwargs):
  """see tqdm.tqdm for arguments"""
  if file is None:

tqdm/utils.py

@@ -4,7 +4,6 @@
 import os
 import re
 import sys
-from ast import literal_eval as safe_eval
 from functools import partial, partialmethod, wraps
 from inspect import signature
 # TODO consider using wcswidth third-party package for 0-width characters
@@ -32,9 +31,12 @@
  colorama.init()
 
 
-def envwrap(prefix, case_sensitive=False, literal_eval=False, is_method=False):
+def envwrap(prefix, types=None, is_method=False):
  """
  Override parameter defaults via `os.environ[prefix + param_name]`.
+ Maps UPPER_CASE env vars map to lower_case param names.
+ camelCase isn't supported (because Windows ignores case).
+
  Precedence (highest first):
  - call (`foo(a=3)`)
  - environ (`FOO_A=2`)
@@ -44,11 +46,10 @@ def envwrap(prefix, case_sensitive=False, literal_eval=False, is_method=False):
  ----------
  prefix : str
  Env var prefix, e.g. "FOO_"
- case_sensitive : bool, optional
- If (default: False), treat env var "FOO_Some_ARG" as "FOO_some_arg".
- literal_eval : bool, optional
- Whether to `ast.literal_eval` the detected env var overrides.
- Otherwise if (default: False), infer types from function signature.
+ types : dict, optional
+ Fallback mappings `{'param_name': type, ...}` if types cannot be
+ inferred from function signature.
+ Consider using `types=collections.defaultdict(lambda: ast.literal_eval)`.
  is_method : bool, optional
  Whether to use `functools.partialmethod`. If (default: False) use `functools.partial`.
 
@@ -65,25 +66,34 @@ def test(a=1, b=2, c=3):
  received: a=42, b=2, c=99
  ```
  """
+ if types is None:
+ types = {}
  i = len(prefix)
- env_overrides = {k[i:] if case_sensitive else k[i:].lower(): v
- for k, v in os.environ.items() if k.startswith(prefix)}
+ env_overrides = {k[i:].lower(): v for k, v in os.environ.items() if k.startswith(prefix)}
  part = partialmethod if is_method else partial
 
  def wrap(func):
  params = signature(func).parameters
+ # ignore unknown env vars
  overrides = {k: v for k, v in env_overrides.items() if k in params}
- if literal_eval:
- return part(func, **{k: safe_eval(v) for k, v in overrides.items()})
- # use `func` signature to infer env override `type` (fallback to `str`)
+ # infer overrides' `type`s
  for k in overrides:
  param = params[k]
- if param.annotation is not param.empty:
- typ = param.annotation
- # TODO: parse type in {Union, Any, Optional, ...}
+ if param.annotation is not param.empty: # typehints
+ for typ in getattr(param.annotation, '__args__', (param.annotation,)):
+ try:
+ overrides[k] = typ(overrides[k])
+ except Exception:
+ pass
+ else:
+ break
+ elif param.default is not None: # type of default value
+ overrides[k] = type(param.default)(overrides[k])
  else:
- typ = str if param.default is None else type(param.default)
- overrides[k] = typ(overrides[k])
+ try: # `types` fallback
+ overrides[k] = types[k](overrides[k])
+ except KeyError: # keep unconverted (`str`)
+ pass
  return part(func, **overrides)
  return wrap