diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 56b122907..d0ba34355 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -69,16 +69,13 @@ jobs: echo "$HOME/bin" >> $GITHUB_PATH - name: tox run: | - TIMEOUT=10m - if [[ "${{ matrix.python }}" = "2.7" ]]; then - TIMEOUT=15m - elif [[ "py${{ matrix.python }}-${{ matrix.os }}" = "py3.8-ubuntu" ]]; then - export TOXENV="py38-tf,py38-tf-keras" # full + if [[ "py${{ matrix.python }}-${{ matrix.os }}" = "py3.11-ubuntu" ]]; then + export TOXENV="py311-tf,py311-tf-keras" # full fi if [[ "${{ matrix.os }}" != "ubuntu" ]]; then - tox -e py${PYVER/./} # basic + tox -e py${PYVER/./} # basic else - timeout $TIMEOUT tox || timeout $TIMEOUT tox || timeout $TIMEOUT tox + timeout 5m tox || timeout 5m tox || timeout 5m tox fi env: PYVER: ${{ matrix.python }} diff --git a/.meta/.readme.rst b/.meta/.readme.rst index 4e325bcd2..a181ddc6d 100644 --- a/.meta/.readme.rst +++ b/.meta/.readme.rst @@ -291,6 +291,12 @@ The most common issues relate to excessive output on multiple lines, instead of a neat one-line progress bar. - Consoles in general: require support for carriage return (``CR``, ``\r``). + + * Some cloud logging consoles which don't support ``\r`` properly + (`cloudwatch `__, + `K8s `__) may benefit from + ``export TQDM_POSITION=-1``. + - Nested progress bars: * Consoles in general: require support for moving cursors up to the @@ -327,13 +333,11 @@ of a neat one-line progress bar. * The same applies to ``itertools``. * Some useful convenience functions can be found under ``tqdm.contrib``. -- `Hanging pipes in python2 `__: - when using ``tqdm`` on the CLI, you may need to use Python 3.5+ for correct - buffering. - `No intermediate output in docker-compose `__: use ``docker-compose run`` instead of ``docker-compose up`` and ``tty: true``. + - Overriding defaults via environment variables: - e.g. in CI jobs, ``export TQDM_MININTERVAL=5`` to avoid log spam. + e.g. in CI/cloud jobs, ``export TQDM_MININTERVAL=5`` to avoid log spam. This override logic is handled by the ``tqdm.utils.envwrap`` decorator (useful independent of ``tqdm``). @@ -349,7 +353,7 @@ Documentation class tqdm(): """{DOC_tqdm}""" - @envwrap("TQDM_", is_method=True) # override defaults via env vars + @envwrap("TQDM_") # override defaults via env vars def __init__(self, iterable=None, desc=None, total=None, leave=True, file=None, ncols=None, mininterval=0.1, maxinterval=10.0, miniters=None, ascii=None, disable=False, diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1b987ee21..533a8ab45 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -97,9 +97,9 @@ versions of Python.) Note: to install all versions of the Python interpreter that are specified in [tox.ini](https://github.com/tqdm/tqdm/blob/master/tox.ini), -you can use `MiniConda` to install a minimal setup. You must also make sure -that each distribution has an alias to call the Python interpreter: -`python27` for Python 2.7's interpreter, `python32` for Python 3.2's, etc. +you can use `MiniConda` to install a minimal setup. You must also ensure +that each distribution has an alias to call the Python interpreter +(e.g. `python311` for Python 3.11's interpreter). ### Alternative unit tests with pytest diff --git a/README.rst b/README.rst index defa9ad35..e62a34639 100644 --- a/README.rst +++ b/README.rst @@ -291,6 +291,12 @@ The most common issues relate to excessive output on multiple lines, instead of a neat one-line progress bar. - Consoles in general: require support for carriage return (``CR``, ``\r``). + + * Some cloud logging consoles which don't support ``\r`` properly + (`cloudwatch `__, + `K8s `__) may benefit from + ``export TQDM_POSITION=-1``. + - Nested progress bars: * Consoles in general: require support for moving cursors up to the @@ -327,13 +333,11 @@ of a neat one-line progress bar. * The same applies to ``itertools``. * Some useful convenience functions can be found under ``tqdm.contrib``. -- `Hanging pipes in python2 `__: - when using ``tqdm`` on the CLI, you may need to use Python 3.5+ for correct - buffering. - `No intermediate output in docker-compose `__: use ``docker-compose run`` instead of ``docker-compose up`` and ``tty: true``. + - Overriding defaults via environment variables: - e.g. in CI jobs, ``export TQDM_MININTERVAL=5`` to avoid log spam. + e.g. in CI/cloud jobs, ``export TQDM_MININTERVAL=5`` to avoid log spam. This override logic is handled by the ``tqdm.utils.envwrap`` decorator (useful independent of ``tqdm``). @@ -353,7 +357,7 @@ Documentation progressbar every time a value is requested. """ - @envwrap("TQDM_", is_method=True) # override defaults via env vars + @envwrap("TQDM_") # override defaults via env vars def __init__(self, iterable=None, desc=None, total=None, leave=True, file=None, ncols=None, mininterval=0.1, maxinterval=10.0, miniters=None, ascii=None, disable=False, diff --git a/asv.conf.json b/asv.conf.json index d826f85b4..27e7a89b1 100644 --- a/asv.conf.json +++ b/asv.conf.json @@ -6,7 +6,7 @@ "environment_type": "virtualenv", "build_command": ["PIP_NO_BUILD_ISOLATION=false python -mpip wheel --no-deps --no-index -w {build_cache_dir} ."], "show_commit_url": "https://github.com/tqdm/tqdm/commit/", - // "pythons": ["2.7", "3.6"], + // "pythons": ["3.7", "3.11"], // "conda_channels": ["conda-forge", "defaults"], "matrix": { "alive-progress": [""], diff --git a/examples/async_coroutines.py b/examples/async_coroutines.py index 40f4f249d..3e3190508 100644 --- a/examples/async_coroutines.py +++ b/examples/async_coroutines.py @@ -1,6 +1,4 @@ -""" -Asynchronous examples using `asyncio`, `async` and `await` on `python>=3.7`. -""" +"""Asynchronous examples using `asyncio`, `async` and `await`.""" import asyncio from tqdm.asyncio import tqdm, trange diff --git a/examples/parallel_bars.py b/examples/parallel_bars.py index 5b969732c..b7e149413 100644 --- a/examples/parallel_bars.py +++ b/examples/parallel_bars.py @@ -21,8 +21,7 @@ def progresser(n, auto_position=True, write_safe=False, blocking=True, progress= sleep(interval) # NB: may not clear instances with higher `position` upon completion # since this worker may not know about other bars #796 - if write_safe: - # we think we know about other bars (currently only py3 threading) + if write_safe: # we think we know about other bars if n == 6: tqdm.write("n == 6 completed") return n + 1 diff --git a/tests/tests_asyncio.py b/tests/tests_asyncio.py index ddfa1b1b0..bdef569fa 100644 --- a/tests/tests_asyncio.py +++ b/tests/tests_asyncio.py @@ -1,4 +1,4 @@ -"""Tests `tqdm.asyncio` on `python>=3.7`.""" +"""Tests `tqdm.asyncio`.""" import asyncio from functools import partial from sys import platform diff --git a/tests/tests_utils.py b/tests/tests_utils.py index 5fb20dcea..6cf1e6cf9 100644 --- a/tests/tests_utils.py +++ b/tests/tests_utils.py @@ -1,41 +1,51 @@ -from pytest import mark +from ast import literal_eval +from collections import defaultdict +from typing import Union # py<3.10 -from tqdm.utils import IS_WIN, envwrap +from tqdm.utils import envwrap def test_envwrap(monkeypatch): - """Test envwrap overrides""" - env_a = 42 - env_c = 1337 - monkeypatch.setenv('FUNC_A', str(env_a)) - monkeypatch.setenv('FUNC_TyPe_HiNt', str(env_c)) + """Test @envwrap (basic)""" + monkeypatch.setenv('FUNC_A', "42") + monkeypatch.setenv('FUNC_TyPe_HiNt', "1337") monkeypatch.setenv('FUNC_Unused', "x") @envwrap("FUNC_") def func(a=1, b=2, type_hint: int = None): return a, b, type_hint - assert (env_a, 2, 1337) == func(), "expected env override" - assert (99, 2, 1337) == func(a=99), "expected manual override" + assert (42, 2, 1337) == func() + assert (99, 2, 1337) == func(a=99) - env_literal = 3.14159 - monkeypatch.setenv('FUNC_literal', str(env_literal)) - @envwrap("FUNC_", literal_eval=True) - def another_func(literal="some_string"): - return literal +def test_envwrap_types(monkeypatch): + """Test @envwrap(types)""" + monkeypatch.setenv('FUNC_notype', "3.14159") - assert env_literal == another_func() + @envwrap("FUNC_", types=defaultdict(lambda: literal_eval)) + def func(notype=None): + return notype + assert 3.14159 == func() -@mark.skipif(IS_WIN, reason="no lowercase environ on Windows") -def test_envwrap_case(monkeypatch): - """Test envwrap case-sensitive overrides""" - env_liTeRaL = 3.14159 - monkeypatch.setenv('FUNC_liTeRaL', str(env_liTeRaL)) + monkeypatch.setenv('FUNC_number', "1") + monkeypatch.setenv('FUNC_string', "1") - @envwrap("FUNC_", literal_eval=True, case_sensitive=True) - def func(liTeRaL="some_string"): - return liTeRaL + @envwrap("FUNC_", types={'number': int}) + def nofallback(number=None, string=None): + return number, string - assert env_liTeRaL == func() + assert 1, "1" == nofallback() + + +def test_envwrap_annotations(monkeypatch): + """Test @envwrap with typehints""" + monkeypatch.setenv('FUNC_number', "1.1") + monkeypatch.setenv('FUNC_string', "1.1") + + @envwrap("FUNC_") + def annotated(number: Union[int, float] = None, string: int = None): + return number, string + + assert 1.1, "1.1" == annotated() diff --git a/tox.ini b/tox.ini index 4b63a438f..cd5dc1b07 100644 --- a/tox.ini +++ b/tox.ini @@ -45,7 +45,7 @@ deps= numpy pandas rich - !py311-tf: tensorflow!=2.5.0 + tf: tensorflow!=2.5.0 keras: keras commands= pytest --cov=tqdm --cov-report= -W=ignore tests_notebook.ipynb --nbval --current-env --sanitize-with=.meta/nbval.ini diff --git a/tqdm/notebook.py b/tqdm/notebook.py index 0a14b4599..0f531ab94 100644 --- a/tqdm/notebook.py +++ b/tqdm/notebook.py @@ -10,6 +10,7 @@ # import compatibility functions and utilities import re import sys +from html import escape from weakref import proxy # to inherit from the tqdm class @@ -58,12 +59,6 @@ except ImportError: pass - # HTML encoding - try: # Py3 - from html import escape - except ImportError: # Py2 - from cgi import escape - __author__ = {"github.com/": ["lrq3000", "casperdcl", "alexanderkuk"]} __all__ = ['tqdm_notebook', 'tnrange', 'tqdm', 'trange'] WARN_NOIPYW = ("IProgress not found. Please update jupyter and ipywidgets." diff --git a/tqdm/std.py b/tqdm/std.py index 218a561f6..9ba8e8506 100644 --- a/tqdm/std.py +++ b/tqdm/std.py @@ -949,13 +949,15 @@ def wrapper(*args, **kwargs): elif _Rolling_and_Expanding is not None: _Rolling_and_Expanding.progress_apply = inner_generator() - @envwrap("TQDM_", is_method=True) # override defaults via env vars + # override defaults via env vars + @envwrap("TQDM_", is_method=True, types={'total': float, 'ncols': int, 'miniters': float, + 'position': int, 'nrows': int}) def __init__(self, iterable=None, desc=None, total=None, leave=True, file=None, ncols=None, mininterval=0.1, maxinterval=10.0, miniters=None, ascii=None, disable=False, unit='it', unit_scale=False, dynamic_ncols=False, smoothing=0.3, bar_format=None, initial=0, position=None, postfix=None, unit_divisor=1000, write_bytes=False, - lock_args=None, nrows=None, colour=None, delay=0, gui=False, + lock_args=None, nrows=None, colour=None, delay=0.0, gui=False, **kwargs): """see tqdm.tqdm for arguments""" if file is None: diff --git a/tqdm/utils.py b/tqdm/utils.py index 249a73960..5a70819c3 100644 --- a/tqdm/utils.py +++ b/tqdm/utils.py @@ -4,7 +4,6 @@ import os import re import sys -from ast import literal_eval as safe_eval from functools import partial, partialmethod, wraps from inspect import signature # TODO consider using wcswidth third-party package for 0-width characters @@ -32,9 +31,12 @@ colorama.init() -def envwrap(prefix, case_sensitive=False, literal_eval=False, is_method=False): +def envwrap(prefix, types=None, is_method=False): """ Override parameter defaults via `os.environ[prefix + param_name]`. + Maps UPPER_CASE env vars map to lower_case param names. + camelCase isn't supported (because Windows ignores case). + Precedence (highest first): - call (`foo(a=3)`) - environ (`FOO_A=2`) @@ -44,11 +46,10 @@ def envwrap(prefix, case_sensitive=False, literal_eval=False, is_method=False): ---------- prefix : str Env var prefix, e.g. "FOO_" - case_sensitive : bool, optional - If (default: False), treat env var "FOO_Some_ARG" as "FOO_some_arg". - literal_eval : bool, optional - Whether to `ast.literal_eval` the detected env var overrides. - Otherwise if (default: False), infer types from function signature. + types : dict, optional + Fallback mappings `{'param_name': type, ...}` if types cannot be + inferred from function signature. + Consider using `types=collections.defaultdict(lambda: ast.literal_eval)`. is_method : bool, optional Whether to use `functools.partialmethod`. If (default: False) use `functools.partial`. @@ -65,25 +66,34 @@ def test(a=1, b=2, c=3): received: a=42, b=2, c=99 ``` """ + if types is None: + types = {} i = len(prefix) - env_overrides = {k[i:] if case_sensitive else k[i:].lower(): v - for k, v in os.environ.items() if k.startswith(prefix)} + env_overrides = {k[i:].lower(): v for k, v in os.environ.items() if k.startswith(prefix)} part = partialmethod if is_method else partial def wrap(func): params = signature(func).parameters + # ignore unknown env vars overrides = {k: v for k, v in env_overrides.items() if k in params} - if literal_eval: - return part(func, **{k: safe_eval(v) for k, v in overrides.items()}) - # use `func` signature to infer env override `type` (fallback to `str`) + # infer overrides' `type`s for k in overrides: param = params[k] - if param.annotation is not param.empty: - typ = param.annotation - # TODO: parse type in {Union, Any, Optional, ...} + if param.annotation is not param.empty: # typehints + for typ in getattr(param.annotation, '__args__', (param.annotation,)): + try: + overrides[k] = typ(overrides[k]) + except Exception: + pass + else: + break + elif param.default is not None: # type of default value + overrides[k] = type(param.default)(overrides[k]) else: - typ = str if param.default is None else type(param.default) - overrides[k] = typ(overrides[k]) + try: # `types` fallback + overrides[k] = types[k](overrides[k]) + except KeyError: # keep unconverted (`str`) + pass return part(func, **overrides) return wrap