From 756bffc3c395126e7289f83aa9cf5028a6c7bf15 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Fri, 21 Jan 2022 14:40:21 -0800 Subject: [PATCH] Release 0.14.4 backports (#1177) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add `cmake/` to MANIFEST.in Forgot to add in #1128, which is then missing in Python sdist files for pip from-source builds. * Add failing test * Read gridSpacing also from long doubles * Python: 3.10 Add support for Python 3.10. * CI: Python 3.10 on Alpine Linux musllinux uses the musl libc on an Alpine stack, especially for docker-ish containers. Let's add coverage while also covering CPython 3.10 * Fix: Python Iteration Repr Typo Fix a typo in the `__repr__` of `Iteration` in Python. * Remove invalid records from our data structure entirely Until now, the half-parsed records were not deleted. Our sanity checks in `Series::~Series()` caught up on that and threw errors. * Fix doxygen? No idea why this PR triggers that error * Don't apply compression operators multipe times Happened so far in variable-based iteration encoding * ICC 19.1.2: CXX17 Work-Arounds (Variant) The `Attribute` constructors with implicit variant conversion sometimes do not work in ICC 19.1.2. ``` openPMD-api/src/RecordComponent.cpp(226): error: no instance of constructor "openPMD::Attribute::Attribute" matches the argument list argument types are: (openPMD::Extent) Attribute a(getExtent()); ^ openPMD-api/include/openPMD/backend/Attribute.hpp(50): note: this candidate was rejected because arguments do not match class Attribute : ^ openPMD-api/include/openPMD/backend/Attribute.hpp(50): note: this candidate was rejected because arguments do not match class Attribute : ^ openPMD-api/include/openPMD/backend/Attribute.hpp(79): note: this candidate was rejected because arguments do not match Attribute(resource r) : Variant(std::move(r)) ^ ``` Same work-around as for NVCC in #1103 * CMake 3.22+: Policy CMP0127 Fix a warning with CMake 3.22+. We use simple syntax in cmake_dependent_option, so we are compatible with the extended syntax in CMake 3.22+: https://cmake.org/cmake/help/v3.22/policy/CMP0127.html * Docs: Minor Warnings Fix minor warnings in the docs: - double reference (make 2nd ref annonymous) - misaligned reference * Docs: Fix HDF5 JSON File Fix misnamed filename and misnamed prefix in text. * Docs: Update Citation & Add BibTeX Add `.bib` blocks for easier citation of standard and API. * Docs: Fix CLI Highlighting Fix a highlighting issue for `openPMD-ls` in the docs. Explicitly use `python3` for module calls. * CMake: Warning Flags First in CXXFLAGS If we append them, then we overwrite flags like `-Wno-...` from environment variables. * ADIOS2 fixes: Incompatibilites w/ `master` Referes to post-`2.7.1` `master` branch of ADIOS2. Catch new ADIOS2 error type: string getCompressionOperator Dirtily catch ADIOS2 exception in automatically_deactivate_span Cleanly destruct a Series that errored during flushing Problem was that IO tasks that do not return cleanly but throw an exception must be removed from the IO queue, otherwise the task will be tried again during destruction. Usually, this means that anything will happen. SerialIOTest: Remove Comments (resolved) Print warning when removing task from IO queue std::string and fix warning * Automatically deactivate span-based API when operators are present * Add .pre-commit-config.yaml * Release: 0.14.4 Co-authored-by: Franz Pöschel --- .github/workflows/linux.yml | 26 +++ .pre-commit-config.yaml | 27 +++ CHANGELOG.rst | 52 +++++- CITATION.cff | 2 +- CMakeLists.txt | 22 ++- Dockerfile | 19 +- MANIFEST.in | 1 + NEWS.rst | 6 + README.md | 2 +- docs/source/citation.rst | 48 ++++- docs/source/conf.py | 4 +- docs/source/details/backendconfig.rst | 7 +- docs/source/details/{json.json => hdf5.json} | 0 docs/source/details/mpi.rst | 2 +- docs/source/dev/dependencies.rst | 2 +- docs/source/index.rst | 2 +- docs/source/utilities/cli.rst | 6 +- include/openPMD/IO/ADIOS/ADIOS2Auxiliary.hpp | 5 +- include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp | 9 + include/openPMD/IO/AbstractIOHandlerImpl.hpp | 9 +- include/openPMD/IO/IOTask.hpp | 9 + include/openPMD/backend/Container.hpp | 11 ++ include/openPMD/version.hpp | 2 +- setup.py | 5 +- src/IO/ADIOS/ADIOS2IOHandler.cpp | 61 ++++++- src/IO/HDF5/HDF5IOHandler.cpp | 7 +- src/IO/IOTask.cpp | 83 +++++++++ src/Mesh.cpp | 2 + src/ParticleSpecies.cpp | 1 + src/RecordComponent.cpp | 6 +- src/binding/python/Iteration.cpp | 2 +- test/CoreTest.cpp | 6 +- test/SerialIOTest.cpp | 182 ++++++++++++++++++- 33 files changed, 590 insertions(+), 38 deletions(-) create mode 100644 .pre-commit-config.yaml rename docs/source/details/{json.json => hdf5.json} (100%) diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 06383feabb..e1e1b6edec 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -229,6 +229,32 @@ jobs: cmake --build build --parallel 2 ctest --test-dir build --output-on-failure + musllinux_py10: + runs-on: ubuntu-20.04 + if: github.event.pull_request.draft == false + container: + image: quay.io/pypa/musllinux_1_1_x86_64 + steps: + - uses: actions/checkout@v2 + - name: Install + run: | + apk update + apk add hdf5-dev + python3.10 -m pip install numpy + - name: Build + env: {CXXFLAGS: -Werror -Wno-deprecated-declarations} + run: | + share/openPMD/download_samples.sh build + chmod u-w build/samples/git-sample/*.h5 + cmake -S . -B build \ + -DopenPMD_USE_PYTHON=ON \ + -DopenPMD_USE_MPI=OFF \ + -DopenPMD_USE_HDF5=ON \ + -DopenPMD_USE_INVASIVE_TESTS=ON \ + -DPython_EXECUTABLE=$(which python3.10) + cmake --build build --parallel 2 + ctest --test-dir build --output-on-failure + conda_ompi_all: runs-on: ubuntu-20.04 steps: diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000000..7dc5e9e03f --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,27 @@ +# To use: +# +# pre-commit run -a +# +# Or: +# +# pre-commit install # (runs every time you commit in git) +# +# To update this file: +# +# pre-commit autoupdate +# +# See https://pre-commit.com for more information + +# Do not check/format anything from third parties +exclude: '^share/openPMD/thirdParty' + +# See https://pre-commit.com/hooks.html for more hooks +repos: +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.1.0 + hooks: + - id: mixed-line-ending + - id: check-json + - id: check-toml + - id: check-yaml + - id: check-added-large-files diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 8542ef49ea..105aa79808 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -3,7 +3,57 @@ Changelog ========= -0.14.2 +0.14.4 +------ +**Date:** 2022-01-21 + +Increased Compatibility & Python Install Bug + +This release fixes various read/parsing bugs and increases compatibility with upcoming versions of ADIOS and old releases of Intel ``icpc``. +An installation issue for pip-based installs from source in the last release was fixed and Python 3.10 support added. +Various documentation and installation warnings have been fixed. + +Changes to "0.14.3" +^^^^^^^^^^^^^^^^^^^ + +Bug Fixes +""""""""" + +- ADIOS2: + + - automatically deactivate ``span`` based ``Put`` API when operators are present #1155 + - solve incompatibilities w/ post-``2.7.1`` ``master``-branch #1166 +- ICC 19.1.2: C++17 work-arounds (``variant``) #1157 +- Don't apply compression operators multiple times in variable-based iteration encoding #1152 +- Reading/parsing: + + - remove invalid records from data structures entirely #1150 + - fix grid spacing with type long double #1137 +- Python: + + - fix ``Iteration`` ``__repr__`` typo #1149 + - add ``cmake/`` to ``MANIFEST.in`` #1140 + +Other +""""" + +- add simple ``.pre-commit-config.yaml`` +- Python: + + - support Python 3.10 #1139 +- CMake: + + - warning flags first in ``CXXFLAGS`` #1172 + - add policy CMP0127 (v3.22+) #1165 +- Docs: + + - fix CLI highlighting #1171 + - update citation & add BibTeX #1168 + - fix HDF5 JSON File #1169 + - minor warnings #1170 + + +0.14.3 ------ **Date:** 2021-11-03 diff --git a/CITATION.cff b/CITATION.cff index 911d132fda..9bb5879e7b 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -25,7 +25,7 @@ contact: orcid: https://orcid.org/0000-0003-1943-7141 email: axelhuebl@lbl.gov title: "openPMD-api: C++ & Python API for Scientific I/O with openPMD" -version: 0.14.3 +version: 0.14.4 repository-code: https://github.com/openPMD/openPMD-api doi: 10.14278/rodare.27 license: LGPL-3.0-or-later diff --git a/CMakeLists.txt b/CMakeLists.txt index 851eaa4b4d..c8eeec006f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -2,7 +2,7 @@ # cmake_minimum_required(VERSION 3.15.0) -project(openPMD VERSION 0.14.3) # LANGUAGES CXX +project(openPMD VERSION 0.14.4) # LANGUAGES CXX # the openPMD "markup"/"schema" standard version set(openPMD_STANDARD_VERSION 1.1.0) @@ -18,6 +18,13 @@ if(POLICY CMP0074) cmake_policy(SET CMP0074 NEW) endif() +# We use simple syntax in cmake_dependent_option, so we are compatible with the +# extended syntax in CMake 3.22+ +# https://cmake.org/cmake/help/v3.22/policy/CMP0127.html +if(POLICY CMP0127) + cmake_policy(SET CMP0127 NEW) +endif() + # Project structure ########################################################### # @@ -883,23 +890,22 @@ if(CMAKE_SOURCE_DIR STREQUAL PROJECT_SOURCE_DIR) # /usr/lib/llvm-6.0/lib/clang/6.0.0/lib/linux/libclang_rt.ubsan_minimal-x86_64.so # at runtime when used with symbol-hidden code (e.g. pybind11 module) - #set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Weverything") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wextra -Wpedantic -Wshadow -Woverloaded-virtual -Wextra-semi -Wunreachable-code") + set(CMAKE_CXX_FLAGS "-Wall -Wextra -Wpedantic -Wshadow -Woverloaded-virtual -Wextra-semi -Wunreachable-code ${CMAKE_CXX_FLAGS}") elseif("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Intel") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -w3 -wd193,383,1572") + set(CMAKE_CXX_FLAGS "-w3 -wd193,383,1572 ${CMAKE_CXX_FLAGS}") elseif("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wextra -Wpedantic -Wshadow -Woverloaded-virtual -Wunreachable-code") + set(CMAKE_CXX_FLAGS "-Wall -Wextra -Wpedantic -Wshadow -Woverloaded-virtual -Wunreachable-code ${CMAKE_CXX_FLAGS}") elseif("${CMAKE_CXX_COMPILER_ID}" STREQUAL "MSVC") # Warning C4503: "decorated name length exceeded, name was truncated" # Symbols longer than 4096 chars are truncated (and hashed instead) - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -wd4503") + set(CMAKE_CXX_FLAGS "-wd4503 ${CMAKE_CXX_FLAGS}") # Warning C4244: "conversion from 'X' to 'Y', possible loss of data" - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -wd4244") + set(CMAKE_CXX_FLAGS "-wd4244 ${CMAKE_CXX_FLAGS}") # Yes, you should build against the same C++ runtime and with same # configuration (Debug/Release). MSVC does inconvenient choices for their # developers, so be it. (Our Windows-users use conda-forge builds, which # are consistent.) - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -wd4251") + set(CMAKE_CXX_FLAGS "-wd4251 ${CMAKE_CXX_FLAGS}") endif() endif() diff --git a/Dockerfile b/Dockerfile index 0986b1a1cb..6a65bc6380 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,8 +5,8 @@ FROM quay.io/pypa/manylinux2010_x86_64 as build-env # FROM quay.io/pypa/manylinux1_x86_64 as build-env ENV DEBIAN_FRONTEND noninteractive -# Python 3.6-3.9 via "36m 37m 38 39" -ARG PY_VERSIONS="36m 37m 38 39" +# Python 3.6-3.10 via "36m 37m 38 39 310" +ARG PY_VERSIONS="36m 37m 38 39 310" # static libs need relocatable symbols for linking to shared python lib ENV CFLAGS="-fPIC ${CFLAGS}" @@ -188,6 +188,21 @@ RUN python3.9 -c "import openpmd_api as io; print(io.__version__); print( RUN python3.9 -m openpmd_api.ls --help RUN openpmd-ls --help +# test in fresh env: Debian:Bullseye + Python 3.10 +FROM debian:bullseye +ENV DEBIAN_FRONTEND noninteractive +COPY --from=build-env /wheelhouse/openPMD_api-*-cp310-cp310-manylinux2010_x86_64.whl . +RUN apt-get update \ + && apt-get install -y --no-install-recommends python3.10 python3-distutils ca-certificates curl \ + && rm -rf /var/lib/apt/lists/* +RUN python3.10 --version \ + && curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py \ + && python3.10 get-pip.py \ + && python3.10 -m pip install openPMD_api-*-cp310-cp310-manylinux2010_x86_64.whl +RUN python3.10 -c "import openpmd_api as io; print(io.__version__); print(io.variants)" +RUN python3.10 -m openpmd_api.ls --help +RUN openpmd-ls --help + # copy binary artifacts (wheels) FROM quay.io/pypa/manylinux2010_x86_64 diff --git a/MANIFEST.in b/MANIFEST.in index e17a33eeb3..ce02012bf8 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -2,6 +2,7 @@ include README.md COPYING COPYING.LESSER include pyproject.toml include requirements.txt global-include CMakeLists.txt *.cmake *.in +recursive-include cmake * recursive-include include * recursive-include src * recursive-include share * diff --git a/NEWS.rst b/NEWS.rst index 58e27d495e..36d8077c14 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -3,6 +3,12 @@ Upgrade Guide ============= +0.15.0 +------ + +Python 3.10 is now supported. + + 0.14.0 ------ diff --git a/README.md b/README.md index f9d684d451..8f251467e3 100644 --- a/README.md +++ b/README.md @@ -119,7 +119,7 @@ while those can be built either with or without: Optional language bindings: * Python: - * Python 3.6 - 3.9 + * Python 3.6 - 3.10 * pybind11 2.6.2+ * numpy 1.15+ * mpi4py 2.1+ (optional, for MPI) diff --git a/docs/source/citation.rst b/docs/source/citation.rst index f5f8a0d5e9..dca1e51a57 100644 --- a/docs/source/citation.rst +++ b/docs/source/citation.rst @@ -18,6 +18,29 @@ The most general reference to openPMD is: *"openPMD: A meta data standard for particle and mesh based data,"* `DOI:10.5281/zenodo.591699 `_ (2015) +The equivalent BibTeX code is: + +.. code-block:: bibtex + + @misc{openPMDstandard, + author = {Huebl, Axel and + Lehe, R{\'e}mi and + Vay, Jean-Luc and + Grote, David P. and + Sbalzarini, Ivo and + Kuschel, Stephan and + Sagan, David and + P{\'e}rez, Fr{\'e}d{\'e}ric and + Koller, Fabian and + Bussmann, Michael}, + title = {{openPMD: A meta data standard for particle and mesh based data}}, + year = 2015, + publisher = {Zenodo}, + doi = {10.5281/zenodo.591699}, + url = {https://www.openPMD.org}, + howpublished = {https://github.com/openPMD} + } + Since the openPMD-standard is an actively evolving meta data schema, a specific version of the openPMD standard might be used in your work. You can select a version-specific DOI from the `release page `_ and add the version number to the cited title, e.g. @@ -34,13 +57,13 @@ openPMD-api is a **software library** that provides a reference implementation o It targets both desktop as well as high-performance computing environments. It is good scientific practice to document all used software, including transient dependencies, with versions in, e.g. a methods section of a publication. -As a software citation, you almost always want to refer to a *specific version* of openPMD-api in your work, as illustrated for version 0.10.3: +As a software citation, you almost always want to refer to a *specific version* of openPMD-api in your work, as illustrated for version 0.14.3: .. tip:: - Fabian Koller, Franz Poeschel, Junmin Gu, and Axel Huebl. - *"openPMD-api 0.10.3: C++ & Python API for Scientific I/O with openPMD,"* - `DOI:10.14278/rodare.209 `_ (2019) + Axel Huebl, Franz Poeschel, Fabian Koller, and Junmin Gu. + *"openPMD-api 0.14.3: C++ & Python API for Scientific I/O with openPMD,"* + `DOI:10.14278/rodare.1234 `_ (2021) A list of all releases and DOIs can be found `on the release page `_. @@ -52,6 +75,23 @@ We also provide a DOI that refers to all releases of openPMD-api: *"openPMD-api: C++ & Python API for Scientific I/O with openPMD"* `DOI:10.14278/rodare.27 `_ (2018) +The equivalent BibTeX code is: + +.. code-block:: bibtex + + @misc{openPMDapi, + author = {Huebl, Axel and + Poeschel, Franz and + Koller, Fabian and + Gu, Junmin}, + title = {{openPMD-api: C++ \& Python API for Scientific I/O with openPMD}}, + month = june, + year = 2018, + doi = {10.14278/rodare.27}, + url = {https://github.com/openPMD/openPMD-api} + } + + Dependent Software ~~~~~~~~~~~~~~~~~~ diff --git a/docs/source/conf.py b/docs/source/conf.py index 530ee1397b..8eacfd47ea 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -83,9 +83,9 @@ # built documents. # # The short X.Y version. -version = u'0.14.3' +version = u'0.14.4' # The full version, including alpha/beta/rc tags. -release = u'0.14.3' +release = u'0.14.4' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/docs/source/details/backendconfig.rst b/docs/source/details/backendconfig.rst index 7d89976731..a2cc6a94e4 100644 --- a/docs/source/details/backendconfig.rst +++ b/docs/source/details/backendconfig.rst @@ -77,6 +77,11 @@ Explanation of the single keys: * ``type`` supported ADIOS operator type, e.g. zfp, sz * ``parameters`` is an associative map of string parameters for the operator (e.g. compression levels) +* ``adios2.use_span_based_put``: The openPMD-api exposes the `span-based Put() API `_ of ADIOS2 via an overload of ``RecordComponent::storeChunk()``. + This API is incompatible with compression operators as described above. + The openPMD-api will automatically use a fallback implementation for the span-based Put() API if any operator is added to a dataset. + This workaround is enabled on a per-dataset level. + The workaround can be completely deactivated by specifying ``{"adios2": {"use_span_based_put": true}}`` or it can alternatively be activated indiscriminately for all datasets by specifying ``{"adios2": {"use_span_based_put": false}}``. Any setting specified under ``adios2.dataset`` is applicable globally as well as on a per-dataset level. Any setting under ``adios2.engine`` is applicable globally only. @@ -94,7 +99,7 @@ A full configuration of the HDF5 backend: All keys found under ``hdf5.dataset`` are applicable globally (future: as well as per dataset). Explanation of the single keys: -* ``adios2.dataset.chunks``: This key contains options for data chunking via `H5Pset_chunk `__. +* ``hdf5.dataset.chunks``: This key contains options for data chunking via `H5Pset_chunk `__. The default is ``"auto"`` for a heuristic. ``"none"`` can be used to disable chunking. Chunking generally improves performance and only needs to be disabled in corner-cases, e.g. when heavily relying on independent, parallel I/O that non-collectively declares data records. diff --git a/docs/source/details/json.json b/docs/source/details/hdf5.json similarity index 100% rename from docs/source/details/json.json rename to docs/source/details/hdf5.json diff --git a/docs/source/details/mpi.rst b/docs/source/details/mpi.rst index 66a51c7f05..5c0f1674fa 100644 --- a/docs/source/details/mpi.rst +++ b/docs/source/details/mpi.rst @@ -43,7 +43,7 @@ Functionality Behavior Description Note that openPMD represents constant record components with attributes, thus inheriting this for ``::makeConstant``. .. [4] We usually open iterations delayed on first access. This first access is usually the ``flush()`` call after a ``storeChunk``/``loadChunk`` operation. If the first access is non-collective, an explicit, collective ``Iteration::open()`` can be used to have the files already open. -Alternatively, iterations might be accessed for the first time by immediate operations such as ``::availableChunks()``. + Alternatively, iterations might be accessed for the first time by immediate operations such as ``::availableChunks()``. .. tip:: diff --git a/docs/source/dev/dependencies.rst b/docs/source/dev/dependencies.rst index d1f3d11ad1..7b9cb6d860 100644 --- a/docs/source/dev/dependencies.rst +++ b/docs/source/dev/dependencies.rst @@ -39,7 +39,7 @@ Optional: language bindings * Python: - * Python 3.6 - 3.9 + * Python 3.6 - 3.10 * pybind11 2.6.2+ * numpy 1.15+ * mpi4py 2.1+ (optional, for MPI) diff --git a/docs/source/index.rst b/docs/source/index.rst index 975b5fbc1c..d68bf9607d 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -42,7 +42,7 @@ openPMD-api version supported openPMD standard versions ======================== =================================== ``2.0.0+`` ``2.0.0+`` (not released yet) ``1.0.0+`` ``1.0.1-1.1.0`` (not released yet) -``0.13.1-0.14.3`` (beta) ``1.0.0-1.1.0`` +``0.13.1-0.14.4`` (beta) ``1.0.0-1.1.0`` ``0.1.0-0.12.0`` (alpha) ``1.0.0-1.1.0`` ======================== =================================== diff --git a/docs/source/utilities/cli.rst b/docs/source/utilities/cli.rst index 797ee94af7..d661793545 100644 --- a/docs/source/utilities/cli.rst +++ b/docs/source/utilities/cli.rst @@ -19,9 +19,9 @@ The syntax of the command line tool is printed via: With some ``pip``-based python installations, you might have to run this as a module: -.. code-block:: python3 +.. code-block:: bash - python -m openpmd_api.ls --help + python3 -m openpmd_api.ls --help ``openpmd-pipe`` ---------------- @@ -48,4 +48,4 @@ With some ``pip``-based python installations, you might have to run this as a mo .. code-block:: bash - python -m openpmd_api.pipe --help \ No newline at end of file + python3 -m openpmd_api.pipe --help diff --git a/include/openPMD/IO/ADIOS/ADIOS2Auxiliary.hpp b/include/openPMD/IO/ADIOS/ADIOS2Auxiliary.hpp index 3c9d0b2548..26cc560dc2 100644 --- a/include/openPMD/IO/ADIOS/ADIOS2Auxiliary.hpp +++ b/include/openPMD/IO/ADIOS/ADIOS2Auxiliary.hpp @@ -105,6 +105,9 @@ namespace detail * @param IO The IO within which to retrieve the attribute. * @param attributeName The full ADIOS name of the attribute. * @param verbose If true, print a warning if not finding the attribute. + * @param voa This function is used by the old and new ADIOS2 schema alike. + * The old one uses ADIOS2 attributes, the new one uses + * ADIOS2 variables. * @return The openPMD datatype corresponding to the type of the attribute. * UNDEFINED if attribute is not found. */ @@ -113,7 +116,7 @@ namespace detail adios2::IO & IO, std::string const & attributeName, bool verbose, - VariableOrAttribute = VariableOrAttribute::Attribute ); + VariableOrAttribute voa = VariableOrAttribute::Attribute ); } // namespace detail #if defined( _MSC_VER ) && !defined( __INTEL_COMPILER ) && !defined( __clang__ ) diff --git a/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp b/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp index 2fa295373f..46826fe7d7 100644 --- a/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp +++ b/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp @@ -236,6 +236,15 @@ class ADIOS2IOHandlerImpl std::string m_engineType; ADIOS2Schema::schema_t m_schema = ADIOS2Schema::schema_0000_00_00; + enum class UseSpan : char + { + Yes, + No, + Auto + }; + + UseSpan m_useSpanBasedPutByDefault = UseSpan::Auto; + enum class AttributeLayout : char { ByAdiosAttributes, diff --git a/include/openPMD/IO/AbstractIOHandlerImpl.hpp b/include/openPMD/IO/AbstractIOHandlerImpl.hpp index c584462d24..619f0ea857 100644 --- a/include/openPMD/IO/AbstractIOHandlerImpl.hpp +++ b/include/openPMD/IO/AbstractIOHandlerImpl.hpp @@ -25,6 +25,7 @@ #include "openPMD/auxiliary/DerefDynamicCast.hpp" #include +#include namespace openPMD @@ -123,8 +124,14 @@ class AbstractIOHandlerImpl availableChunks(i.writable, deref_dynamic_cast< Parameter< O::AVAILABLE_CHUNKS > >(i.parameter.get())); break; } - } catch (unsupported_data_error&) + } catch (...) { + std::cerr + << "[AbstractIOHandlerImpl] IO Task " + << internal::operationAsString( i.operation ) + << " failed with exception. Removing task" + << " from IO queue and passing on the exception." + << std::endl; (*m_handler).m_work.pop(); throw; } diff --git a/include/openPMD/IO/IOTask.hpp b/include/openPMD/IO/IOTask.hpp index 6054edbaa5..d455556e69 100644 --- a/include/openPMD/IO/IOTask.hpp +++ b/include/openPMD/IO/IOTask.hpp @@ -76,6 +76,15 @@ OPENPMDAPI_EXPORT_ENUM_CLASS(Operation) AVAILABLE_CHUNKS //!< Query chunks that can be loaded in a dataset }; // note: if you change the enum members here, please update docs/source/dev/design.rst +namespace internal +{ + /* + * The returned strings are compile-time constants, so no worries about + * pointer validity. + */ + std::string operationAsString( Operation ); +} + struct OPENPMDAPI_EXPORT AbstractParameter { virtual ~AbstractParameter() = default; diff --git a/include/openPMD/backend/Container.hpp b/include/openPMD/backend/Container.hpp index 156e55e292..509d6f230d 100644 --- a/include/openPMD/backend/Container.hpp +++ b/include/openPMD/backend/Container.hpp @@ -366,6 +366,17 @@ class Container : public LegacyAttributable return m_originalContainer.at( std::forward< K >( k ) ); } + /** + * Remove key from the list of accessed keys. + * If the key is not accessed after this again, it will be deleted along + * with all other unaccessed keys upon destruction. + */ + template< typename K > + void forget( K && k ) + { + m_accessedKeys.erase( std::forward< K >( k ) ); + } + ~EraseStaleEntries() { auto & map = *m_originalContainer.m_container; diff --git a/include/openPMD/version.hpp b/include/openPMD/version.hpp index dd264b0682..114221b26d 100644 --- a/include/openPMD/version.hpp +++ b/include/openPMD/version.hpp @@ -29,7 +29,7 @@ */ #define OPENPMDAPI_VERSION_MAJOR 0 #define OPENPMDAPI_VERSION_MINOR 14 -#define OPENPMDAPI_VERSION_PATCH 3 +#define OPENPMDAPI_VERSION_PATCH 4 #define OPENPMDAPI_VERSION_LABEL "" /** @} */ diff --git a/setup.py b/setup.py index 63b6256166..2ab46e545a 100644 --- a/setup.py +++ b/setup.py @@ -156,7 +156,7 @@ def build_extension(self, ext): setup( name='openPMD-api', # note PEP-440 syntax: x.y.zaN but x.y.z.devN - version='0.14.3', + version='0.14.4', author='Axel Huebl, Franz Poeschel, Fabian Koller, Junmin Gu', author_email='axelhuebl@lbl.gov, f.poeschel@hzdr.de', maintainer='Axel Huebl', @@ -178,7 +178,7 @@ def build_extension(self, ext): cmdclass=dict(build_ext=CMakeBuild), # scripts=['openpmd-ls'], zip_safe=False, - python_requires='>=3.6, <3.10', + python_requires='>=3.6, <3.11', # tests_require=['pytest'], install_requires=install_requires, # see: src/bindings/python/cli @@ -211,6 +211,7 @@ def build_extension(self, ext): 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', + 'Programming Language :: Python :: 3.10', ('License :: OSI Approved :: ' 'GNU Lesser General Public License v3 or later (LGPLv3+)'), ], diff --git a/src/IO/ADIOS/ADIOS2IOHandler.cpp b/src/IO/ADIOS/ADIOS2IOHandler.cpp index 23a6efaccb..04d53249e3 100644 --- a/src/IO/ADIOS/ADIOS2IOHandler.cpp +++ b/src/IO/ADIOS/ADIOS2IOHandler.cpp @@ -132,6 +132,14 @@ ADIOS2IOHandlerImpl::init( nlohmann::json cfg ) m_config[ "schema" ].json().get< ADIOS2Schema::schema_t >(); } + if( m_config.json().contains( "use_span_based_put" ) ) + { + m_useSpanBasedPutByDefault = + m_config[ "use_span_based_put" ].json().get< bool >() + ? UseSpan::Yes + : UseSpan::No; + } + auto engineConfig = config( ADIOS2Defaults::str_engine ); if( !engineConfig.json().is_null() ) { @@ -704,6 +712,22 @@ struct GetSpan std::string errorMsg = "ADIOS2: getBufferView()"; }; + +struct HasOperators +{ + template< typename T > + bool operator()( std::string const & name, adios2::IO & IO ) const + { + adios2::Variable< T > variable = IO.InquireVariable< T >( name ); + if( !variable ) + { + return false; + } + return !variable.Operations().empty(); + } + + std::string errorMsg = "ADIOS2: getBufferView()"; +}; } // namespace detail void @@ -721,6 +745,28 @@ ADIOS2IOHandlerImpl::getBufferView( auto file = refreshFileFromParent( writable, /* preferParentFile = */ false ); detail::BufferedActions & ba = getFileData( file, IfFileNotOpen::ThrowError ); + + std::string name = nameOfVariable( writable ); + switch( m_useSpanBasedPutByDefault ) + { + case UseSpan::No: + parameters.out->backendManagedBuffer = false; + return; + case UseSpan::Auto: + { + detail::HasOperators hasOperators; + if( switchAdios2VariableType( + parameters.dtype, hasOperators, name, ba.m_IO ) ) + { + parameters.out->backendManagedBuffer = false; + return; + } + break; + } + case UseSpan::Yes: + break; + } + if( parameters.update ) { detail::I_UpdateSpan &updater = @@ -731,7 +777,6 @@ ADIOS2IOHandlerImpl::getBufferView( else { static detail::GetSpan gs; - std::string name = nameOfVariable( writable ); switchAdios2VariableType( parameters.dtype, gs, this, parameters, ba, name ); } } @@ -1125,11 +1170,21 @@ ADIOS2IOHandlerImpl::getCompressionOperator( std::string const & compression ) try { res = m_ADIOS.DefineOperator( compression, compression ); } - catch ( std::invalid_argument const & ) + catch ( std::invalid_argument const & e ) { std::cerr << "Warning: ADIOS2 backend does not support compression " "method " << compression << ". Continuing without compression." + << "\nOriginal error: " << e.what() + << std::endl; + return auxiliary::Option< adios2::Operator >(); + } + catch(std::string const & s) + { + std::cerr << "Warning: ADIOS2 backend does not support compression " + "method " + << compression << ". Continuing without compression." + << "\nOriginal error: " << s << std::endl; return auxiliary::Option< adios2::Operator >(); } @@ -1578,6 +1633,8 @@ namespace detail { var.SetSelection( { start, count } ); } + // don't add compression operators multiple times + return; } if( !var ) diff --git a/src/IO/HDF5/HDF5IOHandler.cpp b/src/IO/HDF5/HDF5IOHandler.cpp index 09ddc63fd4..56fad7c737 100644 --- a/src/IO/HDF5/HDF5IOHandler.cpp +++ b/src/IO/HDF5/HDF5IOHandler.cpp @@ -346,7 +346,12 @@ HDF5IOHandlerImpl::createDataset(Writable* writable, std::cerr << "[HDF5] Datatype::UNDEFINED caught during dataset creation (serial HDF5)" << std::endl; d = Datatype::BOOL; } - Attribute a(0); + + // note: due to a C++17 issue with ICC 19.1.2 we write the + // T value to variant conversion explicitly + // https://github.com/openPMD/openPMD-api/pull/... + // Attribute a(0); + Attribute a(static_cast(0)); a.dtype = d; std::vector< hsize_t > dims; std::uint64_t num_elements = 1u; diff --git a/src/IO/IOTask.cpp b/src/IO/IOTask.cpp index 0d46627820..074cb24c25 100644 --- a/src/IO/IOTask.cpp +++ b/src/IO/IOTask.cpp @@ -27,4 +27,87 @@ namespace openPMD Writable* getWritable(AttributableInterface* a) { return &a->writable(); } + +namespace internal +{ + std::string + operationAsString( Operation op ) + { + switch( op ) + { + case Operation::CREATE_FILE: + return "CREATE_FILE"; + break; + case Operation::OPEN_FILE: + return "OPEN_FILE"; + break; + case Operation::CLOSE_FILE: + return "CLOSE_FILE"; + break; + case Operation::DELETE_FILE: + return "DELETE_FILE"; + break; + case Operation::CREATE_PATH: + return "CREATE_PATH"; + break; + case Operation::CLOSE_PATH: + return "CLOSE_PATH"; + break; + case Operation::OPEN_PATH: + return "OPEN_PATH"; + break; + case Operation::DELETE_PATH: + return "DELETE_PATH"; + break; + case Operation::LIST_PATHS: + return "LIST_PATHS"; + break; + case Operation::CREATE_DATASET: + return "CREATE_DATASET"; + break; + case Operation::EXTEND_DATASET: + return "EXTEND_DATASET"; + break; + case Operation::OPEN_DATASET: + return "OPEN_DATASET"; + break; + case Operation::DELETE_DATASET: + return "DELETE_DATASET"; + break; + case Operation::WRITE_DATASET: + return "WRITE_DATASET"; + break; + case Operation::READ_DATASET: + return "READ_DATASET"; + break; + case Operation::LIST_DATASETS: + return "LIST_DATASETS"; + break; + case Operation::GET_BUFFER_VIEW: + return "GET_BUFFER_VIEW"; + break; + case Operation::DELETE_ATT: + return "DELETE_ATT"; + break; + case Operation::WRITE_ATT: + return "WRITE_ATT"; + break; + case Operation::READ_ATT: + return "READ_ATT"; + break; + case Operation::LIST_ATTS: + return "LIST_ATTS"; + break; + case Operation::ADVANCE: + return "ADVANCE"; + break; + case Operation::AVAILABLE_CHUNKS: + return "AVAILABLE_CHUNKS"; + break; + default: + return "unknown"; + break; + } + } +} } // openPMD diff --git a/src/Mesh.cpp b/src/Mesh.cpp index e9551c42e6..0e09cfc06b 100644 --- a/src/Mesh.cpp +++ b/src/Mesh.cpp @@ -331,6 +331,8 @@ Mesh::read() setGridSpacing(a.get< std::vector< float > >()); else if( *aRead.dtype == DT::VEC_DOUBLE || *aRead.dtype == DT::DOUBLE ) setGridSpacing(a.get< std::vector< double > >()); + else if( *aRead.dtype == DT::VEC_LONG_DOUBLE || *aRead.dtype == DT::LONG_DOUBLE ) + setGridSpacing(a.get< std::vector< long double > >()); else throw std::runtime_error("Unexpected Attribute datatype for 'gridSpacing'"); diff --git a/src/ParticleSpecies.cpp b/src/ParticleSpecies.cpp index 55e50df5fe..61861b4a1a 100644 --- a/src/ParticleSpecies.cpp +++ b/src/ParticleSpecies.cpp @@ -118,6 +118,7 @@ ParticleSpecies::read() while( ! IOHandler()->m_work.empty() ) IOHandler()->m_work.pop(); + map.forget( record_name ); //(*this)[record_name].erase(RecordComponent::SCALAR); //this->erase(record_name); } diff --git a/src/RecordComponent.cpp b/src/RecordComponent.cpp index d40061c198..05342462e7 100644 --- a/src/RecordComponent.cpp +++ b/src/RecordComponent.cpp @@ -208,7 +208,11 @@ RecordComponent::flush(std::string const& name) aWrite.resource = m_constantValue->getResource(); IOHandler()->enqueue(IOTask(this, aWrite)); aWrite.name = "shape"; - Attribute a(getExtent()); + // note: due to a C++17 issue with ICC 19.1.2 we write the + // T value to variant conversion explicitly + // https://github.com/openPMD/openPMD-api/pull/... + // Attribute a(getExtent()); + Attribute a(static_cast(getExtent())); aWrite.dtype = a.dtype; aWrite.resource = a.getResource(); IOHandler()->enqueue(IOTask(this, aWrite)); diff --git a/src/binding/python/Iteration.cpp b/src/binding/python/Iteration.cpp index 255ed41331..ec85f4fd9a 100644 --- a/src/binding/python/Iteration.cpp +++ b/src/binding/python/Iteration.cpp @@ -35,7 +35,7 @@ void init_Iteration(py::module &m) { .def("__repr__", [](Iteration const & it) { - return ""; + return ""; } ) diff --git a/test/CoreTest.cpp b/test/CoreTest.cpp index 72830b0df2..20ce35e8ed 100644 --- a/test/CoreTest.cpp +++ b/test/CoreTest.cpp @@ -37,7 +37,11 @@ TEST_CASE( "versions_test", "[core]" ) TEST_CASE( "attribute_dtype_test", "[core]" ) { - Attribute a = Attribute(static_cast< char >(' ')); + // note: due to a C++17 issue with ICC 19.1.2 we write the + // T value to variant conversion explicitly + // https://github.com/openPMD/openPMD-api/pull/... + // Attribute a = Attribute(static_cast< char >(' ')); + Attribute a = Attribute(static_cast(static_cast< char >(' '))); REQUIRE(Datatype::CHAR == a.dtype); a = Attribute(static_cast< unsigned char >(' ')); REQUIRE(Datatype::UCHAR == a.dtype); diff --git a/test/SerialIOTest.cpp b/test/SerialIOTest.cpp index 1627e1e9b7..412d8058e2 100644 --- a/test/SerialIOTest.cpp +++ b/test/SerialIOTest.cpp @@ -90,7 +90,7 @@ TEST_CASE( "adios2_char_portability", "[serial][adios2]" ) writeAttribute( "/openPMD", std::string( "1.1.0" ) ); writeAttribute( "/openPMDextension", uint32_t( 0 ) ); writeAttribute( "/software", std::string( "openPMD-api" ) ); - writeAttribute( "/softwareVersion", std::string( "0.14.3" ) ); + writeAttribute( "/softwareVersion", std::string( "0.14.4" ) ); IO.DefineAttribute< uint64_t >( "__openPMD_internal/openPMD2_adios2_schema", 20210209 ); @@ -1167,6 +1167,16 @@ void dtype_test( const std::string & backend ) { s.setAttribute("vecULongLong", std::vector< unsigned long long >({65531u, 65529u})); } + + // long double grid spacing + // should be possible to parse without error upon opening + // the series for reading + { + auto E = s.iterations[ 0 ].meshes[ "E" ]; + E.setGridSpacing( std::vector< long double >{ 1.0, 1.0 } ); + auto E_x = E[ "x" ]; + E_x.makeEmpty< double >( 1 ); + } } Series s = Series("../samples/dtype_test." + backend, Access::READ_ONLY); @@ -4203,6 +4213,176 @@ TEST_CASE( "variableBasedParticleData", "[serial][adios2]" ) } #endif +#if openPMD_HAVE_ADIOS2 +#ifdef ADIOS2_HAVE_BZIP2 +TEST_CASE( "automatically_deactivate_span", "[serial][adios2]" ) +{ + // automatically (de)activate span-based storeChunking + { + Series write( "../samples/span_based.bp", Access::CREATE ); + auto E_uncompressed = write.iterations[ 0 ].meshes[ "E" ][ "x" ]; + auto E_compressed = write.iterations[ 0 ].meshes[ "E" ][ "y" ]; + + Dataset ds{ Datatype::INT, { 10 } }; + + E_uncompressed.resetDataset( ds ); + + std::string compression = R"END( +{ + "adios2": { + "dataset": { + "operators": [ + { + "type": "bzip2" + } + ] + } + } +})END"; + + ds.options = compression; + E_compressed.resetDataset( ds ); + + bool spanWorkaround = false; + E_uncompressed.storeChunk< int >( + { 0 }, { 10 }, [ &spanWorkaround ]( size_t size ) { + spanWorkaround = true; + return std::shared_ptr< int >( + new int[ size ]{}, []( auto * ptr ) { delete[] ptr; } ); + } ); + + REQUIRE( !spanWorkaround ); + + E_compressed.storeChunk< int >( + { 0 }, { 10 }, [ &spanWorkaround ]( size_t size ) { + spanWorkaround = true; + return std::shared_ptr< int >( + new int[ size ]{}, []( auto * ptr ) { delete[] ptr; } ); + } ); + + REQUIRE( spanWorkaround ); + } + + // enable span-based API indiscriminately + { + std::string enable = R"END( +{ + "adios2": { + "use_span_based_put": true + } +})END"; + Series write( "../samples/span_based.bp", Access::CREATE, enable ); + auto E_uncompressed = write.iterations[ 0 ].meshes[ "E" ][ "x" ]; + auto E_compressed = write.iterations[ 0 ].meshes[ "E" ][ "y" ]; + + Dataset ds{ Datatype::INT, { 10 } }; + + E_uncompressed.resetDataset( ds ); + + std::string compression = R"END( +{ + "adios2": { + "dataset": { + "operators": [ + { + "type": "bzip2" + } + ] + } + } +})END"; + + ds.options = compression; + E_compressed.resetDataset( ds ); + + bool spanWorkaround = false; + E_uncompressed.storeChunk< int >( + { 0 }, { 10 }, [ &spanWorkaround ]( size_t size ) { + spanWorkaround = true; + return std::shared_ptr< int >( + new int[ size ]{}, []( auto * ptr ) { delete[] ptr; } ); + } ); + + REQUIRE( !spanWorkaround ); + + try + { + E_compressed.storeChunk< int >( + { 0 }, { 10 }, [ &spanWorkaround ]( size_t size ) { + spanWorkaround = true; + return std::shared_ptr< int >( + new int[ size ]{}, []( auto * ptr ) { delete[] ptr; } ); + } ); + } + catch( std::invalid_argument const & e ) + { + /* + * Using the span-based API in combination with compression is + * unsupported in ADIOS2. + * In newer versions of ADIOS2, an error is thrown. + */ + std::cerr << "Ignoring expected error: " << e.what() << std::endl; + } + + REQUIRE( !spanWorkaround ); + } + + // disable span-based API indiscriminately + { + std::string disable = R"END( +{ + "adios2": { + "use_span_based_put": false + } +})END"; + Series write( "../samples/span_based.bp", Access::CREATE, disable ); + auto E_uncompressed = write.iterations[ 0 ].meshes[ "E" ][ "x" ]; + auto E_compressed = write.iterations[ 0 ].meshes[ "E" ][ "y" ]; + + Dataset ds{ Datatype::INT, { 10 } }; + + E_uncompressed.resetDataset( ds ); + + std::string compression = R"END( +{ + "adios2": { + "dataset": { + "operators": [ + { + "type": "bzip2" + } + ] + } + } +})END"; + + ds.options = compression; + E_compressed.resetDataset( ds ); + + bool spanWorkaround = false; + E_uncompressed.storeChunk< int >( + { 0 }, { 10 }, [ &spanWorkaround ]( size_t size ) { + spanWorkaround = true; + return std::shared_ptr< int >( + new int[ size ]{}, []( auto * ptr ) { delete[] ptr; } ); + } ); + + REQUIRE( spanWorkaround ); + + spanWorkaround = false; + E_compressed.storeChunk< int >( + { 0 }, { 10 }, [ &spanWorkaround ]( size_t size ) { + spanWorkaround = true; + return std::shared_ptr< int >( + new int[ size ]{}, []( auto * ptr ) { delete[] ptr; } ); + } ); + + REQUIRE( spanWorkaround ); + } +} +#endif +#endif + // @todo Upon switching to ADIOS2 2.7.0, test this the other way around also void iterate_nonstreaming_series(