diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 06383feabb..e1e1b6edec 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -229,6 +229,32 @@ jobs: cmake --build build --parallel 2 ctest --test-dir build --output-on-failure + musllinux_py10: + runs-on: ubuntu-20.04 + if: github.event.pull_request.draft == false + container: + image: quay.io/pypa/musllinux_1_1_x86_64 + steps: + - uses: actions/checkout@v2 + - name: Install + run: | + apk update + apk add hdf5-dev + python3.10 -m pip install numpy + - name: Build + env: {CXXFLAGS: -Werror -Wno-deprecated-declarations} + run: | + share/openPMD/download_samples.sh build + chmod u-w build/samples/git-sample/*.h5 + cmake -S . -B build \ + -DopenPMD_USE_PYTHON=ON \ + -DopenPMD_USE_MPI=OFF \ + -DopenPMD_USE_HDF5=ON \ + -DopenPMD_USE_INVASIVE_TESTS=ON \ + -DPython_EXECUTABLE=$(which python3.10) + cmake --build build --parallel 2 + ctest --test-dir build --output-on-failure + conda_ompi_all: runs-on: ubuntu-20.04 steps: diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000000..7dc5e9e03f --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,27 @@ +# To use: +# +# pre-commit run -a +# +# Or: +# +# pre-commit install # (runs every time you commit in git) +# +# To update this file: +# +# pre-commit autoupdate +# +# See https://pre-commit.com for more information + +# Do not check/format anything from third parties +exclude: '^share/openPMD/thirdParty' + +# See https://pre-commit.com/hooks.html for more hooks +repos: +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.1.0 + hooks: + - id: mixed-line-ending + - id: check-json + - id: check-toml + - id: check-yaml + - id: check-added-large-files diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 8542ef49ea..105aa79808 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -3,7 +3,57 @@ Changelog ========= -0.14.2 +0.14.4 +------ +**Date:** 2022-01-21 + +Increased Compatibility & Python Install Bug + +This release fixes various read/parsing bugs and increases compatibility with upcoming versions of ADIOS and old releases of Intel ``icpc``. +An installation issue for pip-based installs from source in the last release was fixed and Python 3.10 support added. +Various documentation and installation warnings have been fixed. + +Changes to "0.14.3" +^^^^^^^^^^^^^^^^^^^ + +Bug Fixes +""""""""" + +- ADIOS2: + + - automatically deactivate ``span`` based ``Put`` API when operators are present #1155 + - solve incompatibilities w/ post-``2.7.1`` ``master``-branch #1166 +- ICC 19.1.2: C++17 work-arounds (``variant``) #1157 +- Don't apply compression operators multiple times in variable-based iteration encoding #1152 +- Reading/parsing: + + - remove invalid records from data structures entirely #1150 + - fix grid spacing with type long double #1137 +- Python: + + - fix ``Iteration`` ``__repr__`` typo #1149 + - add ``cmake/`` to ``MANIFEST.in`` #1140 + +Other +""""" + +- add simple ``.pre-commit-config.yaml`` +- Python: + + - support Python 3.10 #1139 +- CMake: + + - warning flags first in ``CXXFLAGS`` #1172 + - add policy CMP0127 (v3.22+) #1165 +- Docs: + + - fix CLI highlighting #1171 + - update citation & add BibTeX #1168 + - fix HDF5 JSON File #1169 + - minor warnings #1170 + + +0.14.3 ------ **Date:** 2021-11-03 diff --git a/CITATION.cff b/CITATION.cff index 911d132fda..9bb5879e7b 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -25,7 +25,7 @@ contact: orcid: https://orcid.org/0000-0003-1943-7141 email: axelhuebl@lbl.gov title: "openPMD-api: C++ & Python API for Scientific I/O with openPMD" -version: 0.14.3 +version: 0.14.4 repository-code: https://github.com/openPMD/openPMD-api doi: 10.14278/rodare.27 license: LGPL-3.0-or-later diff --git a/CMakeLists.txt b/CMakeLists.txt index 851eaa4b4d..c8eeec006f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -2,7 +2,7 @@ # cmake_minimum_required(VERSION 3.15.0) -project(openPMD VERSION 0.14.3) # LANGUAGES CXX +project(openPMD VERSION 0.14.4) # LANGUAGES CXX # the openPMD "markup"/"schema" standard version set(openPMD_STANDARD_VERSION 1.1.0) @@ -18,6 +18,13 @@ if(POLICY CMP0074) cmake_policy(SET CMP0074 NEW) endif() +# We use simple syntax in cmake_dependent_option, so we are compatible with the +# extended syntax in CMake 3.22+ +# https://cmake.org/cmake/help/v3.22/policy/CMP0127.html +if(POLICY CMP0127) + cmake_policy(SET CMP0127 NEW) +endif() + # Project structure ########################################################### # @@ -883,23 +890,22 @@ if(CMAKE_SOURCE_DIR STREQUAL PROJECT_SOURCE_DIR) # /usr/lib/llvm-6.0/lib/clang/6.0.0/lib/linux/libclang_rt.ubsan_minimal-x86_64.so # at runtime when used with symbol-hidden code (e.g. pybind11 module) - #set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Weverything") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wextra -Wpedantic -Wshadow -Woverloaded-virtual -Wextra-semi -Wunreachable-code") + set(CMAKE_CXX_FLAGS "-Wall -Wextra -Wpedantic -Wshadow -Woverloaded-virtual -Wextra-semi -Wunreachable-code ${CMAKE_CXX_FLAGS}") elseif("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Intel") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -w3 -wd193,383,1572") + set(CMAKE_CXX_FLAGS "-w3 -wd193,383,1572 ${CMAKE_CXX_FLAGS}") elseif("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wextra -Wpedantic -Wshadow -Woverloaded-virtual -Wunreachable-code") + set(CMAKE_CXX_FLAGS "-Wall -Wextra -Wpedantic -Wshadow -Woverloaded-virtual -Wunreachable-code ${CMAKE_CXX_FLAGS}") elseif("${CMAKE_CXX_COMPILER_ID}" STREQUAL "MSVC") # Warning C4503: "decorated name length exceeded, name was truncated" # Symbols longer than 4096 chars are truncated (and hashed instead) - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -wd4503") + set(CMAKE_CXX_FLAGS "-wd4503 ${CMAKE_CXX_FLAGS}") # Warning C4244: "conversion from 'X' to 'Y', possible loss of data" - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -wd4244") + set(CMAKE_CXX_FLAGS "-wd4244 ${CMAKE_CXX_FLAGS}") # Yes, you should build against the same C++ runtime and with same # configuration (Debug/Release). MSVC does inconvenient choices for their # developers, so be it. (Our Windows-users use conda-forge builds, which # are consistent.) - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -wd4251") + set(CMAKE_CXX_FLAGS "-wd4251 ${CMAKE_CXX_FLAGS}") endif() endif() diff --git a/Dockerfile b/Dockerfile index 0986b1a1cb..6a65bc6380 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,8 +5,8 @@ FROM quay.io/pypa/manylinux2010_x86_64 as build-env # FROM quay.io/pypa/manylinux1_x86_64 as build-env ENV DEBIAN_FRONTEND noninteractive -# Python 3.6-3.9 via "36m 37m 38 39" -ARG PY_VERSIONS="36m 37m 38 39" +# Python 3.6-3.10 via "36m 37m 38 39 310" +ARG PY_VERSIONS="36m 37m 38 39 310" # static libs need relocatable symbols for linking to shared python lib ENV CFLAGS="-fPIC ${CFLAGS}" @@ -188,6 +188,21 @@ RUN python3.9 -c "import openpmd_api as io; print(io.__version__); print( RUN python3.9 -m openpmd_api.ls --help RUN openpmd-ls --help +# test in fresh env: Debian:Bullseye + Python 3.10 +FROM debian:bullseye +ENV DEBIAN_FRONTEND noninteractive +COPY --from=build-env /wheelhouse/openPMD_api-*-cp310-cp310-manylinux2010_x86_64.whl . +RUN apt-get update \ + && apt-get install -y --no-install-recommends python3.10 python3-distutils ca-certificates curl \ + && rm -rf /var/lib/apt/lists/* +RUN python3.10 --version \ + && curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py \ + && python3.10 get-pip.py \ + && python3.10 -m pip install openPMD_api-*-cp310-cp310-manylinux2010_x86_64.whl +RUN python3.10 -c "import openpmd_api as io; print(io.__version__); print(io.variants)" +RUN python3.10 -m openpmd_api.ls --help +RUN openpmd-ls --help + # copy binary artifacts (wheels) FROM quay.io/pypa/manylinux2010_x86_64 diff --git a/MANIFEST.in b/MANIFEST.in index e17a33eeb3..ce02012bf8 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -2,6 +2,7 @@ include README.md COPYING COPYING.LESSER include pyproject.toml include requirements.txt global-include CMakeLists.txt *.cmake *.in +recursive-include cmake * recursive-include include * recursive-include src * recursive-include share * diff --git a/NEWS.rst b/NEWS.rst index 58e27d495e..36d8077c14 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -3,6 +3,12 @@ Upgrade Guide ============= +0.15.0 +------ + +Python 3.10 is now supported. + + 0.14.0 ------ diff --git a/README.md b/README.md index f9d684d451..8f251467e3 100644 --- a/README.md +++ b/README.md @@ -119,7 +119,7 @@ while those can be built either with or without: Optional language bindings: * Python: - * Python 3.6 - 3.9 + * Python 3.6 - 3.10 * pybind11 2.6.2+ * numpy 1.15+ * mpi4py 2.1+ (optional, for MPI) diff --git a/docs/source/citation.rst b/docs/source/citation.rst index f5f8a0d5e9..dca1e51a57 100644 --- a/docs/source/citation.rst +++ b/docs/source/citation.rst @@ -18,6 +18,29 @@ The most general reference to openPMD is: *"openPMD: A meta data standard for particle and mesh based data,"* `DOI:10.5281/zenodo.591699 `_ (2015) +The equivalent BibTeX code is: + +.. code-block:: bibtex + + @misc{openPMDstandard, + author = {Huebl, Axel and + Lehe, R{\'e}mi and + Vay, Jean-Luc and + Grote, David P. and + Sbalzarini, Ivo and + Kuschel, Stephan and + Sagan, David and + P{\'e}rez, Fr{\'e}d{\'e}ric and + Koller, Fabian and + Bussmann, Michael}, + title = {{openPMD: A meta data standard for particle and mesh based data}}, + year = 2015, + publisher = {Zenodo}, + doi = {10.5281/zenodo.591699}, + url = {https://www.openPMD.org}, + howpublished = {https://github.com/openPMD} + } + Since the openPMD-standard is an actively evolving meta data schema, a specific version of the openPMD standard might be used in your work. You can select a version-specific DOI from the `release page `_ and add the version number to the cited title, e.g. @@ -34,13 +57,13 @@ openPMD-api is a **software library** that provides a reference implementation o It targets both desktop as well as high-performance computing environments. It is good scientific practice to document all used software, including transient dependencies, with versions in, e.g. a methods section of a publication. -As a software citation, you almost always want to refer to a *specific version* of openPMD-api in your work, as illustrated for version 0.10.3: +As a software citation, you almost always want to refer to a *specific version* of openPMD-api in your work, as illustrated for version 0.14.3: .. tip:: - Fabian Koller, Franz Poeschel, Junmin Gu, and Axel Huebl. - *"openPMD-api 0.10.3: C++ & Python API for Scientific I/O with openPMD,"* - `DOI:10.14278/rodare.209 `_ (2019) + Axel Huebl, Franz Poeschel, Fabian Koller, and Junmin Gu. + *"openPMD-api 0.14.3: C++ & Python API for Scientific I/O with openPMD,"* + `DOI:10.14278/rodare.1234 `_ (2021) A list of all releases and DOIs can be found `on the release page `_. @@ -52,6 +75,23 @@ We also provide a DOI that refers to all releases of openPMD-api: *"openPMD-api: C++ & Python API for Scientific I/O with openPMD"* `DOI:10.14278/rodare.27 `_ (2018) +The equivalent BibTeX code is: + +.. code-block:: bibtex + + @misc{openPMDapi, + author = {Huebl, Axel and + Poeschel, Franz and + Koller, Fabian and + Gu, Junmin}, + title = {{openPMD-api: C++ \& Python API for Scientific I/O with openPMD}}, + month = june, + year = 2018, + doi = {10.14278/rodare.27}, + url = {https://github.com/openPMD/openPMD-api} + } + + Dependent Software ~~~~~~~~~~~~~~~~~~ diff --git a/docs/source/conf.py b/docs/source/conf.py index 530ee1397b..8eacfd47ea 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -83,9 +83,9 @@ # built documents. # # The short X.Y version. -version = u'0.14.3' +version = u'0.14.4' # The full version, including alpha/beta/rc tags. -release = u'0.14.3' +release = u'0.14.4' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/docs/source/details/backendconfig.rst b/docs/source/details/backendconfig.rst index 7d89976731..a2cc6a94e4 100644 --- a/docs/source/details/backendconfig.rst +++ b/docs/source/details/backendconfig.rst @@ -77,6 +77,11 @@ Explanation of the single keys: * ``type`` supported ADIOS operator type, e.g. zfp, sz * ``parameters`` is an associative map of string parameters for the operator (e.g. compression levels) +* ``adios2.use_span_based_put``: The openPMD-api exposes the `span-based Put() API `_ of ADIOS2 via an overload of ``RecordComponent::storeChunk()``. + This API is incompatible with compression operators as described above. + The openPMD-api will automatically use a fallback implementation for the span-based Put() API if any operator is added to a dataset. + This workaround is enabled on a per-dataset level. + The workaround can be completely deactivated by specifying ``{"adios2": {"use_span_based_put": true}}`` or it can alternatively be activated indiscriminately for all datasets by specifying ``{"adios2": {"use_span_based_put": false}}``. Any setting specified under ``adios2.dataset`` is applicable globally as well as on a per-dataset level. Any setting under ``adios2.engine`` is applicable globally only. @@ -94,7 +99,7 @@ A full configuration of the HDF5 backend: All keys found under ``hdf5.dataset`` are applicable globally (future: as well as per dataset). Explanation of the single keys: -* ``adios2.dataset.chunks``: This key contains options for data chunking via `H5Pset_chunk `__. +* ``hdf5.dataset.chunks``: This key contains options for data chunking via `H5Pset_chunk `__. The default is ``"auto"`` for a heuristic. ``"none"`` can be used to disable chunking. Chunking generally improves performance and only needs to be disabled in corner-cases, e.g. when heavily relying on independent, parallel I/O that non-collectively declares data records. diff --git a/docs/source/details/json.json b/docs/source/details/hdf5.json similarity index 100% rename from docs/source/details/json.json rename to docs/source/details/hdf5.json diff --git a/docs/source/details/mpi.rst b/docs/source/details/mpi.rst index 66a51c7f05..5c0f1674fa 100644 --- a/docs/source/details/mpi.rst +++ b/docs/source/details/mpi.rst @@ -43,7 +43,7 @@ Functionality Behavior Description Note that openPMD represents constant record components with attributes, thus inheriting this for ``::makeConstant``. .. [4] We usually open iterations delayed on first access. This first access is usually the ``flush()`` call after a ``storeChunk``/``loadChunk`` operation. If the first access is non-collective, an explicit, collective ``Iteration::open()`` can be used to have the files already open. -Alternatively, iterations might be accessed for the first time by immediate operations such as ``::availableChunks()``. + Alternatively, iterations might be accessed for the first time by immediate operations such as ``::availableChunks()``. .. tip:: diff --git a/docs/source/dev/dependencies.rst b/docs/source/dev/dependencies.rst index d1f3d11ad1..7b9cb6d860 100644 --- a/docs/source/dev/dependencies.rst +++ b/docs/source/dev/dependencies.rst @@ -39,7 +39,7 @@ Optional: language bindings * Python: - * Python 3.6 - 3.9 + * Python 3.6 - 3.10 * pybind11 2.6.2+ * numpy 1.15+ * mpi4py 2.1+ (optional, for MPI) diff --git a/docs/source/index.rst b/docs/source/index.rst index 975b5fbc1c..d68bf9607d 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -42,7 +42,7 @@ openPMD-api version supported openPMD standard versions ======================== =================================== ``2.0.0+`` ``2.0.0+`` (not released yet) ``1.0.0+`` ``1.0.1-1.1.0`` (not released yet) -``0.13.1-0.14.3`` (beta) ``1.0.0-1.1.0`` +``0.13.1-0.14.4`` (beta) ``1.0.0-1.1.0`` ``0.1.0-0.12.0`` (alpha) ``1.0.0-1.1.0`` ======================== =================================== diff --git a/docs/source/utilities/cli.rst b/docs/source/utilities/cli.rst index 797ee94af7..d661793545 100644 --- a/docs/source/utilities/cli.rst +++ b/docs/source/utilities/cli.rst @@ -19,9 +19,9 @@ The syntax of the command line tool is printed via: With some ``pip``-based python installations, you might have to run this as a module: -.. code-block:: python3 +.. code-block:: bash - python -m openpmd_api.ls --help + python3 -m openpmd_api.ls --help ``openpmd-pipe`` ---------------- @@ -48,4 +48,4 @@ With some ``pip``-based python installations, you might have to run this as a mo .. code-block:: bash - python -m openpmd_api.pipe --help \ No newline at end of file + python3 -m openpmd_api.pipe --help diff --git a/include/openPMD/IO/ADIOS/ADIOS2Auxiliary.hpp b/include/openPMD/IO/ADIOS/ADIOS2Auxiliary.hpp index 3c9d0b2548..26cc560dc2 100644 --- a/include/openPMD/IO/ADIOS/ADIOS2Auxiliary.hpp +++ b/include/openPMD/IO/ADIOS/ADIOS2Auxiliary.hpp @@ -105,6 +105,9 @@ namespace detail * @param IO The IO within which to retrieve the attribute. * @param attributeName The full ADIOS name of the attribute. * @param verbose If true, print a warning if not finding the attribute. + * @param voa This function is used by the old and new ADIOS2 schema alike. + * The old one uses ADIOS2 attributes, the new one uses + * ADIOS2 variables. * @return The openPMD datatype corresponding to the type of the attribute. * UNDEFINED if attribute is not found. */ @@ -113,7 +116,7 @@ namespace detail adios2::IO & IO, std::string const & attributeName, bool verbose, - VariableOrAttribute = VariableOrAttribute::Attribute ); + VariableOrAttribute voa = VariableOrAttribute::Attribute ); } // namespace detail #if defined( _MSC_VER ) && !defined( __INTEL_COMPILER ) && !defined( __clang__ ) diff --git a/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp b/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp index 2fa295373f..46826fe7d7 100644 --- a/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp +++ b/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp @@ -236,6 +236,15 @@ class ADIOS2IOHandlerImpl std::string m_engineType; ADIOS2Schema::schema_t m_schema = ADIOS2Schema::schema_0000_00_00; + enum class UseSpan : char + { + Yes, + No, + Auto + }; + + UseSpan m_useSpanBasedPutByDefault = UseSpan::Auto; + enum class AttributeLayout : char { ByAdiosAttributes, diff --git a/include/openPMD/IO/AbstractIOHandlerImpl.hpp b/include/openPMD/IO/AbstractIOHandlerImpl.hpp index c584462d24..619f0ea857 100644 --- a/include/openPMD/IO/AbstractIOHandlerImpl.hpp +++ b/include/openPMD/IO/AbstractIOHandlerImpl.hpp @@ -25,6 +25,7 @@ #include "openPMD/auxiliary/DerefDynamicCast.hpp" #include +#include namespace openPMD @@ -123,8 +124,14 @@ class AbstractIOHandlerImpl availableChunks(i.writable, deref_dynamic_cast< Parameter< O::AVAILABLE_CHUNKS > >(i.parameter.get())); break; } - } catch (unsupported_data_error&) + } catch (...) { + std::cerr + << "[AbstractIOHandlerImpl] IO Task " + << internal::operationAsString( i.operation ) + << " failed with exception. Removing task" + << " from IO queue and passing on the exception." + << std::endl; (*m_handler).m_work.pop(); throw; } diff --git a/include/openPMD/IO/IOTask.hpp b/include/openPMD/IO/IOTask.hpp index 6054edbaa5..d455556e69 100644 --- a/include/openPMD/IO/IOTask.hpp +++ b/include/openPMD/IO/IOTask.hpp @@ -76,6 +76,15 @@ OPENPMDAPI_EXPORT_ENUM_CLASS(Operation) AVAILABLE_CHUNKS //!< Query chunks that can be loaded in a dataset }; // note: if you change the enum members here, please update docs/source/dev/design.rst +namespace internal +{ + /* + * The returned strings are compile-time constants, so no worries about + * pointer validity. + */ + std::string operationAsString( Operation ); +} + struct OPENPMDAPI_EXPORT AbstractParameter { virtual ~AbstractParameter() = default; diff --git a/include/openPMD/backend/Container.hpp b/include/openPMD/backend/Container.hpp index 156e55e292..509d6f230d 100644 --- a/include/openPMD/backend/Container.hpp +++ b/include/openPMD/backend/Container.hpp @@ -366,6 +366,17 @@ class Container : public LegacyAttributable return m_originalContainer.at( std::forward< K >( k ) ); } + /** + * Remove key from the list of accessed keys. + * If the key is not accessed after this again, it will be deleted along + * with all other unaccessed keys upon destruction. + */ + template< typename K > + void forget( K && k ) + { + m_accessedKeys.erase( std::forward< K >( k ) ); + } + ~EraseStaleEntries() { auto & map = *m_originalContainer.m_container; diff --git a/include/openPMD/version.hpp b/include/openPMD/version.hpp index dd264b0682..114221b26d 100644 --- a/include/openPMD/version.hpp +++ b/include/openPMD/version.hpp @@ -29,7 +29,7 @@ */ #define OPENPMDAPI_VERSION_MAJOR 0 #define OPENPMDAPI_VERSION_MINOR 14 -#define OPENPMDAPI_VERSION_PATCH 3 +#define OPENPMDAPI_VERSION_PATCH 4 #define OPENPMDAPI_VERSION_LABEL "" /** @} */ diff --git a/setup.py b/setup.py index 63b6256166..2ab46e545a 100644 --- a/setup.py +++ b/setup.py @@ -156,7 +156,7 @@ def build_extension(self, ext): setup( name='openPMD-api', # note PEP-440 syntax: x.y.zaN but x.y.z.devN - version='0.14.3', + version='0.14.4', author='Axel Huebl, Franz Poeschel, Fabian Koller, Junmin Gu', author_email='axelhuebl@lbl.gov, f.poeschel@hzdr.de', maintainer='Axel Huebl', @@ -178,7 +178,7 @@ def build_extension(self, ext): cmdclass=dict(build_ext=CMakeBuild), # scripts=['openpmd-ls'], zip_safe=False, - python_requires='>=3.6, <3.10', + python_requires='>=3.6, <3.11', # tests_require=['pytest'], install_requires=install_requires, # see: src/bindings/python/cli @@ -211,6 +211,7 @@ def build_extension(self, ext): 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', + 'Programming Language :: Python :: 3.10', ('License :: OSI Approved :: ' 'GNU Lesser General Public License v3 or later (LGPLv3+)'), ], diff --git a/src/IO/ADIOS/ADIOS2IOHandler.cpp b/src/IO/ADIOS/ADIOS2IOHandler.cpp index 23a6efaccb..04d53249e3 100644 --- a/src/IO/ADIOS/ADIOS2IOHandler.cpp +++ b/src/IO/ADIOS/ADIOS2IOHandler.cpp @@ -132,6 +132,14 @@ ADIOS2IOHandlerImpl::init( nlohmann::json cfg ) m_config[ "schema" ].json().get< ADIOS2Schema::schema_t >(); } + if( m_config.json().contains( "use_span_based_put" ) ) + { + m_useSpanBasedPutByDefault = + m_config[ "use_span_based_put" ].json().get< bool >() + ? UseSpan::Yes + : UseSpan::No; + } + auto engineConfig = config( ADIOS2Defaults::str_engine ); if( !engineConfig.json().is_null() ) { @@ -704,6 +712,22 @@ struct GetSpan std::string errorMsg = "ADIOS2: getBufferView()"; }; + +struct HasOperators +{ + template< typename T > + bool operator()( std::string const & name, adios2::IO & IO ) const + { + adios2::Variable< T > variable = IO.InquireVariable< T >( name ); + if( !variable ) + { + return false; + } + return !variable.Operations().empty(); + } + + std::string errorMsg = "ADIOS2: getBufferView()"; +}; } // namespace detail void @@ -721,6 +745,28 @@ ADIOS2IOHandlerImpl::getBufferView( auto file = refreshFileFromParent( writable, /* preferParentFile = */ false ); detail::BufferedActions & ba = getFileData( file, IfFileNotOpen::ThrowError ); + + std::string name = nameOfVariable( writable ); + switch( m_useSpanBasedPutByDefault ) + { + case UseSpan::No: + parameters.out->backendManagedBuffer = false; + return; + case UseSpan::Auto: + { + detail::HasOperators hasOperators; + if( switchAdios2VariableType( + parameters.dtype, hasOperators, name, ba.m_IO ) ) + { + parameters.out->backendManagedBuffer = false; + return; + } + break; + } + case UseSpan::Yes: + break; + } + if( parameters.update ) { detail::I_UpdateSpan &updater = @@ -731,7 +777,6 @@ ADIOS2IOHandlerImpl::getBufferView( else { static detail::GetSpan gs; - std::string name = nameOfVariable( writable ); switchAdios2VariableType( parameters.dtype, gs, this, parameters, ba, name ); } } @@ -1125,11 +1170,21 @@ ADIOS2IOHandlerImpl::getCompressionOperator( std::string const & compression ) try { res = m_ADIOS.DefineOperator( compression, compression ); } - catch ( std::invalid_argument const & ) + catch ( std::invalid_argument const & e ) { std::cerr << "Warning: ADIOS2 backend does not support compression " "method " << compression << ". Continuing without compression." + << "\nOriginal error: " << e.what() + << std::endl; + return auxiliary::Option< adios2::Operator >(); + } + catch(std::string const & s) + { + std::cerr << "Warning: ADIOS2 backend does not support compression " + "method " + << compression << ". Continuing without compression." + << "\nOriginal error: " << s << std::endl; return auxiliary::Option< adios2::Operator >(); } @@ -1578,6 +1633,8 @@ namespace detail { var.SetSelection( { start, count } ); } + // don't add compression operators multiple times + return; } if( !var ) diff --git a/src/IO/HDF5/HDF5IOHandler.cpp b/src/IO/HDF5/HDF5IOHandler.cpp index 09ddc63fd4..56fad7c737 100644 --- a/src/IO/HDF5/HDF5IOHandler.cpp +++ b/src/IO/HDF5/HDF5IOHandler.cpp @@ -346,7 +346,12 @@ HDF5IOHandlerImpl::createDataset(Writable* writable, std::cerr << "[HDF5] Datatype::UNDEFINED caught during dataset creation (serial HDF5)" << std::endl; d = Datatype::BOOL; } - Attribute a(0); + + // note: due to a C++17 issue with ICC 19.1.2 we write the + // T value to variant conversion explicitly + // https://github.com/openPMD/openPMD-api/pull/... + // Attribute a(0); + Attribute a(static_cast(0)); a.dtype = d; std::vector< hsize_t > dims; std::uint64_t num_elements = 1u; diff --git a/src/IO/IOTask.cpp b/src/IO/IOTask.cpp index 0d46627820..074cb24c25 100644 --- a/src/IO/IOTask.cpp +++ b/src/IO/IOTask.cpp @@ -27,4 +27,87 @@ namespace openPMD Writable* getWritable(AttributableInterface* a) { return &a->writable(); } + +namespace internal +{ + std::string + operationAsString( Operation op ) + { + switch( op ) + { + case Operation::CREATE_FILE: + return "CREATE_FILE"; + break; + case Operation::OPEN_FILE: + return "OPEN_FILE"; + break; + case Operation::CLOSE_FILE: + return "CLOSE_FILE"; + break; + case Operation::DELETE_FILE: + return "DELETE_FILE"; + break; + case Operation::CREATE_PATH: + return "CREATE_PATH"; + break; + case Operation::CLOSE_PATH: + return "CLOSE_PATH"; + break; + case Operation::OPEN_PATH: + return "OPEN_PATH"; + break; + case Operation::DELETE_PATH: + return "DELETE_PATH"; + break; + case Operation::LIST_PATHS: + return "LIST_PATHS"; + break; + case Operation::CREATE_DATASET: + return "CREATE_DATASET"; + break; + case Operation::EXTEND_DATASET: + return "EXTEND_DATASET"; + break; + case Operation::OPEN_DATASET: + return "OPEN_DATASET"; + break; + case Operation::DELETE_DATASET: + return "DELETE_DATASET"; + break; + case Operation::WRITE_DATASET: + return "WRITE_DATASET"; + break; + case Operation::READ_DATASET: + return "READ_DATASET"; + break; + case Operation::LIST_DATASETS: + return "LIST_DATASETS"; + break; + case Operation::GET_BUFFER_VIEW: + return "GET_BUFFER_VIEW"; + break; + case Operation::DELETE_ATT: + return "DELETE_ATT"; + break; + case Operation::WRITE_ATT: + return "WRITE_ATT"; + break; + case Operation::READ_ATT: + return "READ_ATT"; + break; + case Operation::LIST_ATTS: + return "LIST_ATTS"; + break; + case Operation::ADVANCE: + return "ADVANCE"; + break; + case Operation::AVAILABLE_CHUNKS: + return "AVAILABLE_CHUNKS"; + break; + default: + return "unknown"; + break; + } + } +} } // openPMD diff --git a/src/Mesh.cpp b/src/Mesh.cpp index e9551c42e6..0e09cfc06b 100644 --- a/src/Mesh.cpp +++ b/src/Mesh.cpp @@ -331,6 +331,8 @@ Mesh::read() setGridSpacing(a.get< std::vector< float > >()); else if( *aRead.dtype == DT::VEC_DOUBLE || *aRead.dtype == DT::DOUBLE ) setGridSpacing(a.get< std::vector< double > >()); + else if( *aRead.dtype == DT::VEC_LONG_DOUBLE || *aRead.dtype == DT::LONG_DOUBLE ) + setGridSpacing(a.get< std::vector< long double > >()); else throw std::runtime_error("Unexpected Attribute datatype for 'gridSpacing'"); diff --git a/src/ParticleSpecies.cpp b/src/ParticleSpecies.cpp index 55e50df5fe..61861b4a1a 100644 --- a/src/ParticleSpecies.cpp +++ b/src/ParticleSpecies.cpp @@ -118,6 +118,7 @@ ParticleSpecies::read() while( ! IOHandler()->m_work.empty() ) IOHandler()->m_work.pop(); + map.forget( record_name ); //(*this)[record_name].erase(RecordComponent::SCALAR); //this->erase(record_name); } diff --git a/src/RecordComponent.cpp b/src/RecordComponent.cpp index d40061c198..05342462e7 100644 --- a/src/RecordComponent.cpp +++ b/src/RecordComponent.cpp @@ -208,7 +208,11 @@ RecordComponent::flush(std::string const& name) aWrite.resource = m_constantValue->getResource(); IOHandler()->enqueue(IOTask(this, aWrite)); aWrite.name = "shape"; - Attribute a(getExtent()); + // note: due to a C++17 issue with ICC 19.1.2 we write the + // T value to variant conversion explicitly + // https://github.com/openPMD/openPMD-api/pull/... + // Attribute a(getExtent()); + Attribute a(static_cast(getExtent())); aWrite.dtype = a.dtype; aWrite.resource = a.getResource(); IOHandler()->enqueue(IOTask(this, aWrite)); diff --git a/src/binding/python/Iteration.cpp b/src/binding/python/Iteration.cpp index 255ed41331..ec85f4fd9a 100644 --- a/src/binding/python/Iteration.cpp +++ b/src/binding/python/Iteration.cpp @@ -35,7 +35,7 @@ void init_Iteration(py::module &m) { .def("__repr__", [](Iteration const & it) { - return ""; + return ""; } ) diff --git a/test/CoreTest.cpp b/test/CoreTest.cpp index 72830b0df2..20ce35e8ed 100644 --- a/test/CoreTest.cpp +++ b/test/CoreTest.cpp @@ -37,7 +37,11 @@ TEST_CASE( "versions_test", "[core]" ) TEST_CASE( "attribute_dtype_test", "[core]" ) { - Attribute a = Attribute(static_cast< char >(' ')); + // note: due to a C++17 issue with ICC 19.1.2 we write the + // T value to variant conversion explicitly + // https://github.com/openPMD/openPMD-api/pull/... + // Attribute a = Attribute(static_cast< char >(' ')); + Attribute a = Attribute(static_cast(static_cast< char >(' '))); REQUIRE(Datatype::CHAR == a.dtype); a = Attribute(static_cast< unsigned char >(' ')); REQUIRE(Datatype::UCHAR == a.dtype); diff --git a/test/SerialIOTest.cpp b/test/SerialIOTest.cpp index 1627e1e9b7..412d8058e2 100644 --- a/test/SerialIOTest.cpp +++ b/test/SerialIOTest.cpp @@ -90,7 +90,7 @@ TEST_CASE( "adios2_char_portability", "[serial][adios2]" ) writeAttribute( "/openPMD", std::string( "1.1.0" ) ); writeAttribute( "/openPMDextension", uint32_t( 0 ) ); writeAttribute( "/software", std::string( "openPMD-api" ) ); - writeAttribute( "/softwareVersion", std::string( "0.14.3" ) ); + writeAttribute( "/softwareVersion", std::string( "0.14.4" ) ); IO.DefineAttribute< uint64_t >( "__openPMD_internal/openPMD2_adios2_schema", 20210209 ); @@ -1167,6 +1167,16 @@ void dtype_test( const std::string & backend ) { s.setAttribute("vecULongLong", std::vector< unsigned long long >({65531u, 65529u})); } + + // long double grid spacing + // should be possible to parse without error upon opening + // the series for reading + { + auto E = s.iterations[ 0 ].meshes[ "E" ]; + E.setGridSpacing( std::vector< long double >{ 1.0, 1.0 } ); + auto E_x = E[ "x" ]; + E_x.makeEmpty< double >( 1 ); + } } Series s = Series("../samples/dtype_test." + backend, Access::READ_ONLY); @@ -4203,6 +4213,176 @@ TEST_CASE( "variableBasedParticleData", "[serial][adios2]" ) } #endif +#if openPMD_HAVE_ADIOS2 +#ifdef ADIOS2_HAVE_BZIP2 +TEST_CASE( "automatically_deactivate_span", "[serial][adios2]" ) +{ + // automatically (de)activate span-based storeChunking + { + Series write( "../samples/span_based.bp", Access::CREATE ); + auto E_uncompressed = write.iterations[ 0 ].meshes[ "E" ][ "x" ]; + auto E_compressed = write.iterations[ 0 ].meshes[ "E" ][ "y" ]; + + Dataset ds{ Datatype::INT, { 10 } }; + + E_uncompressed.resetDataset( ds ); + + std::string compression = R"END( +{ + "adios2": { + "dataset": { + "operators": [ + { + "type": "bzip2" + } + ] + } + } +})END"; + + ds.options = compression; + E_compressed.resetDataset( ds ); + + bool spanWorkaround = false; + E_uncompressed.storeChunk< int >( + { 0 }, { 10 }, [ &spanWorkaround ]( size_t size ) { + spanWorkaround = true; + return std::shared_ptr< int >( + new int[ size ]{}, []( auto * ptr ) { delete[] ptr; } ); + } ); + + REQUIRE( !spanWorkaround ); + + E_compressed.storeChunk< int >( + { 0 }, { 10 }, [ &spanWorkaround ]( size_t size ) { + spanWorkaround = true; + return std::shared_ptr< int >( + new int[ size ]{}, []( auto * ptr ) { delete[] ptr; } ); + } ); + + REQUIRE( spanWorkaround ); + } + + // enable span-based API indiscriminately + { + std::string enable = R"END( +{ + "adios2": { + "use_span_based_put": true + } +})END"; + Series write( "../samples/span_based.bp", Access::CREATE, enable ); + auto E_uncompressed = write.iterations[ 0 ].meshes[ "E" ][ "x" ]; + auto E_compressed = write.iterations[ 0 ].meshes[ "E" ][ "y" ]; + + Dataset ds{ Datatype::INT, { 10 } }; + + E_uncompressed.resetDataset( ds ); + + std::string compression = R"END( +{ + "adios2": { + "dataset": { + "operators": [ + { + "type": "bzip2" + } + ] + } + } +})END"; + + ds.options = compression; + E_compressed.resetDataset( ds ); + + bool spanWorkaround = false; + E_uncompressed.storeChunk< int >( + { 0 }, { 10 }, [ &spanWorkaround ]( size_t size ) { + spanWorkaround = true; + return std::shared_ptr< int >( + new int[ size ]{}, []( auto * ptr ) { delete[] ptr; } ); + } ); + + REQUIRE( !spanWorkaround ); + + try + { + E_compressed.storeChunk< int >( + { 0 }, { 10 }, [ &spanWorkaround ]( size_t size ) { + spanWorkaround = true; + return std::shared_ptr< int >( + new int[ size ]{}, []( auto * ptr ) { delete[] ptr; } ); + } ); + } + catch( std::invalid_argument const & e ) + { + /* + * Using the span-based API in combination with compression is + * unsupported in ADIOS2. + * In newer versions of ADIOS2, an error is thrown. + */ + std::cerr << "Ignoring expected error: " << e.what() << std::endl; + } + + REQUIRE( !spanWorkaround ); + } + + // disable span-based API indiscriminately + { + std::string disable = R"END( +{ + "adios2": { + "use_span_based_put": false + } +})END"; + Series write( "../samples/span_based.bp", Access::CREATE, disable ); + auto E_uncompressed = write.iterations[ 0 ].meshes[ "E" ][ "x" ]; + auto E_compressed = write.iterations[ 0 ].meshes[ "E" ][ "y" ]; + + Dataset ds{ Datatype::INT, { 10 } }; + + E_uncompressed.resetDataset( ds ); + + std::string compression = R"END( +{ + "adios2": { + "dataset": { + "operators": [ + { + "type": "bzip2" + } + ] + } + } +})END"; + + ds.options = compression; + E_compressed.resetDataset( ds ); + + bool spanWorkaround = false; + E_uncompressed.storeChunk< int >( + { 0 }, { 10 }, [ &spanWorkaround ]( size_t size ) { + spanWorkaround = true; + return std::shared_ptr< int >( + new int[ size ]{}, []( auto * ptr ) { delete[] ptr; } ); + } ); + + REQUIRE( spanWorkaround ); + + spanWorkaround = false; + E_compressed.storeChunk< int >( + { 0 }, { 10 }, [ &spanWorkaround ]( size_t size ) { + spanWorkaround = true; + return std::shared_ptr< int >( + new int[ size ]{}, []( auto * ptr ) { delete[] ptr; } ); + } ); + + REQUIRE( spanWorkaround ); + } +} +#endif +#endif + // @todo Upon switching to ADIOS2 2.7.0, test this the other way around also void iterate_nonstreaming_series(