diff --git a/.github/workflows/python-app.yml b/.github/workflows/python-app.yml index 62275e3..d713f51 100644 --- a/.github/workflows/python-app.yml +++ b/.github/workflows/python-app.yml @@ -17,7 +17,7 @@ jobs: matrix: python-version: [ "3.7", "3.8", "3.9", "3.10" ] poetry-version: [ "1.1.14" ] - os: [ ubuntu-18.04, macos-latest, windows-latest ] + os: [ ubuntu-22.04, macos-latest, windows-latest ] runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v2 @@ -39,8 +39,10 @@ jobs: # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide poetry run flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics - name: Test with pytest + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | - poetry run pytest -m "not regression" + poetry run pytest -m "not regression and not token" - name: netrc-gen uses: extractions/netrc@v1 with: @@ -50,5 +52,41 @@ jobs: - name: Regression Test with pytest env: PODAAC_LOGLEVEL: "DEBUG" + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | poetry run pytest -o log_cli=true --log-cli-level=DEBUG -m "regression" + + auth_regression: + needs: build + strategy: + fail-fast: false + max-parallel: 1 + matrix: + python-version: [ "3.7", "3.8", "3.9", "3.10" ] + poetry-version: [ "1.1.14" ] + os: [ ubuntu-22.04, macos-latest, windows-latest ] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v3 + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + - name: Install Poetry + uses: abatilo/actions-poetry@v2.0.0 + with: + poetry-version: ${{ matrix.poetry-version }} + - name: Install dependencies + run: | + poetry install + - name: netrc-gen + uses: extractions/netrc@v1 + with: + machine: urs.earthdata.nasa.gov + username: ${{ secrets.EDL_OPS_USERNAME }} + password: ${{ secrets.EDL_OPS_PASSWORD }} + - name: Regression Test with pytest + env: + PODAAC_LOGLEVEL: "DEBUG" + run: | + poetry run pytest -o log_cli=true --log-cli-level=DEBUG -m "token" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index eba32fa..2697eb1 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -44,7 +44,7 @@ jobs: fail-fast: false matrix: python-version: [ "3.7", "3.8", "3.9", "3.10" ] - os: [ ubuntu-18.04, macos-latest, windows-latest ] + os: [ ubuntu-22.04, macos-latest, windows-latest ] runs-on: ${{ matrix.os }} steps: - name: Set up Python diff --git a/CHANGELOG.md b/CHANGELOG.md index 9431f5c..7e29718 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,14 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) +## 1.13.0 +### Added +- Added --dry-run option to subscriber and downloader to view the files that _would_ be downloaded without actuall downloading them. [102](https://github.com/podaac/data-subscriber/issues/102) +- Added new feature allowing regex to be used in `--extension` `-e` options. For example using -e `PTM_\\d+` would match data files like `filename.PTM_1`, `filename.PTM_2` and `filename.PTM_10`, instead of specifying all possible combinations (`-e PTM_1, -e PTM_2, ..., -e PMT_10`) [115](https://github.com/podaac/data-subscriber/issues/115) +- Added check for updated version [70](https://github.com/podaac/data-subscriber/issues/70) +- Removed CMR Token from log messages [127](https://github.com/podaac/data-subscriber/issues/127) + + ## 1.12.0 ### Fixed - Added EDL based token downloading, removing CMR tokens [98](https://github.com/podaac/data-subscriber/issues/98), diff --git a/Downloader.md b/Downloader.md index f83d737..ebe0dd2 100644 --- a/Downloader.md +++ b/Downloader.md @@ -6,7 +6,7 @@ For installation and dependency information, please see the [top-level README](R ``` $> podaac-data-downloader -h -usage: PO.DAAC bulk-data downloader [-h] -c COLLECTION -d OUTPUTDIRECTORY [--cycle SEARCH_CYCLES] [-sd STARTDATE] [-ed ENDDATE] [-f] [-b BBOX] [-dc] [-dydoy] [-dymd] [-dy] [--offset OFFSET] [-e EXTENSIONS] [--granule-name GRANULE] [--process PROCESS_CMD] [--version] [--verbose] [-p PROVIDER] [--limit LIMIT] +usage: PO.DAAC bulk-data downloader [-h] -c COLLECTION -d OUTPUTDIRECTORY [--cycle SEARCH_CYCLES] [-sd STARTDATE] [-ed ENDDATE] [-f] [-b BBOX] [-dc] [-dydoy] [-dymd] [-dy] [--offset OFFSET] [-e EXTENSIONS] [-gr GRANULENAME] [--process PROCESS_CMD] [--version] [--verbose] [-p PROVIDER] [--limit LIMIT] [--dry-run] optional arguments: -h, --help show this help message and exit @@ -17,44 +17,39 @@ optional arguments: --cycle SEARCH_CYCLES Cycle number for determining downloads. can be repeated for multiple cycles -sd STARTDATE, --start-date STARTDATE - The ISO date time before which data should be retrieved. For Example, --start-date 2021-01-14T00:00:00Z + The ISO date time after which data should be retrieved. For Example, --start-date 2021-01-14T00:00:00Z -ed ENDDATE, --end-date ENDDATE - The ISO date time after which data should be retrieved. For Example, --end-date 2021-01-14T00:00:00Z - -f, --force - Flag to force downloading files that are listed in CMR query, even if the file exists and checksum matches + The ISO date time before which data should be retrieved. For Example, --end-date 2021-01-14T00:00:00Z + -f, --force Flag to force downloading files that are listed in CMR query, even if the file exists and checksum matches -b BBOX, --bounds BBOX - The bounding rectangle to filter result in. Format is W Longitude,S Latitude,E Longitude,N Latitude without - spaces. Due to an issue with parsing arguments, to use this command, please use the -b="-180,-90,180,90" syntax - when calling from the command line. Default: "-180,-90,180,90". + The bounding rectangle to filter result in. Format is W Longitude,S Latitude,E Longitude,N Latitude without spaces. Due to an issue with parsing arguments, to use this command, please use the -b="-180,-90,180,90" syntax when calling from the command line. + Default: "-180,-90,180,90". -dc Flag to use cycle number for directory where data products will be downloaded. -dydoy Flag to use start time (Year/DOY) of downloaded data for directory where data products will be downloaded. - -dymd Flag to use start time (Year/Month/Day) of downloaded data for directory where data products will be - downloaded. + -dymd Flag to use start time (Year/Month/Day) of downloaded data for directory where data products will be downloaded. -dy Flag to use start time (Year) of downloaded data for directory where data products will be downloaded. --offset OFFSET Flag used to shift timestamp. Units are in hours, e.g. 10 or -10. -e EXTENSIONS, --extensions EXTENSIONS - The extensions of products to download. Default is [.nc, .h5, .zip, .tar.gz] - -gr GRANULE, --granule-name GRANULE - The name of the granule to download. Only one granule name can be specified. Script will download all files matching similar granule name sans extension. + Regexps of extensions of products to download. Default is [.nc, .h5, .zip, .tar.gz, .tiff] + -gr GRANULENAME, --granule-name GRANULENAME + Flag to download specific granule from a collection. This parameter can only be used if you know the granule name. Only one granule name can be supplied --process PROCESS_CMD Processing command to run on each downloaded file (e.g., compression). Can be specified multiple times. --version Display script version information and exit. --verbose Verbose mode. -p PROVIDER, --provider PROVIDER Specify a provider for collection search. Default is POCLOUD. - --limit LIMIT Integer limit for number of granules to download. Useful in testing. Defaults to 2000 + --limit LIMIT Integer limit for number of granules to download. Useful in testing. Defaults to no limit. + --dry-run Search and identify files to download, but do not actually download them ``` -##Run the Script - ## Step 2: Run the Script Usage: ``` -usage: PO.DAAC bulk-data downloader [-h] -c COLLECTION -d OUTPUTDIRECTORY [--cycle SEARCH_CYCLES] [-sd STARTDATE] [-ed ENDDATE] [-f] - [-b BBOX] [-dc] [-dydoy] [-dymd] [-dy] [--offset OFFSET] [-e EXTENSIONS] [--process PROCESS_CMD] - [--version] [--verbose] [-p PROVIDER] [--limit LIMIT] +usage: PO.DAAC bulk-data downloader [-h] -c COLLECTION -d OUTPUTDIRECTORY [--cycle SEARCH_CYCLES] [-sd STARTDATE] [-ed ENDDATE] [-f] [-b BBOX] [-dc] [-dydoy] [-dymd] [-dy] [--offset OFFSET] [-e EXTENSIONS] [-gr GRANULENAME] [--process PROCESS_CMD] [--version] [--verbose] + [-p PROVIDER] [--limit LIMIT] [--dry-run] ``` To run the script, the following parameters are required: @@ -215,17 +210,27 @@ podaac-data-downloader -c VIIRS_N20-OSPO-L2P-v2.61 -d ./data -b="-180,-90,180,90 ### Setting extensions -Some collections have many files. To download a specific set of files, you can set the extensions on which downloads are filtered. By default, ".nc", ".h5", and ".zip" files are downloaded by default. +Some collections have many files. To download a specific set of files, you can set the extensions on which downloads are filtered. By default, ".nc", ".h5", and ".zip" files are downloaded by default. The `-e` option is a regular expression check so you can do advanced things like `-e PTM_\\d+` to match `PTM_` followed by one or more digits- useful when the ending of a file has no suffix and has a number (1-12 for PTM, in this example) ``` -e EXTENSIONS, --extensions EXTENSIONS - The extensions of products to download. Default is [.nc, .h5, .zip] + Regexps of extensions of products to download. Default is [.nc, .h5, .zip, .tar.gz, .tiff] ``` An example of the -e usage- note the -e option is additive: ``` podaac-data-subscriber -c VIIRS_N20-OSPO-L2P-v2.61 -d ./data -e .nc -e .h5 -sd 2020-06-01T00:46:02Z -ed 2020-07-01T00:46:02Z ``` + +One may also specify a regular expression to select files. For example, the following are equivalent: + +`podaac-data-subscriber -c VIIRS_N20-OSPO-L2P-v2.61 -d ./data -e PTM_1, -e PTM_2, ..., -e PMT_10 -sd 2020-06-01T00:46:02Z -ed 2020-07-01T00:46:02Z` + +and + +`podaac-data-subscriber -c VIIRS_N20-OSPO-L2P-v2.61 -d ./data -e PTM_\\d+ -sd 2020-06-01T00:46:02Z -ed 2020-07-01T00:46:02Z` + + ### run a post download process Using the `--process` option, you can run a simple command agaisnt the "just" downloaded file. This will take the format of " ". This means you can run a command like `--process gzip` to gzip all downloaded files. We do not support more advanced processes at this time (piping, running a process on a directory, etc). diff --git a/README.md b/README.md index 31c4591..e32c6be 100644 --- a/README.md +++ b/README.md @@ -37,16 +37,16 @@ pip install podaac-data-subscriber you should now have access to the downloader and subscriber Command line interfaces: ``` -$> podaac-data-subscriber -h -usage: podaac_data_subscriber.py [-h] -c COLLECTION -d OUTPUTDIRECTORY [-sd STARTDATE] [-ed ENDDATE] [-b BBOX] [-dc] [-dydoy] [-dymd] [-dy] [--offset OFFSET] [-m MINUTES] - [-e EXTENSIONS] [--process PROCESS_CMD] [--version] [--verbose] [-p PROVIDER] +$> usage: PO.DAAC data subscriber [-h] -c COLLECTION -d OUTPUTDIRECTORY [-f] [-sd STARTDATE] [-ed ENDDATE] [-b BBOX] [-dc] [-dydoy] [-dymd] [-dy] [--offset OFFSET] [-m MINUTES] + [-e EXTENSIONS] [--process PROCESS_CMD] [--version] [--verbose] [-p PROVIDER] [--dry-run] ... ``` ``` -$> podaac-data-downloader -h -usage: PO.DAAC bulk-data downloader [-h] -c COLLECTION -d OUTPUTDIRECTORY [--cycle SEARCH_CYCLES] [-sd STARTDATE] [-ed ENDDATE] [-b BBOX] [-dc] [-dydoy] [-dymd] [-dy] [--offset OFFSET] [-e EXTENSIONS] [--process PROCESS_CMD] [--version] [--verbose] [-p PROVIDER] [--limit LIMIT] +$> usage: PO.DAAC bulk-data downloader [-h] -c COLLECTION -d OUTPUTDIRECTORY [--cycle SEARCH_CYCLES] [-sd STARTDATE] [-ed ENDDATE] [-f] [-b BBOX] [-dc] [-dydoy] [-dymd] [-dy] + [--offset OFFSET] [-e EXTENSIONS] [-gr GRANULENAME] [--process PROCESS_CMD] [--version] [--verbose] [-p PROVIDER] [--limit LIMIT] [--dry-run] + ... ``` @@ -114,6 +114,14 @@ export PODAAC_LOGLEVEL=DEBUG And then run the script. This should give you more verbose output on URL requests to CMR, tokens, etc. +### OTHER OPTIONS + +The podaac downloader and subscriber make calls to github for checking recent releases. Unauthenticated requests are limited to 60 per hour. If you start seeing errors like: +``` +releases_json = {'documentation_url': 'https://docs.github.com/rest/overview/resources-in-the-rest-api#rate-limiting', 'message': "API... here's the good news: Authenticated requests get a higher rate limit. Check out the documentation for more details.)"} +``` +You'll want to set the environment variable GITHUB_TOKEN to a github personal access token- this allows for up to 5000 calls per hour. This requires a free github account. Most users will not run in to this issue. + ### In need of Help? The PO.DAAC User Services Office is the primary point of contact for answering your questions concerning data and information held by the PO.DAAC. User Services staff members are knowledgeable about both the data ordering system and the data products themselves. We answer questions about data, route requests to other DAACs, and direct questions we cannot answer to the appropriate information source. diff --git a/Subscriber.md b/Subscriber.md index 0cd5e3f..f3afc1b 100644 --- a/Subscriber.md +++ b/Subscriber.md @@ -6,43 +6,42 @@ For installation and dependency information, please see the [top-level README](R ``` $> podaac-data-subscriber -h -usage: PO.DAAC data subscriber [-h] -c COLLECTION -d OUTPUTDIRECTORY [-f] [-sd STARTDATE] [-ed ENDDATE] [-b BBOX] [-dc] [-dydoy] [-dymd] [-dy] [--offset OFFSET] [-m MINUTES] [-e EXTENSIONS] [--process PROCESS_CMD] [--version] [--verbose] [-p PROVIDER] - -optional arguments: - -h, --help show this help message and exit - -c COLLECTION, --collection-shortname COLLECTION - The collection shortname for which you want to retrieve data. - -d OUTPUTDIRECTORY, --data-dir OUTPUTDIRECTORY - The directory where data products will be downloaded. - -f, --force Flag to force downloading files that are listed in CMR query, even if the file exists and checksum matches - -sd STARTDATE, --start-date STARTDATE - The ISO date time before which data should be retrieved. For Example, --start-date 2021-01-14T00:00:00Z - -ed ENDDATE, --end-date ENDDATE - The ISO date time after which data should be retrieved. For Example, --end-date 2021-01-14T00:00:00Z - -b BBOX, --bounds BBOX - The bounding rectangle to filter result in. Format is W Longitude,S Latitude,E Longitude,N Latitude without spaces. Due to an issue with parsing arguments, to use this command, please use the -b="-180,-90,180,90" syntax when calling from the command line. Default: "-180,-90,180,90". - -dc Flag to use cycle number for directory where data products will be downloaded. - -dydoy Flag to use start time (Year/DOY) of downloaded data for directory where data products will be downloaded. - -dymd Flag to use start time (Year/Month/Day) of downloaded data for directory where data products will be downloaded. - -dy Flag to use start time (Year) of downloaded data for directory where data products will be downloaded. - --offset OFFSET Flag used to shift timestamp. Units are in hours, e.g. 10 or -10. - -m MINUTES, --minutes MINUTES - How far back in time, in minutes, should the script look for data. If running this script as a cron, this value should be equal to or greater than how often your cron runs (default: 60 minutes). - -e EXTENSIONS, --extensions EXTENSIONS - The extensions of products to download. Default is [.nc, .h5, .zip] - --process PROCESS_CMD - Processing command to run on each downloaded file (e.g., compression). Can be specified multiple times. - --version Display script version information and exit. - --verbose Verbose mode. - -p PROVIDER, --provider PROVIDER - Specify a provider for collection search. Default is POCLOUD. +-h, --help show this help message and exit +-c COLLECTION, --collection-shortname COLLECTION + The collection shortname for which you want to retrieve data. +-d OUTPUTDIRECTORY, --data-dir OUTPUTDIRECTORY + The directory where data products will be downloaded. +-f, --force Flag to force downloading files that are listed in CMR query, even if the file exists and checksum matches +-sd STARTDATE, --start-date STARTDATE + The ISO date time after which data should be retrieved. For Example, --start-date 2021-01-14T00:00:00Z +-ed ENDDATE, --end-date ENDDATE + The ISO date time before which data should be retrieved. For Example, --end-date 2021-01-14T00:00:00Z +-b BBOX, --bounds BBOX + The bounding rectangle to filter result in. Format is W Longitude,S Latitude,E Longitude,N Latitude without spaces. Due to an issue with parsing arguments, to use this command, please use the -b="-180,-90,180,90" syntax when calling from the command line. + Default: "-180,-90,180,90". +-dc Flag to use cycle number for directory where data products will be downloaded. +-dydoy Flag to use start time (Year/DOY) of downloaded data for directory where data products will be downloaded. +-dymd Flag to use start time (Year/Month/Day) of downloaded data for directory where data products will be downloaded. +-dy Flag to use start time (Year) of downloaded data for directory where data products will be downloaded. +--offset OFFSET Flag used to shift timestamp. Units are in hours, e.g. 10 or -10. +-m MINUTES, --minutes MINUTES + How far back in time, in minutes, should the script look for data. If running this script as a cron, this value should be equal to or greater than how often your cron runs. +-e EXTENSIONS, --extensions EXTENSIONS + Regexps of extensions of products to download. Default is [.nc, .h5, .zip, .tar.gz, .tiff] +--process PROCESS_CMD + Processing command to run on each downloaded file (e.g., compression). Can be specified multiple times. +--version Display script version information and exit. +--verbose Verbose mode. +-p PROVIDER, --provider PROVIDER + Specify a provider for collection search. Default is POCLOUD. +--dry-run Search and identify files to download, but do not actually download them ``` ## Run the Script Usage: ``` -usage: podaac_data_subscriber.py [-h] -c COLLECTION -d OUTPUTDIRECTORY [-f] [-sd STARTDATE] [-ed ENDDATE] [-b BBOX] [-dc] [-dydoy] [-dymd] [-dy] [--offset OFFSET] [-m MINUTES] [-e EXTENSIONS] [--version] [--verbose] [-p PROVIDER] +usage: PO.DAAC data subscriber [-h] -c COLLECTION -d OUTPUTDIRECTORY [-f] [-sd STARTDATE] [-ed ENDDATE] [-b BBOX] [-dc] [-dydoy] [-dymd] [-dy] [--offset OFFSET] [-m MINUTES] [-e EXTENSIONS] [--process PROCESS_CMD] [--version] [--verbose] [-p PROVIDER] [--dry-run] ``` To run the script, the following parameters are required: @@ -189,17 +188,26 @@ podaac-data-subscriber -c VIIRS_N20-OSPO-L2P-v2.61 -d ./data -b="-180,-90,180,90 ### Setting extensions -Some collections have many files. To download a specific set of files, you can set the extensions on which downloads are filtered. By default, ".nc", ".h5", and ".zip" files are downloaded by default. +Some collections have many files. To download a specific set of files, you can set the extensions on which downloads are filtered. By default, ".nc", ".h5", and ".zip" files are downloaded by default. The `-e` option is a regular expression check so you can do advanced things like `-e PTM_\\d+` to match `PTM_` followed by one or more digits- useful when the ending of a file has no suffix and has a number (1-12 for PTM, in this example) ``` -e EXTENSIONS, --extensions EXTENSIONS - The extensions of products to download. Default is [.nc, .h5, .zip] + Regexps of extensions of products to download. Default is [.nc, .h5, .zip, .tar.gz, .tiff] ``` An example of the -e usage- note the -e option is additive: ``` podaac-data-subscriber -c VIIRS_N20-OSPO-L2P-v2.61 -d ./data -e .nc -e .h5 ``` + +One may also specify a regular expression to select files. For example, the following are equivalent: + +`podaac-data-subscriber -c VIIRS_N20-OSPO-L2P-v2.61 -d ./data -e PTM_1, -e PTM_2, ..., -e PMT_10 -sd 2020-06-01T00:46:02Z -ed 2020-07-01T00:46:02Z` + +and + +`podaac-data-subscriber -c VIIRS_N20-OSPO-L2P-v2.61 -d ./data -e PTM_\\d+ -sd 2020-06-01T00:46:02Z -ed 2020-07-01T00:46:02Z` + ### run a post download process Using the `--process` option, you can run a simple command agaisnt the "just" downloaded file. This will take the format of " ". This means you can run a command like `--process gzip` to gzip all downloaded files. We do not support more advanced processes at this time (piping, running a process on a directory, etc). diff --git a/poetry.lock b/poetry.lock index 72a359c..6f6b375 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,20 +1,22 @@ [[package]] name = "attrs" -version = "22.1.0" +version = "22.2.0" description = "Classes Without Boilerplate" category = "dev" optional = false -python-versions = ">=3.5" +python-versions = ">=3.6" [package.extras] -dev = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "mypy (>=0.900,!=0.940)", "pytest-mypy-plugins", "zope.interface", "furo", "sphinx", "sphinx-notfound-page", "pre-commit", "cloudpickle"] -docs = ["furo", "sphinx", "zope.interface", "sphinx-notfound-page"] -tests = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "mypy (>=0.900,!=0.940)", "pytest-mypy-plugins", "zope.interface", "cloudpickle"] -tests_no_zope = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "mypy (>=0.900,!=0.940)", "pytest-mypy-plugins", "cloudpickle"] +cov = ["attrs", "coverage-enable-subprocess", "coverage[toml] (>=5.3)"] +dev = ["attrs"] +docs = ["furo", "sphinx", "myst-parser", "zope.interface", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier"] +tests = ["attrs", "zope.interface"] +tests-no-zope = ["hypothesis", "pympler", "pytest (>=4.3.0)", "pytest-xdist", "cloudpickle", "mypy (>=0.971,<0.990)", "pytest-mypy-plugins"] +tests_no_zope = ["hypothesis", "pympler", "pytest (>=4.3.0)", "pytest-xdist", "cloudpickle", "mypy (>=0.971,<0.990)", "pytest-mypy-plugins"] [[package]] name = "certifi" -version = "2022.9.24" +version = "2022.12.7" description = "Python package for providing Mozilla's CA Bundle." category = "main" optional = false @@ -22,22 +24,30 @@ python-versions = ">=3.6" [[package]] name = "charset-normalizer" -version = "2.1.1" +version = "3.0.1" description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." category = "main" optional = false -python-versions = ">=3.6.0" - -[package.extras] -unicode_backport = ["unicodedata2"] +python-versions = "*" [[package]] name = "colorama" -version = "0.4.5" +version = "0.4.6" description = "Cross-platform colored terminal text." category = "dev" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" + +[[package]] +name = "exceptiongroup" +version = "1.1.0" +description = "Backport of PEP 654 (exception groups)" +category = "dev" +optional = false +python-versions = ">=3.7" + +[package.extras] +test = ["pytest (>=6)"] [[package]] name = "flake8" @@ -79,11 +89,11 @@ testing = ["pytest (>=4.6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytes [[package]] name = "iniconfig" -version = "1.1.1" -description = "iniconfig: brain-dead simple config-ini parsing" +version = "2.0.0" +description = "brain-dead simple config-ini parsing" category = "dev" optional = false -python-versions = "*" +python-versions = ">=3.7" [[package]] name = "mccabe" @@ -95,14 +105,11 @@ python-versions = "*" [[package]] name = "packaging" -version = "21.3" +version = "23.0" description = "Core utilities for Python packages" category = "dev" optional = false -python-versions = ">=3.6" - -[package.dependencies] -pyparsing = ">=2.0.2,<3.0.5 || >3.0.5" +python-versions = ">=3.7" [[package]] name = "pluggy" @@ -119,14 +126,6 @@ importlib-metadata = {version = ">=0.12", markers = "python_version < \"3.8\""} testing = ["pytest-benchmark", "pytest"] dev = ["tox", "pre-commit"] -[[package]] -name = "py" -version = "1.11.0" -description = "library with cross-python path, ini-parsing, io, code, log facilities" -category = "dev" -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" - [[package]] name = "pycodestyle" version = "2.8.0" @@ -143,20 +142,9 @@ category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" -[[package]] -name = "pyparsing" -version = "3.0.9" -description = "pyparsing module - Classes and methods to define and execute parsing grammars" -category = "dev" -optional = false -python-versions = ">=3.6.8" - -[package.extras] -diagrams = ["railroad-diagrams", "jinja2"] - [[package]] name = "pytest" -version = "7.1.3" +version = "7.2.1" description = "pytest: simple powerful testing with Python" category = "dev" optional = false @@ -165,19 +153,19 @@ python-versions = ">=3.7" [package.dependencies] attrs = ">=19.2.0" colorama = {version = "*", markers = "sys_platform == \"win32\""} +exceptiongroup = {version = ">=1.0.0rc8", markers = "python_version < \"3.11\""} importlib-metadata = {version = ">=0.12", markers = "python_version < \"3.8\""} iniconfig = "*" packaging = "*" pluggy = ">=0.12,<2.0" -py = ">=1.8.2" -tomli = ">=1.0.0" +tomli = {version = ">=1.0.0", markers = "python_version < \"3.11\""} [package.extras] testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2.7.2)", "requests", "xmlschema"] [[package]] name = "pytest-mock" -version = "3.9.0" +version = "3.10.0" description = "Thin-wrapper around the mock package for easier use with pytest" category = "dev" optional = false @@ -191,7 +179,7 @@ dev = ["pre-commit", "tox", "pytest-asyncio"] [[package]] name = "requests" -version = "2.28.1" +version = "2.28.2" description = "Python HTTP for Humans." category = "main" optional = false @@ -199,7 +187,7 @@ python-versions = ">=3.7, <4" [package.dependencies] certifi = ">=2017.4.17" -charset-normalizer = ">=2,<3" +charset-normalizer = ">=2,<4" idna = ">=2.5,<4" urllib3 = ">=1.21.1,<1.27" @@ -228,7 +216,7 @@ python-versions = ">=3.7" [[package]] name = "typing-extensions" -version = "4.3.0" +version = "4.4.0" description = "Backported and Experimental Type Hints for Python 3.7+" category = "dev" optional = false @@ -236,11 +224,11 @@ python-versions = ">=3.7" [[package]] name = "urllib3" -version = "1.26.12" +version = "1.26.14" description = "HTTP library with thread-safe connection pooling, file post, and more." category = "main" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*, <4" +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" [package.extras] brotli = ["brotlicffi (>=0.8.0)", "brotli (>=1.0.9)", "brotlipy (>=0.6.0)"] @@ -249,15 +237,15 @@ socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] [[package]] name = "zipp" -version = "3.8.1" +version = "3.12.0" description = "Backport of pathlib-compatible object wrapper for zip files" category = "dev" optional = false python-versions = ">=3.7" [package.extras] -docs = ["sphinx", "jaraco.packaging (>=9)", "rst.linker (>=1.9)", "jaraco.tidelift (>=1.4)"] -testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-cov", "pytest-enabler (>=1.3)", "jaraco.itertools", "func-timeout", "pytest-black (>=0.3.7)", "pytest-mypy (>=0.9.1)"] +docs = ["sphinx (>=3.5)", "jaraco.packaging (>=9)", "rst.linker (>=1.9)", "furo", "sphinx-lint", "jaraco.tidelift (>=1.4)"] +testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "flake8 (<5)", "pytest-cov", "pytest-enabler (>=1.3)", "jaraco.itertools", "func-timeout", "jaraco.functools", "more-itertools", "pytest-black (>=0.3.7)", "pytest-mypy (>=0.9.1)", "pytest-flake8"] [metadata] lock-version = "1.1" @@ -266,20 +254,110 @@ content-hash = "c5ece7741408cb266fe803842b66f646317dc3a384e9c54ecbe66a14ce895fed [metadata.files] attrs = [ - {file = "attrs-22.1.0-py2.py3-none-any.whl", hash = "sha256:86efa402f67bf2df34f51a335487cf46b1ec130d02b8d39fd248abfd30da551c"}, - {file = "attrs-22.1.0.tar.gz", hash = "sha256:29adc2665447e5191d0e7c568fde78b21f9672d344281d0c6e1ab085429b22b6"}, + {file = "attrs-22.2.0-py3-none-any.whl", hash = "sha256:29e95c7f6778868dbd49170f98f8818f78f3dc5e0e37c0b1f474e3561b240836"}, + {file = "attrs-22.2.0.tar.gz", hash = "sha256:c9227bfc2f01993c03f68db37d1d15c9690188323c067c641f1a35ca58185f99"}, ] certifi = [ - {file = "certifi-2022.9.24-py3-none-any.whl", hash = "sha256:90c1a32f1d68f940488354e36370f6cca89f0f106db09518524c88d6ed83f382"}, - {file = "certifi-2022.9.24.tar.gz", hash = "sha256:0d9c601124e5a6ba9712dbc60d9c53c21e34f5f641fe83002317394311bdce14"}, + {file = "certifi-2022.12.7-py3-none-any.whl", hash = "sha256:4ad3232f5e926d6718ec31cfc1fcadfde020920e278684144551c91769c7bc18"}, + {file = "certifi-2022.12.7.tar.gz", hash = "sha256:35824b4c3a97115964b408844d64aa14db1cc518f6562e8d7261699d1350a9e3"}, ] charset-normalizer = [ - {file = "charset-normalizer-2.1.1.tar.gz", hash = "sha256:5a3d016c7c547f69d6f81fb0db9449ce888b418b5b9952cc5e6e66843e9dd845"}, - {file = "charset_normalizer-2.1.1-py3-none-any.whl", hash = "sha256:83e9a75d1911279afd89352c68b45348559d1fc0506b054b346651b5e7fee29f"}, + {file = "charset-normalizer-3.0.1.tar.gz", hash = "sha256:ebea339af930f8ca5d7a699b921106c6e29c617fe9606fa7baa043c1cdae326f"}, + {file = "charset_normalizer-3.0.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:88600c72ef7587fe1708fd242b385b6ed4b8904976d5da0893e31df8b3480cb6"}, + {file = "charset_normalizer-3.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:c75ffc45f25324e68ab238cb4b5c0a38cd1c3d7f1fb1f72b5541de469e2247db"}, + {file = "charset_normalizer-3.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:db72b07027db150f468fbada4d85b3b2729a3db39178abf5c543b784c1254539"}, + {file = "charset_normalizer-3.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:62595ab75873d50d57323a91dd03e6966eb79c41fa834b7a1661ed043b2d404d"}, + {file = "charset_normalizer-3.0.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ff6f3db31555657f3163b15a6b7c6938d08df7adbfc9dd13d9d19edad678f1e8"}, + {file = "charset_normalizer-3.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:772b87914ff1152b92a197ef4ea40efe27a378606c39446ded52c8f80f79702e"}, + {file = "charset_normalizer-3.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:70990b9c51340e4044cfc394a81f614f3f90d41397104d226f21e66de668730d"}, + {file = "charset_normalizer-3.0.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:292d5e8ba896bbfd6334b096e34bffb56161c81408d6d036a7dfa6929cff8783"}, + {file = "charset_normalizer-3.0.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:2edb64ee7bf1ed524a1da60cdcd2e1f6e2b4f66ef7c077680739f1641f62f555"}, + {file = "charset_normalizer-3.0.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:31a9ddf4718d10ae04d9b18801bd776693487cbb57d74cc3458a7673f6f34639"}, + {file = "charset_normalizer-3.0.1-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:44ba614de5361b3e5278e1241fda3dc1838deed864b50a10d7ce92983797fa76"}, + {file = "charset_normalizer-3.0.1-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:12db3b2c533c23ab812c2b25934f60383361f8a376ae272665f8e48b88e8e1c6"}, + {file = "charset_normalizer-3.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:c512accbd6ff0270939b9ac214b84fb5ada5f0409c44298361b2f5e13f9aed9e"}, + {file = "charset_normalizer-3.0.1-cp310-cp310-win32.whl", hash = "sha256:502218f52498a36d6bf5ea77081844017bf7982cdbe521ad85e64cabee1b608b"}, + {file = "charset_normalizer-3.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:601f36512f9e28f029d9481bdaf8e89e5148ac5d89cffd3b05cd533eeb423b59"}, + {file = "charset_normalizer-3.0.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:0298eafff88c99982a4cf66ba2efa1128e4ddaca0b05eec4c456bbc7db691d8d"}, + {file = "charset_normalizer-3.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a8d0fc946c784ff7f7c3742310cc8a57c5c6dc31631269876a88b809dbeff3d3"}, + {file = "charset_normalizer-3.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:87701167f2a5c930b403e9756fab1d31d4d4da52856143b609e30a1ce7160f3c"}, + {file = "charset_normalizer-3.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:14e76c0f23218b8f46c4d87018ca2e441535aed3632ca134b10239dfb6dadd6b"}, + {file = "charset_normalizer-3.0.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0c0a590235ccd933d9892c627dec5bc7511ce6ad6c1011fdf5b11363022746c1"}, + {file = "charset_normalizer-3.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8c7fe7afa480e3e82eed58e0ca89f751cd14d767638e2550c77a92a9e749c317"}, + {file = "charset_normalizer-3.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:79909e27e8e4fcc9db4addea88aa63f6423ebb171db091fb4373e3312cb6d603"}, + {file = "charset_normalizer-3.0.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8ac7b6a045b814cf0c47f3623d21ebd88b3e8cf216a14790b455ea7ff0135d18"}, + {file = "charset_normalizer-3.0.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:72966d1b297c741541ca8cf1223ff262a6febe52481af742036a0b296e35fa5a"}, + {file = "charset_normalizer-3.0.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:f9d0c5c045a3ca9bedfc35dca8526798eb91a07aa7a2c0fee134c6c6f321cbd7"}, + {file = "charset_normalizer-3.0.1-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:5995f0164fa7df59db4746112fec3f49c461dd6b31b841873443bdb077c13cfc"}, + {file = "charset_normalizer-3.0.1-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:4a8fcf28c05c1f6d7e177a9a46a1c52798bfe2ad80681d275b10dcf317deaf0b"}, + {file = "charset_normalizer-3.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:761e8904c07ad053d285670f36dd94e1b6ab7f16ce62b9805c475b7aa1cffde6"}, + {file = "charset_normalizer-3.0.1-cp311-cp311-win32.whl", hash = "sha256:71140351489970dfe5e60fc621ada3e0f41104a5eddaca47a7acb3c1b851d6d3"}, + {file = "charset_normalizer-3.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:9ab77acb98eba3fd2a85cd160851816bfce6871d944d885febf012713f06659c"}, + {file = "charset_normalizer-3.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:84c3990934bae40ea69a82034912ffe5a62c60bbf6ec5bc9691419641d7d5c9a"}, + {file = "charset_normalizer-3.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:74292fc76c905c0ef095fe11e188a32ebd03bc38f3f3e9bcb85e4e6db177b7ea"}, + {file = "charset_normalizer-3.0.1-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c95a03c79bbe30eec3ec2b7f076074f4281526724c8685a42872974ef4d36b72"}, + {file = "charset_normalizer-3.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f4c39b0e3eac288fedc2b43055cfc2ca7a60362d0e5e87a637beac5d801ef478"}, + {file = "charset_normalizer-3.0.1-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:df2c707231459e8a4028eabcd3cfc827befd635b3ef72eada84ab13b52e1574d"}, + {file = "charset_normalizer-3.0.1-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:93ad6d87ac18e2a90b0fe89df7c65263b9a99a0eb98f0a3d2e079f12a0735837"}, + {file = "charset_normalizer-3.0.1-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:59e5686dd847347e55dffcc191a96622f016bc0ad89105e24c14e0d6305acbc6"}, + {file = "charset_normalizer-3.0.1-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:cd6056167405314a4dc3c173943f11249fa0f1b204f8b51ed4bde1a9cd1834dc"}, + {file = "charset_normalizer-3.0.1-cp36-cp36m-musllinux_1_1_ppc64le.whl", hash = "sha256:083c8d17153ecb403e5e1eb76a7ef4babfc2c48d58899c98fcaa04833e7a2f9a"}, + {file = "charset_normalizer-3.0.1-cp36-cp36m-musllinux_1_1_s390x.whl", hash = "sha256:f5057856d21e7586765171eac8b9fc3f7d44ef39425f85dbcccb13b3ebea806c"}, + {file = "charset_normalizer-3.0.1-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:7eb33a30d75562222b64f569c642ff3dc6689e09adda43a082208397f016c39a"}, + {file = "charset_normalizer-3.0.1-cp36-cp36m-win32.whl", hash = "sha256:95dea361dd73757c6f1c0a1480ac499952c16ac83f7f5f4f84f0658a01b8ef41"}, + {file = "charset_normalizer-3.0.1-cp36-cp36m-win_amd64.whl", hash = "sha256:eaa379fcd227ca235d04152ca6704c7cb55564116f8bc52545ff357628e10602"}, + {file = "charset_normalizer-3.0.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:3e45867f1f2ab0711d60c6c71746ac53537f1684baa699f4f668d4c6f6ce8e14"}, + {file = "charset_normalizer-3.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cadaeaba78750d58d3cc6ac4d1fd867da6fc73c88156b7a3212a3cd4819d679d"}, + {file = "charset_normalizer-3.0.1-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:911d8a40b2bef5b8bbae2e36a0b103f142ac53557ab421dc16ac4aafee6f53dc"}, + {file = "charset_normalizer-3.0.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:503e65837c71b875ecdd733877d852adbc465bd82c768a067badd953bf1bc5a3"}, + {file = "charset_normalizer-3.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a60332922359f920193b1d4826953c507a877b523b2395ad7bc716ddd386d866"}, + {file = "charset_normalizer-3.0.1-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:16a8663d6e281208d78806dbe14ee9903715361cf81f6d4309944e4d1e59ac5b"}, + {file = "charset_normalizer-3.0.1-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:a16418ecf1329f71df119e8a65f3aa68004a3f9383821edcb20f0702934d8087"}, + {file = "charset_normalizer-3.0.1-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:9d9153257a3f70d5f69edf2325357251ed20f772b12e593f3b3377b5f78e7ef8"}, + {file = "charset_normalizer-3.0.1-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:02a51034802cbf38db3f89c66fb5d2ec57e6fe7ef2f4a44d070a593c3688667b"}, + {file = "charset_normalizer-3.0.1-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:2e396d70bc4ef5325b72b593a72c8979999aa52fb8bcf03f701c1b03e1166918"}, + {file = "charset_normalizer-3.0.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:11b53acf2411c3b09e6af37e4b9005cba376c872503c8f28218c7243582df45d"}, + {file = "charset_normalizer-3.0.1-cp37-cp37m-win32.whl", hash = "sha256:0bf2dae5291758b6f84cf923bfaa285632816007db0330002fa1de38bfcb7154"}, + {file = "charset_normalizer-3.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:2c03cc56021a4bd59be889c2b9257dae13bf55041a3372d3295416f86b295fb5"}, + {file = "charset_normalizer-3.0.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:024e606be3ed92216e2b6952ed859d86b4cfa52cd5bc5f050e7dc28f9b43ec42"}, + {file = "charset_normalizer-3.0.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:4b0d02d7102dd0f997580b51edc4cebcf2ab6397a7edf89f1c73b586c614272c"}, + {file = "charset_normalizer-3.0.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:358a7c4cb8ba9b46c453b1dd8d9e431452d5249072e4f56cfda3149f6ab1405e"}, + {file = "charset_normalizer-3.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:81d6741ab457d14fdedc215516665050f3822d3e56508921cc7239f8c8e66a58"}, + {file = "charset_normalizer-3.0.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8b8af03d2e37866d023ad0ddea594edefc31e827fee64f8de5611a1dbc373174"}, + {file = "charset_normalizer-3.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9cf4e8ad252f7c38dd1f676b46514f92dc0ebeb0db5552f5f403509705e24753"}, + {file = "charset_normalizer-3.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e696f0dd336161fca9adbb846875d40752e6eba585843c768935ba5c9960722b"}, + {file = "charset_normalizer-3.0.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c22d3fe05ce11d3671297dc8973267daa0f938b93ec716e12e0f6dee81591dc1"}, + {file = "charset_normalizer-3.0.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:109487860ef6a328f3eec66f2bf78b0b72400280d8f8ea05f69c51644ba6521a"}, + {file = "charset_normalizer-3.0.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:37f8febc8ec50c14f3ec9637505f28e58d4f66752207ea177c1d67df25da5aed"}, + {file = "charset_normalizer-3.0.1-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:f97e83fa6c25693c7a35de154681fcc257c1c41b38beb0304b9c4d2d9e164479"}, + {file = "charset_normalizer-3.0.1-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:a152f5f33d64a6be73f1d30c9cc82dfc73cec6477ec268e7c6e4c7d23c2d2291"}, + {file = "charset_normalizer-3.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:39049da0ffb96c8cbb65cbf5c5f3ca3168990adf3551bd1dee10c48fce8ae820"}, + {file = "charset_normalizer-3.0.1-cp38-cp38-win32.whl", hash = "sha256:4457ea6774b5611f4bed5eaa5df55f70abde42364d498c5134b7ef4c6958e20e"}, + {file = "charset_normalizer-3.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:e62164b50f84e20601c1ff8eb55620d2ad25fb81b59e3cd776a1902527a788af"}, + {file = "charset_normalizer-3.0.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:8eade758719add78ec36dc13201483f8e9b5d940329285edcd5f70c0a9edbd7f"}, + {file = "charset_normalizer-3.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:8499ca8f4502af841f68135133d8258f7b32a53a1d594aa98cc52013fff55678"}, + {file = "charset_normalizer-3.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:3fc1c4a2ffd64890aebdb3f97e1278b0cc72579a08ca4de8cd2c04799a3a22be"}, + {file = "charset_normalizer-3.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:00d3ffdaafe92a5dc603cb9bd5111aaa36dfa187c8285c543be562e61b755f6b"}, + {file = "charset_normalizer-3.0.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c2ac1b08635a8cd4e0cbeaf6f5e922085908d48eb05d44c5ae9eabab148512ca"}, + {file = "charset_normalizer-3.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f6f45710b4459401609ebebdbcfb34515da4fc2aa886f95107f556ac69a9147e"}, + {file = "charset_normalizer-3.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ae1de54a77dc0d6d5fcf623290af4266412a7c4be0b1ff7444394f03f5c54e3"}, + {file = "charset_normalizer-3.0.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3b590df687e3c5ee0deef9fc8c547d81986d9a1b56073d82de008744452d6541"}, + {file = "charset_normalizer-3.0.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:ab5de034a886f616a5668aa5d098af2b5385ed70142090e2a31bcbd0af0fdb3d"}, + {file = "charset_normalizer-3.0.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:9cb3032517f1627cc012dbc80a8ec976ae76d93ea2b5feaa9d2a5b8882597579"}, + {file = "charset_normalizer-3.0.1-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:608862a7bf6957f2333fc54ab4399e405baad0163dc9f8d99cb236816db169d4"}, + {file = "charset_normalizer-3.0.1-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:0f438ae3532723fb6ead77e7c604be7c8374094ef4ee2c5e03a3a17f1fca256c"}, + {file = "charset_normalizer-3.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:356541bf4381fa35856dafa6a965916e54bed415ad8a24ee6de6e37deccf2786"}, + {file = "charset_normalizer-3.0.1-cp39-cp39-win32.whl", hash = "sha256:39cf9ed17fe3b1bc81f33c9ceb6ce67683ee7526e65fde1447c772afc54a1bb8"}, + {file = "charset_normalizer-3.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:0a11e971ed097d24c534c037d298ad32c6ce81a45736d31e0ff0ad37ab437d59"}, + {file = "charset_normalizer-3.0.1-py3-none-any.whl", hash = "sha256:7e189e2e1d3ed2f4aebabd2d5b0f931e883676e51c7624826e0a4e5fe8a0bf24"}, ] colorama = [ - {file = "colorama-0.4.5-py2.py3-none-any.whl", hash = "sha256:854bf444933e37f5824ae7bfc1e98d5bce2ebe4160d46b5edf346a89358e99da"}, - {file = "colorama-0.4.5.tar.gz", hash = "sha256:e6c6b4334fc50988a639d9b98aa429a0b57da6e17b9a44f0451f930b6967b7a4"}, + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, +] +exceptiongroup = [ + {file = "exceptiongroup-1.1.0-py3-none-any.whl", hash = "sha256:327cbda3da756e2de031a3107b81ab7b3770a602c4d16ca618298c526f4bec1e"}, + {file = "exceptiongroup-1.1.0.tar.gz", hash = "sha256:bcb67d800a4497e1b404c2dd44fca47d3b7a5e5433dbab67f96c1a685cdfdf23"}, ] flake8 = [ {file = "flake8-4.0.1-py2.py3-none-any.whl", hash = "sha256:479b1304f72536a55948cb40a32dce8bb0ffe3501e26eaf292c7e60eb5e0428d"}, @@ -294,25 +372,21 @@ importlib-metadata = [ {file = "importlib_metadata-4.2.0.tar.gz", hash = "sha256:b7e52a1f8dec14a75ea73e0891f3060099ca1d8e6a462a4dff11c3e119ea1b31"}, ] iniconfig = [ - {file = "iniconfig-1.1.1-py2.py3-none-any.whl", hash = "sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3"}, - {file = "iniconfig-1.1.1.tar.gz", hash = "sha256:bc3af051d7d14b2ee5ef9969666def0cd1a000e121eaea580d4a313df4b37f32"}, + {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, + {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, ] mccabe = [ {file = "mccabe-0.6.1-py2.py3-none-any.whl", hash = "sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42"}, {file = "mccabe-0.6.1.tar.gz", hash = "sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f"}, ] packaging = [ - {file = "packaging-21.3-py3-none-any.whl", hash = "sha256:ef103e05f519cdc783ae24ea4e2e0f508a9c99b2d4969652eed6a2e1ea5bd522"}, - {file = "packaging-21.3.tar.gz", hash = "sha256:dd47c42927d89ab911e606518907cc2d3a1f38bbd026385970643f9c5b8ecfeb"}, + {file = "packaging-23.0-py3-none-any.whl", hash = "sha256:714ac14496c3e68c99c29b00845f7a2b85f3bb6f1078fd9f72fd20f0570002b2"}, + {file = "packaging-23.0.tar.gz", hash = "sha256:b6ad297f8907de0fa2fe1ccbd26fdaf387f5f47c7275fedf8cce89f99446cf97"}, ] pluggy = [ {file = "pluggy-1.0.0-py2.py3-none-any.whl", hash = "sha256:74134bbf457f031a36d68416e1509f34bd5ccc019f0bcc952c7b909d06b37bd3"}, {file = "pluggy-1.0.0.tar.gz", hash = "sha256:4224373bacce55f955a878bf9cfa763c1e360858e330072059e10bad68531159"}, ] -py = [ - {file = "py-1.11.0-py2.py3-none-any.whl", hash = "sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378"}, - {file = "py-1.11.0.tar.gz", hash = "sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719"}, -] pycodestyle = [ {file = "pycodestyle-2.8.0-py2.py3-none-any.whl", hash = "sha256:720f8b39dde8b293825e7ff02c475f3077124006db4f440dcbc9a20b76548a20"}, {file = "pycodestyle-2.8.0.tar.gz", hash = "sha256:eddd5847ef438ea1c7870ca7eb78a9d47ce0cdb4851a5523949f2601d0cbbe7f"}, @@ -321,21 +395,17 @@ pyflakes = [ {file = "pyflakes-2.4.0-py2.py3-none-any.whl", hash = "sha256:3bb3a3f256f4b7968c9c788781e4ff07dce46bdf12339dcda61053375426ee2e"}, {file = "pyflakes-2.4.0.tar.gz", hash = "sha256:05a85c2872edf37a4ed30b0cce2f6093e1d0581f8c19d7393122da7e25b2b24c"}, ] -pyparsing = [ - {file = "pyparsing-3.0.9-py3-none-any.whl", hash = "sha256:5026bae9a10eeaefb61dab2f09052b9f4307d44aee4eda64b309723d8d206bbc"}, - {file = "pyparsing-3.0.9.tar.gz", hash = "sha256:2b020ecf7d21b687f219b71ecad3631f644a47f01403fa1d1036b0c6416d70fb"}, -] pytest = [ - {file = "pytest-7.1.3-py3-none-any.whl", hash = "sha256:1377bda3466d70b55e3f5cecfa55bb7cfcf219c7964629b967c37cf0bda818b7"}, - {file = "pytest-7.1.3.tar.gz", hash = "sha256:4f365fec2dff9c1162f834d9f18af1ba13062db0c708bf7b946f8a5c76180c39"}, + {file = "pytest-7.2.1-py3-none-any.whl", hash = "sha256:c7c6ca206e93355074ae32f7403e8ea12163b1163c976fee7d4d84027c162be5"}, + {file = "pytest-7.2.1.tar.gz", hash = "sha256:d45e0952f3727241918b8fd0f376f5ff6b301cc0777c6f9a556935c92d8a7d42"}, ] pytest-mock = [ - {file = "pytest-mock-3.9.0.tar.gz", hash = "sha256:c899a0dcc8a5f22930acd020b500abd5f956911f326864a3b979e4866e14da82"}, - {file = "pytest_mock-3.9.0-py3-none-any.whl", hash = "sha256:1a1b9264224d026932d6685a0f9cef3b61d91563c3e74af9fe5afb2767e13812"}, + {file = "pytest-mock-3.10.0.tar.gz", hash = "sha256:fbbdb085ef7c252a326fd8cdcac0aa3b1333d8811f131bdcc701002e1be7ed4f"}, + {file = "pytest_mock-3.10.0-py3-none-any.whl", hash = "sha256:f4c973eeae0282963eb293eb173ce91b091a79c1334455acfac9ddee8a1c784b"}, ] requests = [ - {file = "requests-2.28.1-py3-none-any.whl", hash = "sha256:8fefa2a1a1365bf5520aac41836fbee479da67864514bdb821f31ce07ce65349"}, - {file = "requests-2.28.1.tar.gz", hash = "sha256:7c5599b102feddaa661c826c56ab4fee28bfd17f5abca1ebbe3e7f19d7c97983"}, + {file = "requests-2.28.2-py3-none-any.whl", hash = "sha256:64299f4909223da747622c030b781c0d7811e359c37124b4bd368fb8c6518baa"}, + {file = "requests-2.28.2.tar.gz", hash = "sha256:98b1b2782e3c6c4904938b84c0eb932721069dfdb9134313beff7c83c2df24bf"}, ] tenacity = [ {file = "tenacity-8.1.0-py3-none-any.whl", hash = "sha256:35525cd47f82830069f0d6b73f7eb83bc5b73ee2fff0437952cedf98b27653ac"}, @@ -346,14 +416,14 @@ tomli = [ {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, ] typing-extensions = [ - {file = "typing_extensions-4.3.0-py3-none-any.whl", hash = "sha256:25642c956049920a5aa49edcdd6ab1e06d7e5d467fc00e0506c44ac86fbfca02"}, - {file = "typing_extensions-4.3.0.tar.gz", hash = "sha256:e6d2677a32f47fc7eb2795db1dd15c1f34eff616bcaf2cfb5e997f854fa1c4a6"}, + {file = "typing_extensions-4.4.0-py3-none-any.whl", hash = "sha256:16fa4864408f655d35ec496218b85f79b3437c829e93320c7c9215ccfd92489e"}, + {file = "typing_extensions-4.4.0.tar.gz", hash = "sha256:1511434bb92bf8dd198c12b1cc812e800d4181cfcb867674e0f8279cc93087aa"}, ] urllib3 = [ - {file = "urllib3-1.26.12-py2.py3-none-any.whl", hash = "sha256:b930dd878d5a8afb066a637fbb35144fe7901e3b209d1cd4f524bd0e9deee997"}, - {file = "urllib3-1.26.12.tar.gz", hash = "sha256:3fa96cf423e6987997fc326ae8df396db2a8b7c667747d47ddd8ecba91f4a74e"}, + {file = "urllib3-1.26.14-py2.py3-none-any.whl", hash = "sha256:75edcdc2f7d85b137124a6c3c9fc3933cdeaa12ecb9a6a959f22797a0feca7e1"}, + {file = "urllib3-1.26.14.tar.gz", hash = "sha256:076907bf8fd355cde77728471316625a4d2f7e713c125f51953bb5b3eecf4f72"}, ] zipp = [ - {file = "zipp-3.8.1-py3-none-any.whl", hash = "sha256:47c40d7fe183a6f21403a199b3e4192cca5774656965b0a4988ad2f8feb5f009"}, - {file = "zipp-3.8.1.tar.gz", hash = "sha256:05b45f1ee8f807d0cc928485ca40a07cb491cf092ff587c0df9cb1fd154848d2"}, + {file = "zipp-3.12.0-py3-none-any.whl", hash = "sha256:9eb0a4c5feab9b08871db0d672745b53450d7f26992fd1e4653aa43345e97b86"}, + {file = "zipp-3.12.0.tar.gz", hash = "sha256:73efd63936398aac78fd92b6f4865190119d6c91b531532e798977ea8dd402eb"}, ] diff --git a/pyproject.toml b/pyproject.toml index f1bbb6b..0cfffb3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "podaac-data-subscriber" -version = "1.12.0" +version = "1.13.0" description = "PO.DAAC Data Subscriber Command Line Tool" authors = ["PO.DAAC "] readme = "README.md" @@ -27,7 +27,8 @@ podaac-data-downloader = 'subscriber.podaac_data_downloader:main' [tool.pytest.ini_options] markers = [ - "regression: marks a test as a regression, requires netrc file (deselect with '-m \"not regresion\"')" + "regression: marks a test as a regression, requires netrc file (deselect with '-m \"not regresion\"')", + "token: marks a test as a token regression, requires netrc file and relies on enterprise URS (deselect with '-m \"not token\"')" ] [build-system] diff --git a/subscriber/podaac_access.py b/subscriber/podaac_access.py index 22d4e6e..3c19d7a 100644 --- a/subscriber/podaac_access.py +++ b/subscriber/podaac_access.py @@ -2,8 +2,10 @@ import logging import netrc import subprocess +import re from datetime import datetime from http.cookiejar import CookieJar +import os from os import makedirs from os.path import isdir, basename, join, splitext from urllib import request @@ -18,6 +20,7 @@ from datetime import datetime import time from requests.auth import HTTPBasicAuth +from packaging import version @@ -27,8 +30,8 @@ import tenacity from datetime import datetime -__version__ = "1.12.0" -extensions = [".nc", ".h5", ".zip", ".tar.gz", ".tiff"] +__version__ = "1.13.0" +extensions = ["\\.nc", "\\.h5", "\\.zip", "\\.tar.gz", "\\.tiff"] edl = "urs.earthdata.nasa.gov" cmr = "cmr.earthdata.nasa.gov" token_url = "https://" + edl + "/api/users" @@ -343,7 +346,9 @@ def download_file(remote_file, output_path, retries=3): logging.warning(f'Error downloading {remote_file}. Retrying download.') # back off on sleep time each error... time.sleep(r) - if r >= retries: + # range is exclusive, so range(3): 0,1,2 so retries will + # never be >= 3; need to subtract 1 (doh) + if r >= retries-1: failed = True else: #downlaoded fie without 503 @@ -531,6 +536,11 @@ def create_citation(collection_json, access_date): year = datetime.strptime(release_date, "%Y-%m-%dT%H:%M:%S.000Z").year return citation_template.format(creator=creator, year=year, title=title, version=version, doi_authority=doi_authority, doi=doi, access_date=access_date) +def search_extension(extension, filename): + if re.search(extension + "$", filename) is not None: + return True + return False + def create_citation_file(short_name, provider, data_path, token=None, verbose=False): # get collection umm-c METADATA params = [ @@ -550,3 +560,33 @@ def create_citation_file(short_name, provider, data_path, token=None, verbose=Fa with open(data_path + "/" + short_name + ".citation.txt", "w") as text_file: text_file.write(citation) + +def get_latest_release(): + github_url = "https://api.github.com/repos/podaac/data-subscriber/releases" + headers = {} + ghtoken = os.environ.get('GITHUB_TOKEN', None) + if ghtoken is not None: + headers = {"Authorization": "Bearer " + ghtoken} + + releases_json = requests.get(github_url, headers=headers).json() + latest_release = get_latest_release_from_json(releases_json) + return latest_release + +def release_is_current(latest_release, this_version): + return not (version.parse(this_version) < version.parse(latest_release)) + +def get_latest_release_from_json(releases_json): + releases = [] + for x in releases_json: + releases.append(x['tag_name']) + sorted(releases, key=lambda x: version.Version(x)).reverse() + return releases[0] + + +def check_for_latest(): + try: + latest_version = get_latest_release() + if not release_is_current(latest_version,__version__): + print(f'You are currently using version {__version__} of the PO.DAAC Data Subscriber/Downloader. Please run:\n\n pip install podaac-data-subscriber --upgrade \n\n to upgrade to the latest version.') + except: + print("Error checking for new version of the po.daac data subscriber. Continuing") diff --git a/subscriber/podaac_data_downloader.py b/subscriber/podaac_data_downloader.py index 6f51bca..6334f68 100755 --- a/subscriber/podaac_data_downloader.py +++ b/subscriber/podaac_data_downloader.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 import argparse import logging -import os +import os, re import sys from datetime import datetime, timedelta from os import makedirs @@ -10,6 +10,7 @@ from urllib.request import urlretrieve from subscriber import podaac_access as pa +from subscriber import token_formatter __version__ = pa.__version__ @@ -86,7 +87,7 @@ def create_parser(): help="Flag used to shift timestamp. Units are in hours, e.g. 10 or -10.") # noqa E501 parser.add_argument("-e", "--extensions", dest="extensions", - help="The extensions of products to download. Default is [.nc, .h5, .zip, .tar.gz]", + help="Regexps of extensions of products to download. Default is [.nc, .h5, .zip, .tar.gz, .tiff]", default=None, action='append') # noqa E501 # Get specific granule from the search @@ -107,6 +108,8 @@ def create_parser(): parser.add_argument("--limit", dest="limit", default=None, type=int, help="Integer limit for number of granules to download. Useful in testing. Defaults to no limit.") # noqa E501 + parser.add_argument("--dry-run", dest="dry_run", action="store_true", help="Search and identify files to download, but do not actually download them") # noqa E501 + return parser @@ -253,7 +256,7 @@ def run(args=None): filtered_downloads = [] for f in downloads: for extension in extensions: - if f.lower().endswith(extension): + if pa.search_extension(extension, f): filtered_downloads.append(f) downloads = filtered_downloads @@ -264,9 +267,19 @@ def run(args=None): logging.info("Found " + str(len(downloads)) + " total files to download") if download_limit: logging.info("Limiting downloads to " + str(args.limit) + " total files") + if args.verbose: logging.info("Downloading files with extensions: " + str(extensions)) + if args.dry_run: + logging.info("Dry-run option specified. Listing Downloads.") + for download in downloads[:download_limit]: + logging.info(download) + logging.info("Dry-run option specific. Exiting.") + return + + + # NEED TO REFACTOR THIS, A LOT OF STUFF in here # Finish by downloading the files to the data directory in a loop. # Overwrite `.update` with a new timestamp on success. @@ -321,10 +334,14 @@ def run(args=None): def main(): + log_format = '[%(asctime)s] {%(filename)s:%(lineno)d} %(levelname)s - %(message)s' log_level = os.environ.get('PODAAC_LOGLEVEL', 'INFO').upper() logging.basicConfig(stream=sys.stdout, - format='[%(asctime)s] {%(filename)s:%(lineno)d} %(levelname)s - %(message)s', + format=log_format, level=log_level) + + for handler in logging.root.handlers: + handler.setFormatter(token_formatter.TokenFormatter(log_format)) logging.debug("Log level set to " + log_level) try: @@ -335,4 +352,5 @@ def main(): if __name__ == '__main__': + pa.check_for_latest() main() diff --git a/subscriber/podaac_data_subscriber.py b/subscriber/podaac_data_subscriber.py index eb103ac..52ec5d4 100755 --- a/subscriber/podaac_data_subscriber.py +++ b/subscriber/podaac_data_subscriber.py @@ -14,7 +14,7 @@ # Accounts are free to create and take just a moment to set up. import argparse import logging -import os +import os, re import sys from datetime import datetime, timedelta from os import makedirs @@ -23,6 +23,8 @@ from urllib.request import urlretrieve from subscriber import podaac_access as pa +from subscriber import token_formatter + __version__ = pa.__version__ @@ -92,7 +94,7 @@ def create_parser(): help="How far back in time, in minutes, should the script look for data. If running this script as a cron, this value should be equal to or greater than how often your cron runs.", type=int, default=None) # noqa E501 parser.add_argument("-e", "--extensions", dest="extensions", - help="The extensions of products to download. Default is [.nc, .h5, .zip]", default=None, + help="Regexps of extensions of products to download. Default is [.nc, .h5, .zip, .tar.gz, .tiff]", default=None, action='append') # noqa E501 parser.add_argument("--process", dest="process_cmd", help="Processing command to run on each downloaded file (e.g., compression). Can be specified multiple times.", @@ -104,6 +106,8 @@ def create_parser(): parser.add_argument("-p", "--provider", dest="provider", default='POCLOUD', help="Specify a provider for collection search. Default is POCLOUD.") # noqa E501 + parser.add_argument("--dry-run", dest="dry_run", action="store_true", help="Search and identify files to download, but do not actually download them") # noqa E501 + return parser @@ -260,7 +264,7 @@ def run(args=None): filtered_downloads = [] for f in downloads: for extension in extensions: - if f.lower().endswith(extension): + if pa.search_extension(extension, f): filtered_downloads.append(f) downloads = filtered_downloads @@ -272,6 +276,14 @@ def run(args=None): if args.verbose: logging.info("Downloading files with extensions: " + str(extensions)) + if args.dry_run: + logging.info("Dry-run option specified. Listing Downloads.") + for download in downloads: + logging.info(download) + logging.info("Dry-run option specific. Exiting.") + return + + # NEED TO REFACTOR THIS, A LOT OF STUFF in here # Finish by downloading the files to the data directory in a loop. # Overwrite `.update` with a new timestamp on success. @@ -328,10 +340,13 @@ def run(args=None): def main(): + log_format = '[%(asctime)s] {%(filename)s:%(lineno)d} %(levelname)s - %(message)s' log_level = os.environ.get('PODAAC_LOGLEVEL', 'INFO').upper() logging.basicConfig(stream=sys.stdout, - format='[%(asctime)s] {%(filename)s:%(lineno)d} %(levelname)s - %(message)s', + format=log_format, level=log_level) + for handler in logging.root.handlers: + handler.setFormatter(token_formatter.TokenFormatter(log_format)) logging.debug("Log level set to " + log_level) try: @@ -342,4 +357,5 @@ def main(): if __name__ == '__main__': + pa.check_for_latest() main() diff --git a/subscriber/token_formatter.py b/subscriber/token_formatter.py new file mode 100644 index 0000000..3bb4128 --- /dev/null +++ b/subscriber/token_formatter.py @@ -0,0 +1,14 @@ +import logging +import re + + +class TokenFormatter(logging.Formatter): + """Formatter that removes sensitive information in urls.""" + @staticmethod + def _filter(s): + + return re.sub(r'token=(.*)\??', r'token=****', s) + + def format(self, record): + original = logging.Formatter.format(self, record) + return self._filter(original) diff --git a/tests/releases.json b/tests/releases.json new file mode 100644 index 0000000..76840cd --- /dev/null +++ b/tests/releases.json @@ -0,0 +1,431 @@ +[ + { + "url": "https://api.github.com/repos/podaac/data-subscriber/releases/85798710", + "assets_url": "https://api.github.com/repos/podaac/data-subscriber/releases/85798710/assets", + "upload_url": "https://uploads.github.com/repos/podaac/data-subscriber/releases/85798710/assets{?name,label}", + "html_url": "https://github.com/podaac/data-subscriber/releases/tag/1.12.0", + "id": 85798710, + "author": { + "login": "mike-gangl", + "id": 59702631, + "node_id": "MDQ6VXNlcjU5NzAyNjMx", + "avatar_url": "https://avatars.githubusercontent.com/u/59702631?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/mike-gangl", + "html_url": "https://github.com/mike-gangl", + "followers_url": "https://api.github.com/users/mike-gangl/followers", + "following_url": "https://api.github.com/users/mike-gangl/following{/other_user}", + "gists_url": "https://api.github.com/users/mike-gangl/gists{/gist_id}", + "starred_url": "https://api.github.com/users/mike-gangl/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/mike-gangl/subscriptions", + "organizations_url": "https://api.github.com/users/mike-gangl/orgs", + "repos_url": "https://api.github.com/users/mike-gangl/repos", + "events_url": "https://api.github.com/users/mike-gangl/events{/privacy}", + "received_events_url": "https://api.github.com/users/mike-gangl/received_events", + "type": "User", + "site_admin": false + }, + "node_id": "RE_kwDOFdnRzs4FHS82", + "tag_name": "1.12.0", + "target_commitish": "main", + "name": "Subscriber 1.12.0", + "draft": false, + "prerelease": false, + "created_at": "2022-12-12T22:14:24Z", + "published_at": "2022-12-12T23:04:11Z", + "assets": [], + "tarball_url": "https://api.github.com/repos/podaac/data-subscriber/tarball/1.12.0", + "zipball_url": "https://api.github.com/repos/podaac/data-subscriber/zipball/1.12.0", + "body": "## 1.12.0\r\n### Fixed\r\n- Added EDL based token downloading, removing CMR tokens [98](https://github.com/podaac/data-subscriber/issues/98),\r\n### Added\r\n- Added ability to download by filename [109](https://github.com/podaac/data-subscriber/issues/109) and additional regression testing" + }, + { + "url": "https://api.github.com/repos/podaac/data-subscriber/releases/76177645", + "assets_url": "https://api.github.com/repos/podaac/data-subscriber/releases/76177645/assets", + "upload_url": "https://uploads.github.com/repos/podaac/data-subscriber/releases/76177645/assets{?name,label}", + "html_url": "https://github.com/podaac/data-subscriber/releases/tag/1.11.0", + "id": 76177645, + "author": { + "login": "mike-gangl", + "id": 59702631, + "node_id": "MDQ6VXNlcjU5NzAyNjMx", + "avatar_url": "https://avatars.githubusercontent.com/u/59702631?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/mike-gangl", + "html_url": "https://github.com/mike-gangl", + "followers_url": "https://api.github.com/users/mike-gangl/followers", + "following_url": "https://api.github.com/users/mike-gangl/following{/other_user}", + "gists_url": "https://api.github.com/users/mike-gangl/gists{/gist_id}", + "starred_url": "https://api.github.com/users/mike-gangl/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/mike-gangl/subscriptions", + "organizations_url": "https://api.github.com/users/mike-gangl/orgs", + "repos_url": "https://api.github.com/users/mike-gangl/repos", + "events_url": "https://api.github.com/users/mike-gangl/events{/privacy}", + "received_events_url": "https://api.github.com/users/mike-gangl/received_events", + "type": "User", + "site_admin": false + }, + "node_id": "RE_kwDOFdnRzs4EimDt", + "tag_name": "1.12.0-rc2", + "target_commitish": "main", + "name": "Subscriber 1.11.0", + "draft": false, + "prerelease": false, + "created_at": "2022-09-02T17:47:28Z", + "published_at": "2022-09-02T19:27:17Z", + "assets": [], + "tarball_url": "https://api.github.com/repos/podaac/data-subscriber/tarball/1.11.0", + "zipball_url": "https://api.github.com/repos/podaac/data-subscriber/zipball/1.11.0", + "body": "## 1.11.0\r\n### Fixed\r\n- Fixed an issue where token-refresh was expecting a dictionary, not a list of tuples\r\n- Fixed issues where token was not propagated to downloader CMR query [94](https://github.com/podaac/data-subscriber/issues/94)\r\n- Fixed an issue with 503 errors on data download not being re-tried. [97](https://github.com/podaac/data-subscriber/issues/9797)\r\n- added \".tiff\" to default extensions to address #[100](https://github.com/podaac/data-subscriber/issues/100)\r\n- removed erroneous 'warning' message on not downloading all data to close [99](https://github.com/podaac/data-subscriber/issues/99)\r\n- updated help documentation for start/end times to close [79](https://github.com/podaac/data-subscriber/issues/79)\r\n### Added\r\n- Added citation file creation when data are downloaded [91](https://github.com/podaac/data-subscriber/issues/91). Required some updates to the regression testing." + }, + { + "url": "https://api.github.com/repos/podaac/data-subscriber/releases/70187121", + "assets_url": "https://api.github.com/repos/podaac/data-subscriber/releases/70187121/assets", + "upload_url": "https://uploads.github.com/repos/podaac/data-subscriber/releases/70187121/assets{?name,label}", + "html_url": "https://github.com/podaac/data-subscriber/releases/tag/1.10.2", + "id": 70187121, + "author": { + "login": "mike-gangl", + "id": 59702631, + "node_id": "MDQ6VXNlcjU5NzAyNjMx", + "avatar_url": "https://avatars.githubusercontent.com/u/59702631?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/mike-gangl", + "html_url": "https://github.com/mike-gangl", + "followers_url": "https://api.github.com/users/mike-gangl/followers", + "following_url": "https://api.github.com/users/mike-gangl/following{/other_user}", + "gists_url": "https://api.github.com/users/mike-gangl/gists{/gist_id}", + "starred_url": "https://api.github.com/users/mike-gangl/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/mike-gangl/subscriptions", + "organizations_url": "https://api.github.com/users/mike-gangl/orgs", + "repos_url": "https://api.github.com/users/mike-gangl/repos", + "events_url": "https://api.github.com/users/mike-gangl/events{/privacy}", + "received_events_url": "https://api.github.com/users/mike-gangl/received_events", + "type": "User", + "site_admin": false + }, + "node_id": "RE_kwDOFdnRzs4ELvhx", + "tag_name": "1.10.2", + "target_commitish": "main", + "name": "Subscriber 1.10.2", + "draft": false, + "prerelease": false, + "created_at": "2022-06-22T23:40:00Z", + "published_at": "2022-06-22T23:41:15Z", + "assets": [], + "tarball_url": "https://api.github.com/repos/podaac/data-subscriber/tarball/1.10.2", + "zipball_url": "https://api.github.com/repos/podaac/data-subscriber/zipball/1.10.2", + "body": "## [1.10.2]\r\n### Fixed\r\n- Fixed an issue where using a default global bounding box prevented download of data that didn't use the horizontal spatial domain [87](https://github.com/podaac/data-subscriber/issues/87)\r\n- Fixed limit option not being respected. [86](https://github.com/podaac/data-subscriber/issues/86)" + }, + { + "url": "https://api.github.com/repos/podaac/data-subscriber/releases/69558165", + "assets_url": "https://api.github.com/repos/podaac/data-subscriber/releases/69558165/assets", + "upload_url": "https://uploads.github.com/repos/podaac/data-subscriber/releases/69558165/assets{?name,label}", + "html_url": "https://github.com/podaac/data-subscriber/releases/tag/1.10.1", + "id": 69558165, + "author": { + "login": "mike-gangl", + "id": 59702631, + "node_id": "MDQ6VXNlcjU5NzAyNjMx", + "avatar_url": "https://avatars.githubusercontent.com/u/59702631?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/mike-gangl", + "html_url": "https://github.com/mike-gangl", + "followers_url": "https://api.github.com/users/mike-gangl/followers", + "following_url": "https://api.github.com/users/mike-gangl/following{/other_user}", + "gists_url": "https://api.github.com/users/mike-gangl/gists{/gist_id}", + "starred_url": "https://api.github.com/users/mike-gangl/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/mike-gangl/subscriptions", + "organizations_url": "https://api.github.com/users/mike-gangl/orgs", + "repos_url": "https://api.github.com/users/mike-gangl/repos", + "events_url": "https://api.github.com/users/mike-gangl/events{/privacy}", + "received_events_url": "https://api.github.com/users/mike-gangl/received_events", + "type": "User", + "site_admin": false + }, + "node_id": "RE_kwDOFdnRzs4EJV-V", + "tag_name": "1.10.1", + "target_commitish": "main", + "name": "Subscriber 1.10.1", + "draft": false, + "prerelease": false, + "created_at": "2022-06-15T19:50:57Z", + "published_at": "2022-06-15T20:09:30Z", + "assets": [], + "tarball_url": "https://api.github.com/repos/podaac/data-subscriber/tarball/1.10.1", + "zipball_url": "https://api.github.com/repos/podaac/data-subscriber/zipball/1.10.1", + "body": "# Changelog\r\nAll notable changes to this project will be documented in this file.\r\n\r\nThe format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)\r\n\r\n## [1.10.1]\r\n### Fixed\r\n- Support for SHA-256 and SHA-512 checksums" + }, + { + "url": "https://api.github.com/repos/podaac/data-subscriber/releases/67586638", + "assets_url": "https://api.github.com/repos/podaac/data-subscriber/releases/67586638/assets", + "upload_url": "https://uploads.github.com/repos/podaac/data-subscriber/releases/67586638/assets{?name,label}", + "html_url": "https://github.com/podaac/data-subscriber/releases/tag/1.10.0", + "id": 67586638, + "author": { + "login": "mike-gangl", + "id": 59702631, + "node_id": "MDQ6VXNlcjU5NzAyNjMx", + "avatar_url": "https://avatars.githubusercontent.com/u/59702631?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/mike-gangl", + "html_url": "https://github.com/mike-gangl", + "followers_url": "https://api.github.com/users/mike-gangl/followers", + "following_url": "https://api.github.com/users/mike-gangl/following{/other_user}", + "gists_url": "https://api.github.com/users/mike-gangl/gists{/gist_id}", + "starred_url": "https://api.github.com/users/mike-gangl/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/mike-gangl/subscriptions", + "organizations_url": "https://api.github.com/users/mike-gangl/orgs", + "repos_url": "https://api.github.com/users/mike-gangl/repos", + "events_url": "https://api.github.com/users/mike-gangl/events{/privacy}", + "received_events_url": "https://api.github.com/users/mike-gangl/received_events", + "type": "User", + "site_admin": false + }, + "node_id": "RE_kwDOFdnRzs4EB0pO", + "tag_name": "1.10.0", + "target_commitish": "main", + "name": "Subscriber 1.10.0", + "draft": false, + "prerelease": false, + "created_at": "2022-05-23T18:52:49Z", + "published_at": "2022-05-23T18:56:07Z", + "assets": [], + "tarball_url": "https://api.github.com/repos/podaac/data-subscriber/tarball/1.10.0", + "zipball_url": "https://api.github.com/repos/podaac/data-subscriber/zipball/1.10.0", + "body": "## [1.10.0]\r\n### Changed\r\n- Changed minimum supported python version to 3.7, down from 3.8.\r\n" + }, + { + "url": "https://api.github.com/repos/podaac/data-subscriber/releases/67304549", + "assets_url": "https://api.github.com/repos/podaac/data-subscriber/releases/67304549/assets", + "upload_url": "https://uploads.github.com/repos/podaac/data-subscriber/releases/67304549/assets{?name,label}", + "html_url": "https://github.com/podaac/data-subscriber/releases/tag/1.9.1-r2", + "id": 67304549, + "author": { + "login": "mike-gangl", + "id": 59702631, + "node_id": "MDQ6VXNlcjU5NzAyNjMx", + "avatar_url": "https://avatars.githubusercontent.com/u/59702631?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/mike-gangl", + "html_url": "https://github.com/mike-gangl", + "followers_url": "https://api.github.com/users/mike-gangl/followers", + "following_url": "https://api.github.com/users/mike-gangl/following{/other_user}", + "gists_url": "https://api.github.com/users/mike-gangl/gists{/gist_id}", + "starred_url": "https://api.github.com/users/mike-gangl/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/mike-gangl/subscriptions", + "organizations_url": "https://api.github.com/users/mike-gangl/orgs", + "repos_url": "https://api.github.com/users/mike-gangl/repos", + "events_url": "https://api.github.com/users/mike-gangl/events{/privacy}", + "received_events_url": "https://api.github.com/users/mike-gangl/received_events", + "type": "User", + "site_admin": false + }, + "node_id": "RE_kwDOFdnRzs4EAvxl", + "tag_name": "1.9.1-r2", + "target_commitish": "main", + "name": "Subscriber 1.9.1", + "draft": false, + "prerelease": false, + "created_at": "2022-05-19T17:23:04Z", + "published_at": "2022-05-19T17:27:11Z", + "assets": [], + "tarball_url": "https://api.github.com/repos/podaac/data-subscriber/tarball/1.9.1-r2", + "zipball_url": "https://api.github.com/repos/podaac/data-subscriber/zipball/1.9.1-r2", + "body": "# Changelog\r\nAll notable changes to this project will be documented in this file.\r\n\r\nThe format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)\r\n\r\n## [1.9.1]\r\n### Changed\r\n- Switched to [poetry](https://python-poetry.org/) as the build tool for the project\r\n" + }, + { + "url": "https://api.github.com/repos/podaac/data-subscriber/releases/65602913", + "assets_url": "https://api.github.com/repos/podaac/data-subscriber/releases/65602913/assets", + "upload_url": "https://uploads.github.com/repos/podaac/data-subscriber/releases/65602913/assets{?name,label}", + "html_url": "https://github.com/podaac/data-subscriber/releases/tag/1.9.0", + "id": 65602913, + "author": { + "login": "mike-gangl", + "id": 59702631, + "node_id": "MDQ6VXNlcjU5NzAyNjMx", + "avatar_url": "https://avatars.githubusercontent.com/u/59702631?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/mike-gangl", + "html_url": "https://github.com/mike-gangl", + "followers_url": "https://api.github.com/users/mike-gangl/followers", + "following_url": "https://api.github.com/users/mike-gangl/following{/other_user}", + "gists_url": "https://api.github.com/users/mike-gangl/gists{/gist_id}", + "starred_url": "https://api.github.com/users/mike-gangl/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/mike-gangl/subscriptions", + "organizations_url": "https://api.github.com/users/mike-gangl/orgs", + "repos_url": "https://api.github.com/users/mike-gangl/repos", + "events_url": "https://api.github.com/users/mike-gangl/events{/privacy}", + "received_events_url": "https://api.github.com/users/mike-gangl/received_events", + "type": "User", + "site_admin": false + }, + "node_id": "RE_kwDOFdnRzs4D6QVh", + "tag_name": "1.9.0", + "target_commitish": "main", + "name": "Subscriber 1.9.0", + "draft": false, + "prerelease": false, + "created_at": "2022-04-28T16:40:35Z", + "published_at": "2022-04-28T16:49:59Z", + "assets": [], + "tarball_url": "https://api.github.com/repos/podaac/data-subscriber/tarball/1.9.0", + "zipball_url": "https://api.github.com/repos/podaac/data-subscriber/zipball/1.9.0", + "body": "# Changelog\r\nAll notable changes to this project will be documented in this file.\r\n\r\nThe format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)\r\n\r\n## [1.9.0]\r\n### Added\r\n- check if file exists before downloading a file. [17](https://github.com/podaac/data-subscriber/issues/17)\r\n- added automated regression testing\r\n### Changed\r\n- Implemented Search After CMR interface to allow granule listings > 2000 [15](https://github.com/podaac/data-subscriber/issues/15)\r\n- Retry CMR queries on server error using random exponential backoff max 60 seconds and 10 retries\r\n- Refresh token if CMR returns 401 error\r\n- Converted print statements to log statements\r\n### Deprecated\r\n### Removed\r\n### Fixed\r\n### Security" + }, + { + "url": "https://api.github.com/repos/podaac/data-subscriber/releases/60633077", + "assets_url": "https://api.github.com/repos/podaac/data-subscriber/releases/60633077/assets", + "upload_url": "https://uploads.github.com/repos/podaac/data-subscriber/releases/60633077/assets{?name,label}", + "html_url": "https://github.com/podaac/data-subscriber/releases/tag/1.8.0", + "id": 60633077, + "author": { + "login": "mike-gangl", + "id": 59702631, + "node_id": "MDQ6VXNlcjU5NzAyNjMx", + "avatar_url": "https://avatars.githubusercontent.com/u/59702631?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/mike-gangl", + "html_url": "https://github.com/mike-gangl", + "followers_url": "https://api.github.com/users/mike-gangl/followers", + "following_url": "https://api.github.com/users/mike-gangl/following{/other_user}", + "gists_url": "https://api.github.com/users/mike-gangl/gists{/gist_id}", + "starred_url": "https://api.github.com/users/mike-gangl/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/mike-gangl/subscriptions", + "organizations_url": "https://api.github.com/users/mike-gangl/orgs", + "repos_url": "https://api.github.com/users/mike-gangl/repos", + "events_url": "https://api.github.com/users/mike-gangl/events{/privacy}", + "received_events_url": "https://api.github.com/users/mike-gangl/received_events", + "type": "User", + "site_admin": false + }, + "node_id": "RE_kwDOFdnRzs4DnS_1", + "tag_name": "1.8.0", + "target_commitish": "main", + "name": "Subscriber 1.8.0", + "draft": false, + "prerelease": false, + "created_at": "2022-02-28T18:22:48Z", + "published_at": "2022-02-28T18:25:14Z", + "assets": [], + "tarball_url": "https://api.github.com/repos/podaac/data-subscriber/tarball/1.8.0", + "zipball_url": "https://api.github.com/repos/podaac/data-subscriber/zipball/1.8.0", + "body": "### Added\r\n- limit to set limit of downloads- useful for testing\r\n- cycle based downloads to the podaac-data-downloader. [41](https://github.com/podaac/data-subscriber/issues/41)\r\n- conftest.py added to force module inclusion for pytest\r\n- podaac-data-downloader script for bulk data downloading\r\n### Changed\r\n- created library of common access mechanisms to split between subscriber and downloader capabilities\r\n- added .tar.gz to list of default extensions. [40](https://github.com/podaac/data-subscriber/issues/40)\r\n- Ignore error if destination directory already exists. [46](https://github.com/podaac/data-subscriber/issues/46)\r\n- Updated the naming convention of .update file. [44](https://github.com/podaac/data-subscriber/issues/44)\r\n- one of -m, -sd, or -ed must be given to subscriber. Previously -m 60 was the default if nothing was specified.\r\n### Deprecated\r\n- use of \".update\" file naming convention. This will still work, but will be renamed to .update__COLLECTIONNAME after a successful run. the \".update\" file will need to be manually cleaned up. See [issue 44](https://github.com/podaac/data-subscriber/issues/44)\r\n### Removed\r\n### Fixed\r\n- issue where only specifying an end data cause issues in subscriber. [39](https://github.com/podaac/data-subscriber/issues/39)\r\n### Security" + }, + { + "url": "https://api.github.com/repos/podaac/data-subscriber/releases/58477196", + "assets_url": "https://api.github.com/repos/podaac/data-subscriber/releases/58477196/assets", + "upload_url": "https://uploads.github.com/repos/podaac/data-subscriber/releases/58477196/assets{?name,label}", + "html_url": "https://github.com/podaac/data-subscriber/releases/tag/1.7.2", + "id": 58477196, + "author": { + "login": "mike-gangl", + "id": 59702631, + "node_id": "MDQ6VXNlcjU5NzAyNjMx", + "avatar_url": "https://avatars.githubusercontent.com/u/59702631?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/mike-gangl", + "html_url": "https://github.com/mike-gangl", + "followers_url": "https://api.github.com/users/mike-gangl/followers", + "following_url": "https://api.github.com/users/mike-gangl/following{/other_user}", + "gists_url": "https://api.github.com/users/mike-gangl/gists{/gist_id}", + "starred_url": "https://api.github.com/users/mike-gangl/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/mike-gangl/subscriptions", + "organizations_url": "https://api.github.com/users/mike-gangl/orgs", + "repos_url": "https://api.github.com/users/mike-gangl/repos", + "events_url": "https://api.github.com/users/mike-gangl/events{/privacy}", + "received_events_url": "https://api.github.com/users/mike-gangl/received_events", + "type": "User", + "site_admin": false + }, + "node_id": "RE_kwDOFdnRzs4DfEqM", + "tag_name": "1.7.2", + "target_commitish": "main", + "name": "Subscriber 1.7.2", + "draft": false, + "prerelease": false, + "created_at": "2022-02-02T00:48:22Z", + "published_at": "2022-02-02T00:49:57Z", + "assets": [], + "tarball_url": "https://api.github.com/repos/podaac/data-subscriber/tarball/1.7.2", + "zipball_url": "https://api.github.com/repos/podaac/data-subscriber/zipball/1.7.2", + "body": "## [1.7.2]\r\n### Added\r\n### Changed\r\n- Made number of files to download a non-verbose default printout. [33](https://github.com/podaac/data-subscriber/issues/33)\r\n### Deprecated\r\n### Removed\r\n### Fixed\r\n### Security\r\n" + }, + { + "url": "https://api.github.com/repos/podaac/data-subscriber/releases/58470446", + "assets_url": "https://api.github.com/repos/podaac/data-subscriber/releases/58470446/assets", + "upload_url": "https://uploads.github.com/repos/podaac/data-subscriber/releases/58470446/assets{?name,label}", + "html_url": "https://github.com/podaac/data-subscriber/releases/tag/1.7.1", + "id": 58470446, + "author": { + "login": "mike-gangl", + "id": 59702631, + "node_id": "MDQ6VXNlcjU5NzAyNjMx", + "avatar_url": "https://avatars.githubusercontent.com/u/59702631?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/mike-gangl", + "html_url": "https://github.com/mike-gangl", + "followers_url": "https://api.github.com/users/mike-gangl/followers", + "following_url": "https://api.github.com/users/mike-gangl/following{/other_user}", + "gists_url": "https://api.github.com/users/mike-gangl/gists{/gist_id}", + "starred_url": "https://api.github.com/users/mike-gangl/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/mike-gangl/subscriptions", + "organizations_url": "https://api.github.com/users/mike-gangl/orgs", + "repos_url": "https://api.github.com/users/mike-gangl/repos", + "events_url": "https://api.github.com/users/mike-gangl/events{/privacy}", + "received_events_url": "https://api.github.com/users/mike-gangl/received_events", + "type": "User", + "site_admin": false + }, + "node_id": "RE_kwDOFdnRzs4DfDAu", + "tag_name": "1.7.1", + "target_commitish": "main", + "name": "Subscriber 1.7.1", + "draft": false, + "prerelease": false, + "created_at": "2022-02-01T22:09:40Z", + "published_at": "2022-02-01T22:11:41Z", + "assets": [], + "tarball_url": "https://api.github.com/repos/podaac/data-subscriber/tarball/1.7.1", + "zipball_url": "https://api.github.com/repos/podaac/data-subscriber/zipball/1.7.1", + "body": "# Changelog\r\nAll notable changes to this project will be documented in this file.\r\n\r\nThe format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)\r\n\r\n## [1.7.1]\r\n### Added\r\n- Auto build and deploy to pypi on tag/release.\r\n### Changed\r\n### Deprecated\r\n### Removed\r\n### Fixed\r\n### Security\r\n\r\n## [1.7.0]\r\n### Added\r\n- Added ability to call a process on downlaoded files. [Thank to Joe Sapp](https://github.com/sappjw).\r\n### Changed\r\n- Turned -e option into 'additive' mode (multiple -e options allowed.) [Thanks to Joe Sapp](https://github.com/sappjw)\r\n### Deprecated\r\n### Removed\r\n### Fixed\r\n- issue not being able to find granuleUR [#28](https://github.com/podaac/data-subscriber/issues/28)\r\n### Security\r\n\r\n## [1.6.1]\r\n### Added\r\n- added warning for more than 2k granules\r\n### Changed\r\n### Deprecated\r\n### Removed\r\n### Fixed\r\n- strip newline characters from .update to fix https://github.com/podaac/data-subscriber/issues/25\r\n### Security\r\n\r\n## [1.6.0]\r\n### Added\r\n- added --offset flag for timestamp shift when creating DOY folder - (resolves https://github.com/podaac/data-subscriber/issues/23)\r\n### Changed\r\n### Deprecated\r\n### Removed\r\n### Fixed\r\n### Security\r\n\r\n## [1.5.0] - 2021-10-12\r\n### Added\r\n- added ability to change the provider using the -p/--provider flag. Default is 'POCLOUD'\r\n- added pyproject info and setup.py fixes to enable pypi pushes\r\n### Changed\r\n- added pytest and flake8 fixes for automated builds\r\n### Deprecated\r\n### Removed\r\n### Fixed\r\n### Security\r\n\r\n## [1.4.0] - 2021-10-05\r\n### Added\r\n### Changed\r\n- changed changing created_at to updated_since to allow for re-download of updated granules based on collection redeliveries - (resolves https://github.com/podaac/data-subscriber/issues/18)\r\n### Deprecated\r\n### Removed\r\n### Fixed\r\n### Security\r\n\r\n\r\n## [1.3.0] - 2021-08-26\r\n### Added\r\n- added additional non-flat output directory option of -dy - (resolves https://github.com/podaac/data-subscriber/issues/13)\r\n### Changed\r\n### Deprecated\r\n### Removed\r\n### Fixed\r\n### Security\r\n\r\n## [1.2.0] - 2021-08-15\r\n### Added\r\n- Added logging capability using the SUBSCRIBER_LOGLEVEL environment variable\r\n- Added -st and -ed flags and respect the .update flag\r\n### Changed\r\n### Deprecated\r\n### Removed\r\n- removed the -ds flag as it caused confusion.\r\n### Fixed\r\n### Security\r\n\r\n## [1.1.2] - 2021-06-20\r\n### Added\r\n- added default layouts for non-flat output directories - (resolves https://github.com/podaac/data-subscriber/issues/6)\r\n- Added logging capability using the SUBSCRIBER_LOGLEVEL environment variable\r\n- added additional non-flat output directory option of -dy - (resolves https://github.com/podaac/data-subscriber/issues/13)\r\n### Changed\r\n### Deprecated\r\n### Removed\r\n### Fixed\r\n### Security\r\n\r\n## [1.1.1] - 2021-06-06\r\n### Added\r\n### Changed\r\n### Deprecated\r\n### Removed\r\n### Fixed\r\n### Security\r\n- updated urllib3>=1.26.5 (fixes https://github.com/advisories/GHSA-q2q7-5pp4-w6pg)\r\n\r\n## [1.1.0] - 2021-05-28\r\n### Added\r\n- User Agent to request so we can determine better support posture based on metrics\r\n### Changed\r\n### Deprecated\r\n### Removed\r\n### Fixed\r\n### Security\r\n\r\n\r\n## [1.0.0] - 2021-05-13\r\n### Added\r\n- data subscriber functionality\r\n### Changed\r\n### Deprecated\r\n### Removed\r\n### Fixed\r\n### Security" + }, + { + "url": "https://api.github.com/repos/podaac/data-subscriber/releases/58470099", + "assets_url": "https://api.github.com/repos/podaac/data-subscriber/releases/58470099/assets", + "upload_url": "https://uploads.github.com/repos/podaac/data-subscriber/releases/58470099/assets{?name,label}", + "html_url": "https://github.com/podaac/data-subscriber/releases/tag/1.7.0", + "id": 58470099, + "author": { + "login": "mike-gangl", + "id": 59702631, + "node_id": "MDQ6VXNlcjU5NzAyNjMx", + "avatar_url": "https://avatars.githubusercontent.com/u/59702631?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/mike-gangl", + "html_url": "https://github.com/mike-gangl", + "followers_url": "https://api.github.com/users/mike-gangl/followers", + "following_url": "https://api.github.com/users/mike-gangl/following{/other_user}", + "gists_url": "https://api.github.com/users/mike-gangl/gists{/gist_id}", + "starred_url": "https://api.github.com/users/mike-gangl/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/mike-gangl/subscriptions", + "organizations_url": "https://api.github.com/users/mike-gangl/orgs", + "repos_url": "https://api.github.com/users/mike-gangl/repos", + "events_url": "https://api.github.com/users/mike-gangl/events{/privacy}", + "received_events_url": "https://api.github.com/users/mike-gangl/received_events", + "type": "User", + "site_admin": false + }, + "node_id": "RE_kwDOFdnRzs4DfC7T", + "tag_name": "1.7.0", + "target_commitish": "main", + "name": "Subscriber 1.7.0", + "draft": false, + "prerelease": false, + "created_at": "2022-02-01T22:01:53Z", + "published_at": "2022-02-01T22:05:17Z", + "assets": [], + "tarball_url": "https://api.github.com/repos/podaac/data-subscriber/tarball/1.7.0", + "zipball_url": "https://api.github.com/repos/podaac/data-subscriber/zipball/1.7.0", + "body": "# Changelog\r\nAll notable changes to this project will be documented in this file.\r\n\r\nThe format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)\r\n\r\n## [1.7.0]\r\n### Added\r\n- Added ability to call a process on downlaoded files. [Thank to Joe Sapp](https://github.com/sappjw).\r\n### Changed\r\n- Turned -e option into 'additive' mode (multiple -e options allowed.) [Thanks to Joe Sapp](https://github.com/sappjw)\r\n### Deprecated\r\n### Removed\r\n### Fixed\r\n- issue not being able to find granuleUR [#28](https://github.com/podaac/data-subscriber/issues/28)\r\n### Security" + } +] diff --git a/tests/test_downloader_regression.py b/tests/test_downloader_regression.py index a41c67b..027ea50 100644 --- a/tests/test_downloader_regression.py +++ b/tests/test_downloader_regression.py @@ -37,6 +37,17 @@ def test_downloader_limit_MUR(): assert len([name for name in os.listdir('./MUR25-JPL-L4-GLOB-v04.2') if os.path.isfile('./MUR25-JPL-L4-GLOB-v04.2/' + name) and "citation.txt" not in name ])==1 shutil.rmtree('./MUR25-JPL-L4-GLOB-v04.2') +# Test the downlaoder on SWOT Simulated single file download +@pytest.mark.regression +def test_downloader_limit_dry_run(): + shutil.rmtree('./MUR25-JPL-L4-GLOB-v04.2', ignore_errors=True) + args2 = create_downloader_args('-c MUR25-JPL-L4-GLOB-v04.2 -d ./MUR25-JPL-L4-GLOB-v04.2 --dry-run -sd 2020-01-01T00:00:00Z -ed 2020-01-30T00:00:00Z --limit 1 --verbose'.split()) + pdd.run(args2) + # So running the test in parallel, sometimes we get a 401 on the token... + # Let's ensure we're only looking for data files here + assert len([name for name in os.listdir('./MUR25-JPL-L4-GLOB-v04.2') if os.path.isfile('./MUR25-JPL-L4-GLOB-v04.2/' + name) and "citation.txt" not in name ])==0 + shutil.rmtree('./MUR25-JPL-L4-GLOB-v04.2') + #Test the downlaoder on MUR25 data for start/stop/, yyyy/mmm/dd dir structure, # and offset. Running it a second time to ensure it downlaods the files again- diff --git a/tests/test_subscriber.py b/tests/test_subscriber.py index 983cdce..f40994c 100644 --- a/tests/test_subscriber.py +++ b/tests/test_subscriber.py @@ -9,6 +9,9 @@ import json import tempfile from os.path import exists +from packaging import version + + def test_temporal_range(): @@ -82,8 +85,8 @@ def test_search_after(): 'bounding_box': "-180,-90,180,90", } results = pa.get_search_results(params, True) - assert results['hits'] == 3748 - assert len(results['items']) == 3748 + assert results['hits'] == 3751 + assert len(results['items']) == 3751 def test_update_format_change(cleanup_update_test): print("Running Test") @@ -198,7 +201,10 @@ def test_param_update(): def test_downloader_retry(mocker): mck = mocker.patch('subscriber.podaac_access.urlretrieve', side_effect=HTTPError("url", 503, "msg", None, None)) - pa.download_file("myUrl", "outputPath") + try: + pa.download_file("myUrl", "outputPath") + except Exception: + pass assert mck.call_count == 3 def validate(args): @@ -206,3 +212,32 @@ def validate(args): args2 = parser.parse_args(args) pa.validate(args2) return args2 + +def test_check_updates(): + version.parse(pa.get_latest_release()) + +def test_compare_release(): + tag="1.11.0" + assert pa.release_is_current(tag,"1.11.0") + assert pa.release_is_current(tag,"2.10.0") + assert pa.release_is_current(tag,"1.11.1") + + assert not pa.release_is_current(tag,"1.10.0") + assert not pa.release_is_current(tag,"1.10.5") + assert not pa.release_is_current(tag,"0.9000.5") + +def test_extensions(): + assert pa.search_extension('\\.tiff', "myfile.tiff") == True + assert pa.search_extension('\\.tiff', "myfile.tif") == False + assert pa.search_extension('\\.tiff', "myfile.gtiff") == False + assert pa.search_extension('PTM_\\d+', "myfile.PTM_1") == True + assert pa.search_extension('PTM_\\d+', "myfile.PTM_10") == True + assert pa.search_extension('PTM_\\d+', "myfile.PTM_09") == True + assert pa.search_extension('PTM_\\d+', "myfile.PTM_9") == True + + +def test_get_latest_release_from_json(): + f = open('tests/releases.json') + release_json = json.load(f) + latest_release = pa.get_latest_release_from_json(release_json) + assert latest_release == "1.12.0" diff --git a/tests/test_subscriber_regression.py b/tests/test_subscriber_regression.py index ffa3685..d94e0ac 100644 --- a/tests/test_subscriber_regression.py +++ b/tests/test_subscriber_regression.py @@ -23,6 +23,14 @@ def test_subscriber_ecco_only_enddate(): assert exists('./ECCO_L4_ATM_STATE_05DEG_DAILY_V4R4/1992/003/ATM_SURFACE_TEMP_HUM_WIND_PRES_day_mean_1992-01-03_ECCO_V4r4_latlon_0p50deg.nc') shutil.rmtree('./ECCO_L4_ATM_STATE_05DEG_DAILY_V4R4') +# Test to ensure nothing is downloaded via dry-run +@pytest.mark.regression +def test_subscriber_ecco_dry_run(): + args2 = create_args('-c ECCO_L4_ATM_STATE_05DEG_DAILY_V4R4 -ed 1992-01-03T00:00:00Z -d ./ECCO_L4_ATM_STATE_05DEG_DAILY_V4R4 -dydoy --dry-run'.split()) + pds.run(args2) + assert len(os.listdir('./ECCO_L4_ATM_STATE_05DEG_DAILY_V4R4')) == 0 + shutil.rmtree('./ECCO_L4_ATM_STATE_05DEG_DAILY_V4R4') + # test to download S6 data by start/stop time, and bbox, and put it in the # cycle based directory structure @pytest.mark.regression diff --git a/tests/test_token_regression.py b/tests/test_token_regression.py index 8eb0641..b4242b6 100644 --- a/tests/test_token_regression.py +++ b/tests/test_token_regression.py @@ -6,14 +6,15 @@ import shutil from pathlib import Path -@pytest.mark.regression +@pytest.mark.token def setup_function(method): # Deletes all known tokens tokens = pa.list_tokens(pa.token_url) for x in tokens: pa.delete_token(pa.token_url, x) -@pytest.mark.regression + +@pytest.mark.token def teardown_function(method): # Deletes all known tokens tokens = pa.list_tokens(pa.token_url) @@ -23,7 +24,7 @@ def teardown_function(method): # REGRESSION TEST CURRENTLY REQUIRES A .NETRC file for CMR/Data Download # token API can be found here: https://wiki.earthdata.nasa.gov/display/EL/API+Documentation # explore https://urs.earthdata.nasa.gov/documentation/for_integrators/api_documentation#/oauth/token -@pytest.mark.regression +@pytest.mark.token def test_list_tokens(): tokens = pa.list_tokens(pa.token_url) assert len(tokens) == 0 @@ -31,7 +32,7 @@ def test_list_tokens(): tokens = pa.list_tokens(pa.token_url) assert len(tokens) == 1 -@pytest.mark.regression +@pytest.mark.token def test_edl_getToken(): token = pa.get_token(pa.token_url) assert token != ""