Merge branch 'sodadata:main' into main

.github/workflows/main.workflow.yml

@@ -44,7 +44,7 @@ jobs:
  - "mysql"
  - "duckdb"
  - "dask"
- - "vertica"
+ # - "vertica"
 
  env:
  DATA_SOURCE: ${{ matrix.data-source }}
@@ -80,9 +80,6 @@ jobs:
  MYSQL_ROOT_PASSWORD: sodacore
  # TODO: add appropriate vars
  SPARK_DF_HOST: ${{ secrets.SPARK_DF_HOST }}
- VERTICA_USERNAME: dbadmin
- VERTICA_PASSWORD: password
- VERTICA_DATABASE: vmart
 
  steps:
  - uses: actions/checkout@v3

.github/workflows/pr.workflow.yml

@@ -34,7 +34,7 @@ jobs:
  - "mysql"
  - "duckdb"
  - "dask"
- - "vertica"
+
 
  env:
  DATA_SOURCE: ${{ matrix.data-source }}
@@ -61,10 +61,7 @@ jobs:
  MYSQL_PASSWORD: sodacore
  MYSQL_ROOT_PASSWORD: sodacore
  SPARK_DF_HOST: ${{ secrets.SPARK_DF_HOST }}
- VERTICA_USERNAME: dbadmin
- VERTICA_PASSWORD: password
- VERTICA_DATABASE: vmart
-
+
  steps:
  - uses: actions/checkout@v3
 
@@ -84,8 +81,8 @@ jobs:
 
  - name: Test with tox
  run: |
- tox -- soda -k soda/core
- tox -- soda -k soda/${{ matrix.data-source }}
+ tox --exit-and-dump-after 3600 -- soda -k soda/core
+ tox --exit-and-dump-after 3600 -- soda -k soda/${{ matrix.data-source }}
  env:
  test_data_source: ${{ matrix.data-source }}
 
@@ -116,7 +113,7 @@ jobs:
 
  - name: Test with tox
  run: |
- tox -- soda -k soda/core
+ tox --exit-and-dump-after 3600 -- soda -k soda/core
  env:
  test_data_source: postgres
  WESTMALLE: BETTER_THAN_LA_TRAPPE
@@ -148,4 +145,4 @@ jobs:
 
  - name: Test with tox
  run: |
- tox -- soda -k soda/scientific
+ tox --exit-and-dump-after 3600 -- soda -k soda/scientific

.pre-commit-config.yaml

@@ -24,7 +24,7 @@ repos:
  - id: autoflake
  args: ["--in-place", "--remove-all-unused-imports"]
  - repo: https://github.com/asottile/pyupgrade
- rev: v3.4.0
+ rev: v3.10.1
  hooks:
  - id: pyupgrade
  args: [--py37-plus]
@@ -35,7 +35,7 @@ repos:
  additional_dependencies: [toml]
  name: Sort imports using isort
  - repo: https://github.com/psf/black
- rev: 23.3.0
+ rev: 23.7.0
  hooks:
  - id: black
  name: Run black formatter

README.md

@@ -18,28 +18,33 @@
 
 Soda Core is a free, open-source, command-line tool and Python library that enables you to use the Soda Checks Language to turn user-defined input into aggregated SQL queries. 
 
-When it runs a scan on a dataset, Soda Core executes the checks to find invalid, missing, or unexpected data. When your Soda Checks fail, they surface the data that you defined as “bad”.
+When it runs a scan on a dataset, Soda Core executes the checks to find invalid, missing, or unexpected data. When your Soda Checks fail, they surface the data that you defined as bad-quality.
 
+#### Soda Library 
 
+Consider using **[Soda Library](https://docs.soda.io/soda/quick-start-sip.html)**, an extension of Soda Core that offers more features and functionality, and enables you to connect to a [Soda Cloud](https://docs.soda.io/soda-cloud/overview.html) account to collaborate with your team on data quality.
+[Install Soda Library](https://docs.soda.io/soda-library/install.html) and get started with a 45-day free trial.
+
+<br />
 
 ## Get started
 
-Soda Core currently supports connections to several data sources. See [Compatibility](https://docs.soda.io/soda-core/installation.html#compatibility) for a complete list.
+Soda Core currently supports connections to several data sources. See [Compatibility](/docs/installation.md#compatibility) for a complete list.
 
 **Requirements**
 * Python 3.8 or greater
 * Pip 21.0 or greater
 
 
 **Install and run**
-1. To get started, use the install command, replacing `soda-core-postgres` with the package that matches your data source. See [Install Soda Core](https://docs.soda.io/soda-core/installation.html#install) for a complete list.<br />
+1. To get started, use the install command, replacing `soda-core-postgres` with the package that matches your data source. See [Install Soda Core](/docs/installation.md) for a complete list.<br />
  ```shell
  pip install soda-core-postgres
  ```
 
-2. Prepare a `configuration.yml` file to connect to your data source. Then, write data quality checks in a `checks.yml` file. See [Configure Soda Core](https://docs.soda.io/soda-core/configuration.html#configuration-instructions).
+2. Prepare a `configuration.yml` file to connect to your data source. Then, write data quality checks in a `checks.yml` file. See [Configure Soda Core](/docs/configuration.md).
 
-3. Run a scan to review checks that passed, failed, or warned during a scan. See [Run a Soda Core scan](https://docs.soda.io/soda-core/scan-core.html).
+3. Run a scan to review checks that passed, failed, or warned during a scan. See [Run a Soda Core scan](/docs/scan-core.md).
  ```shell
  soda scan -d your_datasource -c configuration.yml checks.yml
  ```
@@ -81,6 +86,6 @@ checks for dim_department_group:
 
 ## Documentation
 
-* [Soda Core](https://docs.soda.io/soda-core/overview-main.html)
+* [Soda Core](/docs/overview-main.md)
 * [Soda Checks Language (SodaCL)](https://docs.soda.io/soda-cl/soda-cl-overview.html)
 

dev-requirements.in

@@ -1,16 +1,16 @@
-pip-tools
-pytest
-python-dotenv
-tox
-tox-docker
-pytest-html
-pytest-cov
-Faker
-tbump
+pip-tools~=6.5
+pytest~=7.0
+python-dotenv~=1.0
+tox~=4.6
+tox-docker~=4.1
+pytest-html~=3.1
+pytest-cov~=3.0
+faker~=13.3
+tbump~=6.7
 black==22.6.0
-typing_extensions>=4.3.0
-urllib3
-pygments
-readme-renderer
+typing_extensions>=4.3.0,<5
+urllib3~=1.26
+pygments~=2.11
+readme-renderer~=32.0
 certifi>=2022.12.07
 wheel>=0.38.1

dev-requirements.txt

@@ -1,147 +1,154 @@
 #
-# This file is autogenerated by pip-compile with Python 3.9
+# This file is autogenerated by pip-compile with Python 3.10
 # by the following command:
 #
 # pip-compile dev-requirements.in
 #
-attrs==21.4.0
- # via pytest
 black==22.6.0
  # via -r dev-requirements.in
-bleach==4.1.0
+bleach==6.0.0
  # via readme-renderer
-certifi==2022.12.7
+build==0.10.0
+ # via pip-tools
+cachetools==5.3.1
+ # via tox
+certifi==2023.5.7
  # via
  # -r dev-requirements.in
  # requests
-charset-normalizer==2.0.12
+chardet==5.1.0
+ # via tox
+charset-normalizer==3.2.0
  # via requests
-cli-ui==0.16.0
+cli-ui==0.17.2
  # via tbump
-click==8.0.4
+click==8.1.5
  # via
  # black
  # pip-tools
-colorama==0.4.4
- # via cli-ui
+colorama==0.4.6
+ # via
+ # cli-ui
+ # tox
 contextlib2==21.6.0
  # via schema
-coverage[toml]==6.3.2
+coverage[toml]==7.2.7
  # via pytest-cov
-distlib==0.3.4
+distlib==0.3.6
  # via virtualenv
 docker==5.0.3
  # via tox-docker
 docopt==0.6.2
  # via tbump
-docutils==0.18.1
+docutils==0.20.1
  # via readme-renderer
-faker==13.3.0
+exceptiongroup==1.1.2
+ # via pytest
+faker==13.16.0
  # via -r dev-requirements.in
-filelock==3.6.0
+filelock==3.12.2
  # via
  # tox
  # virtualenv
-idna==3.3
+idna==3.4
  # via requests
-iniconfig==1.1.1
+iniconfig==2.0.0
  # via pytest
-mypy-extensions==0.4.3
+mypy-extensions==1.0.0
  # via black
-packaging==21.3
+packaging==23.1
  # via
- # bleach
+ # build
+ # pyproject-api
  # pytest
  # tox
-pathspec==0.9.0
+ # tox-docker
+pathspec==0.11.1
  # via black
-pep517==0.12.0
- # via pip-tools
-pip-tools==6.5.1
+pip-tools==6.14.0
  # via -r dev-requirements.in
-platformdirs==2.5.1
+platformdirs==3.8.1
  # via
  # black
+ # tox
  # virtualenv
-pluggy==1.0.0
+pluggy==1.2.0
  # via
  # pytest
  # tox
 py==1.11.0
- # via
- # pytest
- # tox
-pygments==2.11.2
+ # via pytest-html
+pygments==2.15.1
  # via
  # -r dev-requirements.in
  # readme-renderer
-pyparsing==3.0.7
- # via packaging
-pytest==7.0.1
+pyproject-api==1.5.3
+ # via tox
+pyproject-hooks==1.0.0
+ # via build
+pytest==7.4.0
  # via
  # -r dev-requirements.in
  # pytest-cov
  # pytest-html
  # pytest-metadata
 pytest-cov==3.0.0
  # via -r dev-requirements.in
-pytest-html==3.1.1
+pytest-html==3.2.0
  # via -r dev-requirements.in
-pytest-metadata==1.11.0
+pytest-metadata==3.0.0
  # via pytest-html
 python-dateutil==2.8.2
  # via faker
-python-dotenv==0.19.2
+python-dotenv==1.0.0
  # via -r dev-requirements.in
 readme-renderer==32.0
  # via -r dev-requirements.in
-requests==2.27.1
+requests==2.31.0
  # via docker
 schema==0.7.5
  # via tbump
 six==1.16.0
  # via
  # bleach
  # python-dateutil
- # tox
- # virtualenv
-tabulate==0.8.9
+tabulate==0.8.10
  # via cli-ui
-tbump==6.7.0
+tbump==6.10.0
  # via -r dev-requirements.in
-toml==0.10.2
- # via tox
 tomli==2.0.1
  # via
  # black
+ # build
  # coverage
- # pep517
+ # pip-tools
+ # pyproject-api
+ # pyproject-hooks
  # pytest
-tomlkit==0.10.0
+ # tox
+tomlkit==0.11.8
  # via tbump
-tox==3.24.5
+tox==4.6.4
  # via
  # -r dev-requirements.in
  # tox-docker
-tox-docker==3.1.0
+tox-docker==4.1.0
  # via -r dev-requirements.in
-typing-extensions==4.3.0
- # via
- # -r dev-requirements.in
- # black
-unidecode==1.3.3
+typing-extensions==4.7.1
+ # via -r dev-requirements.in
+unidecode==1.3.6
  # via cli-ui
-urllib3==1.26.8
+urllib3==1.26.16
  # via
  # -r dev-requirements.in
  # requests
-virtualenv==20.13.2
+virtualenv==20.23.1
  # via tox
 webencodings==0.5.1
  # via bleach
-websocket-client==1.3.1
+websocket-client==1.6.1
  # via docker
-wheel==0.38.4
+wheel==0.40.0
  # via
  # -r dev-requirements.in
  # pip-tools