Merge pull request #4 from offbi/hotfix/greedy_table_replace

Hotfix

CHANGELOG.md

@@ -1,3 +1,9 @@
+0.1.1 - 2021-02-08
+==================
+- Fixed problem with dashes in global and cmd flags. Use plus sign istead.
+- Fixed problem with greedy replace script table name.
+- Documentation fixes.
+
 0.1.0 - 2020-12-23
 ==================
 - First written

HOOKS.md

@@ -34,7 +34,7 @@
 Modifiers:
  * [`generate-missing-sources`](https://github.com/offbi/pre-commit-dbt/blob/main/HOOKS.md#generate-missing-sources): If any source is missing this hook tries to create it.
  * [`generate-model-properties-file`](https://github.com/offbi/pre-commit-dbt/blob/main/HOOKS.md#generate-model-properties-file): Generate model properties file.
- * [`unify-column-descriptions`](https://github.com/offbi/pre-commit-dbt/blob/main/HOOKS.md#unify-column-descriptions): Unify column descriptions across all models.
+ * [`unify-column-description`](https://github.com/offbi/pre-commit-dbt/blob/main/HOOKS.md#unify-column-description): Unify column descriptions across all models.
  * [`replace-script-table-names`](https://github.com/offbi/pre-commit-dbt/blob/main/HOOKS.md#replace-script-table-names): Replace table names with `source()` or `ref()` macros in the script.
  * [`remove-script-semicolon`](https://github.com/offbi/pre-commit-dbt/blob/main/HOOKS.md#remove-script-semicolon): Remove the semicolon at the end of the script.
 
@@ -237,7 +237,7 @@ repos:
  rev: v0.1.0
  hooks:
  - id: check-model-has-meta-keys
- args: ['--meta-keys', 'foo', 'bar', "--"]
+  args: ['--meta-keys', 'foo', 'bar', "--"]
 ```
 
 :warning: do not forget to include `--` as the last argument. Otherwise `pre-commit` would not be able to separate a list of files with args.
@@ -326,8 +326,8 @@ repos:
 - repo: https://github.com/offbi/pre-commit-dbt
  rev: v0.1.0
  hooks:
- - id: check-model-has-tests-by-type
- args: ["--tests", "unique=1", "data=1", "--"]
+ - id: check-model-has-tests-by-name
+  args: ["--tests", "unique=1", "data=1", "--"]
 ```
 
 :warning: do not forget to include `--` as the last argument. Otherwise `pre-commit` would not be able to separate a list of files with args.
@@ -368,8 +368,8 @@ repos:
 - repo: https://github.com/offbi/pre-commit-dbt
  rev: v0.1.0
  hooks:
- - id: check-model-has-tests-by-name
- args: ["--tests", "schema=1", "data=1", "--"]
+ - id: check-model-has-tests-by-type
+  args: ["--tests", "schema=1", "data=1", "--"]
 ```
 
 :warning: do not forget to include `--` as the last argument. Otherwise `pre-commit` would not be able to separate a list of files with args.
@@ -411,7 +411,7 @@ repos:
  rev: v0.1.0
  hooks:
  - id: check-model-has-tests
- args: ["--test-cnt", "2", "--"]
+  args: ["--test-cnt", "2", "--"]
 ```
 
 :warning: do not forget to include `--` as the last argument. Otherwise `pre-commit` would not be able to separate a list of files with args.
@@ -453,7 +453,7 @@ repos:
  rev: v0.1.0
  hooks:
  - id: check-model-tags
- args: ["--tags", "foo", "bar", "--"]
+  args: ["--tags", "foo", "bar", "--"]
 ```
 
 :warning: do not forget to include `--` as the last argument. Otherwise `pre-commit` would not be able to separate a list of files with args.
@@ -698,7 +698,7 @@ repos:
  rev: v0.1.0
  hooks:
  - id: check-source-has-freshness
- args: ["--freshness", "error_after", "warn_after", "--"]
+  args: ["--freshness", "error_after", "warn_after", "--"]
 ```
 
 :warning: do not forget to include `--` as the last argument. Otherwise `pre-commit` would not be able to separate a list of files with args.
@@ -769,7 +769,7 @@ repos:
  rev: v0.1.0
  hooks:
  - id: check-source-has-meta-keys
- args: ['--meta-keys', 'foo', 'bar', "--"]
+  args: ['--meta-keys', 'foo', 'bar', "--"]
 ```
 
 :warning: do not forget to include `--` as the last argument. Otherwise `pre-commit` would not be able to separate a list of files with args.
@@ -807,8 +807,8 @@ repos:
 - repo: https://github.com/offbi/pre-commit-dbt
  rev: v0.1.0
  hooks:
- - id: check-source-has-tests-by-type
- args: ["--tests", "unique=1", "data=1", "--"]
+ - id: check-source-has-tests-by-name
+  args: ["--tests", "unique=1", "data=1", "--"]
 ```
 
 :warning: do not forget to include `--` as the last argument. Otherwise `pre-commit` would not be able to separate a list of files with args.
@@ -847,8 +847,8 @@ repos:
 - repo: https://github.com/offbi/pre-commit-dbt
  rev: v0.1.0
  hooks:
- - id: check-source-has-tests-by-name
- args: ["--tests", "schema=1", "data=1", "--"]
+ - id: check-source-has-tests-by-type
+  args: ["--tests", "schema=1", "data=1", "--"]
 ```
 
 :warning: do not forget to include `--` as the last argument. Otherwise `pre-commit` would not be able to separate a list of files with args.
@@ -888,7 +888,7 @@ repos:
  rev: v0.1.0
  hooks:
  - id: check-source-has-tests
- args: ["--tests", "2", "--"]
+  args: ["--tests", "2", "--"]
 ```
 
 :warning: do not forget to include `--` as the last argument. Otherwise `pre-commit` would not be able to separate a list of files with args.
@@ -928,7 +928,7 @@ repos:
  rev: v0.1.0
  hooks:
  - id: check-source-tags
- args: ["--tags", "foo", "bar", "--"]
+  args: ["--tags", "foo", "bar", "--"]
 ```
 
 :warning: do not forget to include `--` as the last argument. Otherwise `pre-commit` would not be able to separate a list of files with args.
@@ -969,7 +969,7 @@ repos:
  rev: v0.1.0
  hooks:
  - id: generate-missing-sources
- args: ["--schema-file", "models/schema.yml", "--"]
+  args: ["--schema-file", "models/schema.yml", "--"]
 ```
 
 :warning: do not forget to include `--` as the last argument. Otherwise `pre-commit` would not be able to separate a list of files with args.
@@ -1009,7 +1009,7 @@ Unfortunately, this hook breaks your formatting.
 
 -----
 
-### `unify-column-descriptions`
+### `unify-column-description`
 
 Unify column descriptions across all models.
 
@@ -1023,8 +1023,8 @@ repos:
 - repo: https://github.com/offbi/pre-commit-dbt
  rev: v0.1.0
  hooks:
- - id: generate-missing-sources
- args: ["--ignore", "foo", "--"]
+ - id: unify-column-description
+  args: ["--ignore", "foo", "--"]
 ```
 
 :warning: do not forget to include `--` as the last argument. Otherwise `pre-commit` would not be able to separate a list of files with args.
@@ -1070,7 +1070,6 @@ repos:
  rev: v0.1.0
  hooks:
  - id: replace-script-table-names
- args: ["--ignore", "foo", "--"]
 ```
 
 :warning: do not forget to include `--` as the last argument. Otherwise `pre-commit` would not be able to separate a list of files with args.
@@ -1115,8 +1114,8 @@ repos:
 - repo: https://github.com/offbi/pre-commit-dbt
  rev: v0.1.0
  hooks:
- - id: replace-script-table-names
- args: ["--properties-file", "/models/{schema}/{name}.yml", "--"]
+ - id: generate-model-properties-file
+  args: ["--properties-file", "/models/{schema}/{name}.yml", "--"]
 ```
 
 :warning: do not forget to include `--` as the last argument. Otherwise `pre-commit` would not be able to separate a list of files with args.
@@ -1202,8 +1201,8 @@ Run the` dbt compile` command. Generates executable SQL from source model, test,
 
 #### Arguments
 
-`--global-flags`: Global dbt flags applicable to all subcommands.</br>
-`--cmd-flags`: Command-specific dbt flags.</br>
+`--global-flags`: Global dbt flags applicable to all subcommands. Instead of dash `-` please use `+`.</br>
+`--cmd-flags`: Command-specific dbt flags. Instead of dash `-` please use `+`.</br>
 `--model-prefix`: Prefix dbt selector, for selecting parents.</br>
 `--model-postfix`: Postfix dbt selector, for selecting children.
 
@@ -1214,7 +1213,7 @@ repos:
  rev: v0.1.0
  hooks:
  - id: dbt-compile
- args: ["--model-prefix". "+", "--"]
+  args: ["--model-prefix". "+", "--"]
 ```
 
 :warning: do not forget to include `--` as the last argument. Otherwise `pre-commit` would not be able to separate a list of files with args.
@@ -1256,8 +1255,8 @@ Run `dbt run` command. Executes compiled SQL model files.
 
 #### Arguments
 
-`--global-flags`: Global dbt flags applicable to all subcommands.</br>
-`--cmd-flags`: Command-specific dbt flags.</br>
+`--global-flags`: Global dbt flags applicable to all subcommands. Instead of dash `-` please use `+`.</br>
+`--cmd-flags`: Command-specific dbt flags. Instead of dash `-` please use `+`.</br>
 `--model-prefix`: Prefix dbt selector, for selecting parents.</br>
 `--model-postfix`: Postfix dbt selector, for selecting children.
 
@@ -1268,7 +1267,7 @@ repos:
  rev: v0.1.0
  hooks:
  - id: dbt-run
- args: ["--model-prefix". "+", "--"]
+  args: ["--model-prefix". "+", "--"]
 ```
 
 :warning: do not forget to include `--` as the last argument. Otherwise `pre-commit` would not be able to separate a list of files with args.
@@ -1280,8 +1279,8 @@ Run `dbt test` command. Runs tests on data in deployed models.
 
 #### Arguments
 
-`--global-flags`: Global dbt flags applicable to all subcommands.</br>
-`--cmd-flags`: Command-specific dbt flags.</br>
+`--global-flags`: Global dbt flags applicable to all subcommands. Instead of dash `-` please use `+`.</br>
+`--cmd-flags`: Command-specific dbt flags. Instead of dash `-` please use `+`.</br>
 `--model-prefix`: Prefix dbt selector, for selecting parents.</br>
 `--model-postfix`: Postfix dbt selector, for selecting children.
 
@@ -1292,10 +1291,9 @@ repos:
  rev: v0.1.0
  hooks:
  - id: dbt-test
- args: ["--model-prefix", "+", "--"]
+  args: ["--model-prefix", "+", "--"]
 ```
 
 :warning: do not forget to include `--` as the last argument. Otherwise `pre-commit` would not be able to separate a list of files with args.
 
 -----
-

README.md

@@ -65,7 +65,7 @@ If this is the case, `pre-commit-dbt` is here to help you!
 Modifiers:
  * [`generate-missing-sources`](https://github.com/offbi/pre-commit-dbt/blob/main/HOOKS.md#generate-missing-sources): If any source is missing this hook tries to create it.
  * [`generate-model-properties-file`](https://github.com/offbi/pre-commit-dbt/blob/main/HOOKS.md#generate-model-properties-file): Generate model properties file.
- * [`unify-column-descriptions`](https://github.com/offbi/pre-commit-dbt/blob/main/HOOKS.md#unify-column-descriptions): Unify column descriptions across all models.
+ * [`unify-column-description`](https://github.com/offbi/pre-commit-dbt/blob/main/HOOKS.md#unify-column-description): Unify column descriptions across all models.
  * [`replace-script-table-names`](https://github.com/offbi/pre-commit-dbt/blob/main/HOOKS.md#replace-script-table-names): Replace table names with `source()` or `ref()` macros in the script.
  * [`remove-script-semicolon`](https://github.com/offbi/pre-commit-dbt/blob/main/HOOKS.md#remove-script-semicolon): Remove the semicolon at the end of the script.
 

pre_commit_dbt/check_script_has_no_table_name.py

@@ -44,7 +44,9 @@ def has_table_name(sql: str, filename: str) -> Tuple[int, Set[str]]:
  for prev, cur, nxt in prev_cur_next_iter(sql_split):
  if prev in ["from", "join"] and cur not in IGNORE_WORDS:
  tables.add(cur.lower() if cur else cur)
- if cur == "as" and nxt == "(" and prev not in IGNORE_WORDS: # pragma: no mutate
+ if (
+ cur.lower() == "as" and nxt and nxt[0] == "(" and prev not in IGNORE_WORDS
+ ): # pragma: no mutate
  cte.add(prev.lower() if prev else prev)
 
  table_names = tables.difference(cte)

pre_commit_dbt/dbt_compile.py

@@ -6,6 +6,7 @@
 from pre_commit_dbt.utils import add_dbt_cmd_args
 from pre_commit_dbt.utils import add_dbt_cmd_model_args
 from pre_commit_dbt.utils import add_filenames_args
+from pre_commit_dbt.utils import get_flags
 from pre_commit_dbt.utils import paths_to_dbt_models
 from pre_commit_dbt.utils import run_dbt_cmd
 
@@ -17,8 +18,8 @@ def prepare_cmd(
  prefix: str = "",
  postfix: str = "",
 ) -> List[str]:
- global_flags = global_flags or []
- cmd_flags = cmd_flags or []
+ global_flags = get_flags(global_flags)
+ cmd_flags = get_flags(cmd_flags)
  dbt_models = paths_to_dbt_models(paths, prefix, postfix)
  cmd = ["dbt", *global_flags, "compile", "-m", *dbt_models, *cmd_flags]
  return cmd

pre_commit_dbt/dbt_docs_generate.py

@@ -4,15 +4,16 @@
 from typing import Sequence
 
 from pre_commit_dbt.utils import add_dbt_cmd_args
+from pre_commit_dbt.utils import get_flags
 from pre_commit_dbt.utils import run_dbt_cmd
 
 
 def docs_generate_cmd(
  global_flags: Optional[Sequence[str]] = None,
  cmd_flags: Optional[Sequence[str]] = None,
 ) -> List[str]:
- global_flags = global_flags or []
- cmd_flags = cmd_flags or []
+ global_flags = get_flags(global_flags)
+ cmd_flags = get_flags(cmd_flags)
  cmd = ["dbt", *global_flags, "docs", "generate", *cmd_flags]
  return cmd
 

pre_commit_dbt/dbt_run.py

@@ -6,6 +6,7 @@
 from pre_commit_dbt.utils import add_dbt_cmd_args
 from pre_commit_dbt.utils import add_dbt_cmd_model_args
 from pre_commit_dbt.utils import add_filenames_args
+from pre_commit_dbt.utils import get_flags
 from pre_commit_dbt.utils import paths_to_dbt_models
 from pre_commit_dbt.utils import run_dbt_cmd
 
@@ -17,8 +18,8 @@ def prepare_cmd(
  prefix: str = "",
  postfix: str = "",
 ) -> List[str]:
- global_flags = global_flags or []
- cmd_flags = cmd_flags or []
+ global_flags = get_flags(global_flags)
+ cmd_flags = get_flags(cmd_flags)
  dbt_models = paths_to_dbt_models(paths, prefix, postfix)
  cmd = ["dbt", *global_flags, "run", "-m", *dbt_models, *cmd_flags]
  return cmd

pre_commit_dbt/dbt_test.py

@@ -6,6 +6,7 @@
 from pre_commit_dbt.utils import add_dbt_cmd_args
 from pre_commit_dbt.utils import add_dbt_cmd_model_args
 from pre_commit_dbt.utils import add_filenames_args
+from pre_commit_dbt.utils import get_flags
 from pre_commit_dbt.utils import paths_to_dbt_models
 from pre_commit_dbt.utils import run_dbt_cmd
 
@@ -17,8 +18,8 @@ def prepare_cmd(
  prefix: str = "",
  postfix: str = "",
 ) -> List[str]:
- global_flags = global_flags or []
- cmd_flags = cmd_flags or []
+ global_flags = get_flags(global_flags)
+ cmd_flags = get_flags(cmd_flags)
  dbt_models = paths_to_dbt_models(paths, prefix, postfix)
  cmd = ["dbt", *global_flags, "test", "-m", *dbt_models, *cmd_flags]
  return cmd

pre_commit_dbt/replace_script_table_names.py

@@ -98,7 +98,9 @@ def main(argv: Optional[Sequence[str]] = None) -> int:
  get_unknown_source(tables),
  )
  for replacement in to_replace:
- sql = re.sub(*replacement, sql, re.IGNORECASE)
+ old = r"([\\\s\n\r\t])" + replacement[0] + r"([\\\s\n\r\t])"
+ new = r"\1" + replacement[1] + r"\2"
+ sql = re.sub(old, new, sql, re.IGNORECASE)
  file.write_text(sql, encoding="utf-8")
 
  return status_code

pre_commit_dbt/utils.py

@@ -118,6 +118,13 @@ def get_models(
  yield Model(key, node.get("name"), filename, node) # pragma: no mutate
 
 
+def get_flags(flags: Optional[Sequence[str]] = None) -> List[str]:
+ if flags:
+ return [flag.replace("+", "-") for flag in flags if flag]
+ else:
+ return []
+
+
 def get_model_schemas(
  yml_files: Sequence[Path], filenames: Set[str], all_schemas: bool = False
 ) -> Generator[ModelSchema, None, None]:
@@ -250,12 +257,12 @@ def add_dbt_cmd_args(parser: argparse.ArgumentParser) -> NoReturn:
  parser.add_argument(
  "--global-flags",
  nargs="*",
- help="Global dbt flags applicable to all subcommands.",
+ help="Global dbt flags applicable to all subcommands. Instead of dash `-` please use `+`.",
  )
  parser.add_argument(
  "--cmd-flags",
  nargs="*",
- help="Command-specific dbt flags.",
+ help="Command-specific dbt flags. Instead of dash `-` please use `+`.",
  )
 
 

tests/unit/test_dbt_compile.py

@@ -34,20 +34,20 @@ def test_dbt_compile_error():
  (["/aa/bb/cc.txt"], None, None, ["dbt", "compile", "-m", "cc"]),
  (
  ["/aa/bb/cc.txt"],
- ["--debug", "--no-write-json"],
+ ["++debug", "++no-write-json"],
  None,
  ["dbt", "--debug", "--no-write-json", "compile", "-m", "cc"],
  ),
  (
  ["/aa/bb/cc.txt"],
  None,
- ["-t", "prod"],
+ ["+t", "prod"],
  ["dbt", "compile", "-m", "cc", "-t", "prod"],
  ),
  (
  ["/aa/bb/cc.txt"],
  "",
- ["-t", "prod"],
+ ["+t", "prod"],
  ["dbt", "compile", "-m", "cc", "-t", "prod"],
  ),
  ],