Add sql backend + sqlfluff linter (#20854)

build-support/bin/generate_builtin_lockfiles.py

@@ -52,6 +52,7 @@
 from pants.backend.python.typecheck.pytype.subsystem import Pytype
 from pants.backend.scala.lint.scalafmt.subsystem import ScalafmtSubsystem
 from pants.backend.scala.subsystems.scalatest import Scalatest
+from pants.backend.sql.lint.sqlfluff.subsystem import Sqlfluff
 from pants.backend.terraform.dependency_inference import TerraformHcl2Parser
 from pants.backend.tools.semgrep.subsystem import SemgrepSubsystem
 from pants.backend.tools.yamllint.subsystem import Yamllint
@@ -132,6 +133,7 @@ class JvmTool(Tool[JvmToolBase]):
  PythonTool(SemgrepSubsystem, "pants.backend.experimental.tools.semgrep"),
  PythonTool(Setuptools, "pants.backend.python"),
  PythonTool(SetuptoolsSCM, "pants.backend.python"),
+ PythonTool(Sqlfluff, "pants.backend.experimental.sql.lint.sqlfluff"),
  PythonTool(TerraformHcl2Parser, "pants.backend.experimental.terraform"),
  PythonTool(TwineSubsystem, "pants.backend.python"),
  PythonTool(Yamllint, "pants.backend.experimental.tools.yamllint"),

docs/docs/ad-hoc-tools/_category_.json

@@ -1,4 +1,4 @@
 {
  "label": "Ad-Hoc Tools",
- "position": 12
+ "position": 13
 }

docs/docs/contributions/_category_.json

@@ -1,4 +1,4 @@
 {
  "label": "Contributions",
- "position": 15
+ "position": 16
 }

docs/docs/releases/_category_.json

@@ -1,4 +1,4 @@
 {
  "label": "Releases",
- "position": 14
+ "position": 15
 }

docs/docs/sql/_category_.json

@@ -0,0 +1,4 @@
+{
+ "label": "SQL",
+ "position": 12
+}

docs/docs/sql/index.mdx

@@ -0,0 +1,54 @@
+---
+ title: SQL Overview
+ sidebar_position: 999
+---
+
+---
+
+:::caution SQL support is in alpha stage
+Pants is currently building support for SQL. Simple use cases might be
+supported, but many options are missing.
+
+Please share feedback for what you need to use Pants with your SQL queries by
+either [opening a GitHub
+issue](https://github.com/pantsbuild/pants/issues/new/choose) or [joining our
+Slack](/community/getting-help)!
+:::
+
+## Initial setup
+
+First, activate the relevant backend in `pants.toml`:
+
+```toml title="pants.toml"
+[GLOBAL]
+backend_packages = [
+ ...
+ "pants.backend.experimental.sql",
+ ...
+]
+```
+
+The SQL backend adds [`sql_source`](../../reference/targets/sql_source.mdx) and
+[`sql_sources`](../../reference/targets/sql_sources.mdx) target types for SQL
+files. The [`sql_source`](../../reference/targets/sql_source.mdx) behaves like
+[`resource`](../../reference/targets/resource.mdx), so you can use it directly
+without wrappers. The `tailor` goal will automatically generate the targets for
+your .sql files.
+
+## Enable sqlfluff linter
+
+To enable the linter activate the relevant backend in `pants.toml`:
+
+```toml title="pants.toml"
+[GLOBAL]
+backend_packages = [
+ ...
+ "pants.backend.experimental.sql.lint.sqlfluff",
+ ...
+]
+```
+
+You can run the linter via `lint` goal:
+```
+pants lint --only=sqlfluff ::
+```

docs/docs/writing-plugins/_category_.json

@@ -1,4 +1,4 @@
 {
  "label": "Writing Plugins",
- "position": 13
+ "position": 14
 }

docs/notes/2.22.x.md

@@ -24,6 +24,12 @@ If you encounter such discrepancies, and you can't resolve them easily, please [
 
 ### Backends
 
+#### NEW: SQL
+
+A new experimental `SQL` backend was added along with the [sqlfluff
+linter](https://www.pantsbuild.org/2.22/reference/subsystems/sqlfluff). See
+docs [here](https://www.pantsbuild.org/2.22/docs/sql).
+
 #### JVM
 
 ##### Scala

src/python/pants/backend/experimental/sql/lint/sqlfluff/register.py

@@ -0,0 +1,12 @@
+# Copyright 2024 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+from pants.backend.sql.lint.sqlfluff import rules as sqlfluff_rules
+from pants.backend.sql.lint.sqlfluff import skip_field, subsystem
+
+
+def rules():
+ return [
+ *subsystem.rules(),
+ *sqlfluff_rules.rules(),
+ *skip_field.rules(),
+ ]

src/python/pants/backend/experimental/sql/register.py

@@ -0,0 +1,15 @@
+# Copyright 2024 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+from pants.backend.sql import tailor
+from pants.backend.sql.target_types import SqlSourcesGeneratorTarget, SqlSourceTarget
+
+
+def target_types():
+ return [
+ SqlSourceTarget,
+ SqlSourcesGeneratorTarget,
+ ]
+
+
+def rules():
+ return [*tailor.rules()]

src/python/pants/backend/sql/BUILD

@@ -0,0 +1,9 @@
+# Copyright 2024 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+python_sources(
+ sources=[
+ "tailor.py",
+ "target_types.py",
+ ],
+)

src/python/pants/backend/sql/lint/sqlfluff/BUILD

@@ -0,0 +1,11 @@
+# Copyright 2024 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+resource(name="lockfile", source="sqlfluff.lock")
+
+python_sources(
+ overrides={"subsystem.py": {"dependencies": [":lockfile"]}},
+)
+
+python_tests(
+ name="tests",
+)

src/python/pants/backend/sql/lint/sqlfluff/rules.py

@@ -0,0 +1,144 @@
+# Copyright 2024 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any, Tuple
+
+from typing_extensions import assert_never
+
+from pants.backend.python.util_rules import pex
+from pants.backend.python.util_rules.pex import PexRequest, VenvPex, VenvPexProcess
+from pants.backend.sql.lint.sqlfluff.subsystem import Sqlfluff, SqlfluffFieldSet, SqlfluffMode
+from pants.core.goals.fix import FixResult, FixTargetsRequest
+from pants.core.goals.fmt import FmtResult, FmtTargetsRequest
+from pants.core.goals.lint import LintResult, LintTargetsRequest
+from pants.core.util_rules.config_files import ConfigFiles, ConfigFilesRequest
+from pants.core.util_rules.partitions import PartitionerType
+from pants.core.util_rules.source_files import SourceFiles, SourceFilesRequest
+from pants.engine.fs import Digest, MergeDigests
+from pants.engine.internals.native_engine import Snapshot
+from pants.engine.process import FallibleProcessResult
+from pants.engine.rules import Get, MultiGet, collect_rules, rule
+from pants.util.logging import LogLevel
+from pants.util.meta import classproperty
+from pants.util.strutil import pluralize
+
+
+class SqlfluffFixRequest(FixTargetsRequest):
+ field_set_type = SqlfluffFieldSet
+ tool_subsystem = Sqlfluff
+ partitioner_type = PartitionerType.DEFAULT_SINGLE_PARTITION
+
+ # We don't need to include automatically added lint rules for this SqlfluffFixRequest,
+ # because these lint rules are already checked by SqlfluffLintRequest.
+ enable_lint_rules = False
+
+
+class SqlfluffLintRequest(LintTargetsRequest):
+ field_set_type = SqlfluffFieldSet
+ tool_subsystem = Sqlfluff
+ partitioner_type = PartitionerType.DEFAULT_SINGLE_PARTITION
+
+
+class SqlfluffFormatRequest(FmtTargetsRequest):
+ field_set_type = SqlfluffFieldSet
+ tool_subsystem = Sqlfluff
+ partitioner_type = PartitionerType.DEFAULT_SINGLE_PARTITION
+
+ @classproperty
+ def tool_name(cls) -> str:
+ return "sqlfluff format"
+
+ @classproperty
+ def tool_id(cls) -> str:
+ return "sqlfluff-format"
+
+
+@dataclass(frozen=True)
+class _RunSqlfluffRequest:
+ snapshot: Snapshot
+ mode: SqlfluffMode
+
+
+@rule(level=LogLevel.DEBUG)
+async def run_sqlfluff(
+ request: _RunSqlfluffRequest,
+ sqlfluff: Sqlfluff,
+) -> FallibleProcessResult:
+ sqlfluff_pex_get = Get(VenvPex, PexRequest, sqlfluff.to_pex_request())
+
+ config_files_get = Get(
+ ConfigFiles, ConfigFilesRequest, sqlfluff.config_request(request.snapshot.dirs)
+ )
+
+ sqlfluff_pex, config_files = await MultiGet(sqlfluff_pex_get, config_files_get)
+
+ input_digest = await Get(
+ Digest,
+ MergeDigests((request.snapshot.digest, config_files.snapshot.digest)),
+ )
+
+ initial_args: Tuple[str, ...] = ()
+ if request.mode is SqlfluffMode.FMT:
+ initial_args = ("format",)
+ elif request.mode is SqlfluffMode.FIX:
+ initial_args = ("fix", *sqlfluff.fix_args)
+ elif request.mode is SqlfluffMode.LINT:
+ initial_args = ("lint",)
+ else:
+ assert_never(request.mode)
+
+ conf_args = ["--config", sqlfluff.config] if sqlfluff.config else []
+
+ result = await Get(
+ FallibleProcessResult,
+ VenvPexProcess(
+ sqlfluff_pex,
+ argv=(*initial_args, *conf_args, *sqlfluff.args, *request.snapshot.files),
+ input_digest=input_digest,
+ output_files=request.snapshot.files,
+ description=f"Run sqlfluff {' '.join(initial_args)} on {pluralize(len(request.snapshot.files), 'file')}.",
+ level=LogLevel.DEBUG,
+ ),
+ )
+ return result
+
+
+@rule(desc="Fix with sqlfluff fix", level=LogLevel.DEBUG)
+async def sqlfluff_fix(request: SqlfluffFixRequest.Batch, sqlfluff: Sqlfluff) -> FixResult:
+ result = await Get(
+ FallibleProcessResult, _RunSqlfluffRequest(snapshot=request.snapshot, mode=SqlfluffMode.FIX)
+ )
+ return await FixResult.create(request, result)
+
+
+@rule(desc="Lint with sqlfluff lint", level=LogLevel.DEBUG)
+async def sqlfluff_lint(request: SqlfluffLintRequest.Batch[SqlfluffFieldSet, Any]) -> LintResult:
+ source_files = await Get(
+ SourceFiles, SourceFilesRequest(field_set.source for field_set in request.elements)
+ )
+ result = await Get(
+ FallibleProcessResult,
+ _RunSqlfluffRequest(snapshot=source_files.snapshot, mode=SqlfluffMode.LINT),
+ )
+ return LintResult.create(request, result)
+
+
+@rule(desc="Format with sqlfluff format", level=LogLevel.DEBUG)
+async def sqlfluff_fmt(request: SqlfluffFormatRequest.Batch, sqlfluff: Sqlfluff) -> FmtResult:
+ result = await Get(
+ FallibleProcessResult,
+ _RunSqlfluffRequest(snapshot=request.snapshot, mode=SqlfluffMode.FMT),
+ )
+ return await FmtResult.create(request, result)
+
+
+def rules():
+ return [
+ *collect_rules(),
+ *SqlfluffLintRequest.rules(),
+ *SqlfluffFixRequest.rules(),
+ *SqlfluffFormatRequest.rules(),
+ *pex.rules(),
+ ]