Support for subqueries in order_bys and group_bys (#607)

integration_test/myxql/test_helper.exs

@@ -113,7 +113,9 @@ excludes = [
  # MySQL doesn't support indexed parameters
  :placeholders,
  # MySQL doesn't support specifying columns for ON DELETE SET NULL
- :on_delete_nilify_column_list
+ :on_delete_nilify_column_list,
+ # MySQL doesnt' support anything except a single column in DISTINCT
+ :multicolumn_distinct
 ]
 
 if Version.match?(version, ">= 8.0.0") do

integration_test/sql/subquery.exs

@@ -112,4 +112,42 @@ defmodule Ecto.Integration.SubQueryTest do
  select: fragment("? + ?", p.visits, ^1)
  assert [12, 12] = TestRepo.all(query)
  end
+
+ @tag :subquery_in_order_by
+ test "subqueries in order by" do
+ TestRepo.insert!(%Post{visits: 10, title: "hello"})
+ TestRepo.insert!(%Post{visits: 11, title: "hello"})
+
+ query = from p in Post, as: :p, order_by: [asc: exists(from p in Post, where: p.visits > parent_as(:p).visits)]
+
+ assert [%{visits: 11}, %{visits: 10}] = TestRepo.all(query)
+ end
+
+ @tag :multicolumn_distinct
+ @tag :subquery_in_distinct
+ test "subqueries in distinct" do
+ TestRepo.insert!(%Post{visits: 10, title: "hello1"})
+ TestRepo.insert!(%Post{visits: 10, title: "hello2"})
+ TestRepo.insert!(%Post{visits: 11, title: "hello"})
+
+ query = from p in Post, as: :p, distinct: exists(from p in Post, where: p.visits > parent_as(:p).visits), order_by: [asc: :title]
+
+ assert [%{title: "hello"}, %{title: "hello1"}] = TestRepo.all(query)
+ end
+
+ @tag :subquery_in_group_by
+ test "subqueries in group by" do
+ TestRepo.insert!(%Post{visits: 10, title: "hello1"})
+ TestRepo.insert!(%Post{visits: 10, title: "hello2"})
+ TestRepo.insert!(%Post{visits: 11, title: "hello"})
+
+ query = from p in Post, as: :p, select: sum(p.visits), group_by: exists(from p in Post, where: p.visits > parent_as(:p).visits), order_by: [sum(p.visits)]
+
+ query
+ |> TestRepo.all()
+ |> Enum.map(&Decimal.new/1)
+ |> Enum.zip([Decimal.new(11), Decimal.new(20)])
+ |> Enum.all?(fn {a, b} -> Decimal.eq?(a, b) end)
+ |> assert()
+ end
 end

integration_test/tds/test_helper.exs

@@ -57,7 +57,13 @@ ExUnit.start(
  # MSSQL can't reference aliased columns in ORDER BY expressions
  :selected_as_with_order_by_expression,
  # MSSQL doesn't support specifying columns for ON DELETE SET NULL
- :on_delete_nilify_column_list
+ :on_delete_nilify_column_list,
+ # MSSQL doesnt' support anything except a single column in DISTINCT
+ :multicolumn_distinct,
+ # MSSQL doesnt' support subqueries in group by or in distinct
+ :subquery_in_group_by,
+ :subquery_in_distinct,
+ :subquery_in_order_by
  ]
 )
 

lib/ecto/adapters/myxql/connection.ex

@@ -75,7 +75,7 @@ if Code.ensure_loaded?(MyXQL) do
  ## Query
 
  @parent_as __MODULE__
- alias Ecto.Query.{BooleanExpr, JoinExpr, QueryExpr, WithExpr}
+ alias Ecto.Query.{BooleanExpr, ByExpr, JoinExpr, QueryExpr, WithExpr}
 
  @impl true
  def all(query, as_prefix \\ []) do
@@ -319,10 +319,10 @@ if Code.ensure_loaded?(MyXQL) do
  end
 
  defp distinct(nil, _sources, _query), do: []
- defp distinct(%QueryExpr{expr: true}, _sources, _query), do: "DISTINCT "
- defp distinct(%QueryExpr{expr: false}, _sources, _query), do: []
+ defp distinct(%ByExpr{expr: true}, _sources, _query), do: "DISTINCT "
+ defp distinct(%ByExpr{expr: false}, _sources, _query), do: []
 
- defp distinct(%QueryExpr{expr: exprs}, _sources, query) when is_list(exprs) do
+ defp distinct(%ByExpr{expr: exprs}, _sources, query) when is_list(exprs) do
  error!(query, "DISTINCT with multiple columns is not supported by MySQL")
  end
 
@@ -511,8 +511,8 @@ if Code.ensure_loaded?(MyXQL) do
  defp group_by(%{group_bys: group_bys} = query, sources) do
  [
  " GROUP BY "
- | Enum.map_intersperse(group_bys, ", ", fn %QueryExpr{expr: expr} ->
- Enum.map_intersperse(expr, ", ", &expr(&1, sources, query))
+ | Enum.map_intersperse(group_bys, ", ", fn %ByExpr{expr: expr} ->
+ Enum.map_intersperse(expr, ", ", &top_level_expr(&1, sources, query))
  end)
  ]
  end
@@ -549,14 +549,14 @@ if Code.ensure_loaded?(MyXQL) do
  defp order_by(%{order_bys: order_bys} = query, sources) do
  [
  " ORDER BY "
- | Enum.map_intersperse(order_bys, ", ", fn %QueryExpr{expr: expr} ->
+ | Enum.map_intersperse(order_bys, ", ", fn %ByExpr{expr: expr} ->
  Enum.map_intersperse(expr, ", ", &order_by_expr(&1, sources, query))
  end)
  ]
  end
 
  defp order_by_expr({dir, expr}, sources, query) do
- str = expr(expr, sources, query)
+ str = top_level_expr(expr, sources, query)
 
  case dir do
  :asc -> str
@@ -627,6 +627,21 @@ if Code.ensure_loaded?(MyXQL) do
  [?(, expr(expr, sources, query), ?)]
  end
 
+ defp top_level_expr(%Ecto.SubQuery{query: query}, sources, parent_query) do
+ combinations =
+ Enum.map(query.combinations, fn {type, combination_query} ->
+ {type, put_in(combination_query.aliases[@parent_as], {parent_query, sources})}
+ end)
+
+ query = put_in(query.combinations, combinations)
+ query = put_in(query.aliases[@parent_as], {parent_query, sources})
+ [all(query, subquery_as_prefix(sources))]
+ end
+
+ defp top_level_expr(other, sources, parent_query) do
+ expr(other, sources, parent_query)
+ end
+
  defp expr({:^, [], [_ix]}, _sources, _query) do
  ~c"?"
  end
@@ -687,15 +702,8 @@ if Code.ensure_loaded?(MyXQL) do
  error!(query, "MySQL adapter does not support aggregate filters")
  end
 
- defp expr(%Ecto.SubQuery{query: query}, sources, parent_query) do
- combinations =
- Enum.map(query.combinations, fn {type, combination_query} ->
- {type, put_in(combination_query.aliases[@parent_as], {parent_query, sources})}
- end)
-
- query = put_in(query.combinations, combinations)
- query = put_in(query.aliases[@parent_as], {parent_query, sources})
- [?(, all(query, subquery_as_prefix(sources)), ?)]
+ defp expr(%Ecto.SubQuery{} = subquery, sources, parent_query) do
+ [?(, top_level_expr(subquery, sources, parent_query), ?)]
  end
 
  defp expr({:fragment, _, [kw]}, _sources, query) when is_list(kw) or tuple_size(kw) == 3 do
@@ -790,7 +798,13 @@ if Code.ensure_loaded?(MyXQL) do
  [op_to_binary(left, sources, query), op | op_to_binary(right, sources, query)]
 
  {:fun, fun} ->
- [fun, ?(, modifier, Enum.map_intersperse(args, ", ", &expr(&1, sources, query)), ?)]
+ [
+ fun,
+ ?(,
+ modifier,
+ Enum.map_intersperse(args, ", ", &top_level_expr(&1, sources, query)),
+ ?)
+ ]
  end
  end
 

lib/ecto/adapters/postgres/connection.ex

@@ -155,7 +155,7 @@ if Code.ensure_loaded?(Postgrex) do
  end
 
  @parent_as __MODULE__
- alias Ecto.Query.{BooleanExpr, JoinExpr, QueryExpr, WithExpr}
+ alias Ecto.Query.{BooleanExpr, ByExpr, JoinExpr, QueryExpr, WithExpr}
 
  @impl true
  def all(query, as_prefix \\ []) do
@@ -551,11 +551,11 @@ if Code.ensure_loaded?(Postgrex) do
  end
 
  defp distinct(nil, _, _), do: {[], []}
- defp distinct(%QueryExpr{expr: []}, _, _), do: {[], []}
- defp distinct(%QueryExpr{expr: true}, _, _), do: {" DISTINCT", []}
- defp distinct(%QueryExpr{expr: false}, _, _), do: {[], []}
+ defp distinct(%ByExpr{expr: []}, _, _), do: {[], []}
+ defp distinct(%ByExpr{expr: true}, _, _), do: {" DISTINCT", []}
+ defp distinct(%ByExpr{expr: false}, _, _), do: {[], []}
 
- defp distinct(%QueryExpr{expr: exprs}, sources, query) do
+ defp distinct(%ByExpr{expr: exprs}, sources, query) do
  {[
  " DISTINCT ON (",
  Enum.map_intersperse(exprs, ", ", fn {_, expr} -> expr(expr, sources, query) end),
@@ -772,7 +772,7 @@ if Code.ensure_loaded?(Postgrex) do
  [
  " GROUP BY "
  | Enum.map_intersperse(group_bys, ", ", fn
- %QueryExpr{expr: expr} ->
+ %ByExpr{expr: expr} ->
  Enum.map_intersperse(expr, ", ", &expr(&1, sources, query))
  end)
  ]

lib/ecto/adapters/tds/connection.ex

@@ -149,7 +149,7 @@ if Code.ensure_loaded?(Tds) do
 
  @parent_as __MODULE__
  alias Ecto.Query
- alias Ecto.Query.{BooleanExpr, JoinExpr, QueryExpr, WithExpr}
+ alias Ecto.Query.{BooleanExpr, ByExpr, JoinExpr, QueryExpr, WithExpr}
 
  @impl true
  def all(query, as_prefix \\ []) do
@@ -390,10 +390,10 @@ if Code.ensure_loaded?(Tds) do
  end
 
  defp distinct(nil, _sources, _query), do: []
- defp distinct(%QueryExpr{expr: true}, _sources, _query), do: "DISTINCT "
- defp distinct(%QueryExpr{expr: false}, _sources, _query), do: []
+ defp distinct(%ByExpr{expr: true}, _sources, _query), do: "DISTINCT "
+ defp distinct(%ByExpr{expr: false}, _sources, _query), do: []
 
- defp distinct(%QueryExpr{expr: exprs}, _sources, query) when is_list(exprs) do
+ defp distinct(%ByExpr{expr: exprs}, _sources, query) when is_list(exprs) do
  error!(
  query,
  "DISTINCT with multiple columns is not supported by MsSQL. " <>
@@ -584,8 +584,8 @@ if Code.ensure_loaded?(Tds) do
  defp group_by(%{group_bys: group_bys} = query, sources) do
  [
  " GROUP BY "
- | Enum.map_intersperse(group_bys, ", ", fn %QueryExpr{expr: expr} ->
- Enum.map_intersperse(expr, ", ", &expr(&1, sources, query))
+ | Enum.map_intersperse(group_bys, ", ", fn %ByExpr{expr: expr} ->
+ Enum.map_intersperse(expr, ", ", &top_level_expr(&1, sources, query))
  end)
  ]
  end
@@ -595,14 +595,14 @@ if Code.ensure_loaded?(Tds) do
  defp order_by(%{order_bys: order_bys} = query, sources) do
  [
  " ORDER BY "
- | Enum.map_intersperse(order_bys, ", ", fn %QueryExpr{expr: expr} ->
+ | Enum.map_intersperse(order_bys, ", ", fn %ByExpr{expr: expr} ->
  Enum.map_intersperse(expr, ", ", &order_by_expr(&1, sources, query))
  end)
  ]
  end
 
  defp order_by_expr({dir, expr}, sources, query) do
- str = expr(expr, sources, query)
+ str = top_level_expr(expr, sources, query)
 
  case dir do
  :asc -> str
@@ -708,6 +708,21 @@ if Code.ensure_loaded?(Tds) do
  [?(, expr(expr, sources, query), ?)]
  end
 
+ defp top_level_expr(%Ecto.SubQuery{query: query}, sources, parent_query) do
+ combinations =
+ Enum.map(query.combinations, fn {type, combination_query} ->
+ {type, put_in(combination_query.aliases[@parent_as], {parent_query, sources})}
+ end)
+
+ query = put_in(query.combinations, combinations)
+ query = put_in(query.aliases[@parent_as], {parent_query, sources})
+ [all(query, subquery_as_prefix(sources))]
+ end
+
+ defp top_level_expr(other, sources, parent_query) do
+ expr(other, sources, parent_query)
+ end
+
  # :^ - represents parameter ix is index number
  defp expr({:^, [], [idx]}, _sources, _query) do
  "@#{idx + 1}"
@@ -787,15 +802,8 @@ if Code.ensure_loaded?(Tds) do
  error!(query, "Tds adapter does not support aggregate filters")
  end
 
- defp expr(%Ecto.SubQuery{query: query}, sources, parent_query) do
- combinations =
- Enum.map(query.combinations, fn {type, combination_query} ->
- {type, put_in(combination_query.aliases[@parent_as], {parent_query, sources})}
- end)
-
- query = put_in(query.combinations, combinations)
- query = put_in(query.aliases[@parent_as], {parent_query, sources})
- [?(, all(query, subquery_as_prefix(sources)), ?)]
+ defp expr(%Ecto.SubQuery{} = subquery, sources, parent_query) do
+ [?(, top_level_expr(subquery, sources, parent_query), ?)]
  end
 
  defp expr({:fragment, _, [kw]}, _sources, query) when is_list(kw) or tuple_size(kw) == 3 do
@@ -873,7 +881,13 @@ if Code.ensure_loaded?(Tds) do
  [op_to_binary(left, sources, query), op | op_to_binary(right, sources, query)]
 
  {:fun, fun} ->
- [fun, ?(, modifier, Enum.map_intersperse(args, ", ", &expr(&1, sources, query)), ?)]
+ [
+ fun,
+ ?(,
+ modifier,
+ Enum.map_intersperse(args, ", ", &top_level_expr(&1, sources, query)),
+ ?)
+ ]
  end
  end
 

mix.lock

@@ -4,8 +4,8 @@
  "decimal": {:hex, :decimal, "2.1.1", "5611dca5d4b2c3dd497dec8f68751f1f1a54755e8ed2a966c2633cf885973ad6", [:mix], [], "hexpm", "53cfe5f497ed0e7771ae1a475575603d77425099ba5faef9394932b35020ffcc"},
  "deep_merge": {:hex, :deep_merge, "1.0.0", "b4aa1a0d1acac393bdf38b2291af38cb1d4a52806cf7a4906f718e1feb5ee961", [:mix], [], "hexpm", "ce708e5f094b9cd4e8f2be4f00d2f4250c4095be93f8cd6d018c753894885430"},
  "earmark_parser": {:hex, :earmark_parser, "1.4.39", "424642f8335b05bb9eb611aa1564c148a8ee35c9c8a8bba6e129d51a3e3c6769", [:mix], [], "hexpm", "06553a88d1f1846da9ef066b87b57c6f605552cfbe40d20bd8d59cc6bde41944"},
- "ecto": {:git, "https://github.com/elixir-ecto/ecto.git", "bed81b9a69f3425147fb57df1b3dd5fb4c95792c", []},
- "ex_doc": {:hex, :ex_doc, "0.32.2", "f60bbeb6ccbe75d005763e2a328e6f05e0624232f2393bc693611c2d3ae9fa0e", [:mix], [{:earmark_parser, "~> 1.4.39", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_c, ">= 0.1.0", [hex: :makeup_c, repo: "hexpm", optional: true]}, {:makeup_elixir, "~> 0.14 or ~> 1.0", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1 or ~> 1.0", [hex: :makeup_erlang, repo: "hexpm", optional: false]}, {:makeup_html, ">= 0.1.0", [hex: :makeup_html, repo: "hexpm", optional: true]}], "hexpm", "a4480305cdfe7fdfcbb77d1092c76161626d9a7aa4fb698aee745996e34602df"},
+ "ecto": {:git, "https://github.com/elixir-ecto/ecto.git", "a898915d2f16dbf1257b8c83a11a1ae07193de42", []},
+ "ex_doc": {:hex, :ex_doc, "0.33.0", "690562b153153c7e4d455dc21dab86e445f66ceba718defe64b0ef6f0bd83ba0", [:mix], [{:earmark_parser, "~> 1.4.39", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_c, ">= 0.1.0", [hex: :makeup_c, repo: "hexpm", optional: true]}, {:makeup_elixir, "~> 0.14 or ~> 1.0", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1 or ~> 1.0", [hex: :makeup_erlang, repo: "hexpm", optional: false]}, {:makeup_html, ">= 0.1.0", [hex: :makeup_html, repo: "hexpm", optional: true]}], "hexpm", "3f69adc28274cb51be37d09b03e4565232862a4b10288a3894587b0131412124"},
  "jason": {:hex, :jason, "1.4.1", "af1504e35f629ddcdd6addb3513c3853991f694921b1b9368b0bd32beb9f1b63", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "fbb01ecdfd565b56261302f7e1fcc27c4fb8f32d56eab74db621fc154604a7a1"},
  "makeup": {:hex, :makeup, "1.1.2", "9ba8837913bdf757787e71c1581c21f9d2455f4dd04cfca785c70bbfff1a76a3", [:mix], [{:nimble_parsec, "~> 1.2.2 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "cce1566b81fbcbd21eca8ffe808f33b221f9eee2cbc7a1706fc3da9ff18e6cac"},
  "makeup_elixir": {:hex, :makeup_elixir, "0.16.2", "627e84b8e8bf22e60a2579dad15067c755531fea049ae26ef1020cad58fe9578", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.2.3 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "41193978704763f6bbe6cc2758b84909e62984c7752b3784bd3c218bb341706b"},