Expand segments in queries (#4982)

* Refactor segments model

* Fix inconsistent code

* Remove superfluous error case

* Beautify Plausible.Segments module

* Expand segments in filters

* Add tests

* Generate types

* Remove extraneous newlines

* Move Segment filters logic away from QueryParser

* Add moduledoc

* Add tests for /v2/query-internal-test

* Refactor max segment filters count to module attribute

* Add more parser tests and unify asserts in query

assets/js/types/query-api.d.ts

@@ -64,7 +64,7 @@ export type CustomPropertyFilterDimensions = string;
 export type GoalDimension = "event:goal";
 export type TimeDimensions = "time" | "time:month" | "time:week" | "time:day" | "time:hour";
 export type FilterTree = FilterEntry | FilterAndOr | FilterNot;
-export type FilterEntry = FilterWithoutGoals | FilterWithGoals | FilterWithPattern;
+export type FilterEntry = FilterWithoutGoals | FilterWithGoals | FilterWithPattern | FilterForSegment;
 /**
  * @minItems 3
  * @maxItems 4
@@ -115,6 +115,11 @@ export type FilterWithPattern = [
  * filter operation
  */
 export type FilterOperationRegex = "matches" | "matches_not";
+/**
+ * @minItems 3
+ * @maxItems 3
+ */
+export type FilterForSegment = ["is", "segment", number[]];
 /**
  * @minItems 2
  * @maxItems 2

lib/plausible/segments/filters.ex

@@ -0,0 +1,108 @@
+defmodule Plausible.Segments.Filters do
+  @moduledoc """
+  This module contains functions that enable resolving segments in filters.
+  """
+  alias Plausible.Segments
+  alias Plausible.Stats.Filters
+
+  @max_segment_filters_count 10
+
+  @doc """
+  Finds unique segment IDs used in query filters.
+
+  ## Examples
+    iex> get_segment_ids([[:not, [:is, "segment", [10, 20]]], [:contains, "visit:entry_page", ["blog"]]])
+    {:ok, [10, 20]}
+
+    iex> get_segment_ids([[:and, [[:is, "segment", Enum.to_list(1..6)], [:is, "segment", Enum.to_list(1..6)]]]])
+    {:error, "Invalid filters. You can only use up to 10 segment filters in a query."}
+  """
+  def get_segment_ids(filters) do
+    ids =
+      filters
+      |> Filters.traverse()
+      |> Enum.flat_map(fn
+        {[_operation, "segment", clauses], _depth} -> clauses
+        _ -> []
+      end)
+
+    if length(ids) > @max_segment_filters_count do
+      {:error,
+       "Invalid filters. You can only use up to #{@max_segment_filters_count} segment filters in a query."}
+    else
+      {:ok, Enum.uniq(ids)}
+    end
+  end
+
+  def preload_needed_segments(%Plausible.Site{} = site, filters) do
+    with {:ok, segment_ids} <- get_segment_ids(filters),
+         {:ok, segments} <-
+           Segments.get_many(
+             site,
+             segment_ids,
+             fields: [:id, :segment_data]
+           ),
+         {:ok, segments_by_id} <-
+           {:ok,
+            Enum.into(
+              segments,
+              %{},
+              fn %Segments.Segment{id: id, segment_data: segment_data} ->
+                case Filters.QueryParser.parse_filters(segment_data["filters"]) do
+                  {:ok, filters} -> {id, filters}
+                  _ -> {id, nil}
+                end
+              end
+            )},
+         :ok <-
+           if(Enum.any?(segment_ids, fn id -> is_nil(Map.get(segments_by_id, id)) end),
+             do: {:error, "Invalid filters. Some segments don't exist or aren't accessible."},
+             else: :ok
+           ) do
+      {:ok, segments_by_id}
+    end
+  end
+
+  defp replace_segment_with_filter_tree([_, "segment", clauses], preloaded_segments) do
+    if length(clauses) === 1 do
+      [[:and, Map.get(preloaded_segments, Enum.at(clauses, 0))]]
+    else
+      [[:or, Enum.map(clauses, fn id -> [:and, Map.get(preloaded_segments, id)] end)]]
+    end
+  end
+
+  defp replace_segment_with_filter_tree(_filter, _preloaded_segments) do
+    nil
+  end
+
+  @doc """
+  ## Examples
+
+    iex> resolve_segments([[:is, "visit:entry_page", ["/home"]]], %{})
+    {:ok, [[:is, "visit:entry_page", ["/home"]]]}
+
+    iex> resolve_segments([[:is, "visit:entry_page", ["/home"]], [:is, "segment", [1]]], %{1 => [[:contains, "visit:entry_page", ["blog"]], [:is, "visit:country", ["PL"]]]})
+    {:ok, [
+      [:is, "visit:entry_page", ["/home"]],
+      [:and, [[:contains, "visit:entry_page", ["blog"]], [:is, "visit:country", ["PL"]]]]
+    ]}
+
+    iex> resolve_segments([[:is, "segment", [1, 2]]], %{1 => [[:contains, "event:goal", ["Singup"]], [:is, "visit:country", ["PL"]]], 2 => [[:contains, "event:goal", ["Sauna"]], [:is, "visit:country", ["EE"]]]})
+    {:ok, [
+      [:or, [
+        [:and, [[:contains, "event:goal", ["Singup"]], [:is, "visit:country", ["PL"]]]],
+        [:and, [[:contains, "event:goal", ["Sauna"]], [:is, "visit:country", ["EE"]]]]]
+      ]
+    ]}
+  """
+  def resolve_segments(original_filters, preloaded_segments) do
+    if map_size(preloaded_segments) > 0 do
+      {:ok,
+       Filters.transform_filters(original_filters, fn f ->
+         replace_segment_with_filter_tree(f, preloaded_segments)
+       end)}
+    else
+      {:ok, original_filters}
+    end
+  end
+end

lib/plausible/segments/segments.ex

@@ -37,6 +37,21 @@ defmodule Plausible.Segments do
     end
   end
 
+  @spec get_many(Plausible.Site.t(), list(pos_integer()), Keyword.t()) ::
+          {:ok, [Segment.t()]}
+  def get_many(%Plausible.Site{} = site, segment_ids, opts) when is_list(segment_ids) do
+    fields = Keyword.get(opts, :fields, [:id])
+
+    query =
+      from(segment in Segment,
+        select: ^fields,
+        where: segment.site_id == ^site.id,
+        where: segment.id in ^segment_ids
+      )
+
+    {:ok, Repo.all(query)}
+  end
+
   @spec get_one(pos_integer(), Plausible.Site.t(), atom(), pos_integer() | nil) ::
           {:ok, Segment.t()}
           | error_not_enough_permissions()

lib/plausible/stats/filters/filters.ex

@@ -57,7 +57,7 @@ defmodule Plausible.Stats.Filters do
 
   Returns an empty list when argument type is unexpected (e.g. `nil`).
 
-  ### Examples:
+  ## Examples:
 
       iex> Filters.parse("visit:browser!=Chrome")
       [[:is_not, "visit:browser", ["Chrome"]]]
@@ -128,6 +128,13 @@ defmodule Plausible.Stats.Filters do
   Transformer will receive each node (filter, and/or/not subtree) of
   query and must return a list of nodes to replace it with or nil
   to ignore and look deeper.
+
+  ## Examples
+    iex> Filters.transform_filters([[:is, "visit:os", ["Linux"]], [:and, [[:is, "segment", [1]], [:is, "segment", [2]]]]], fn
+    ...>    [_, "segment", _] -> [[:is, "segment", ["changed"]]]
+    ...>    _ -> nil
+    ...>  end)
+    [[:is, "visit:os", ["Linux"]], [:and, [[:is, "segment", ["changed"]], [:is, "segment", ["changed"]]]]]
   """
   def transform_filters(filters, transformer) do
     filters
@@ -146,15 +153,15 @@ defmodule Plausible.Stats.Filters do
 
       # Reached a leaf node, return existing value
       {nil, filter} ->
-        [[filter]]
+        [filter]
 
       # Transformer returned a value - don't transform that subtree
       {transformed_filters, _filter} ->
         transformed_filters
     end
   end
 
-  defp traverse(filters, depth \\ -1) do
+  def traverse(filters, depth \\ -1) do
     filters
     |> Enum.flat_map(&traverse_tree(&1, depth + 1))
   end

lib/plausible/stats/filters/query_parser.ex

@@ -34,6 +34,10 @@ defmodule Plausible.Stats.Filters.QueryParser do
          utc_time_range = raw_time_range |> DateTimeRange.to_timezone("Etc/UTC"),
          {:ok, metrics} <- parse_metrics(Map.get(params, "metrics", [])),
          {:ok, filters} <- parse_filters(Map.get(params, "filters", [])),
+         {:ok, preloaded_segments} <-
+           Plausible.Segments.Filters.preload_needed_segments(site, filters),
+         {:ok, filters} <-
+           Plausible.Segments.Filters.resolve_segments(filters, preloaded_segments),
          {:ok, dimensions} <- parse_dimensions(Map.get(params, "dimensions", [])),
          {:ok, order_by} <- parse_order_by(Map.get(params, "order_by")),
          {:ok, include} <- parse_include(site, Map.get(params, "include", %{})),
@@ -161,7 +165,10 @@ defmodule Plausible.Stats.Filters.QueryParser do
            "Invalid visit:country filter, visit:country needs to be a valid 2-letter country code."}
         end
 
-      {_, true} ->
+      {"segment", _} when all_integers? ->
+        {:ok, list}
+
+      {_, true} when filter_key !== "segment" ->
         {:ok, list}
 
       _ ->
@@ -396,6 +403,9 @@ defmodule Plausible.Stats.Filters.QueryParser do
           {:error, error_message}
         end
 
+      "segment" ->
+        {:ok, filter_key}
+
       _ ->
         {:error, error_message}
     end

priv/json-schemas/query-api-schema.json

@@ -98,13 +98,11 @@
                   "minItems": 2,
                   "maxItems": 2,
                   "items": {
-                      "type": "string",
-                      "format": "date"
+                    "type": "string",
+                    "format": "date"
                   },
                   "description": "If custom period. A list of two ISO8601 dates or timestamps to compare against.",
-                  "examples": [
-                    ["2024-01-01", "2024-01-31"]
-                  ]
+                  "examples": [["2024-01-01", "2024-01-31"]]
                 }
               },
               "required": ["mode", "date_range"],
@@ -439,11 +437,35 @@
         }
       ]
     },
+    "filter_for_segment": {
+      "type": "array",
+      "additionalItems": false,
+      "minItems": 3,
+      "maxItems": 3,
+      "items": [
+        {
+          "const": "is"
+        },
+        {
+          "const": "segment"
+        },
+        {
+          "type": "array",
+          "items": {
+            "type": ["integer"]
+          }
+        }
+      ]
+    },
     "filter_entry": {
       "oneOf": [
         { "$ref": "#/definitions/filter_without_goals" },
         { "$ref": "#/definitions/filter_with_goals" },
-        { "$ref": "#/definitions/filter_with_pattern" }
+        { "$ref": "#/definitions/filter_with_pattern" },
+        {
+          "$ref": "#/definitions/filter_for_segment",
+          "$comment": "only :internal"
+        }
       ]
     },
     "filter_tree": {

test/plausible/segments/filters_test.exs

@@ -0,0 +1,5 @@
+defmodule Plausible.Segments.FiltersTest do
+  use ExUnit.Case, async: true
+
+  doctest Plausible.Segments.Filters, import: true
+end