From 5d05853209baa213cae0703a07dfd2745ebe8755 Mon Sep 17 00:00:00 2001 From: brian Date: Mon, 13 Jan 2025 23:54:46 +0000 Subject: [PATCH] a pagination system that works --- lib/ppr_api/_residential_sales.ex | 205 ---------------- lib/ppr_api/pagination/cursor.ex | 48 ++++ lib/ppr_api/residential_sales.ex | 231 ++++++++---------- .../live/residential_sale_live/index.ex | 1 - test/ppr_api/residential_sales_test.exs | 9 +- 5 files changed, 158 insertions(+), 336 deletions(-) delete mode 100644 lib/ppr_api/_residential_sales.ex create mode 100644 lib/ppr_api/pagination/cursor.ex diff --git a/lib/ppr_api/_residential_sales.ex b/lib/ppr_api/_residential_sales.ex deleted file mode 100644 index 3df7773..0000000 --- a/lib/ppr_api/_residential_sales.ex +++ /dev/null @@ -1,205 +0,0 @@ -# defmodule PprApi.ResidentialSales do -# import Ecto.Query, warn: false -# alias PprApi.Repo -# alias PprApi.Fetches -# alias PprApi.Fetches.Fetch -# alias PprApi.ResidentialSales.ResidentialSale -# alias PprApi.FingerprintHelper - -# def list_residential_sales(opts \\ []) do -# opts = parse_opts(opts) - -# entries = -# ResidentialSale -# |> apply_cursor(opts) -# |> apply_order(opts) -# |> apply_limit(opts) -# |> Repo.all() - -# entries = -# case opts["cursor"] do -# {_value, "before"} -> Enum.reverse(entries) -# _ -> entries -# end - -# after_entry = hd(Enum.reverse(entries)) -# before_entry = hd(entries) - -# %{ -# entries: entries, -# metadata: %{ -# after_cursor: encode_cursor(after_entry), -# before_cursor: encode_cursor(before_entry), -# limit: opts["limit"], -# sort: -# opts["sort"] -# |> Tuple.to_list() -# |> Enum.join("-"), -# total_rows: total_residential_sales() -# } -# } -# end - -# @doc """ -# Returns the date of the most recent residential sale. -# """ -# def latest_sale_date do -# ResidentialSale -# |> select([s], max(s.date_of_sale)) -# |> Repo.one() -# end - -# @doc """ -# Returns the current count of residential sales. -# """ -# def total_residential_sales do -# case Fetches.get_latest_successful_fetch() do -# {%Fetch{} = fetch} -> fetch.total_rows -# _ -> 0 -# end -# end - -# @doc """ -# Gets a single residential_sale. -# Raises `Ecto.NoResultsError` if the Residential sale does not exist. -# """ -# def get_residential_sale!(id), do: Repo.get!(ResidentialSale, id) - -# @doc """ -# Inserts residential sales in batches of 1000. -# """ -# def upsert_rows(rows) do -# batch_size = 1000 - -# rows -# |> Enum.chunk_every(batch_size) -# |> Enum.reduce(0, fn chunk, acc -> -# inserted = upsert_batch(chunk) -# acc + inserted -# end) -# end - -# defp upsert_batch(rows) do -# now = DateTime.utc_now() |> DateTime.truncate(:second) - -# rows_with_fingerprints = -# Enum.map(rows, fn row -> -# row -# |> Map.put(:fingerprint, FingerprintHelper.compute_fingerprint(row)) -# |> Map.put(:inserted_at, now) -# |> Map.put(:updated_at, now) -# end) - -# # Remove duplicate fingerprints within the batch -# unique_rows = remove_duplicate_fingerprints(rows_with_fingerprints) - -# # Perform the bulk upsert, counting only inserted rows -# {rows_inserted, _} = -# Repo.insert_all( -# ResidentialSale, -# unique_rows, -# on_conflict: [set: [updated_at: now]], -# conflict_target: :fingerprint -# ) - -# rows_inserted -# end - -# # Helper to remove duplicate fingerprints, keeping the first occurrence -# defp remove_duplicate_fingerprints(rows_with_fingerprints) do -# rows_with_fingerprints -# |> Enum.reduce(%{}, fn row, acc -> -# Map.put_new(acc, row.fingerprint, row) -# end) -# |> Map.values() -# end - -# # parsing stuff -# # -# # e.g. %{"before" => "706867", "limit" => "100", "sort" => "date-desc"} - -# defp parse_opts(opts) do -# opts -# |> parse_cursor() -# |> parse_limit() -# |> parse_sort() -# end - -# defp parse_cursor(opts) do -# case Enum.find(["after", "before"], &Map.has_key?(opts, &1)) do -# nil -> -# opts - -# key -> -# opts -# |> Map.delete(key) -# |> Map.put("cursor", {opts[key], key}) -# end -# end - -# defp parse_limit(opts) when is_map(opts) do -# opts -# |> Map.put("limit", parse_limit(opts["limit"])) -# end - -# defp parse_limit(limit) when is_number(limit) do -# limit -# end - -# defp parse_limit(limit) when is_binary(limit) do -# case Integer.parse(limit) do -# {num, _} -> num -# end -# end - -# defp parse_sort(opts) when is_map(opts) do -# opts -# |> Map.update("sort", nil, fn sort -> -# case String.split(sort, "-") do -# [field, direction] -> {field, direction} -# _ -> nil -# end -# end) -# end - -# defp apply_cursor(query, %{ -# "cursor" => {cursor_value, cursor_direction}, -# "sort" => {_sort_field, sort_direction} -# }) do -# case determine_operator(sort_direction, cursor_direction) do -# :lt -> query |> where([s], s.id < ^cursor_value) -# :gt -> query |> where([s], s.id > ^cursor_value) -# end -# end - -# defp apply_cursor(query, _opts) do -# query -# end - -# defp determine_operator(sort_direction, cursor_direction) do -# case {sort_direction, cursor_direction} do -# {"asc", "before"} -> :lt -# {"asc", "after"} -> :gt -# {"desc", "before"} -> :gt -# {"desc", "after"} -> :lt -# end -# end - -# defp apply_order(query, opts) do -# query |> order_by(desc: :id) -# end - -# defp apply_limit(query, opts) do -# query |> limit(^opts["limit"]) -# end - -# defp encode_cursor(entry) do -# entry.id -# end - -# defp decode_cursor(cursor) do -# cursor -# |> Integer.parse() -# |> elem(0) -# end -# end diff --git a/lib/ppr_api/pagination/cursor.ex b/lib/ppr_api/pagination/cursor.ex new file mode 100644 index 0000000..838458e --- /dev/null +++ b/lib/ppr_api/pagination/cursor.ex @@ -0,0 +1,48 @@ +defmodule PprApi.Pagination.Cursor do + alias PprApi.ResidentialSales.ResidentialSale + + def encode_cursor(nil, _opts), do: nil + + # "date" sort => "YYYY-MM-DD-" + def encode_cursor(%ResidentialSale{id: id, date_of_sale: date}, %{"sort" => {"date", _dir}}) do + # Convert date to ISO8601, then combine with ID + value_str = Date.to_iso8601(date) + combined = "#{value_str}|#{id}" + Base.url_encode64(combined, padding: false) + end + + # "price" sort => "-" + def encode_cursor(%ResidentialSale{id: id, price_in_euros: price}, %{"sort" => {"price", _dir}}) do + value_str = to_string(price) + combined = "#{value_str}|#{id}" + Base.url_encode64(combined, padding: false) + end + + def encode_cursor(_entry, _opts), do: nil + + def decode_cursor(nil, _field), do: nil + + def decode_cursor(encoded, "date") do + with {:ok, decoded} <- Base.url_decode64(encoded, padding: false), + [val_str, raw_id] <- String.split(decoded, "|", parts: 2), + {:ok, date} <- Date.from_iso8601(val_str), + {id, ""} <- Integer.parse(raw_id) do + {date, id} + else + _ -> nil + end + end + + def decode_cursor(encoded, "price") do + with {:ok, decoded} <- Base.url_decode64(encoded, padding: false), + [val_str, raw_id] <- String.split(decoded, "|", parts: 2), + {price, ""} <- Integer.parse(val_str), + {id, ""} <- Integer.parse(raw_id) do + {price, id} + else + _ -> nil + end + end + + def decode_cursor(_encoded, _field), do: nil +end diff --git a/lib/ppr_api/residential_sales.ex b/lib/ppr_api/residential_sales.ex index 8fa3749..29bac2c 100644 --- a/lib/ppr_api/residential_sales.ex +++ b/lib/ppr_api/residential_sales.ex @@ -5,53 +5,27 @@ defmodule PprApi.ResidentialSales do alias PprApi.Fetches.Fetch alias PprApi.ResidentialSales.ResidentialSale alias PprApi.FingerprintHelper + alias PprApi.Pagination.Cursor @doc """ Lists residential sales with keyset pagination. - Accepts options like: + Options: %{ "sort" => "date-desc" | "date-asc" | "price-desc" | "price-asc", "after" => | "before" => , "limit" => "10" | "1000" } - - The keyset cursor is a Base64: - - For a date sort, `sort_value` is the "YYYY-MM-DD" string. - - For a price sort, `sort_value` is the stringified price in euros. """ def list_residential_sales(opts \\ []) do opts = parse_opts(opts) - entries = - ResidentialSale - |> apply_cursor(opts) - |> apply_order(opts) - |> apply_limit(opts) - |> Repo.all() - |> apply_corrective_flip(opts) - - has_next_page? = check_if_next_page?(entries, opts) - has_prev_page? = check_if_prev_page?(entries, opts) - - after_entry = - if has_next_page? do - List.last(entries) - else - nil - end - - before_entry = - if has_prev_page? do - List.first(entries) - else - nil - end + {before_entry, entries, after_entry} = get_sales_and_peek(opts) %{ entries: entries, metadata: %{ - after_cursor: encode_cursor(after_entry, opts), - before_cursor: encode_cursor(before_entry, opts), + after_cursor: Cursor.encode_cursor(after_entry, opts), + before_cursor: Cursor.encode_cursor(before_entry, opts), limit: opts["limit"], sort: opts["sort"] @@ -63,7 +37,7 @@ defmodule PprApi.ResidentialSales do end @doc """ - Returns the date of the most recent residential sale. + Returns the date of the most recent sale. """ def latest_sale_date do ResidentialSale @@ -72,7 +46,7 @@ defmodule PprApi.ResidentialSales do end @doc """ - Returns the current count of residential sales. + Current count of residential sales. """ def total_residential_sales do case Fetches.get_latest_successful_fetch() do @@ -82,13 +56,13 @@ defmodule PprApi.ResidentialSales do end @doc """ - Gets a single residential_sale. - Raises `Ecto.NoResultsError` if the Residential sale does not exist. + Gets a single residential sale by ID. + Raises if it doesn’t exist. """ def get_residential_sale!(id), do: Repo.get!(ResidentialSale, id) @doc """ - Inserts residential sales in batches of 1000. + Inserts sales in batches of 1000. Skips duplicates. """ def upsert_rows(rows) do batch_size = 1000 @@ -112,10 +86,10 @@ defmodule PprApi.ResidentialSales do |> Map.put(:updated_at, now) end) - # Remove duplicate fingerprints within the batch + # remove duplicate fingerprints within the batch unique_rows = remove_duplicate_fingerprints(rows_with_fingerprints) - # Perform the bulk upsert, counting only inserted rows + # perform the bulk upsert {rows_inserted, _} = Repo.insert_all( ResidentialSale, @@ -127,7 +101,6 @@ defmodule PprApi.ResidentialSales do rows_inserted end - # Helper to remove duplicate fingerprints, keeping the first occurrence defp remove_duplicate_fingerprints(rows_with_fingerprints) do rows_with_fingerprints |> Enum.reduce(%{}, fn row, acc -> @@ -164,13 +137,13 @@ defmodule PprApi.ResidentialSales do defp parse_cursor(%{"after" => value, "sort" => {sort_field, _sort_direction}} = opts) do opts |> Map.delete("after") - |> Map.put("cursor", {decode_cursor(value, sort_field), "after"}) + |> Map.put("cursor", {Cursor.decode_cursor(value, sort_field), "after"}) end defp parse_cursor(%{"before" => value, "sort" => {sort_field, _sort_direction}} = opts) do opts |> Map.delete("before") - |> Map.put("cursor", {decode_cursor(value, sort_field), "before"}) + |> Map.put("cursor", {Cursor.decode_cursor(value, sort_field), "before"}) end defp parse_cursor(opts), do: opts @@ -190,6 +163,39 @@ defmodule PprApi.ResidentialSales do end end + # The Actual Query + + # if i'm ascending and paginating forwards (after), i want to get the entries + # that are greater than the cursor value. + # + # if i'm ascending and paginating backwards (before), i want to get the entries + # that are less than (but closest to) the cursor value. + # + # if i'm descending and paginating forwards (after), i want to get the entries + # that are less than the cursor value + # + # if i'm descending and paginating backwards (before), i want to get the entries + # that are greater than (but closest to) the cursor value + # + # before and after should follow the sort order, so should be independent of the + # cursor direction. (i.e. if i've pressed previous, then the previous button should + # still show me further in the previous direction.) + + defp get_sales_and_peek(opts) do + entries = + ResidentialSale + |> apply_cursor(opts) + |> apply_order(opts) + |> apply_limit(opts) + |> Repo.all() + |> apply_corrective_flip(opts) + + before_entry = set_peek_entry(:before, entries, opts) + after_entry = set_peek_entry(:after, entries, opts) + + {before_entry, entries, after_entry} + end + defp apply_cursor(query, %{ "cursor" => {{cursor_value, cursor_id}, direction}, "sort" => {"date", order} @@ -277,7 +283,6 @@ defmodule PprApi.ResidentialSales do query |> limit(^opts["limit"]) end - # APPLY CORRECTIVE FLIP # if we're paginating backwards, we need to flip the order of the results defp apply_corrective_flip(entries, %{"cursor" => {_cursor, "before"}}) do Enum.reverse(entries) @@ -285,99 +290,67 @@ defmodule PprApi.ResidentialSales do defp apply_corrective_flip(entries, _opts), do: entries - # ENCODE & DECODE - - defp encode_cursor(nil, _opts), do: nil + # PEEK QUERIES - # "date" sort => "YYYY-MM-DD-" - defp encode_cursor(%ResidentialSale{id: id, date_of_sale: date}, %{"sort" => {"date", _dir}}) do - # Convert date to ISO8601, then combine with ID - value_str = Date.to_iso8601(date) - combined = "#{value_str}|#{id}" - Base.url_encode64(combined, padding: false) - end + defp set_peek_entry(_dir, [], _opts), do: nil - # "price" sort => "-" - defp encode_cursor(%ResidentialSale{id: id, price_in_euros: price}, %{"sort" => {"price", _dir}}) do - value_str = to_string(price) - combined = "#{value_str}|#{id}" - Base.url_encode64(combined, padding: false) + defp set_peek_entry(:before, [first_entry | _], %{"sort" => {field, order}}) do + if has_entry_in_direction?(field, order, first_entry, :before), + do: first_entry, + else: nil end - # peek functions - - defp check_if_next_page?([], _opts), do: false - - defp check_if_next_page?(entries, opts) do - # 1) Get the last entry in the current page + defp set_peek_entry(:after, entries, %{"sort" => {field, order}}) do last_entry = List.last(entries) - encoded_cursor = encode_cursor(last_entry, opts) - - # 2) Build new opts with "after" that last entry - new_opts = - opts - |> Map.delete("cursor") - # just to be safe - |> Map.delete("before") - |> Map.put("after", encoded_cursor) - - # 3) Run the same pipeline with limit(1) - ResidentialSale - |> apply_cursor(new_opts) - |> apply_order(new_opts) - |> limit(1) - |> Repo.exists?() - end - - defp check_if_prev_page?([], _opts), do: false - defp check_if_prev_page?(entries, opts) do - # 1) Get the first entry in the current page - first_entry = List.first(entries) - encoded_cursor = encode_cursor(first_entry, opts) - - # 2) Build new opts with "before" that first entry - new_opts = - opts - |> Map.delete("cursor") - # just to be safe - |> Map.delete("after") - |> Map.put("before", encoded_cursor) - - # 3) Run the same pipeline with limit(1) - ResidentialSale - |> apply_cursor(new_opts) - |> apply_order(new_opts) - |> limit(1) - |> Repo.exists?() - end - - # fallback if sort or fields are unexpected - defp encode_cursor(_entry, _opts), do: nil - - defp decode_cursor(nil, _field), do: nil - - defp decode_cursor(encoded, "date") do - with {:ok, decoded} <- Base.url_decode64(encoded, padding: false), - [val_str, raw_id] <- String.split(decoded, "|", parts: 2), - {:ok, date} <- Date.from_iso8601(val_str), - {id, ""} <- Integer.parse(raw_id) do - {date, id} - else - _ -> nil - end - end - - defp decode_cursor(encoded, "price") do - with {:ok, decoded} <- Base.url_decode64(encoded, padding: false), - [val_str, raw_id] <- String.split(decoded, "|", parts: 2), - {price, ""} <- Integer.parse(val_str), - {id, ""} <- Integer.parse(raw_id) do - {price, id} - else - _ -> nil - end + if has_entry_in_direction?(field, order, last_entry, :after), + do: last_entry, + else: nil end - defp decode_cursor(_encoded, _field), do: nil + defp has_entry_in_direction?( + "date", + order, + %ResidentialSale{id: id, date_of_sale: date}, + direction + ), + do: + has_entry_in_direction_date(ResidentialSale, order, date, id, direction) + |> Repo.exists?() + + defp has_entry_in_direction?( + "price", + order, + %ResidentialSale{id: id, price_in_euros: price}, + direction + ), + do: + has_entry_in_direction_price(ResidentialSale, order, price, id, direction) + |> Repo.exists?() + + # date + defp has_entry_in_direction_date(queryable, "asc", date, id, :before), + do: less_than_date(queryable, date, id) + + defp has_entry_in_direction_date(queryable, "asc", date, id, :after), + do: greater_than_date(queryable, date, id) + + defp has_entry_in_direction_date(queryable, "desc", date, id, :before), + do: greater_than_date(queryable, date, id) + + defp has_entry_in_direction_date(queryable, "desc", date, id, :after), + do: less_than_date(queryable, date, id) + + # price + defp has_entry_in_direction_price(queryable, "asc", price, id, :before), + do: less_than_price(queryable, price, id) + + defp has_entry_in_direction_price(queryable, "asc", price, id, :after), + do: greater_than_price(queryable, price, id) + + defp has_entry_in_direction_price(queryable, "desc", price, id, :before), + do: greater_than_price(queryable, price, id) + + defp has_entry_in_direction_price(queryable, "desc", price, id, :after), + do: less_than_price(queryable, price, id) end diff --git a/lib/ppr_api_web/live/residential_sale_live/index.ex b/lib/ppr_api_web/live/residential_sale_live/index.ex index 2344040..1bb956b 100644 --- a/lib/ppr_api_web/live/residential_sale_live/index.ex +++ b/lib/ppr_api_web/live/residential_sale_live/index.ex @@ -1,7 +1,6 @@ defmodule PprApiWeb.ResidentialSaleLive.Index do use PprApiWeb, :live_view alias PprApi.ResidentialSales - alias PprApi.Pagination def mount(params, _session, socket) do opts = diff --git a/test/ppr_api/residential_sales_test.exs b/test/ppr_api/residential_sales_test.exs index 1588882..6c5d274 100644 --- a/test/ppr_api/residential_sales_test.exs +++ b/test/ppr_api/residential_sales_test.exs @@ -9,7 +9,14 @@ defmodule PprApi.ResidentialSalesTest do sale1 = residential_sale_fixture(%{date_of_sale: ~D[2023-01-01]}) sale2 = residential_sale_fixture(%{date_of_sale: ~D[2023-01-02]}) - %{entries: sales, metadata: _metadata} = ResidentialSales.list_residential_sales() + %{entries: sales, metadata: _metadata} = + ResidentialSales.list_residential_sales(%{ + "sort" => "date-desc", + "before" => nil, + "after" => nil, + "limit" => 2 + }) + assert [sale2.id, sale1.id] == Enum.map(sales, & &1.id) end end