From fd31e85c42d6226d2a3bff48c56131c320bc72d0 Mon Sep 17 00:00:00 2001 From: Paul Lamb Date: Sat, 26 Dec 2015 10:23:58 -0600 Subject: [PATCH 01/16] add credo for static analysis, simplify insert pattern matching on bulk insert version, change count & length variable names so they don't ever get confused with Kernel.count and Kernel.length --- lib/moebius/document_query.ex | 4 ++-- lib/moebius/query.ex | 14 +++++++------- mix.exs | 3 ++- mix.lock | 4 +++- test/moebius/basic_select_test.exs | 2 +- 5 files changed, 15 insertions(+), 12 deletions(-) diff --git a/lib/moebius/document_query.ex b/lib/moebius/document_query.ex index 273c5cc..ef8b201 100644 --- a/lib/moebius/document_query.ex +++ b/lib/moebius/document_query.ex @@ -170,11 +170,11 @@ defmodule Moebius.DocumentQuery do @doc """ Alias for Query limit """ - def limit(cmd, length), do: Moebius.Query.limit(cmd, length) + def limit(cmd, len), do: Moebius.Query.limit(cmd, len) @doc """ Alias for Query offset """ - def offset(cmd, length), do: Moebius.Query.offset(cmd, length) + def offset(cmd, len), do: Moebius.Query.offset(cmd, len) @doc """ Alias for function """ diff --git a/lib/moebius/query.ex b/lib/moebius/query.ex index 8c7a9f2..1220828 100644 --- a/lib/moebius/query.ex +++ b/lib/moebius/query.ex @@ -399,19 +399,19 @@ defmodule Moebius.Query do result = db(:people) |> insert(data) ``` """ - def insert(cmd, [[h | t] | rest]) do - records = [[h | t] | rest] - [first | rest] = records + #def insert(cmd, records) when is_list(records) and not is_tuple(hd(records)) do + def insert(cmd, [[hd | _] | _] = records) when is_tuple(hd) do # need a single definitive column map to arrest and roll back Tx if # and of the inputs are malformed (different cols vs. vals) - column_map = Keyword.keys(first) + column_map = records |> hd |> Keyword.keys transaction fn(pid) -> bulk_insert_batch(cmd, records, [], column_map) |> Enum.map(fn(cmd) -> execute(cmd, pid) end) |> List.flatten end + end defp bulk_insert_batch(cmd, records, acc, column_map) do @@ -464,7 +464,7 @@ defmodule Moebius.Query do end ``` """ - def insert(cmd, pid, criteria) do + def insert(cmd, pid, criteria) when is_pid(pid) do insert_command(cmd, criteria) |> execute(:single, pid) end @@ -506,7 +506,7 @@ defmodule Moebius.Query do cols = Keyword.keys(criteria) vals = Keyword.values(criteria) - {cols, count} = Enum.map_reduce cols, 1, fn col, acc -> + {cols, col_count} = Enum.map_reduce cols, 1, fn col, acc -> {"#{col} = $#{acc}", acc + 1} end @@ -514,7 +514,7 @@ defmodule Moebius.Query do where = cond do length(cmd.where_columns) > 0 -> - {filters, _count} = Enum.map_reduce cmd.where_columns, count, fn col, acc -> + {filters, _count} = Enum.map_reduce cmd.where_columns, col_count, fn col, acc -> {"#{col} = $#{acc}", acc + 1} end " where " <> Enum.join(filters, " and ") diff --git a/mix.exs b/mix.exs index eebf132..8296709 100644 --- a/mix.exs +++ b/mix.exs @@ -35,7 +35,8 @@ defmodule Moebius.Mixfile do {:poison, "~> 1.5"}, {:json, "~> 0.3.0"}, {:ex_doc, "~> 0.10", only: [:dev, :docs]}, - {:earmark, "~> 0.1.18", only: [:dev, :docs]}] + {:earmark, "~> 0.1.18", only: [:dev, :docs]}, + {:credo, "~> 0.2", only: [:dev, :test]}] end def package do diff --git a/mix.lock b/mix.lock index 00e491a..179b546 100644 --- a/mix.lock +++ b/mix.lock @@ -1,4 +1,6 @@ -%{"combine": {:hex, :combine, "0.5.2"}, +%{"bunt": {:hex, :bunt, "0.1.4"}, + "combine": {:hex, :combine, "0.5.2"}, + "credo": {:hex, :credo, "0.2.4"}, "decimal": {:hex, :decimal, "1.1.0"}, "earmark": {:hex, :earmark, "0.1.18"}, "ex_doc": {:hex, :ex_doc, "0.10.0"}, diff --git a/test/moebius/basic_select_test.exs b/test/moebius/basic_select_test.exs index 349883e..1b5de0c 100644 --- a/test/moebius/basic_select_test.exs +++ b/test/moebius/basic_select_test.exs @@ -6,7 +6,7 @@ defmodule Moebius.BasicSelectTest do setup do db(:logs) |> delete db(:users) |> delete - user = db(:users) |> insert(email: "friend@test.com") + user = db(:users) |> insert([email: "friend@test.com"]) db(:users) |> insert(email: "enemy@test.com") {:ok, res: user} From f675d0a5981590d2fcdf7a9fc2dd7275406998ce Mon Sep 17 00:00:00 2001 From: Paul Lamb Date: Sat, 26 Dec 2015 11:26:41 -0600 Subject: [PATCH 02/16] cleanup pipes so that they start with a a raw value --- lib/moebius/document_query.ex | 12 ++++--- lib/moebius/query.ex | 66 +++++++++++++++++++++++------------ lib/moebius/runner.ex | 4 +-- lib/moebius/transformer.ex | 15 ++++---- 4 files changed, 60 insertions(+), 37 deletions(-) diff --git a/lib/moebius/document_query.ex b/lib/moebius/document_query.ex index ef8b201..a0aa66e 100644 --- a/lib/moebius/document_query.ex +++ b/lib/moebius/document_query.ex @@ -304,8 +304,8 @@ defmodule Moebius.DocumentQuery do @doc """ Deletes a document with the given id """ - def delete(cmd, id), do: delete_command(cmd, id) |> execute(:single) - def delete(cmd, pid, id), do: delete_command(cmd, id) |> execute(pid) + def delete(cmd, id), do: cmd |> delete_command(id) |> execute(:single) + def delete(cmd, pid, id), do: cmd |> delete_command(id) |> execute(pid) @doc """ An alias for `delete/1`, removes a document based on the filter setup. @@ -316,8 +316,8 @@ defmodule Moebius.DocumentQuery do @doc """ Deletes a document based on the filter (if any) """ - def delete(cmd), do: delete_command(cmd) |> execute - def delete(cmd, pid), do: delete_command(cmd) |> execute(pid) + def delete(cmd), do: cmd |> delete_command |> execute + def delete(cmd, pid), do: cmd |> delete_command |> execute(pid) @doc """ @@ -499,7 +499,9 @@ defmodule Moebius.DocumentQuery do end defp handle_row([id, json]) do - decode_json(json) |> Map.put_new(:id, id) + json + |> decode_json + |> Map.put_new(:id, id) end #defp decode_json(json) when is_map(json), do: Moebius.Transformer.to_atom_map(json) diff --git a/lib/moebius/query.ex b/lib/moebius/query.ex index 1220828..bb7de88 100644 --- a/lib/moebius/query.ex +++ b/lib/moebius/query.ex @@ -200,7 +200,8 @@ defmodule Moebius.Query do ``` """ def first(cmd, cols \\ "*") do - res = select_command(cmd, cols) + res = cmd + |> select_command(cols) |> execute(:single) cond do @@ -225,7 +226,8 @@ defmodule Moebius.Query do ``` """ def last(cmd, sort_by) when is_atom(sort_by) do - sort(cmd, sort_by, :desc) + cmd + |> sort(sort_by, :desc) |> select_command |> execute(:single) end @@ -262,7 +264,8 @@ defmodule Moebius.Query do ``` """ def all(cmd, cols \\ "*") do - select_command(cmd, cols) + cmd + |> select_command(cols) |> execute end @@ -407,7 +410,8 @@ defmodule Moebius.Query do column_map = records |> hd |> Keyword.keys transaction fn(pid) -> - bulk_insert_batch(cmd, records, [], column_map) + cmd + |> bulk_insert_batch(records, [], column_map) |> Enum.map(fn(cmd) -> execute(cmd, pid) end) |> List.flatten end @@ -465,7 +469,8 @@ defmodule Moebius.Query do ``` """ def insert(cmd, pid, criteria) when is_pid(pid) do - insert_command(cmd, criteria) + cmd + |> insert_command(cmd, criteria) |> execute(:single, pid) end @@ -482,7 +487,8 @@ defmodule Moebius.Query do ``` """ def insert(cmd, criteria) do - insert_command(cmd, criteria) + cmd + |> insert_command(cmd, criteria) |> execute(:single) end @@ -550,7 +556,8 @@ defmodule Moebius.Query do ``` """ def update(cmd, pid, :single, criteria) when is_list(criteria) do - update_command(cmd, criteria) + cmd + |> update_command(criteria) |> execute(:single, pid) end @@ -571,7 +578,8 @@ defmodule Moebius.Query do ``` """ def update(cmd, :single, criteria) when is_list(criteria) do - update_command(cmd, criteria) + cmd + |> update_command(criteria) |> execute(:single) end @@ -587,7 +595,8 @@ defmodule Moebius.Query do ``` """ def update(cmd, criteria) when is_list(criteria) do - update_command(cmd, criteria) + cmd + |> update_command(criteria) |> execute end @@ -615,7 +624,8 @@ defmodule Moebius.Query do ``` """ def delete(cmd, pid) do - delete_command(cmd) + cmd + |> delete_command |> execute(:single, pid) end @@ -632,7 +642,8 @@ defmodule Moebius.Query do ``` """ def delete(cmd) do - delete_command(cmd) + cmd + |> delete_command |> execute(:single) end @@ -686,7 +697,8 @@ defmodule Moebius.Query do result = sql_file(:simple) """ def sql_file(file) do - sql_file_command(file, []) + file + |> sql_file_command([]) |> execute end @@ -700,7 +712,8 @@ defmodule Moebius.Query do ``` """ def sql_file(file, :single, params) do - sql_file_command(file, params) + file + |> sql_file_command(params) |> execute(:single) end @@ -714,7 +727,8 @@ defmodule Moebius.Query do ``` """ def sql_file(file, params) do - sql_file_command(file, params) + file + |> sql_file_command(params) |> execute end @@ -747,7 +761,8 @@ defmodule Moebius.Query do ``` """ def function(function_name) do - function_command(function_name, []) + function_name + |> function_command([]) |> execute end @@ -765,7 +780,8 @@ defmodule Moebius.Query do ``` """ def function(function_name, :single) do - function_command(function_name, []) + function_name + |> function_command([]) |> execute(:single) end @@ -783,7 +799,8 @@ defmodule Moebius.Query do ``` """ def function(function_name, params) do - function_command(function_name, params) + function_name + |> function_command(params) |> execute end @@ -799,7 +816,8 @@ defmodule Moebius.Query do ``` """ def function(function_name, :single, params) do - function_command(function_name, params) + function_name + |> function_command(params) |> execute(:single) end @@ -848,7 +866,8 @@ defmodule Moebius.Query do Executes a pass-through query and returns a single result as part of a transaction """ def execute(cmd, :single, pid) do - Moebius.Runner.execute(cmd, pid) + cmd + |> Moebius.Runner.execute(pid) |> Moebius.Transformer.to_single end @@ -856,7 +875,8 @@ defmodule Moebius.Query do Executes a command, returning a list of results """ def execute(cmd) do - Moebius.Runner.execute(cmd) + cmd + |> Moebius.Runner.execute |> Moebius.Transformer.to_list end @@ -864,7 +884,8 @@ defmodule Moebius.Query do Executes a pass-through query and returns a single result """ def execute(cmd, :single) do - Moebius.Runner.execute(cmd) + cmd + |> Moebius.Runner.execute |> Moebius.Transformer.to_single end @@ -872,7 +893,8 @@ defmodule Moebius.Query do Executes a command, returning a list of results as part of a transaction """ def execute(cmd, pid) do - Moebius.Runner.execute(cmd, pid) + cmd + |> Moebius.Runner.execute(pid) |> Moebius.Transformer.to_list end diff --git a/lib/moebius/runner.ex b/lib/moebius/runner.ex index e25e92b..2b925c4 100644 --- a/lib/moebius/runner.ex +++ b/lib/moebius/runner.ex @@ -10,8 +10,8 @@ defmodule Moebius.Runner do extensions = [{Postgrex.Extensions.JSON, library: Poison}] Application.get_env(:moebius, :connection) - |> Keyword.update(:extensions, extensions, &(&1 ++ extensions)) - |> Postgrex.Connection.start_link + |> Keyword.update(:extensions, extensions, &(&1 ++ extensions)) + |> Postgrex.Connection.start_link end @doc """ diff --git a/lib/moebius/transformer.ex b/lib/moebius/transformer.ex index 4415829..202a10f 100644 --- a/lib/moebius/transformer.ex +++ b/lib/moebius/transformer.ex @@ -26,13 +26,12 @@ defmodule Moebius.Transformer do @doc """ Coerce a large result set into an array of atom-keyed maps """ - def to_list({:error, err}), - do: {:error, err} - def to_list({:ok, %{rows: nil}}), - do: [] + def to_list({:error, err}), do: {:error, err} + def to_list({:ok, %{rows: nil}}), do: [] def to_list({:ok, %{rows: rows, columns: cols}}) do Enum.map rows, fn(r) -> - zip_columns_and_row({cols, r}) + {cols, r} + |> zip_columns_and_row |> to_map end end @@ -40,15 +39,15 @@ defmodule Moebius.Transformer do @doc """ Coerces a Postgrex.Result into a single atom-keyed map """ - def to_single({:error, err}), - do: {:error, err} + def to_single({:error, err}), do: {:error, err} def to_single({:ok, %{command: :delete, num_rows: count}}), do: %{deleted: count} def to_single({:ok, %{num_rows: count}}) when count == 0, do: nil def to_single({:ok, %{num_rows: count} = res}) when count > 0 do - get_first_result(res) + res + |> get_first_result |> zip_columns_and_row |> to_map end From f6dc5ab29d4f16067f2c014abc9e8144e4af1a68 Mon Sep 17 00:00:00 2001 From: Paul Lamb Date: Sat, 26 Dec 2015 11:28:30 -0600 Subject: [PATCH 03/16] fix pipeline fixes --- lib/moebius/query.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/moebius/query.ex b/lib/moebius/query.ex index bb7de88..999b7da 100644 --- a/lib/moebius/query.ex +++ b/lib/moebius/query.ex @@ -470,7 +470,7 @@ defmodule Moebius.Query do """ def insert(cmd, pid, criteria) when is_pid(pid) do cmd - |> insert_command(cmd, criteria) + |> insert_command(criteria) |> execute(:single, pid) end @@ -488,7 +488,7 @@ defmodule Moebius.Query do """ def insert(cmd, criteria) do cmd - |> insert_command(cmd, criteria) + |> insert_command(criteria) |> execute(:single) end From 55b9aa4487c7feb5ce035fdb372530ab76039cda Mon Sep 17 00:00:00 2001 From: Paul Lamb Date: Sat, 26 Dec 2015 13:25:10 -0600 Subject: [PATCH 04/16] revert a change to a test that should not have been committed --- test/moebius/basic_select_test.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/moebius/basic_select_test.exs b/test/moebius/basic_select_test.exs index 1b5de0c..349883e 100644 --- a/test/moebius/basic_select_test.exs +++ b/test/moebius/basic_select_test.exs @@ -6,7 +6,7 @@ defmodule Moebius.BasicSelectTest do setup do db(:logs) |> delete db(:users) |> delete - user = db(:users) |> insert([email: "friend@test.com"]) + user = db(:users) |> insert(email: "friend@test.com") db(:users) |> insert(email: "enemy@test.com") {:ok, res: user} From ce2e8a2500a61e33f4428725293dc4069e71beeb Mon Sep 17 00:00:00 2001 From: Paul Lamb Date: Sat, 26 Dec 2015 13:41:35 -0600 Subject: [PATCH 05/16] add is_pid guard clauses --- lib/moebius/document_query.ex | 12 ++++++------ lib/moebius/query.ex | 8 ++++---- lib/moebius/runner.ex | 4 ++-- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/lib/moebius/document_query.ex b/lib/moebius/document_query.ex index a0aa66e..486d523 100644 --- a/lib/moebius/document_query.ex +++ b/lib/moebius/document_query.ex @@ -220,8 +220,8 @@ defmodule Moebius.DocumentQuery do res end - def save(cmd, pid, doc) when is_list(doc), do: save(cmd, pid, Enum.into(doc, %{})) - def save(cmd, pid, doc) when is_map(doc) do + def save(cmd, pid, doc) when is_pid(pid) and is_list(doc), do: save(cmd, pid, Enum.into(doc, %{})) + def save(cmd, pid, doc) when is_pid(pid) and is_map(doc) do cmd = %{cmd | pid: pid} try do @@ -299,25 +299,25 @@ defmodule Moebius.DocumentQuery do An alias for `delete/2`, removes a document with the specified ID. """ def remove(cmd, id), do: delete(cmd, id) - def remove(cmd, pid, id), do: delete(cmd, pid, id) + def remove(cmd, pid, id) when is_pid(pid), do: delete(cmd, pid, id) @doc """ Deletes a document with the given id """ def delete(cmd, id), do: cmd |> delete_command(id) |> execute(:single) - def delete(cmd, pid, id), do: cmd |> delete_command(id) |> execute(pid) + def delete(cmd, pid, id) when is_pid(pid), do: cmd |> delete_command(id) |> execute(pid) @doc """ An alias for `delete/1`, removes a document based on the filter setup. """ def remove(cmd), do: delete(cmd) - def remove(cmd, pid), do: delete(cmd, pid) + def remove(cmd, pid) when is_pid(pid), do: delete(cmd, pid) @doc """ Deletes a document based on the filter (if any) """ def delete(cmd), do: cmd |> delete_command |> execute - def delete(cmd, pid), do: cmd |> delete_command |> execute(pid) + def delete(cmd, pid) when is_pid(pid), do: cmd |> delete_command |> execute(pid) @doc """ diff --git a/lib/moebius/query.ex b/lib/moebius/query.ex index 999b7da..1c75984 100644 --- a/lib/moebius/query.ex +++ b/lib/moebius/query.ex @@ -555,7 +555,7 @@ defmodule Moebius.Query do end ``` """ - def update(cmd, pid, :single, criteria) when is_list(criteria) do + def update(cmd, pid, :single, criteria) when is_pid(pid) and is_list(criteria) do cmd |> update_command(criteria) |> execute(:single, pid) @@ -623,7 +623,7 @@ defmodule Moebius.Query do end ``` """ - def delete(cmd, pid) do + def delete(cmd, pid) when is_pid(pid) do cmd |> delete_command |> execute(:single, pid) @@ -865,7 +865,7 @@ defmodule Moebius.Query do @doc """ Executes a pass-through query and returns a single result as part of a transaction """ - def execute(cmd, :single, pid) do + def execute(cmd, :single, pid) when is_pid(pid) do cmd |> Moebius.Runner.execute(pid) |> Moebius.Transformer.to_single @@ -892,7 +892,7 @@ defmodule Moebius.Query do @doc """ Executes a command, returning a list of results as part of a transaction """ - def execute(cmd, pid) do + def execute(cmd, pid) when is_pid(pid) do cmd |> Moebius.Runner.execute(pid) |> Moebius.Transformer.to_list diff --git a/lib/moebius/runner.ex b/lib/moebius/runner.ex index 2b925c4..9a32492 100644 --- a/lib/moebius/runner.ex +++ b/lib/moebius/runner.ex @@ -33,7 +33,7 @@ defmodule Moebius.Runner do Executes a command for a given transaction specified with `pid`. If the execution fails, it will be caught in `Query.transaction/1` and reported back using `{:error, err}`. """ - def execute(cmd, pid) do + def execute(cmd, pid) when is_pid(pid) do case Postgrex.Connection.query(pid, cmd.sql, cmd.params) do {:ok, result} -> {:ok, result} @@ -51,7 +51,7 @@ defmodule Moebius.Runner do pid end - def commit_and_close_transaction(pid) do + def commit_and_close_transaction(pid) when is_pid(pid) do Postgrex.Connection.query(pid, "COMMIT;",[]) Postgrex.Connection.stop(pid) end From 939cee6d9c8ea5dbd4c35a8fb00087c6e87185ee Mon Sep 17 00:00:00 2001 From: Paul Lamb Date: Sat, 26 Dec 2015 13:47:43 -0600 Subject: [PATCH 06/16] remove old method signture comment --- lib/moebius/query.ex | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/moebius/query.ex b/lib/moebius/query.ex index 1c75984..e6ee8a5 100644 --- a/lib/moebius/query.ex +++ b/lib/moebius/query.ex @@ -402,7 +402,6 @@ defmodule Moebius.Query do result = db(:people) |> insert(data) ``` """ - #def insert(cmd, records) when is_list(records) and not is_tuple(hd(records)) do def insert(cmd, [[hd | _] | _] = records) when is_tuple(hd) do # need a single definitive column map to arrest and roll back Tx if From 0e9005cf2ba0775269abbaede67bbafba9437c1f Mon Sep 17 00:00:00 2001 From: Paul Lamb Date: Sat, 26 Dec 2015 14:16:22 -0600 Subject: [PATCH 07/16] add QueryCommand parameter matching --- lib/moebius/query.ex | 74 ++++++++++++++++++++++---------------------- 1 file changed, 37 insertions(+), 37 deletions(-) diff --git a/lib/moebius/query.ex b/lib/moebius/query.ex index e6ee8a5..a64d7ab 100644 --- a/lib/moebius/query.ex +++ b/lib/moebius/query.ex @@ -1,6 +1,7 @@ defmodule Moebius.Query do import Inflex, only: [singularize: 1] + alias Moebius.QueryCommand use Moebius.QueryFilter @moduledoc """ @@ -27,7 +28,7 @@ defmodule Moebius.Query do do: db(Atom.to_string(table)) def db(table), - do: %Moebius.QueryCommand{table_name: table} + do: %QueryCommand{table_name: table} @doc """ @@ -67,7 +68,7 @@ defmodule Moebius.Query do |> find(1) ``` """ - def find(cmd, id) do + def find(%QueryCommand{} = cmd, id) do cmd |> filter(id: id) |> select_command @@ -88,12 +89,12 @@ defmodule Moebius.Query do |> to_list ``` """ - def sort(cmd, cols, direction \\ :asc) + #def sort(%QueryCommand{} = cmd, cols, direction \\ :asc) - def sort(cmd, cols, direction) when is_atom(cols), + def sort(%QueryCommand{} = cmd, cols, direction) when is_atom(cols), do: sort(cmd, Atom.to_string(cols), direction) - def sort(cmd, cols, direction) when is_binary(cols), + def sort(%QueryCommand{} = cmd, cols, direction) when is_binary(cols), do: %{cmd | order: " order by #{cols} #{direction}"} @doc """ @@ -139,7 +140,7 @@ defmodule Moebius.Query do |> to_list ``` """ - def skip(cmd, n), + def skip(%QueryCommand{} = cmd, n), do: offset(cmd, n) @doc """ @@ -159,7 +160,7 @@ defmodule Moebius.Query do #command is a QueryCommand object with all of the pipelined settings applied ``` """ - def select_command(cmd, cols \\ "*") when is_bitstring(cols) do + def select_command(%QueryCommand{} = cmd, cols \\ "*") when is_bitstring(cols) do %{cmd | sql: "select #{cols} from #{cmd.table_name}#{cmd.join}#{cmd.where}#{cmd.order}#{cmd.limit}#{cmd.offset};"} end @@ -174,7 +175,7 @@ defmodule Moebius.Query do #count == 20 """ - def count(cmd) do + def count(%QueryCommand{} = cmd) do res = %{cmd | sql: "select count(1) from #{cmd.table_name}#{cmd.join}#{cmd.where}#{cmd.order}#{cmd.limit}#{cmd.offset};"} |> execute(:single) @@ -199,7 +200,7 @@ defmodule Moebius.Query do |> first("first, last, email") ``` """ - def first(cmd, cols \\ "*") do + def first(%QueryCommand{} = cmd, cols \\ "*") do res = cmd |> select_command(cols) |> execute(:single) @@ -225,7 +226,7 @@ defmodule Moebius.Query do |> last("first, last, email") ``` """ - def last(cmd, sort_by) when is_atom(sort_by) do + def last(%QueryCommand{} = cmd, sort_by) when is_atom(sort_by) do cmd |> sort(sort_by, :desc) |> select_command @@ -246,7 +247,7 @@ defmodule Moebius.Query do |> to_list("first, last, email") ``` """ - def to_list(cmd, cols \\ "*"), + def to_list(%QueryCommand{} = cmd, cols \\ "*"), do: all(cmd, cols) @doc """ @@ -263,7 +264,7 @@ defmodule Moebius.Query do |> all("first, last, email") ``` """ - def all(cmd, cols \\ "*") do + def all(%QueryCommand{} = cmd, cols \\ "*") do cmd |> select_command(cols) |> execute @@ -284,7 +285,7 @@ defmodule Moebius.Query do |> reduce(:sum, :money_spent) ``` """ - def group(cmd, cols) when is_atom(cols), + def group(%QueryCommand{} = cmd, cols) when is_atom(cols), do: group(cmd, Atom.to_string(cols)) @doc """ @@ -301,7 +302,7 @@ defmodule Moebius.Query do |> reduce(:sum, :money_spent) ``` """ - def group(cmd, cols), + def group(%QueryCommand{} = cmd, cols), do: %{cmd | group_by: cols} @doc """ @@ -317,7 +318,7 @@ defmodule Moebius.Query do |> reduce(:sum, :money_spent) ``` """ - def map(cmd, criteria), + def map(%QueryCommand{} = cmd, criteria), do: filter(cmd, criteria) @doc """ @@ -334,10 +335,10 @@ defmodule Moebius.Query do |> reduce(:sum, :money_spent) ``` """ - def reduce(cmd, op, column) when is_atom(column), + def reduce(%QueryCommand{} = cmd, op, column) when is_atom(column), do: reduce(cmd, op, Atom.to_string(column)) - def reduce(cmd, op, column) when is_bitstring(column) do + def reduce(%QueryCommand{} = cmd, op, column) when is_bitstring(column) do sql = cond do cmd.group_by -> "select #{op}(#{column}), #{cmd.group_by} from #{cmd.table_name}#{cmd.join}#{cmd.where} GROUP BY #{cmd.group_by}" @@ -373,7 +374,7 @@ defmodule Moebius.Query do |> run ``` """ - def search(cmd, for: term, in: columns) when is_list columns do + def search(%QueryCommand{} = cmd, for: term, in: columns) when is_list columns do concat_list = Enum.map_join(columns, ", ' ', ", &"#{&1}") sql = """ select *, ts_rank_cd(to_tsvector(concat(#{concat_list})),to_tsquery($1)) as rank from #{cmd.table_name} @@ -402,7 +403,7 @@ defmodule Moebius.Query do result = db(:people) |> insert(data) ``` """ - def insert(cmd, [[hd | _] | _] = records) when is_tuple(hd) do + def insert(%QueryCommand{} = cmd, [[hd | _] | _] = records) when is_tuple(hd) do # need a single definitive column map to arrest and roll back Tx if # and of the inputs are malformed (different cols vs. vals) @@ -417,8 +418,7 @@ defmodule Moebius.Query do end - defp bulk_insert_batch(cmd, records, acc, column_map) do - [first | rest] = records + defp bulk_insert_batch(%QueryCommand{} = cmd, records, acc, column_map) do # 20,000 seems to be the optimal number here. Technically you can go up to 34,464, but I think Postgrex imposes a lower limit, as I # hit a wall at 34,000, but succeeded at 30,000. Perf on 100k records is best at 20,000. @@ -435,7 +435,7 @@ defmodule Moebius.Query do end end - defp bulk_insert_command(cmd, [first | rest]) do + defp bulk_insert_command(%QueryCommand{} = cmd, [first | rest]) do records = [first | rest] cols = cmd.columns vals = Enum.reduce(Enum.reverse(records), [], fn(listitem, acc) -> @@ -467,7 +467,7 @@ defmodule Moebius.Query do end ``` """ - def insert(cmd, pid, criteria) when is_pid(pid) do + def insert(%QueryCommand{} = cmd, pid, criteria) when is_pid(pid) do cmd |> insert_command(criteria) |> execute(:single, pid) @@ -485,7 +485,7 @@ defmodule Moebius.Query do |> insert(email: "test@test.com", first: "Test", last: "User") ``` """ - def insert(cmd, criteria) do + def insert(%QueryCommand{} = cmd, criteria) do cmd |> insert_command(criteria) |> execute(:single) @@ -494,7 +494,7 @@ defmodule Moebius.Query do @doc """ Creates an insert command based on the assembled pipeline """ - def insert_command(cmd, criteria) do + def insert_command(%QueryCommand{} = cmd, criteria) do cols = Keyword.keys(criteria) vals = Keyword.values(criteria) sql = "insert into #{cmd.table_name}(" <> Enum.map_join(cols, ", ", &"#{&1}") <> ")" <> @@ -506,7 +506,7 @@ defmodule Moebius.Query do @doc """ Creates an update command based on the assembled pipeline. """ - def update_command(cmd, criteria) do + def update_command(%QueryCommand{} = cmd, criteria) do cols = Keyword.keys(criteria) vals = Keyword.values(criteria) @@ -554,7 +554,7 @@ defmodule Moebius.Query do end ``` """ - def update(cmd, pid, :single, criteria) when is_pid(pid) and is_list(criteria) do + def update(%QueryCommand{} = cmd, pid, :single, criteria) when is_pid(pid) and is_list(criteria) do cmd |> update_command(criteria) |> execute(:single, pid) @@ -576,7 +576,7 @@ defmodule Moebius.Query do ``` """ - def update(cmd, :single, criteria) when is_list(criteria) do + def update(%QueryCommand{} = cmd, :single, criteria) when is_list(criteria) do cmd |> update_command(criteria) |> execute(:single) @@ -593,7 +593,7 @@ defmodule Moebius.Query do |> update(status: "preferred") ``` """ - def update(cmd, criteria) when is_list(criteria) do + def update(%QueryCommand{} = cmd, criteria) when is_list(criteria) do cmd |> update_command(criteria) |> execute @@ -602,7 +602,7 @@ defmodule Moebius.Query do @doc """ Creates a DELETE command """ - def delete_command(cmd) do + def delete_command(%QueryCommand{} = cmd) do sql = "delete from #{cmd.table_name}" <> cmd.where <> ";" %{cmd | sql: sql, type: :delete} end @@ -622,7 +622,7 @@ defmodule Moebius.Query do end ``` """ - def delete(cmd, pid) when is_pid(pid) do + def delete(%QueryCommand{} = cmd, pid) when is_pid(pid) do cmd |> delete_command |> execute(:single, pid) @@ -640,7 +640,7 @@ defmodule Moebius.Query do ``` """ - def delete(cmd) do + def delete(%QueryCommand{} = cmd) do cmd |> delete_command |> execute(:single) @@ -671,7 +671,7 @@ defmodule Moebius.Query do |> select ``` """ - def join(cmd, table, opts \\ []) do + def join(%QueryCommand{} = cmd, table, opts \\ []) do join_type = Keyword.get(opts, :join, "inner") join_table = Keyword.get(opts, :on, cmd.table_name) foreign_key = Keyword.get(opts, :foreign_key, "#{singularize(join_table)}_id") @@ -864,7 +864,7 @@ defmodule Moebius.Query do @doc """ Executes a pass-through query and returns a single result as part of a transaction """ - def execute(cmd, :single, pid) when is_pid(pid) do + def execute(%QueryCommand{} = cmd, :single, pid) when is_pid(pid) do cmd |> Moebius.Runner.execute(pid) |> Moebius.Transformer.to_single @@ -873,7 +873,7 @@ defmodule Moebius.Query do @doc """ Executes a command, returning a list of results """ - def execute(cmd) do + def execute(%QueryCommand{} = cmd) do cmd |> Moebius.Runner.execute |> Moebius.Transformer.to_list @@ -882,7 +882,7 @@ defmodule Moebius.Query do @doc """ Executes a pass-through query and returns a single result """ - def execute(cmd, :single) do + def execute(%QueryCommand{} = cmd, :single) do cmd |> Moebius.Runner.execute |> Moebius.Transformer.to_single @@ -891,7 +891,7 @@ defmodule Moebius.Query do @doc """ Executes a command, returning a list of results as part of a transaction """ - def execute(cmd, pid) when is_pid(pid) do + def execute(%QueryCommand{} = cmd, pid) when is_pid(pid) do cmd |> Moebius.Runner.execute(pid) |> Moebius.Transformer.to_list From 4b1cd3da87bc5fb37c690b13c8ff61d256abe88f Mon Sep 17 00:00:00 2001 From: Paul Lamb Date: Sat, 26 Dec 2015 14:19:45 -0600 Subject: [PATCH 08/16] reorder insert/2 and remove unused variables to remove compiler warnings --- lib/moebius/query.ex | 94 ++++++++++++++++++++++---------------------- 1 file changed, 47 insertions(+), 47 deletions(-) diff --git a/lib/moebius/query.ex b/lib/moebius/query.ex index a64d7ab..16a18be 100644 --- a/lib/moebius/query.ex +++ b/lib/moebius/query.ex @@ -418,77 +418,43 @@ defmodule Moebius.Query do end - defp bulk_insert_batch(%QueryCommand{} = cmd, records, acc, column_map) do - - # 20,000 seems to be the optimal number here. Technically you can go up to 34,464, but I think Postgrex imposes a lower limit, as I - # hit a wall at 34,000, but succeeded at 30,000. Perf on 100k records is best at 20,000. - max_params = 20000 - cmd = %{ cmd | columns: column_map} - max_records_per_command = div(max_params, length(cmd.columns)) - - { current, next_batch } = Enum.split(records, max_records_per_command) - this_cmd = bulk_insert_command(cmd, current) - case next_batch do - [] -> Enum.reverse([this_cmd | acc]) - _ -> - db(cmd.table_name) |> bulk_insert_batch(next_batch, [this_cmd | acc], column_map) - end - end - - defp bulk_insert_command(%QueryCommand{} = cmd, [first | rest]) do - records = [first | rest] - cols = cmd.columns - vals = Enum.reduce(Enum.reverse(records), [], fn(listitem, acc) -> - Enum.concat(Keyword.values(listitem), acc) end) - - params_sql = elem(Enum.map_reduce(vals, 0, fn(v, acc) -> {"$#{acc + 1}", acc + 1} end),0) - |> Enum.chunk(length(cols), length(cols), []) - |> Enum.map(fn(chunk) -> "(#{Enum.join(chunk, ", ")})" end) - |> Enum.join(", ") - - sql_body = "insert into #{cmd.table_name} (" <> Enum.join(cols, ", ") <> ") " <> - "values #{ params_sql } returning *;" - - %{cmd | columns: cols, sql: sql_body, params: vals, type: :insert} - end - @doc """ - A simple insert that is part of a transaction that returns the inserted record. Create your list of data and send it on in. + A simple insert that that returns the inserted record. Create your list of data and send it on in. - pid: - The process id of the transaction (retrieved from the `transaction` callback) criteria: - A list or map of data to be saved Example: ``` - tranaction fn(pid) -> - new_user = db(:users) - |> insert(pid, email: "test@test.com", first: "Test", last: "User") - end + new_user = db(:users) + |> insert(email: "test@test.com", first: "Test", last: "User") ``` """ - def insert(%QueryCommand{} = cmd, pid, criteria) when is_pid(pid) do + def insert(%QueryCommand{} = cmd, criteria) do cmd |> insert_command(criteria) - |> execute(:single, pid) + |> execute(:single) end @doc """ - A simple insert that that returns the inserted record. Create your list of data and send it on in. + A simple insert that is part of a transaction that returns the inserted record. Create your list of data and send it on in. + pid: - The process id of the transaction (retrieved from the `transaction` callback) criteria: - A list or map of data to be saved Example: ``` - new_user = db(:users) - |> insert(email: "test@test.com", first: "Test", last: "User") + tranaction fn(pid) -> + new_user = db(:users) + |> insert(pid, email: "test@test.com", first: "Test", last: "User") + end ``` """ - def insert(%QueryCommand{} = cmd, criteria) do + def insert(%QueryCommand{} = cmd, pid, criteria) when is_pid(pid) do cmd |> insert_command(criteria) - |> execute(:single) + |> execute(:single, pid) end @doc """ @@ -503,6 +469,40 @@ defmodule Moebius.Query do %{cmd | sql: sql, params: vals, type: :insert} end + defp bulk_insert_batch(%QueryCommand{} = cmd, records, acc, column_map) do + + # 20,000 seems to be the optimal number here. Technically you can go up to 34,464, but I think Postgrex imposes a lower limit, as I + # hit a wall at 34,000, but succeeded at 30,000. Perf on 100k records is best at 20,000. + max_params = 20000 + cmd = %{ cmd | columns: column_map} + max_records_per_command = div(max_params, length(cmd.columns)) + + { current, next_batch } = Enum.split(records, max_records_per_command) + this_cmd = bulk_insert_command(cmd, current) + case next_batch do + [] -> Enum.reverse([this_cmd | acc]) + _ -> + db(cmd.table_name) |> bulk_insert_batch(next_batch, [this_cmd | acc], column_map) + end + end + + defp bulk_insert_command(%QueryCommand{} = cmd, [first | rest]) do + records = [first | rest] + cols = cmd.columns + vals = Enum.reduce(Enum.reverse(records), [], fn(listitem, acc) -> + Enum.concat(Keyword.values(listitem), acc) end) + + params_sql = elem(Enum.map_reduce(vals, 0, fn(_, acc) -> {"$#{acc + 1}", acc + 1} end),0) + |> Enum.chunk(length(cols), length(cols), []) + |> Enum.map(fn(chunk) -> "(#{Enum.join(chunk, ", ")})" end) + |> Enum.join(", ") + + sql_body = "insert into #{cmd.table_name} (" <> Enum.join(cols, ", ") <> ") " <> + "values #{ params_sql } returning *;" + + %{cmd | columns: cols, sql: sql_body, params: vals, type: :insert} + end + @doc """ Creates an update command based on the assembled pipeline. """ From 4153a525928cc5526010a37393b390b114cac560 Mon Sep 17 00:00:00 2001 From: Paul Lamb Date: Sat, 26 Dec 2015 14:56:10 -0600 Subject: [PATCH 09/16] DocumentCommand pattern matching in function signatures, reordering of code and cleanup of unused vars to clean up compiler warnings --- lib/moebius/document_query.ex | 116 +++++++++++++++++++--------------- 1 file changed, 65 insertions(+), 51 deletions(-) diff --git a/lib/moebius/document_query.ex b/lib/moebius/document_query.ex index 486d523..71d4a2e 100644 --- a/lib/moebius/document_query.ex +++ b/lib/moebius/document_query.ex @@ -29,9 +29,11 @@ defmodule Moebius.DocumentQuery do """ import Poison + alias Moebius.DocumentCommand + alias Moebius.Query - def transaction(fun), do: Moebius.Query.transaction(fun) + def transaction(fun), do: Query.transaction(fun) @doc """ Specifies the table or view you want to query. @@ -83,7 +85,7 @@ defmodule Moebius.DocumentQuery do |> first ``` """ - def contains(cmd, criteria) do + def contains(%DocumentCommand{} = cmd, criteria) do map = Enum.into(criteria, %{}) encoded = encode!(map) @@ -104,7 +106,7 @@ defmodule Moebius.DocumentQuery do |> first ``` """ - def filter(cmd, criteria, params \\ []) when is_bitstring(criteria) do + def filter(%DocumentCommand{} = cmd, criteria, params \\ []) when is_bitstring(criteria) do unless is_list(params) do params = [params] end @@ -124,7 +126,7 @@ defmodule Moebius.DocumentQuery do |> to_list ``` """ - def filter(cmd, field, operator, params) do + def filter(%DocumentCommand{} = cmd, field, operator, params) do unless is_list(params) do params = [params] end @@ -144,7 +146,7 @@ defmodule Moebius.DocumentQuery do |> to_list ``` """ - def exists(cmd, field, params) do + def exists(%DocumentCommand{} = cmd, field, params) do unless is_list(params) do params = [params] end @@ -155,7 +157,7 @@ defmodule Moebius.DocumentQuery do @doc """ Creates a SELECT command based on the assembled pipeline. """ - def select_command(cmd) do + def select_command(%DocumentCommand{} = cmd) do sql = """ select id, #{cmd.json_field}::text from #{cmd.table_name} @@ -170,11 +172,11 @@ defmodule Moebius.DocumentQuery do @doc """ Alias for Query limit """ - def limit(cmd, len), do: Moebius.Query.limit(cmd, len) + def limit(%DocumentCommand{} = cmd, len), do: Moebius.Query.limit(cmd, len) @doc """ Alias for Query offset """ - def offset(cmd, len), do: Moebius.Query.offset(cmd, len) + def offset(%DocumentCommand{} = cmd, len), do: Moebius.Query.offset(cmd, len) @doc """ Alias for function """ @@ -191,7 +193,7 @@ defmodule Moebius.DocumentQuery do |> to_list ``` """ - def sort(cmd, cols, direction \\ :asc) do + def sort(%DocumentCommand{} = cmd, cols, direction \\ :asc) do order_column = cols if is_atom(cols) do order_column = Atom.to_string cols @@ -212,16 +214,16 @@ defmodule Moebius.DocumentQuery do |> save(product) ``` """ - def save(cmd, doc) when is_list(doc), do: save(cmd, Enum.into(doc, %{})) - def save(cmd, doc) when is_map(doc) do + def save(%DocumentCommand{} = cmd, doc) when is_list(doc), do: save(cmd, Enum.into(doc, %{})) + def save(%DocumentCommand{} = cmd, doc) when is_map(doc) do pid = Moebius.Runner.open_transaction res = save(cmd, pid, doc) Moebius.Runner.commit_and_close_transaction(pid) res end - def save(cmd, pid, doc) when is_pid(pid) and is_list(doc), do: save(cmd, pid, Enum.into(doc, %{})) - def save(cmd, pid, doc) when is_pid(pid) and is_map(doc) do + def save(%DocumentCommand{} = cmd, pid, doc) when is_pid(pid) and is_list(doc), do: save(cmd, pid, Enum.into(doc, %{})) + def save(%DocumentCommand{} = cmd, pid, doc) when is_pid(pid) and is_map(doc) do cmd = %{cmd | pid: pid} try do @@ -238,7 +240,7 @@ defmodule Moebius.DocumentQuery do end - defp update_search([], cmd), do: [] + defp update_search([], _), do: [] defp update_search(query_result, cmd) do if length(cmd.search_fields) > 0 do @@ -253,13 +255,13 @@ defmodule Moebius.DocumentQuery do query_result end - defp decide_command(cmd, doc) do + defp decide_command(%DocumentCommand{} = cmd, doc) do cond do Map.has_key? doc, :id -> update_command(cmd, Map.delete(doc, :id), doc.id) true -> insert_command(cmd, doc) end end - defp create_document_table(cmd, doc) do + defp create_document_table(%DocumentCommand{} = cmd, _) do sql = """ create table #{cmd.table_name}( id serial primary key not null, @@ -270,7 +272,7 @@ defmodule Moebius.DocumentQuery do ); """ - res = %Moebius.QueryCommand{sql: sql} |> Moebius.Runner.execute(cmd.pid) + %Moebius.QueryCommand{sql: sql} |> Moebius.Runner.execute(cmd.pid) %Moebius.QueryCommand{sql: "create index idx_#{cmd.table_name}_search on #{cmd.table_name} using GIN(search);"} |> Moebius.Runner.execute(cmd.pid) %Moebius.QueryCommand{sql: "create index idx_#{cmd.table_name} on #{cmd.table_name} using GIN(body jsonb_path_ops);"} |> Moebius.Runner.execute(cmd.pid) @@ -290,34 +292,45 @@ defmodule Moebius.DocumentQuery do ``` """ def searchable({:error, err}), do: {:error ,err} - def searchable(cmd, search_params) when is_list(search_params) do + def searchable(%DocumentCommand{} = cmd, search_params) when is_list(search_params) do %{cmd | search_fields: search_params} end + @doc """ + An alias for `delete/1`, removes a document based on the filter setup. + """ + def remove(%DocumentCommand{} = cmd), do: delete(cmd) + @doc """ + An alias for `delete/1`, removes a document based on the filter setup. + """ + def remove(%DocumentCommand{} = cmd, pid) when is_pid(pid), do: delete(cmd, pid) @doc """ An alias for `delete/2`, removes a document with the specified ID. """ - def remove(cmd, id), do: delete(cmd, id) - def remove(cmd, pid, id) when is_pid(pid), do: delete(cmd, pid, id) - + def remove(%DocumentCommand{} = cmd, id), do: delete(cmd, id) @doc """ - Deletes a document with the given id + An alias for `delete/2`, removes a document with the specified ID. """ - def delete(cmd, id), do: cmd |> delete_command(id) |> execute(:single) - def delete(cmd, pid, id) when is_pid(pid), do: cmd |> delete_command(id) |> execute(pid) + def remove(%DocumentCommand{} = cmd, pid, id) when is_pid(pid), do: delete(cmd, pid, id) + @doc """ - An alias for `delete/1`, removes a document based on the filter setup. + Deletes a document based on the filter (if any) """ - def remove(cmd), do: delete(cmd) - def remove(cmd, pid) when is_pid(pid), do: delete(cmd, pid) - + def delete(%DocumentCommand{} = cmd), do: cmd |> delete_command |> execute @doc """ Deletes a document based on the filter (if any) """ - def delete(cmd), do: cmd |> delete_command |> execute - def delete(cmd, pid) when is_pid(pid), do: cmd |> delete_command |> execute(pid) + def delete(%DocumentCommand{} = cmd, pid) when is_pid(pid), do: cmd |> delete_command |> execute(pid) + @doc """ + Deletes a document with the given id + """ + def delete(%DocumentCommand{} = cmd, id), do: cmd |> delete_command(id) |> execute(:single) + @doc """ + Deletes a document with the given id + """ + def delete(%DocumentCommand{} = cmd, pid, id) when is_pid(pid), do: cmd |> delete_command(id) |> execute(pid) @doc """ @@ -332,7 +345,7 @@ defmodule Moebius.DocumentQuery do |> first ``` """ - def first(cmd) do + def first(%DocumentCommand{} = cmd) do res = cmd |> select_command |> execute(:single) @@ -353,7 +366,7 @@ defmodule Moebius.DocumentQuery do |> find(12) ``` """ - def find(cmd, id) when is_integer id do + def find(%DocumentCommand{} = cmd, id) when is_integer id do #no need to param this, it's an integer sql = "select id, #{cmd.json_field}::text from #{cmd.table_name} where id=#{id}" %{cmd | sql: sql} |> execute(:single) @@ -370,7 +383,7 @@ defmodule Moebius.DocumentQuery do |> search("test.com") ``` """ - def search(cmd, term) when is_bitstring(term) do + def search(%DocumentCommand{} = cmd, term) when is_bitstring(term) do sql = """ select id, #{cmd.json_field}::text from #{cmd.table_name} @@ -394,7 +407,7 @@ defmodule Moebius.DocumentQuery do |> search(for: "test.com", in: [:email]) ``` """ - def search(cmd, for: term, in: fields) do + def search(%DocumentCommand{} = cmd, for: term, in: fields) do terms = Enum.map_join(fields, ", ' ', ", &"body -> '#{Atom.to_string(&1)}'") sql = """ @@ -410,9 +423,9 @@ defmodule Moebius.DocumentQuery do @doc """ Executes a query returning a list of items """ - def to_list(cmd), do: all(cmd) + def to_list(%DocumentCommand{} = cmd), do: all(cmd) - def all(cmd) do + def all(%DocumentCommand{} = cmd) do cmd |> select_command |> execute @@ -420,29 +433,30 @@ defmodule Moebius.DocumentQuery do @doc """ - Executes a query returning a single item + Executes a query returning a list of items. """ - def execute(cmd, :single) do + def execute(%DocumentCommand{} = cmd) do cmd |> Moebius.Runner.execute |> parse_json_column(cmd) - |> return_results(:single) + |> return_results() end + @doc """ - Executes a query returning a list of items. + Executes a query returning a single item """ - def execute(cmd) do + def execute(%DocumentCommand{} = cmd, :single) do cmd |> Moebius.Runner.execute |> parse_json_column(cmd) - |> return_results() + |> return_results(:single) end @doc """ Executes a query as part of a transaction returning a list of items """ - def execute(cmd, pid) do + def execute(%DocumentCommand{} = cmd, pid) when is_pid(pid) do cmd |> Moebius.Runner.execute(pid) |> parse_json_column(cmd) @@ -450,7 +464,7 @@ defmodule Moebius.DocumentQuery do end - defp update_command(cmd, change, id) when is_map(change) and is_integer(id) do + defp update_command(%DocumentCommand{} = cmd, change, id) when is_map(change) and is_integer(id) do {:ok, encoded} = JSON.encode(change) sql = """ update #{cmd.table_name} @@ -463,17 +477,17 @@ defmodule Moebius.DocumentQuery do - defp delete_command(cmd, id) when is_integer(id) do + defp delete_command(%DocumentCommand{} = cmd, id) when is_integer(id) do sql = "delete from #{cmd.table_name} where id=#{id} returning id, body::text" %{cmd | sql: sql, type: :delete} end - defp delete_command(cmd) do + defp delete_command(%DocumentCommand{} = cmd) do sql = "delete from #{cmd.table_name} #{cmd.where} returning id, body::text;" %{cmd | sql: sql, type: :delete} end - defp insert_command(cmd, doc) when is_bitstring(doc) do + defp insert_command(%DocumentCommand{} = cmd, doc) when is_bitstring(doc) do sql = """ insert into #{cmd.table_name}(#{cmd.json_field}) VALUES('#{doc}') @@ -482,19 +496,19 @@ defmodule Moebius.DocumentQuery do %{cmd | sql: sql, params: [doc], type: :insert} end - defp insert_command(cmd, doc) when is_list(doc) or is_map(doc) do + defp insert_command(%DocumentCommand{} = cmd, doc) when is_list(doc) or is_map(doc) do {:ok, encoded} = JSON.encode(doc) insert_command(cmd, encoded) end defp return_results({:error, err}), do: {:error, err} + defp return_results(results), do: results defp return_results([results], :single), do: results defp return_results(results, :single), do: results - defp return_results(results), do: results - defp parse_json_column({:error, err}, cmd), do: {:error, err} - defp parse_json_column({:ok, res}, cmd) do + defp parse_json_column({:error, err}, _), do: {:error, err} + defp parse_json_column({:ok, res}, _) do Enum.map(res.rows, &handle_row/1) end From 87a8f84843e1b1bb03217e024a79fe1c23c1f63f Mon Sep 17 00:00:00 2001 From: Paul Lamb Date: Sat, 26 Dec 2015 19:48:42 -0600 Subject: [PATCH 10/16] extract bulk_insert function, refactor insert_command/2 to seperate out concerns --- lib/moebius/query.ex | 34 ++++++++++++++++++---------------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/lib/moebius/query.ex b/lib/moebius/query.ex index 16a18be..f140757 100644 --- a/lib/moebius/query.ex +++ b/lib/moebius/query.ex @@ -403,20 +403,8 @@ defmodule Moebius.Query do result = db(:people) |> insert(data) ``` """ - def insert(%QueryCommand{} = cmd, [[hd | _] | _] = records) when is_tuple(hd) do - - # need a single definitive column map to arrest and roll back Tx if - # and of the inputs are malformed (different cols vs. vals) - column_map = records |> hd |> Keyword.keys - - transaction fn(pid) -> - cmd - |> bulk_insert_batch(records, [], column_map) - |> Enum.map(fn(cmd) -> execute(cmd, pid) end) - |> List.flatten - end - - end + def insert(%QueryCommand{} = cmd, [[hd | _] | _] = records) when is_tuple(hd), + do: bulk_insert(cmd, records) @doc """ A simple insert that that returns the inserted record. Create your list of data and send it on in. @@ -463,12 +451,26 @@ defmodule Moebius.Query do def insert_command(%QueryCommand{} = cmd, criteria) do cols = Keyword.keys(criteria) vals = Keyword.values(criteria) - sql = "insert into #{cmd.table_name}(" <> Enum.map_join(cols, ", ", &"#{&1}") <> ")" <> - " values(" <> Enum.map_join(1..length(cols), ", ", &"$#{&1}") <> ") returning *;" + column_names = Enum.map_join(cols,", ", &"#{&1}") + parameter_placeholders = Enum.map_join(1..length(cols), ", ", &"$#{&1}") + sql = "insert into #{cmd.table_name}(#{column_names}) values(#{parameter_placeholders}) returning *;" %{cmd | sql: sql, params: vals, type: :insert} end + defp bulk_insert(%QueryCommand{} = cmd, records) do + # need a single definitive column map to arrest and roll back Tx if + # and of the inputs are malformed (different cols vs. vals) + column_map = records |> hd |> Keyword.keys + + transaction fn(pid) -> + cmd + |> bulk_insert_batch(records, [], column_map) + |> Enum.map(fn(cmd) -> execute(cmd, pid) end) + |> List.flatten + end + end + defp bulk_insert_batch(%QueryCommand{} = cmd, records, acc, column_map) do # 20,000 seems to be the optimal number here. Technically you can go up to 34,464, but I think Postgrex imposes a lower limit, as I From 94b14ec56ac4e2eabf0816a561fb1ac36dc99044 Mon Sep 17 00:00:00 2001 From: Paul Lamb Date: Sat, 26 Dec 2015 20:34:38 -0600 Subject: [PATCH 11/16] seperate code from string interpolation --- lib/moebius/query.ex | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/moebius/query.ex b/lib/moebius/query.ex index f140757..76439ec 100644 --- a/lib/moebius/query.ex +++ b/lib/moebius/query.ex @@ -524,7 +524,7 @@ defmodule Moebius.Query do {filters, _count} = Enum.map_reduce cmd.where_columns, col_count, fn col, acc -> {"#{col} = $#{acc}", acc + 1} end - " where " <> Enum.join(filters, " and ") + "where " <> Enum.join(filters, " and ") cmd.where -> cmd.where end @@ -535,7 +535,9 @@ defmodule Moebius.Query do length(vals) > 0 -> vals end - sql = "update #{cmd.table_name} set " <> Enum.join(cols, ", ") <> where <> " returning *;" + columns = Enum.join(cols, ", ") + + sql = "update #{cmd.table_name} set #{columns} #{where} returning *;" %{cmd | sql: sql, type: :update, params: params} end From caf13336aa3b02d6b1e56d5db536d22b7a3a90ea Mon Sep 17 00:00:00 2001 From: Paul Lamb Date: Mon, 28 Dec 2015 15:02:31 -0600 Subject: [PATCH 12/16] redo the spaces so that tests pass --- lib/moebius/query.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/moebius/query.ex b/lib/moebius/query.ex index 76439ec..6b6ac98 100644 --- a/lib/moebius/query.ex +++ b/lib/moebius/query.ex @@ -524,7 +524,7 @@ defmodule Moebius.Query do {filters, _count} = Enum.map_reduce cmd.where_columns, col_count, fn col, acc -> {"#{col} = $#{acc}", acc + 1} end - "where " <> Enum.join(filters, " and ") + " where " <> Enum.join(filters, " and ") cmd.where -> cmd.where end @@ -537,7 +537,7 @@ defmodule Moebius.Query do columns = Enum.join(cols, ", ") - sql = "update #{cmd.table_name} set #{columns} #{where} returning *;" + sql = "update #{cmd.table_name} set #{columns}#{where} returning *;" %{cmd | sql: sql, type: :update, params: params} end From dab8d27b5bc8aeea3f4b6050027f228740d0b43a Mon Sep 17 00:00:00 2001 From: Paul Lamb Date: Mon, 28 Dec 2015 15:06:05 -0600 Subject: [PATCH 13/16] update dependencies to current, except for postgrex --- mix.exs | 6 +++--- mix.lock | 17 ++++++++++------- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/mix.exs b/mix.exs index 8296709..80091af 100644 --- a/mix.exs +++ b/mix.exs @@ -30,12 +30,12 @@ defmodule Moebius.Mixfile do defp deps do [{:postgrex, "~> 0.9.1"}, - {:timex, "~> 0.19.4"}, + {:timex, "~> 1.0.0-rc4"}, {:inflex, "~> 1.5.0"}, {:poison, "~> 1.5"}, {:json, "~> 0.3.0"}, - {:ex_doc, "~> 0.10", only: [:dev, :docs]}, - {:earmark, "~> 0.1.18", only: [:dev, :docs]}, + {:ex_doc, "~> 0.11.2", only: [:dev, :docs]}, + {:earmark, "~> 0.2.0", only: [:dev, :docs]}, {:credo, "~> 0.2", only: [:dev, :test]}] end diff --git a/mix.lock b/mix.lock index 179b546..03e505d 100644 --- a/mix.lock +++ b/mix.lock @@ -1,15 +1,18 @@ %{"bunt": {:hex, :bunt, "0.1.4"}, - "combine": {:hex, :combine, "0.5.2"}, + "certifi": {:hex, :certifi, "0.3.0"}, + "combine": {:hex, :combine, "0.6.0"}, + "connection": {:hex, :connection, "1.0.2"}, "credo": {:hex, :credo, "0.2.4"}, - "decimal": {:hex, :decimal, "1.1.0"}, - "earmark": {:hex, :earmark, "0.1.18"}, - "ex_doc": {:hex, :ex_doc, "0.10.0"}, - "hackney": {:hex, :hackney, "1.3.2"}, + "decimal": {:hex, :decimal, "1.1.1"}, + "earmark": {:hex, :earmark, "0.2.0"}, + "ex_doc": {:hex, :ex_doc, "0.11.2"}, + "hackney": {:hex, :hackney, "1.4.7"}, "idna": {:hex, :idna, "1.0.2"}, "inflex": {:hex, :inflex, "1.5.0"}, "json": {:hex, :json, "0.3.2"}, + "mimerl": {:hex, :mimerl, "1.0.2"}, "poison": {:hex, :poison, "1.5.0"}, "postgrex": {:hex, :postgrex, "0.9.1"}, "ssl_verify_hostname": {:hex, :ssl_verify_hostname, "1.0.5"}, - "timex": {:hex, :timex, "0.19.5"}, - "tzdata": {:hex, :tzdata, "0.5.4"}} + "timex": {:hex, :timex, "1.0.0-rc4"}, + "tzdata": {:hex, :tzdata, "0.5.6"}} From 1e08adba8d6cc3d2737cd3f8eb6db5f5d428c2bf Mon Sep 17 00:00:00 2001 From: Paul Lamb Date: Mon, 28 Dec 2015 15:57:42 -0600 Subject: [PATCH 14/16] update postgrex and fix a document query param bug --- lib/moebius/document_query.ex | 2 +- mix.exs | 2 +- mix.lock | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/moebius/document_query.ex b/lib/moebius/document_query.ex index 71d4a2e..056cac2 100644 --- a/lib/moebius/document_query.ex +++ b/lib/moebius/document_query.ex @@ -493,7 +493,7 @@ defmodule Moebius.DocumentQuery do VALUES('#{doc}') RETURNING id, #{cmd.json_field}::text; """ - %{cmd | sql: sql, params: [doc], type: :insert} + %{cmd | sql: sql, params: [], type: :insert} end defp insert_command(%DocumentCommand{} = cmd, doc) when is_list(doc) or is_map(doc) do diff --git a/mix.exs b/mix.exs index 80091af..93d3cdf 100644 --- a/mix.exs +++ b/mix.exs @@ -29,7 +29,7 @@ defmodule Moebius.Mixfile do end defp deps do - [{:postgrex, "~> 0.9.1"}, + [{:postgrex, "~> 0.10"}, {:timex, "~> 1.0.0-rc4"}, {:inflex, "~> 1.5.0"}, {:poison, "~> 1.5"}, diff --git a/mix.lock b/mix.lock index 03e505d..bc74990 100644 --- a/mix.lock +++ b/mix.lock @@ -12,7 +12,7 @@ "json": {:hex, :json, "0.3.2"}, "mimerl": {:hex, :mimerl, "1.0.2"}, "poison": {:hex, :poison, "1.5.0"}, - "postgrex": {:hex, :postgrex, "0.9.1"}, + "postgrex": {:hex, :postgrex, "0.10.0"}, "ssl_verify_hostname": {:hex, :ssl_verify_hostname, "1.0.5"}, "timex": {:hex, :timex, "1.0.0-rc4"}, "tzdata": {:hex, :tzdata, "0.5.6"}} From 7e7c771b83da34955017452d5d6dc688a68a2c4a Mon Sep 17 00:00:00 2001 From: Paul Lamb Date: Wed, 30 Dec 2015 19:39:21 -0600 Subject: [PATCH 15/16] update dependencies: timex 1.0 and credo 0.2.5 --- mix.exs | 4 ++-- mix.lock | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/mix.exs b/mix.exs index 93d3cdf..21febdb 100644 --- a/mix.exs +++ b/mix.exs @@ -30,13 +30,13 @@ defmodule Moebius.Mixfile do defp deps do [{:postgrex, "~> 0.10"}, - {:timex, "~> 1.0.0-rc4"}, + {:timex, "~> 1.0.0"}, {:inflex, "~> 1.5.0"}, {:poison, "~> 1.5"}, {:json, "~> 0.3.0"}, {:ex_doc, "~> 0.11.2", only: [:dev, :docs]}, {:earmark, "~> 0.2.0", only: [:dev, :docs]}, - {:credo, "~> 0.2", only: [:dev, :test]}] + {:credo, "~> 0.2.5", only: [:dev, :test]}] end def package do diff --git a/mix.lock b/mix.lock index bc74990..5f8893c 100644 --- a/mix.lock +++ b/mix.lock @@ -1,8 +1,8 @@ %{"bunt": {:hex, :bunt, "0.1.4"}, "certifi": {:hex, :certifi, "0.3.0"}, - "combine": {:hex, :combine, "0.6.0"}, + "combine": {:hex, :combine, "0.7.0"}, "connection": {:hex, :connection, "1.0.2"}, - "credo": {:hex, :credo, "0.2.4"}, + "credo": {:hex, :credo, "0.2.5"}, "decimal": {:hex, :decimal, "1.1.1"}, "earmark": {:hex, :earmark, "0.2.0"}, "ex_doc": {:hex, :ex_doc, "0.11.2"}, @@ -14,5 +14,5 @@ "poison": {:hex, :poison, "1.5.0"}, "postgrex": {:hex, :postgrex, "0.10.0"}, "ssl_verify_hostname": {:hex, :ssl_verify_hostname, "1.0.5"}, - "timex": {:hex, :timex, "1.0.0-rc4"}, + "timex": {:hex, :timex, "1.0.0"}, "tzdata": {:hex, :tzdata, "0.5.6"}} From a1e117b68eb9a7ea6acf7fa8f602df1523254935 Mon Sep 17 00:00:00 2001 From: Paul Lamb Date: Wed, 30 Dec 2015 19:51:47 -0600 Subject: [PATCH 16/16] remove json dependency and use poison --- lib/moebius/document_query.ex | 4 ++-- mix.exs | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/moebius/document_query.ex b/lib/moebius/document_query.ex index 056cac2..8264d67 100644 --- a/lib/moebius/document_query.ex +++ b/lib/moebius/document_query.ex @@ -465,7 +465,7 @@ defmodule Moebius.DocumentQuery do defp update_command(%DocumentCommand{} = cmd, change, id) when is_map(change) and is_integer(id) do - {:ok, encoded} = JSON.encode(change) + {:ok, encoded} = Poison.encode(change) sql = """ update #{cmd.table_name} set #{cmd.json_field} = '#{encoded}' @@ -497,7 +497,7 @@ defmodule Moebius.DocumentQuery do end defp insert_command(%DocumentCommand{} = cmd, doc) when is_list(doc) or is_map(doc) do - {:ok, encoded} = JSON.encode(doc) + {:ok, encoded} = Poison.encode(doc) insert_command(cmd, encoded) end diff --git a/mix.exs b/mix.exs index 21febdb..48dd24b 100644 --- a/mix.exs +++ b/mix.exs @@ -33,7 +33,6 @@ defmodule Moebius.Mixfile do {:timex, "~> 1.0.0"}, {:inflex, "~> 1.5.0"}, {:poison, "~> 1.5"}, - {:json, "~> 0.3.0"}, {:ex_doc, "~> 0.11.2", only: [:dev, :docs]}, {:earmark, "~> 0.2.0", only: [:dev, :docs]}, {:credo, "~> 0.2.5", only: [:dev, :test]}]