diff --git a/.credo.exs b/.credo.exs index ce2973c..dbdab20 100644 --- a/.credo.exs +++ b/.credo.exs @@ -109,7 +109,7 @@ {Credo.Check.Readability.RedundantBlankLines, []}, {Credo.Check.Readability.Semicolons, []}, {Credo.Check.Readability.SpaceAfterCommas, []}, - # {Credo.Check.Readability.Specs, []}, + {Credo.Check.Readability.Specs, []}, {Credo.Check.Readability.StringSigils, []}, {Credo.Check.Readability.TrailingBlankLine, []}, {Credo.Check.Readability.TrailingWhiteSpace, []}, diff --git a/README.md b/README.md index aa6cacb..4724abc 100644 --- a/README.md +++ b/README.md @@ -46,12 +46,12 @@ if the code it depends on, or the example itself, have not been changed. When the code changes, the example is executed again. -## Tests +## Tests -The examples are created to work with the code base, but they can also serve as a unit test. +The examples are created to work with the code base, but they can also serve as a unit test. To let ExUnit use the examples in your codebase as tests, add a test file in the `test/` folder, and -import the `ExExample.Test` module. +import the `ExExample.Test` module. To run the examples from above, add a file `ny_examples_test.exs` to your `test/` folder and include the following. diff --git a/lib/ex_example.ex b/lib/ex_example.ex index 295606e..8caa4c6 100644 --- a/lib/ex_example.ex +++ b/lib/ex_example.ex @@ -3,17 +3,111 @@ defmodule ExExample do Documentation for `ExExample`. """ alias ExExample.Analyze + alias ExExample.Cache alias ExExample.Executor + ############################################################ + # Types # + ############################################################ + + @typedoc """ + A dependency is a function that will be called by an example. + The format of a dependency is `{{module, function}, arity}` + """ + @type dependency :: {{atom(), atom()}, non_neg_integer()} + + @typedoc """ + """ + @type example :: {atom(), list(dependency)} + + ############################################################ + # Helpers # + ############################################################ + + @doc """ + I return the hidden name of an example. + The hidden name is the example body without modification. + """ + @spec hidden_name({atom(), atom()}) :: {atom(), atom()} + def hidden_name({module, func}) do + {module, String.to_atom("__#{func}__")} + end + + @doc """ + I determine if a module/function pair is an example or not. + + A function is an example if it is defined in a module that has the `__examples__/0` function + implemented, and when the `__examples__()` output lists that function name as being an example. + """ + @spec example?(dependency()) :: boolean() + def example?({{module, func}, _arity}) do + example_module?(module) and Keyword.has_key?(module.__examples__(), func) + end + + @doc """ + I return true if the given module contains examples. + """ + @spec example_module?(atom()) :: boolean + def example_module?(module) do + {:__examples__, 0} in module.__info__(:functions) + end + + @doc """ + I return a list of all dependencies for this example. + Note: this does includes other called modules too (e.g., Enum). + """ + @spec all_dependencies({atom(), atom()}) :: [dependency()] + def all_dependencies({module, func}) do + module.__examples__() + |> Keyword.get(func, []) + end + + @doc """ + I return a list of example dependencies for this example. + Note: this does not include other called modules. + """ + @spec example_dependencies({atom(), atom()}) :: [dependency()] + def example_dependencies({module, func}) do + all_dependencies({module, func}) + |> Enum.filter(&example?/1) + end + + @doc """ + I return a list of examples in the order they should be + executed in. + + I do this by topologically sorting their execution order. + """ + @spec execution_order(atom()) :: [{atom(), atom()}] + def execution_order(module) do + module.__examples__() + |> Enum.reduce(Graph.new(), fn + {function, []}, g -> + Graph.add_vertex(g, {__MODULE__, function}) + + {function, dependencies}, g -> + dependencies + # filter out all non-example dependencies + |> Enum.filter(&example?/1) + |> Enum.reduce(g, fn {{module, func}, _arity}, g -> + Graph.add_edge(g, {module, func}, {module, function}) + end) + end) + |> Graph.topsort() + end + + ############################################################ + # Macros # + ############################################################ + defmacro __using__(_options) do quote do import unquote(__MODULE__) + @behaviour ExExample.Behaviour + # module attribute that holds all the examples - Module.register_attribute(__MODULE__, :example_dependencies, accumulate: true) Module.register_attribute(__MODULE__, :examples, accumulate: true) - Module.register_attribute(__MODULE__, :copies, accumulate: true) - Module.register_attribute(__MODULE__, :copy, accumulate: false) @before_compile unquote(__MODULE__) end @@ -21,61 +115,8 @@ defmodule ExExample do defmacro __before_compile__(_env) do quote do - @doc """ - I return a list of all the dependencies for a given example, - or the list of all dependencies if no argument is given. - """ - def __example_dependencies__, do: @example_dependencies - - def __example_dependencies__(dependee) do - @example_dependencies - |> Enum.find({nil, []}, fn {name, _} -> name == dependee end) - |> elem(1) - end - - @doc """ - I reutrn all the examples in this module. - """ - def __examples__ do - @examples - end - - @doc """ - I run all the examples in this module. - """ - def __run_examples__ do - __sorted__() - |> Enum.each(fn {module, name} -> - apply(module, name, []) - end) - end - - @doc """ - I return a topologically sorted list of examples. - This list is the order in which the examples should be run. - """ - @spec __sorted__() :: list({atom(), atom()}) - def __sorted__ do - __example_dependencies__() - |> Enum.reduce(Graph.new(), fn - {example, []}, g -> - Graph.add_vertex(g, {__MODULE__, example}) - - {example, dependencies}, g -> - dependencies - # filter out all non-example dependencies - |> Enum.filter(&Executor.example?/1) - |> Enum.reduce(g, fn {{module, func}, _arity}, g -> - Graph.add_edge(g, {module, func}, {__MODULE__, example}) - end) - end) - |> Graph.topsort() - end - - def __example_copy__(example_name) do - @copies - |> Keyword.get(example_name, nil) - end + @spec __examples__ :: [ExExample.example()] + def __examples__, do: @examples end end @@ -91,24 +132,22 @@ defmodule ExExample do hidden_example_name = String.to_atom("__#{example_name}__") quote do - # fetch the attribute value, and then clear it for the next examples. - example_copy_tag = Module.get_attribute(unquote(__CALLER__.module), :copy) - Module.delete_attribute(unquote(__CALLER__.module), :copy) - def unquote({hidden_example_name, context, args}) do unquote(body) end - @copies {unquote(example_name), {unquote(__CALLER__.module), example_copy_tag}} - @example_dependencies {unquote(example_name), unquote(called_functions)} - @examples unquote(example_name) + @examples {unquote(example_name), unquote(called_functions)} def unquote(name) do - example_dependencies = __example_dependencies__(unquote(example_name)) - example_copy = __example_copy__(unquote(example_name)) + case Executor.attempt_example({__MODULE__, unquote(example_name)}, []) do + %{result: %Cache.Result{success: :success} = result} -> + result.result + + %{result: %Cache.Result{success: :failed} = result} -> + raise result.result - Executor.maybe_run_example(__MODULE__, unquote(example_name), example_dependencies, - copy: example_copy - ) + %{result: %Cache.Result{success: :skipped} = result} -> + :skipped + end end end end diff --git a/lib/ex_example/analyzer/analyze.ex b/lib/ex_example/analyzer/analyze.ex index ce8f391..85380fb 100644 --- a/lib/ex_example/analyzer/analyze.ex +++ b/lib/ex_example/analyzer/analyze.ex @@ -13,29 +13,21 @@ defmodule ExExample.Analyze do """ defstruct called_functions: [], env: nil, functions: [] + @spec put_call(map(), {atom(), atom()}, non_neg_integer()) :: map() def put_call(state, mod, arg) do %{state | called_functions: [{mod, arg} | state.called_functions]} end + @spec put_def(map(), atom(), non_neg_integer()) :: map() def put_def(state, func, arity) do %{state | functions: [{func, arity} | state.functions]} end end - # ---------------------------------------------------------------------------- - # Compute hash of all modules that the example depends on - - def compile_dependency_hash(dependencies) do - dependencies - |> Enum.map(fn {{module, _func}, _arity} -> - module.__info__(:attributes)[:vsn] - end) - |> :erlang.phash2() - end - # ---------------------------------------------------------------------------- # Exctract function calls from ast + @spec extract_function_calls(tuple(), Macro.Env.t()) :: [{{atom(), atom()}, non_neg_integer()}] def extract_function_calls(ast, env) do state = %State{env: env} # IO.inspect(env) @@ -52,6 +44,7 @@ defmodule ExExample.Analyze do # qualified function call # e.g., Foo.bar() + defp extract_function_call( {{:., _, [{:__aliases__, _, aliases}, func_name]}, _, args} = ast, state @@ -70,6 +63,10 @@ defmodule ExExample.Analyze do end end + defp extract_function_call({{:., _, _args}, _, _} = ast, state) do + {ast, state} + end + # variable in binding # e.g. `x` in `x = 1` defp extract_function_call({_func, _, nil} = ast, state) do diff --git a/lib/ex_example/behaviour.ex b/lib/ex_example/behaviour.ex new file mode 100644 index 0000000..28ca9b1 --- /dev/null +++ b/lib/ex_example/behaviour.ex @@ -0,0 +1,12 @@ +defmodule ExExample.Behaviour do + @moduledoc """ + I help determine when Examples ought to be run again or be copied + + + I do this by defining out a behaviour that is to be used with the + use macro for ExExample + """ + + @callback rerun?(any()) :: boolean() + @callback copy(any()) :: any() +end diff --git a/lib/ex_example/cache.ex b/lib/ex_example/cache.ex deleted file mode 100644 index e69de29..0000000 diff --git a/lib/ex_example/cache/cache.ex b/lib/ex_example/cache/cache.ex index 7c4a3e5..849aeb6 100644 --- a/lib/ex_example/cache/cache.ex +++ b/lib/ex_example/cache/cache.ex @@ -10,6 +10,15 @@ defmodule ExExample.Cache do @cache_name __MODULE__ + @doc """ + I clear the entire cache. + """ + @spec clear() :: :ok + def clear do + Cachex.clear!(@cache_name) + :ok + end + @doc """ I store a result in cache for a given key. """ @@ -22,7 +31,7 @@ defmodule ExExample.Cache do I fetch a previous Result from the cache if it exists. If it does not exist, I return `{:error, :not_found}`. """ - @spec get_result(Key.t()) :: {:ok, any()} | {:error, :no_result} + @spec get_result(Key.t()) :: {:ok, Result.t()} | {:error, :no_result} def get_result(%Key{} = key) do case Cachex.get(@cache_name, key) do {:ok, nil} -> diff --git a/lib/ex_example/cache/result.ex b/lib/ex_example/cache/result.ex index 558b085..4d71b60 100644 --- a/lib/ex_example/cache/result.ex +++ b/lib/ex_example/cache/result.ex @@ -15,5 +15,6 @@ defmodule ExExample.Cache.Result do field(:key, Key.t()) field(:success, :failed | :success | :skipped) field(:result, term()) + field(:cached, boolean(), default: true) end end diff --git a/lib/ex_example/executor.ex b/lib/ex_example/executor.ex index 38dadc6..b2ed6cd 100644 --- a/lib/ex_example/executor.ex +++ b/lib/ex_example/executor.ex @@ -5,20 +5,154 @@ defmodule ExExample.Executor do I contain logic to determine if a cachd result should be used, computation should be done again, or if an example should be skipped. """ - alias ExExample.Cache require Logger - @type dependency :: {{atom(), atom()}, non_neg_integer()} + + alias ExExample.Cache + alias ExExample.Run + + ############################################################ + # API # + ############################################################ + + @spec print_dependencies(ExExample.Run.t()) :: binary() + def print_dependencies(run) do + output = + if run.success != [] do + run.success + |> Enum.map_join( + ", ", + fn {{mod, func}, _arity} -> " 🟢 #{inspect(mod)}.#{Atom.to_string(func)}" end + ) + else + "" + end + + output = + if run.failed != [] do + run.success + |> Enum.map_join( + ", ", + fn {{mod, func}, _arity} -> + " 🔴 #{inspect(mod)}.#{Atom.to_string(func)}" + end + ) + |> Kernel.<>(output) + else + output + end + + output = + if run.no_cache != [] do + run.success + |> Enum.map_join( + ", ", + fn {{mod, func}, _arity} -> + " ⚪️ #{inspect(mod)}.#{Atom.to_string(func)}" + end + ) + |> Kernel.<>(output) + else + output + end + + output = + if run.skipped != [] do + run.success + |> Enum.map_join( + ", ", + fn {{mod, func}, _arity} -> + " ⚪️ #{inspect(mod)}.#{Atom.to_string(func)}" + end + ) + |> Kernel.<>(output) + else + output + end + + if output == "", do: "", else: "\n" <> output + end + + @spec print_run(ExExample.Run.t()) :: :ok + def print_run(%Run{result: %Cache.Result{success: :success} = result} = run) do + cached = if result.cached, do: "(cached) ", else: "" + + IO.puts(""" + 🟢 #{cached}#{inspect(run.key.module)}.#{Atom.to_string(run.key.function)}\ + #{print_dependencies(run)}\ + """) + + :ok + end + + def print_run(%Run{result: %Cache.Result{success: :skipped} = result} = run) do + cached = if result.cached, do: "(cached) ", else: "" + + IO.puts(""" + ⚪️ #{cached}#{inspect(run.key.module)}.#{Atom.to_string(run.key.function)}\ + #{print_dependencies(run)}\ + """) + + :ok + end + + def print_run(%Run{result: %Cache.Result{success: :failed} = result} = run) do + cached = if result.cached, do: "(cached) ", else: "" + + IO.puts(""" + 🔴 #{cached}#{inspect(run.key.module)}.#{Atom.to_string(run.key.function)}\ + #{print_dependencies(run)}\ + """) + + :ok + end + + @spec pretty_run(atom()) :: :ok + def pretty_run(module) do + module + |> ExExample.execution_order() + |> Enum.map(&attempt_example(&1, [])) + |> Enum.each(&print_run/1) + + :ok + end @doc """ - I determine if a module/function pair is an example or not. + I return the last known result of an example invocation. + If the example has not been run yet I return an error. + """ + @spec last_result(ExExample.dependency()) :: :success | :skipped | :failed | :no_cache + def last_result({{module, func}, _arity}) do + deps_hash = dependency_hash({module, func}) + + key = %Cache.Key{module: module, function: func, arguments: [], deps_hash: deps_hash} + + case Cache.get_result(key) do + {:ok, result} -> + result.success - A function is an example if it is defined in a module that has the `__examples__/0` function - implemented, and when the `__examples__()` output lists that function name as being an example. + {:error, :no_result} -> + :no_cache + end + end + + @doc """ + Given an example, I return a map of all its dependencies + that failed, succeeded, were skipped, or have not run yet. """ - @spec example?(dependency) :: boolean() - def example?({{module, func}, _arity}) do - {:__examples__, 0} in module.__info__(:functions) and func in module.__examples__() + @spec dependency_results({atom(), atom()}) :: %{ + success: [ExExample.dependency()], + skipped: [ExExample.dependency()], + failed: [ExExample.dependency()], + no_cache: [ExExample.dependency()] + } + def dependency_results({module, func}) do + results = + {module, func} + |> ExExample.example_dependencies() + |> Enum.group_by(&last_result/1) + + Map.merge(%{success: [], skipped: [], failed: [], no_cache: []}, results) end @doc """ @@ -26,97 +160,120 @@ defmodule ExExample.Executor do This hash can be used to determine of an example was run with an older version of a dependency. """ - def deps_hash(dependencies) do - dependencies + @spec dependency_hash({atom(), atom()}) :: non_neg_integer() + def dependency_hash({module, func}) do + {module, func} + |> ExExample.all_dependencies() |> Enum.map(fn {{module, _func}, _arity} -> - module.__info__(:attributes)[:vsn] + {module, module.__info__(:attributes)[:vsn]} end) + |> Enum.uniq() |> :erlang.phash2() end @doc """ - I run an example, iff all its dependencies have succeeded. + I run all the examples in the given module. + I use the cache for each invocation. + """ + @spec run_all_examples(atom()) :: [Run.t()] + def run_all_examples(module) do + module + |> ExExample.execution_order() + |> Enum.map(&attempt_example(&1, [])) + end - If all the dependencies of this example executed succesfully, - I will execute the example. + @doc """ + I attempt to run an example. - If any of the example its dependencies either failed or were skipped, - I will skip the example. + I return a struct that holds the result, the key, and a list of all + the dependencies and their previous result. """ - @spec maybe_run_example(atom(), atom(), list(dependency), Keyword.t()) :: any() - def maybe_run_example(module, func, dependencies, copy: copy) do - dependency_results = - dependencies - |> Enum.map(fn {{module, func}, _arity} -> - Cache.state({module, func}) - end) - |> Enum.group_by(& &1) - |> Map.put_new(:success, []) - |> Map.put_new(:skipped, []) - |> Map.put_new(:failed, []) - - deps_hash = deps_hash(dependencies) - - case dependency_results do - %{success: _, failed: [], skipped: []} -> - # check for a cached result - case Cache.get_result(%Cache.Key{module: module, function: func, deps_hash: deps_hash}) do - # cached result, no recompile - {:ok, result} -> - Logger.debug("found cached result for #{inspect(module)}.#{func}") - copy_result(result, copy).result - - {:error, :no_result} -> - Logger.debug("running #{inspect(module)}.#{func} for the first time") - hidden_example_name = String.to_atom("__#{func}__") - - run_example(module, hidden_example_name, [], func, deps_hash) - |> Map.get(:result) - end + @spec attempt_example({atom(), atom()}, [any()]) :: Run.t() + def attempt_example({module, func}, arguments) do + deps_hash = dependency_hash({module, func}) + key = %Cache.Key{module: module, function: func, arguments: arguments, deps_hash: deps_hash} - map -> - Logger.warning( - "skipping #{inspect(module)}.#{func} due to failed or skipped dependencies" - ) + case dependency_results({module, func}) do + # no failures, only no cache or success + %{failed: [], skipped: [], no_cache: no_cache, success: success} -> + result = run_example_with_cache({module, func}, arguments) + %Run{key: key, result: result, no_cache: no_cache, success: success} + + # failures and/or skipped + %{failed: failed, skipped: skipped, no_cache: no_cache, success: success} -> + result = %Cache.Result{key: key, success: :skipped, result: nil, cached: false} + Cache.put_result(result, key) + + %Run{ + key: key, + result: result, + no_cache: no_cache, + success: success, + failed: failed, + skipped: skipped + } + end + end + + @doc """ + I run an example with the cached results. + If there is cached result, I return that. + If there is no result in the cache I run the example. + """ + @spec run_example_with_cache({atom(), atom()}, [any()]) :: Cache.Result.t() + def run_example_with_cache({module, func}, arguments) do + deps_hash = dependency_hash({module, func}) + key = %Cache.Key{module: module, function: func, arguments: arguments, deps_hash: deps_hash} - {:error, :skipped_or_failed, map} + case Cache.get_result(key) do + {:ok, result} -> + if module.rerun?(result.result) do + run_example({module, func}, arguments) + else + %{result | result: module.copy(result.result)} + end + + {:error, :no_result} -> + run_example({module, func}, arguments) end end - # @doc """ - # I run an example in a module and wrap its output in - # something that can be cached. - # """ - @spec run_example(atom(), atom(), list(term()), atom(), any()) :: Cache.Result.t() - defp run_example(module, func, arguments, example_name, deps_hash) do - key = %Cache.Key{ - module: module, - function: example_name, - arguments: arguments, - deps_hash: deps_hash - } + @doc """ + I run an example directly. I do not consult the cache for a previous result. + I return a result of this execution and put it in the cache. + """ + @spec run_example({atom(), atom()}, [any()]) :: Cache.Result.t() + def run_example({module, func}, arguments) do + deps_hash = dependency_hash({module, func}) + key = %Cache.Key{module: module, function: func, arguments: arguments, deps_hash: deps_hash} result = try do - %Cache.Result{key: key, success: :success, result: apply(module, func, [])} + {module, func} = ExExample.hidden_name({module, func}) + result = apply(module, func, arguments) + %Cache.Result{key: key, success: :success, result: result} rescue e -> - Logger.error(inspect(e)) %Cache.Result{key: key, success: :failed, result: e} end - # put the result of this invocation in the cache. + # store the result in the cache Cache.put_result(result, key) - result + %{result | cached: false} end - # @doc """ - # Given a result from a previous invocation and a copy function, I create a copy of the result. - # """ - defp copy_result(%Cache.Result{} = result, {_, nil}), do: result - - defp copy_result(%Cache.Result{} = result, {module, func}) do - %{result | result: apply(module, func, [result.result])} + @doc """ + Given an example, I return a hash of all its dependencies. + This hash can be used to determine of an example was run with + an older version of a dependency. + """ + @spec deps_hash(list(ExExample.dependency())) :: non_neg_integer() + def deps_hash(dependencies) do + dependencies + |> Enum.map(fn {{module, _func}, _arity} -> + module.__info__(:attributes)[:vsn] + end) + |> :erlang.phash2() end end diff --git a/lib/ex_example/run.ex b/lib/ex_example/run.ex new file mode 100644 index 0000000..a329a4f --- /dev/null +++ b/lib/ex_example/run.ex @@ -0,0 +1,23 @@ +defmodule ExExample.Run do + @moduledoc """ + I am the result of running an example. + + I contain meta-data about this particular invocation such as + whether the example was found in cache, the state of its dependencies, + and the key. + """ + alias ExExample.Cache.Key + alias ExExample.Cache.Result + + use TypedStruct + + typedstruct do + field(:cached, boolean(), default: true) + field(:key, Key.t()) + field(:result, Result.t()) + field(:skipped, [ExExample.dependency()], default: []) + field(:failed, [ExExample.dependency()], default: []) + field(:no_cache, [ExExample.dependency()], default: []) + field(:success, [ExExample.dependency()], default: []) + end +end diff --git a/lib/ex_example/tests.ex b/lib/ex_example/tests.ex new file mode 100644 index 0000000..4b5214e --- /dev/null +++ b/lib/ex_example/tests.ex @@ -0,0 +1,25 @@ +defmodule ExExample.Tests do + @moduledoc """ + I generate a test for the given module that runs all of its examples. + """ + defmacro __using__(for: module) do + alias ExExample.Cache + + module_to_test = Macro.expand(module, __CALLER__) + examples = ExExample.execution_order(module_to_test) + + for {mod, func} <- examples do + quote do + test "#{inspect(unquote(mod))}.#{Atom.to_string(unquote(func))}" do + case ExExample.Executor.attempt_example({unquote(mod), unquote(func)}, []) do + %{result: %Cache.Result{success: :failed} = result} -> + raise result.result + + _ -> + :ok + end + end + end + end + end +end diff --git a/lib/examples/stack.ex b/lib/examples/stack.ex index 6861f7a..a837d64 100644 --- a/lib/examples/stack.ex +++ b/lib/examples/stack.ex @@ -11,9 +11,9 @@ defmodule Stack do field(:elements, [any()], default: []) end - @spec create() :: {:ok, t()} - def create do - {:ok, %Stack{}} + @spec create([any()]) :: {:ok, t()} + def create(xs \\ []) do + {:ok, %Stack{elements: xs}} end # yesyesyes diff --git a/lib/examples/stack_examples.ex b/lib/examples/stack_examples.ex index 31e0ed6..b0c1945 100644 --- a/lib/examples/stack_examples.ex +++ b/lib/examples/stack_examples.ex @@ -1,4 +1,5 @@ defmodule Examples.Stack do + # all exmaples must return a Stack @moduledoc """ I contain examples that test the `Stack` implementation. """ @@ -6,21 +7,22 @@ defmodule Examples.Stack do import ExUnit.Assertions - @copy :do_copy example new_stack do - {:ok, stack} = Stack.create() - assert stack == %Stack{} + {:ok, stack} = Stack.create([]) + assert stack == %Stack{elements: []} stack end example empty_stack_should_be_empty do stack = new_stack() assert Stack.empty?(stack) + stack end example push_stack do stack = new_stack() {:ok, stack} = Stack.push(stack, 1) + assert stack == %Stack{elements: [1]} stack end @@ -30,7 +32,11 @@ defmodule Examples.Stack do stack end - def do_copy(stack) do + @spec copy(any()) :: Stack.t() + def copy(stack) do %Stack{elements: stack.elements} end + + @spec rerun?(any()) :: boolean() + def rerun?(_), do: false end diff --git a/test/stack_test.exs b/test/stack_test.exs index 73a60c9..aabe1bd 100644 --- a/test/stack_test.exs +++ b/test/stack_test.exs @@ -1,8 +1,5 @@ defmodule StackTest do use ExUnit.Case doctest ExExample - - test "greets the world" do - Examples.Stack.__run_examples__() - end + use ExExample.Tests, for: Examples.Stack end