From 854d99f7d48265f74fe30b75b9f243b0554cda18 Mon Sep 17 00:00:00 2001 From: Sovetnik <5460740+sovetnik@users.noreply.github.com> Date: Sat, 3 Aug 2024 17:59:53 +0300 Subject: [PATCH] Introduce Clone task --- .tool-versions | 2 +- README.md | 43 ++++++-- lib/mix/tasks/clone.ex | 79 +++++++++++++ lib/mix/tasks/dump.ex | 2 +- lib/umwelt/client.ex | 11 ++ lib/umwelt/client/agent.ex | 93 ++++++++++++++++ lib/umwelt/client/application.ex | 15 +++ lib/umwelt/client/clone.ex | 95 ++++++++++++++++ lib/umwelt/client/fetcher.ex | 24 ++++ lib/umwelt/client/request.ex | 49 +++++++++ lib/umwelt/client/writer.ex | 41 +++++++ mix.exs | 11 +- mix.lock | 13 +++ test/mix/tasks/clone_test.exs | 93 ++++++++++++++++ test/mix/tasks/dump_test.exs | 11 +- test/umwelt/client/agent_test.exs | 165 ++++++++++++++++++++++++++++ test/umwelt/client/request_test.exs | 51 +++++++++ test/umwelt/client/writer_test.exs | 80 ++++++++++++++ test/umwelt/parser_test.exs | 12 ++ 19 files changed, 872 insertions(+), 18 deletions(-) create mode 100644 lib/mix/tasks/clone.ex create mode 100644 lib/umwelt/client.ex create mode 100644 lib/umwelt/client/agent.ex create mode 100644 lib/umwelt/client/application.ex create mode 100644 lib/umwelt/client/clone.ex create mode 100644 lib/umwelt/client/fetcher.ex create mode 100644 lib/umwelt/client/request.ex create mode 100644 lib/umwelt/client/writer.ex create mode 100644 test/mix/tasks/clone_test.exs create mode 100644 test/umwelt/client/agent_test.exs create mode 100644 test/umwelt/client/request_test.exs create mode 100644 test/umwelt/client/writer_test.exs diff --git a/.tool-versions b/.tool-versions index 94515c4..7879e57 100644 --- a/.tool-versions +++ b/.tool-versions @@ -1,2 +1,2 @@ elixir 1.17.2 -erlang 27.0 +erlang 27.0.1 diff --git a/README.md b/README.md index 3b7d7e0..8d14d53 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,6 @@ # Umwelt ![Umwelt CI](https://github.com/sovetnik/umwelt/actions/workflows/elixir.yml/badge.svg?event=push) [![wakatime](https://wakatime.com/badge/user/7542de1a-027f-4ed7-bc4b-c31d4cf9aa2a/project/018c9f92-bb93-4303-816f-bc0799a61194.svg)](https://wakatime.com/badge/user/7542de1a-027f-4ed7-bc4b-c31d4cf9aa2a/project/018c9f92-bb93-4303-816f-bc0799a61194) Client for [umwelt.dev](https://umwelt.dev) -## Implemented actions: - -### Dump - -Extracts Umwelt from Elixir project and dumps it into `root_name.bin` - ## Installation [available in Hex](https://hex.pm/packages/umwelt), the package can be installed @@ -22,24 +16,53 @@ end ## Usage -Right now it is a proof of concept, and in this version parser can parse some business-logic related code, via `mix dump`. +### Dump + +Extracts Umwelt from Elixir project and dumps it into `root_name.bin` In common case, when you want to parse your project and it's name from `Mix.Project.config()[:app]` matches root folder name `lib/root_name`, use: ```bash - mix dump + mix umwelt.dump ``` When you wanna parse another folder in lib, `lib/another_root_name`, use: ```bash - mix dump another_root_name + mix umwelt.dump another_root_name +``` + +### Clone + +Fetch and write all modules from specified phase. + +When your project is ready, you can get its code and specs. +Create a new elixir or phoenix app, add umwelt and pull the code. +```bash + mix new project_name + cd project_name ``` +add umwelt to deps in `mix.exs`. +You have to obtain a token on [profile page](https://umwelt.dev/auth/profile) +```bash + export UMWELT_TOKEN="your_token" + mix umwelt.clone phase_id + mix test --trace +``` +And now you will see all messages for failing tests and can start coding. + +If you want to reduce logger add this to your `config.exs` +``` + config :logger, + compile_time_purge_matching: [ + [application: :umwelt, level_lower_than: :warning] + ] +``` ## Planned Here is the list of planned features: ### Client functions -Set of push/pull/sync mix tasks to sync local code with remote representation on [umwelt.dev](https://umwelt.dev) +Set of pull/push/sync mix tasks to sync local code with remote representation on [umwelt.dev](https://umwelt.dev) ### Unparser Tools for update local code with changes made on web side. diff --git a/lib/mix/tasks/clone.ex b/lib/mix/tasks/clone.ex new file mode 100644 index 0000000..b5e9d17 --- /dev/null +++ b/lib/mix/tasks/clone.ex @@ -0,0 +1,79 @@ +defmodule Mix.Tasks.Umwelt.Clone do + @moduledoc "Clones phase modules and code" + @shortdoc "The code puller" + use Mix.Task + require Logger + + @impl Mix.Task + def run([phase_id]) do + case System.get_env("UMWELT_TOKEN", "no_token") do + "no_token" -> + """ + Token not found in env! + You can get it on umwelt.dev/auth/profile and do + + export UMWELT_TOKEN="token" + + or pass it directly in + + mix clone phase_id "token" + + """ + |> Logger.warning() + + token -> + run([phase_id, token]) + end + end + + @impl Mix.Task + def run([phase_id, token]) do + Umwelt.Client.Application.start(nil, nil) + + Umwelt.Client.Supervisor + |> Process.whereis() + |> Process.monitor() + + %{phase_id: phase_id, token: token} + |> assign_host() + |> assign_port() + |> Umwelt.Client.pull() + + receive do + {:DOWN, _, :process, _, _} -> + Logger.info("Done!") + + other -> + Logger.warning(inspect(other)) + end + end + + defp assign_host(params) do + host = + case Mix.env() do + :dev -> + System.get_env("UMWELT_HOST", "https://umwelt.dev") + + :test -> + "http://localhost" + end + + Map.put(params, :api_host, host) + end + + defp assign_port(params) do + port = + case Mix.env() do + :dev -> + case params.api_host do + "http://localhost" -> 4000 + "https://umwelt.dev" -> 443 + end + + :test -> + Application.get_env(:umwelt, :api_port) + end + + Map.put(params, :port, port) + end +end diff --git a/lib/mix/tasks/dump.ex b/lib/mix/tasks/dump.ex index 05e3fb5..54162dd 100644 --- a/lib/mix/tasks/dump.ex +++ b/lib/mix/tasks/dump.ex @@ -1,4 +1,4 @@ -defmodule Mix.Tasks.Dump do +defmodule Mix.Tasks.Umwelt.Dump do @moduledoc "This task for self-parse umwelt" @shortdoc "The lib parser" diff --git a/lib/umwelt/client.ex b/lib/umwelt/client.ex new file mode 100644 index 0000000..6d0514e --- /dev/null +++ b/lib/umwelt/client.ex @@ -0,0 +1,11 @@ +defmodule Umwelt.Client do + @moduledoc "Client for umwelt.dev" + + alias Umwelt.Client.Clone + + require Logger + + def pull(params) do + GenServer.cast(Clone, {:pull, params}) + end +end diff --git a/lib/umwelt/client/agent.ex b/lib/umwelt/client/agent.ex new file mode 100644 index 0000000..078b585 --- /dev/null +++ b/lib/umwelt/client/agent.ex @@ -0,0 +1,93 @@ +defmodule Umwelt.Client.Agent do + @moduledoc "Keeps pulling metadata" + + use Agent + require Logger + + def start_link(_args) do + Agent.start_link( + fn -> + %{ + modules: %{}, + waiting: [], + fetching: [], + fetched: [], + writing: [], + written: [], + total: 0 + } + end, + name: __MODULE__ + ) + end + + def all_waiting, do: Agent.get(__MODULE__, fn state -> state.waiting end) + + def completed?, + do: Agent.get(__MODULE__, fn state -> state.total == Enum.count(state.written) end) + + def state, do: Agent.get(__MODULE__, fn state -> state end) + def total, do: Agent.get(__MODULE__, fn state -> state.total end) + def ready, do: Agent.get(__MODULE__, fn state -> Enum.count(state.written) end) + + def add_modules(modules) do + Agent.update(__MODULE__, fn state -> + %{ + state + | modules: modules, + waiting: Map.keys(modules), + fetching: [], + fetched: [], + writing: [], + written: [], + total: map_size(modules) + } + end) + end + + def next_waiting do + Agent.get_and_update(__MODULE__, fn state -> + case state.waiting do + [mod_name | modules] -> + { + %{id: state.modules[mod_name], name: mod_name}, + state + |> Map.put(:waiting, modules) + |> Map.put(:fetching, [mod_name | state.fetching]) + } + + [] -> + {nil, state} + end + end) + end + + def update_status(mod_name, :fetched) do + Agent.update(__MODULE__, fn state -> + state + |> Map.put(:fetching, List.delete(state.fetching, mod_name)) + |> Map.put(:fetched, [mod_name | state.fetched]) + end) + end + + def update_status(mod_name, :writing) do + Agent.update(__MODULE__, fn state -> + state + |> Map.put(:writing, [mod_name | state.writing]) + end) + end + + def update_status(mod_name, :written) do + Agent.update(__MODULE__, fn state -> + state + |> Map.put(:writing, List.delete(state.writing, mod_name)) + |> Map.put(:written, [mod_name | state.written]) + end) + + render_progress() + end + + def render_progress() do + ProgressBar.render(ready(), total(), suffix: :count) + end +end diff --git a/lib/umwelt/client/application.ex b/lib/umwelt/client/application.ex new file mode 100644 index 0000000..624bcfd --- /dev/null +++ b/lib/umwelt/client/application.ex @@ -0,0 +1,15 @@ +defmodule Umwelt.Client.Application do + @moduledoc "Client app & Supervisor" + + use Application + + def start(_type, _args) do + children = [ + {Umwelt.Client.Agent, []}, + {Umwelt.Client.Clone, []} + ] + + opts = [strategy: :one_for_one, name: Umwelt.Client.Supervisor] + Supervisor.start_link(children, opts) + end +end diff --git a/lib/umwelt/client/clone.ex b/lib/umwelt/client/clone.ex new file mode 100644 index 0000000..e422dee --- /dev/null +++ b/lib/umwelt/client/clone.ex @@ -0,0 +1,95 @@ +defmodule Umwelt.Client.Clone do + @moduledoc "Clone main process" + + use GenServer + require Logger + + alias Umwelt.Client + + def start_link(_), + do: + GenServer.start_link( + __MODULE__, + %{phase_id: nil, port: nil}, + name: __MODULE__ + ) + + def init(state), do: {:ok, state} + + def handle_cast({:pull, params}, _state) do + send(self(), :start_pulling) + + {:noreply, params} + end + + def handle_info(:start_pulling, state) do + case Client.Request.fetch_modules(state) do + {:ok, modules} -> + Logger.info("Fetching modules: #{inspect(Map.keys(modules))}") + modules |> Client.Agent.add_modules() + + {:error, reason} -> + Logger.error("Failed to fetch modules: #{inspect(reason)}. Stopping...") + Supervisor.stop(Client.Supervisor) + end + + send(self(), :spawn_fetchers) + + {:noreply, state} + end + + def handle_info(:spawn_fetchers, state) do + total = Client.Agent.total() + Logger.debug("Spawning fetchers: #{total}") + + Client.Agent.all_waiting() + |> Enum.each(fn _ -> spawn_fetcher(state) end) + + {:noreply, state} + end + + def handle_info({:fetched, %{name: name, code: code}}, state) do + Client.Agent.update_status(name, :fetched) + spawn_writer(%{name: name, code: code}) + {:noreply, state} + end + + def handle_info({:fetch_failed, module}, state) do + Logger.warning("Respawning failed fetcher for module #{module.name}") + Client.Fetcher.start_link(module) + {:noreply, state} + end + + def handle_info({:written, mod_name}, state) do + Client.Agent.update_status(mod_name, :written) + send(self(), :maybe_stop) + {:noreply, state} + end + + def handle_info(:maybe_stop, state) do + if Client.Agent.completed?() do + Logger.debug("All modules processed. Stopping application.") + Supervisor.stop(Client.Supervisor) + end + + {:noreply, state} + end + + defp spawn_fetcher(state) do + case Client.Agent.next_waiting() do + nil -> + Logger.debug("No more modules to fetch") + :ok + + module -> + Logger.debug("Spawning fetcher for module #{inspect(module.name)}") + Client.Fetcher.start_link(Map.merge(module, state)) + end + end + + defp spawn_writer(%{name: name} = module) do + Logger.debug("Spawning writer for module #{inspect(name)}") + Client.Agent.update_status(name, :writing) + Client.Writer.start_link(module) + end +end diff --git a/lib/umwelt/client/fetcher.ex b/lib/umwelt/client/fetcher.ex new file mode 100644 index 0000000..bb294fc --- /dev/null +++ b/lib/umwelt/client/fetcher.ex @@ -0,0 +1,24 @@ +defmodule Umwelt.Client.Fetcher do + @moduledoc "Code fetcher task" + + use Task + require Logger + + alias Umwelt.Client + + def start_link(module) do + Task.start_link(__MODULE__, :run, [module]) + end + + def run(module) do + case Client.Request.fetch_code(module) do + {:ok, code} -> + Logger.debug("Success fetch #{module.name}") + send(Client.Clone, {:fetched, %{name: module.name, code: code}}) + + {:error, reason} -> + Logger.warning("Fail fetch #{module.name}: #{reason}") + send(Client.Clone, {:fetch_failed, module}) + end + end +end diff --git a/lib/umwelt/client/request.ex b/lib/umwelt/client/request.ex new file mode 100644 index 0000000..a2e5c4e --- /dev/null +++ b/lib/umwelt/client/request.ex @@ -0,0 +1,49 @@ +defmodule Umwelt.Client.Request do + @moduledoc "Umwelt client request" + + require Logger + + def fetch_modules(params) do + url = ~c"#{params.api_host}:#{params.port}/api/trees/#{params.phase_id}" + + case do_request(url, params.token) do + {:ok, {{~c"HTTP/1.1", 200, ~c"OK"}, _headers, body}} -> + {:ok, Jason.decode!(body)["data"]} + + {:ok, {{~c"HTTP/1.1", _status, _reason}, _headers, body}} -> + %{"error" => reason} = Jason.decode!(body) + + {:error, reason} + + {:error, reason} -> + {:error, reason} + end + end + + def fetch_code(params) do + url = ~c"#{params.api_host}:#{params.port}/api/code/#{params.phase_id}/#{params.id}" + + case do_request(url, params.token) do + {:ok, {{~c"HTTP/1.1", 200, ~c"OK"}, _headers, body}} -> + {:ok, Jason.decode!(body)["data"]} + + {:ok, {{~c"HTTP/1.1", _status, _reason}, _headers, body}} -> + %{"error" => reason} = Jason.decode!(body) + {:error, reason} + + {:error, reason} -> + {:error, reason} + end + end + + defp do_request(url, token) do + :httpc.request(:get, {url, headers(token)}, [], []) + end + + defp headers(token) do + [ + {~c"Authorization", ~c"Bearer #{token}"}, + {~c"Accept", ~c"application/json"} + ] + end +end diff --git a/lib/umwelt/client/writer.ex b/lib/umwelt/client/writer.ex new file mode 100644 index 0000000..8c6a9cf --- /dev/null +++ b/lib/umwelt/client/writer.ex @@ -0,0 +1,41 @@ +defmodule Umwelt.Client.Writer do + @moduledoc "File writer task" + + use Task + require Logger + + alias Umwelt.Client + + def start_link(module) do + Task.start_link(__MODULE__, :run, [module]) + end + + def run(module) do + module.code + |> Enum.each(fn {path, code} -> write_to_file(path, code) end) + + send(Client.Clone, {:written, module.name}) + end + + defp write_to_file(path, code) do + path = Path.expand(path, "umwelt_raw") + path |> Path.dirname() |> File.mkdir_p!() + + case File.read(path) do + {:ok, content} when content == code -> + Logger.debug("Write #{path}: identical") + {path, :identical} + + {:ok, content} -> + :ok = File.write!("#{path}_", content) + :ok = File.write!(path, code) + Logger.debug("Write #{path}: created_with_backup") + {path, :created_with_backup} + + {:error, _reason} -> + :ok = File.write!(path, code) + :ok = Logger.debug("Write #{path}: created") + {path, :created} + end + end +end diff --git a/mix.exs b/mix.exs index f0a2a84..f0dacf8 100644 --- a/mix.exs +++ b/mix.exs @@ -4,8 +4,8 @@ defmodule Umwelt.MixProject do def project do [ app: :umwelt, - version: "0.1.8", - elixir: "~> 1.15", + version: "0.2.0", + elixir: "~> 1.17", compilers: [:leex, :yecc] ++ Mix.compilers(), start_permanent: Mix.env() == :prod, deps: deps(), @@ -21,13 +21,18 @@ defmodule Umwelt.MixProject do # Run "mix help compile.app" to learn about applications. def application do [ - extra_applications: [:logger] + extra_applications: [:inets, :logger, :public_key, :ssl], + mod: {Umwelt.Client.Application, []} ] end # Run "mix help deps" to learn about dependencies. defp deps do [ + {:certifi, "~> 2.5"}, + {:jason, "~> 1.2"}, + {:progress_bar, "~> 3.0"}, + {:bypass, "~> 2.1", only: :test}, {:credo, "~> 1.6", only: [:dev, :test], runtime: false}, {:dialyxir, "~> 1.0", only: [:dev, :test], runtime: false}, {:ex_doc, "~> 0.29", only: :dev, runtime: false} diff --git a/mix.lock b/mix.lock index 0736b5c..9329361 100644 --- a/mix.lock +++ b/mix.lock @@ -1,6 +1,12 @@ %{ "bunt": {:hex, :bunt, "1.0.0", "081c2c665f086849e6d57900292b3a161727ab40431219529f13c4ddcf3e7a44", [:mix], [], "hexpm", "dc5f86aa08a5f6fa6b8096f0735c4e76d54ae5c9fa2c143e5a1fc7c1cd9bb6b5"}, + "bypass": {:hex, :bypass, "2.1.0", "909782781bf8e20ee86a9cabde36b259d44af8b9f38756173e8f5e2e1fabb9b1", [:mix], [{:plug, "~> 1.7", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 2.0", [hex: :plug_cowboy, repo: "hexpm", optional: false]}, {:ranch, "~> 1.3", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm", "d9b5df8fa5b7a6efa08384e9bbecfe4ce61c77d28a4282f79e02f1ef78d96b80"}, + "certifi": {:hex, :certifi, "2.13.0", "e52be248590050b2dd33b0bb274b56678f9068e67805dca8aa8b1ccdb016bbf6", [:rebar3], [], "hexpm", "8f3d9533a0f06070afdfd5d596b32e21c6580667a492891851b0e2737bc507a1"}, + "cowboy": {:hex, :cowboy, "2.12.0", "f276d521a1ff88b2b9b4c54d0e753da6c66dd7be6c9fca3d9418b561828a3731", [:make, :rebar3], [{:cowlib, "2.13.0", [hex: :cowlib, repo: "hexpm", optional: false]}, {:ranch, "1.8.0", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm", "8a7abe6d183372ceb21caa2709bec928ab2b72e18a3911aa1771639bef82651e"}, + "cowboy_telemetry": {:hex, :cowboy_telemetry, "0.4.0", "f239f68b588efa7707abce16a84d0d2acf3a0f50571f8bb7f56a15865aae820c", [:rebar3], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "7d98bac1ee4565d31b62d59f8823dfd8356a169e7fcbb83831b8a5397404c9de"}, + "cowlib": {:hex, :cowlib, "2.13.0", "db8f7505d8332d98ef50a3ef34b34c1afddec7506e4ee4dd4a3a266285d282ca", [:make, :rebar3], [], "hexpm", "e1e1284dc3fc030a64b1ad0d8382ae7e99da46c3246b815318a4b848873800a4"}, "credo": {:hex, :credo, "1.7.7", "771445037228f763f9b2afd612b6aa2fd8e28432a95dbbc60d8e03ce71ba4446", [:mix], [{:bunt, "~> 0.2.1 or ~> 1.0", [hex: :bunt, repo: "hexpm", optional: false]}, {:file_system, "~> 0.2 or ~> 1.0", [hex: :file_system, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "8bc87496c9aaacdc3f90f01b7b0582467b69b4bd2441fe8aae3109d843cc2f2e"}, + "decimal": {:hex, :decimal, "2.1.1", "5611dca5d4b2c3dd497dec8f68751f1f1a54755e8ed2a966c2633cf885973ad6", [:mix], [], "hexpm", "53cfe5f497ed0e7771ae1a475575603d77425099ba5faef9394932b35020ffcc"}, "dialyxir": {:hex, :dialyxir, "1.4.3", "edd0124f358f0b9e95bfe53a9fcf806d615d8f838e2202a9f430d59566b6b53b", [:mix], [{:erlex, ">= 0.2.6", [hex: :erlex, repo: "hexpm", optional: false]}], "hexpm", "bf2cfb75cd5c5006bec30141b131663299c661a864ec7fbbc72dfa557487a986"}, "earmark_parser": {:hex, :earmark_parser, "1.4.39", "424642f8335b05bb9eb611aa1564c148a8ee35c9c8a8bba6e129d51a3e3c6769", [:mix], [], "hexpm", "06553a88d1f1846da9ef066b87b57c6f605552cfbe40d20bd8d59cc6bde41944"}, "erlex": {:hex, :erlex, "0.2.6", "c7987d15e899c7a2f34f5420d2a2ea0d659682c06ac607572df55a43753aa12e", [:mix], [], "hexpm", "2ed2e25711feb44d52b17d2780eabf998452f6efda104877a3881c2f8c0c0c75"}, @@ -10,5 +16,12 @@ "makeup": {:hex, :makeup, "1.1.2", "9ba8837913bdf757787e71c1581c21f9d2455f4dd04cfca785c70bbfff1a76a3", [:mix], [{:nimble_parsec, "~> 1.2.2 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "cce1566b81fbcbd21eca8ffe808f33b221f9eee2cbc7a1706fc3da9ff18e6cac"}, "makeup_elixir": {:hex, :makeup_elixir, "0.16.2", "627e84b8e8bf22e60a2579dad15067c755531fea049ae26ef1020cad58fe9578", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.2.3 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "41193978704763f6bbe6cc2758b84909e62984c7752b3784bd3c218bb341706b"}, "makeup_erlang": {:hex, :makeup_erlang, "1.0.0", "6f0eff9c9c489f26b69b61440bf1b238d95badae49adac77973cbacae87e3c2e", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "ea7a9307de9d1548d2a72d299058d1fd2339e3d398560a0e46c27dab4891e4d2"}, + "mime": {:hex, :mime, "2.0.6", "8f18486773d9b15f95f4f4f1e39b710045fa1de891fada4516559967276e4dc2", [:mix], [], "hexpm", "c9945363a6b26d747389aac3643f8e0e09d30499a138ad64fe8fd1d13d9b153e"}, "nimble_parsec": {:hex, :nimble_parsec, "1.4.0", "51f9b613ea62cfa97b25ccc2c1b4216e81df970acd8e16e8d1bdc58fef21370d", [:mix], [], "hexpm", "9c565862810fb383e9838c1dd2d7d2c437b3d13b267414ba6af33e50d2d1cf28"}, + "plug": {:hex, :plug, "1.16.1", "40c74619c12f82736d2214557dedec2e9762029b2438d6d175c5074c933edc9d", [:mix], [{:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_crypto, "~> 1.1.1 or ~> 1.2 or ~> 2.0", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "a13ff6b9006b03d7e33874945b2755253841b238c34071ed85b0e86057f8cddc"}, + "plug_cowboy": {:hex, :plug_cowboy, "2.7.1", "87677ffe3b765bc96a89be7960f81703223fe2e21efa42c125fcd0127dd9d6b2", [:mix], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:cowboy_telemetry, "~> 0.3", [hex: :cowboy_telemetry, repo: "hexpm", optional: false]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "02dbd5f9ab571b864ae39418db7811618506256f6d13b4a45037e5fe78dc5de3"}, + "plug_crypto": {:hex, :plug_crypto, "2.1.0", "f44309c2b06d249c27c8d3f65cfe08158ade08418cf540fd4f72d4d6863abb7b", [:mix], [], "hexpm", "131216a4b030b8f8ce0f26038bc4421ae60e4bb95c5cf5395e1421437824c4fa"}, + "progress_bar": {:hex, :progress_bar, "3.0.0", "f54ff038c2ac540cfbb4c2bfe97c75e7116ead044f3c2b10c9f212452194b5cd", [:mix], [{:decimal, "~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}], "hexpm", "6981c2b25ab24aecc91a2dc46623658e1399c21a2ae24db986b90d678530f2b7"}, + "ranch": {:hex, :ranch, "1.8.0", "8c7a100a139fd57f17327b6413e4167ac559fbc04ca7448e9be9057311597a1d", [:make, :rebar3], [], "hexpm", "49fbcfd3682fab1f5d109351b61257676da1a2fdbe295904176d5e521a2ddfe5"}, + "telemetry": {:hex, :telemetry, "1.2.1", "68fdfe8d8f05a8428483a97d7aab2f268aaff24b49e0f599faa091f1d4e7f61c", [:rebar3], [], "hexpm", "dad9ce9d8effc621708f99eac538ef1cbe05d6a874dd741de2e689c47feafed5"}, } diff --git a/test/mix/tasks/clone_test.exs b/test/mix/tasks/clone_test.exs new file mode 100644 index 0000000..373f08a --- /dev/null +++ b/test/mix/tasks/clone_test.exs @@ -0,0 +1,93 @@ +defmodule Mix.Tasks.CloneTest do + use ExUnit.Case + import ExUnit.CaptureLog + + alias Bypass + alias Mix.Tasks.Umwelt.Clone + + setup do + bypass = Bypass.open() + Application.put_env(:umwelt, :api_port, bypass.port) + + :ok = Application.ensure_started(:umwelt) + + on_exit(fn -> File.rm_rf!("umwelt_raw/temp") end) + + {:ok, bypass: bypass} + end + + describe "clone" do + test "interface call" do + Application.put_env(:umwelt, :api_token, "no_token") + + assert capture_log([], fn -> assert :ok == Clone.run([42]) end) =~ "Token not found in env!" + end + + test "success", %{bypass: bypass} do + Bypass.expect_once(bypass, "GET", "/api/trees/23", fn conn -> + Plug.Conn.resp(conn, 200, ~s({"data": { + "Disco": "1", + "Disco.Chaos": "2", + "Disco.Discord": "3" + }})) + end) + + Bypass.expect_once(bypass, "GET", "/api/code/23/1", fn conn -> + Plug.Conn.resp( + conn, + 200, + ~s({"data": {"temp/lib/disco.ex": "defmodule Disco", + "temp/test/disco_test.ex": "defmodule DiscoTest"}}) + ) + end) + + Bypass.expect_once(bypass, "GET", "/api/code/23/2", fn conn -> + Plug.Conn.resp( + conn, + 200, + ~s({"data": {"temp/lib/disco/chaos.ex": "defmodule Disco.Chaos", + "temp/test/disco/chaos_test.ex": "defmodule Disco.ChaosTest"}}) + ) + end) + + # here we check that fetcher respawns after 401 + Bypass.expect_once(bypass, "GET", "/api/code/23/3", fn conn -> + Plug.Conn.resp(conn, 401, ~s({"error": "Unauthorized"})) + end) + + Bypass.expect_once(bypass, "GET", "/api/code/23/3", fn conn -> + Plug.Conn.resp( + conn, + 200, + ~s({"data": {"temp/lib/disco/discord.ex": "defmodule Disco.Discord", + "temp/test/disco/discord_test.ex": "defmodule Disco.DiscordTest"}}) + ) + end) + + assert capture_log([], fn -> assert :ok == Clone.run([23, "token"]) end) =~ "Done!" + + :timer.sleep(666) + assert "defmodule Disco" == File.read!("umwelt_raw/temp/lib/disco.ex") + assert "defmodule DiscoTest" == File.read!("umwelt_raw/temp/test/disco_test.ex") + assert "defmodule Disco.Chaos" == File.read!("umwelt_raw/temp/lib/disco/chaos.ex") + assert "defmodule Disco.ChaosTest" == File.read!("umwelt_raw/temp/test/disco/chaos_test.ex") + assert "defmodule Disco.Discord" == File.read!("umwelt_raw/temp/lib/disco/discord.ex") + + assert "defmodule Disco.DiscordTest" == + File.read!("umwelt_raw/temp/test/disco/discord_test.ex") + + assert {:ok, _} = File.rm_rf("umwelt_raw/temp") + end + + test "when fetch unsuccessful", %{bypass: bypass} do + Bypass.expect_once(bypass, "GET", "/api/trees/42", fn conn -> + Plug.Conn.resp(conn, 401, ~s({"error": "Unauthorized"})) + end) + + assert capture_log([], fn -> assert :ok == Clone.run([42, "bad_token"]) end) =~ + "Failed to fetch modules: \"Unauthorized\". Stopping..." + + Application.stop(:umwelt) + end + end +end diff --git a/test/mix/tasks/dump_test.exs b/test/mix/tasks/dump_test.exs index b6dd6f2..bf278ed 100644 --- a/test/mix/tasks/dump_test.exs +++ b/test/mix/tasks/dump_test.exs @@ -1,11 +1,14 @@ defmodule Mix.Tasks.DumpTest do use ExUnit.Case, async: true + import ExUnit.CaptureIO - alias Mix.Tasks.Dump + alias Mix.Tasks.Umwelt.Dump describe "parse([root_name])" do test "parse app from Mix.Project.config by default" do - assert :ok = Dump.run([]) + assert capture_io([], fn -> + assert :ok == Dump.run([]) + end) =~ "Parsing result saved into umwelt.bin" {:ok, bin} = File.read("umwelt.bin") @@ -15,7 +18,9 @@ defmodule Mix.Tasks.DumpTest do end test "parse app from given root_name" do - assert :ok = Dump.run(["umwelt"]) + assert capture_io([], fn -> + assert :ok == Dump.run(["umwelt"]) + end) =~ "Parsing result saved into umwelt.bin" {:ok, bin} = File.read("umwelt.bin") diff --git a/test/umwelt/client/agent_test.exs b/test/umwelt/client/agent_test.exs new file mode 100644 index 0000000..3f2f519 --- /dev/null +++ b/test/umwelt/client/agent_test.exs @@ -0,0 +1,165 @@ +defmodule Umwelt.Client.AgentTest do + use ExUnit.Case + + alias Umwelt.Client.Agent + + setup do + modules = %{ + "Discordian" => 248_169, + "Discordian.Aftermath" => 248_188, + "Discordian.Bureaucracy" => 248_187, + "Discordian.Chaos" => 248_184, + "Discordian.Confusion" => 248_186, + "Discordian.Discord" => 248_185 + } + + :ok = Application.ensure_started(:umwelt) + + {:ok, modules: modules} + end + + test "initial state is empty", _context do + assert %{ + modules: %{}, + waiting: [], + fetching: [], + fetched: [], + writing: [], + written: [], + total: 0 + } == Agent.state() + end + + test "add_modules/1 adds modules to state", %{modules: modules} do + Agent.add_modules(modules) + + assert %{ + modules: modules, + waiting: Map.keys(modules), + fetching: [], + fetched: [], + writing: [], + written: [], + total: 6 + } == Agent.state() + + Agent.add_modules(%{}) + end + + describe "next_waiting/0" do + test "retrieves and updates the state correctly", %{modules: modules} do + Agent.add_modules(modules) + + assert Agent.next_waiting() == %{id: 248_169, name: "Discordian"} + + assert %{ + modules: modules, + waiting: Map.keys(modules) -- ["Discordian"], + fetching: ["Discordian"], + fetched: [], + writing: [], + written: [], + total: 6 + } == Agent.state() + + Agent.add_modules(%{}) + end + + test "when waiting list is already empty" do + Agent.add_modules(%{"Discordian" => 248_169}) + + assert Agent.next_waiting() == %{id: 248_169, name: "Discordian"} + + assert %{ + modules: %{"Discordian" => 248_169}, + waiting: [], + fetching: ["Discordian"], + fetched: [], + writing: [], + written: [], + total: 1 + } == Agent.state() + + assert Agent.next_waiting() == nil + + assert %{ + modules: %{"Discordian" => 248_169}, + waiting: [], + fetching: ["Discordian"], + fetched: [], + writing: [], + written: [], + total: 1 + } == Agent.state() + + Agent.add_modules(%{}) + end + end + + describe "update_status/2" do + test "updates the module status to :fetched", %{modules: modules} do + Agent.add_modules(modules) + Agent.next_waiting() + + Agent.update_status("Discordian", :fetched) + state = Agent.state() + assert state.fetching == [] + assert state.fetched == ["Discordian"] + Agent.add_modules(%{}) + end + + test "updates the module status to :writing", %{modules: modules} do + Agent.add_modules(modules) + Agent.next_waiting() + Agent.update_status("Discordian", :fetched) + + Agent.update_status("Discordian", :writing) + state = Agent.state() + assert state.writing == ["Discordian"] + Agent.add_modules(%{}) + end + + test "updates the module status to :written", %{modules: modules} do + Agent.add_modules(modules) + Agent.next_waiting() + Agent.update_status("Discordian", :fetched) + Agent.update_status("Discordian", :writing) + + Agent.update_status("Discordian", :written) + state = Agent.state() + assert state.writing == [] + assert state.written == ["Discordian"] + Agent.add_modules(%{}) + end + end + + describe "completed?/0" do + test "returns true when all modules are written", %{modules: modules} do + Agent.add_modules(modules) + + for module_name <- Map.keys(modules) do + Agent.next_waiting() + Agent.update_status(module_name, :fetched) + Agent.update_status(module_name, :writing) + Agent.update_status(module_name, :written) + end + + assert Agent.completed?() + Agent.add_modules(%{}) + end + + test "returns false when not all modules are written", %{modules: modules} do + Agent.add_modules(modules) + + for module_name <- Enum.take(Map.keys(modules), 3) do + Agent.next_waiting() + Agent.update_status(module_name, :fetched) + Agent.update_status(module_name, :writing) + Agent.update_status(module_name, :written) + end + + refute Agent.completed?() + Agent.add_modules(%{}) + end + end +end diff --git a/test/umwelt/client/request_test.exs b/test/umwelt/client/request_test.exs new file mode 100644 index 0000000..619f03a --- /dev/null +++ b/test/umwelt/client/request_test.exs @@ -0,0 +1,51 @@ +defmodule Umwelt.Client.RequestTest do + use ExUnit.Case + + alias Bypass + alias Umwelt.Client.Request + + setup do + bypass = Bypass.open() + :ok = Application.ensure_started(:umwelt) + + {:ok, bypass: bypass} + end + + test "fetch_modules returns modules", %{bypass: bypass} do + Bypass.expect(bypass, fn conn -> + Plug.Conn.resp(conn, 200, ~s({"data": { + "Disco": "1", + "Disco.Chaos": "2", + "Disco.Discord": "3" + }})) + end) + + assert {:ok, + %{ + "Disco" => "1", + "Disco.Chaos" => "2", + "Disco.Discord" => "3" + }} == + Request.fetch_modules(%{ + api_host: "http://localhost", + phase_id: 1, + port: bypass.port, + token: "token" + }) + end + + test "fetch_code returns code files", %{bypass: bypass} do + Bypass.expect(bypass, fn conn -> + Plug.Conn.resp(conn, 200, ~s({"data": {"foo.ex": "code1", "foo_spec.exs": "code2"}})) + end) + + assert {:ok, %{"foo.ex" => "code1", "foo_spec.exs" => "code2"}} == + Request.fetch_code(%{ + api_host: "http://localhost", + id: 23, + phase_id: 5, + port: bypass.port, + token: "token" + }) + end +end diff --git a/test/umwelt/client/writer_test.exs b/test/umwelt/client/writer_test.exs new file mode 100644 index 0000000..6e5f441 --- /dev/null +++ b/test/umwelt/client/writer_test.exs @@ -0,0 +1,80 @@ +defmodule Umwelt.Client.WriterTest do + use ExUnit.Case + + alias Umwelt.Client.{Agent, Writer} + + setup do + module = %{ + code: %{ + "temp/lib/disco/chaos.ex" => "defmodule Disco.Chaos", + "temp/test/disco/chaos_test.exs" => "defmodule Disco.ChaosTest" + }, + name: "Disco.Chaos" + } + + :ok = Application.ensure_started(:umwelt) + + Agent.add_modules(%{"Disco.Chaos" => 23}) + Agent.update_status("Disco.Chaos", :fetched) + + on_exit(fn -> File.rm_rf!("umwelt_raw/temp") end) + {:ok, module: module} + end + + describe "run/1" do + test "writes new files correctly", %{module: module} do + lib_path = "umwelt_raw/temp/lib/disco/chaos.ex" + test_path = "umwelt_raw/temp/test/disco/chaos_test.exs" + + log = capture_log(fn -> Writer.run(module) end) + + :timer.sleep(666) + assert log =~ " #{Path.expand(lib_path)}: created" + assert log =~ " #{Path.expand(test_path)}: created" + + assert File.read!(lib_path) == "defmodule Disco.Chaos" + assert File.read!(test_path) == "defmodule Disco.ChaosTest" + end + + test "writes identical files correctly", %{module: module} do + lib_path = "umwelt_raw/temp/lib/disco/chaos.ex" + test_path = "umwelt_raw/temp/test/disco/chaos_test.exs" + + File.mkdir_p!(Path.dirname(lib_path)) + File.write!(lib_path, "defmodule Disco.Chaos") + File.mkdir_p!(Path.dirname(test_path)) + File.write!(test_path, "defmodule Disco.ChaosTest") + + log = capture_log(fn -> Writer.run(module) end) + + assert log =~ " #{Path.expand(lib_path)}: identical" + assert log =~ " #{Path.expand(test_path)}: identical" + end + + test "creates backup and writes new content", %{module: module} do + lib_path = "umwelt_raw/temp/lib/disco/chaos.ex" + test_path = "umwelt_raw/temp/test/disco/chaos_test.exs" + lib_backup_path = "#{lib_path}_" + test_backup_path = "#{test_path}_" + + File.mkdir_p!(Path.dirname(lib_path)) + File.write!(lib_path, "old lib content") + File.mkdir_p!(Path.dirname(test_path)) + File.write!(test_path, "old test content") + + log = capture_log(fn -> Writer.run(module) end) + + assert log =~ "#{Path.expand(lib_path)}: created_with_backup" + assert log =~ "#{Path.expand(test_path)}: created_with_backup" + + assert File.read!(lib_path) == "defmodule Disco.Chaos" + assert File.read!(test_path) == "defmodule Disco.ChaosTest" + assert File.read!(lib_backup_path) == "old lib content" + assert File.read!(test_backup_path) == "old test content" + end + end + + defp capture_log(fun) do + ExUnit.CaptureLog.capture_log(fn -> fun.() end) + end +end diff --git a/test/umwelt/parser_test.exs b/test/umwelt/parser_test.exs index a14c5c5..f459f80 100644 --- a/test/umwelt/parser_test.exs +++ b/test/umwelt/parser_test.exs @@ -225,6 +225,18 @@ defmodule Umwelt.ParserTest do } == Parser.parse_raw(code) end + + test "parsing error", %{} do + assert {:error, + {[ + opening_delimiter: :do, + expected_delimiter: :end, + line: 1, + column: 9, + end_line: 1, + end_column: 11 + ], "missing terminator: end", ""}} == Parser.parse_raw("def foo do") + end end describe "reading ast" do