From 1e6956474ee791c89c9a62de35c5feae68d66438 Mon Sep 17 00:00:00 2001 From: Sovetnik <5460740+sovetnik@users.noreply.github.com> Date: Sat, 3 Aug 2024 17:59:53 +0300 Subject: [PATCH 1/6] Introduce Clone task --- .github/workflows/elixir.yml | 6 +- .tool-versions | 2 +- README.md | 43 ++++++-- lib/mix/tasks/clone.ex | 79 +++++++++++++ lib/mix/tasks/dump.ex | 2 +- lib/umwelt/client.ex | 11 ++ lib/umwelt/client/agent.ex | 93 ++++++++++++++++ lib/umwelt/client/application.ex | 15 +++ lib/umwelt/client/clone.ex | 95 ++++++++++++++++ lib/umwelt/client/fetcher.ex | 24 ++++ lib/umwelt/client/request.ex | 49 +++++++++ lib/umwelt/client/writer.ex | 41 +++++++ mix.exs | 11 +- mix.lock | 13 +++ test/mix/tasks/clone_test.exs | 93 ++++++++++++++++ test/mix/tasks/dump_test.exs | 11 +- test/umwelt/client/agent_test.exs | 165 ++++++++++++++++++++++++++++ test/umwelt/client/request_test.exs | 51 +++++++++ test/umwelt/client/writer_test.exs | 80 ++++++++++++++ test/umwelt/parser_test.exs | 12 ++ 20 files changed, 875 insertions(+), 21 deletions(-) create mode 100644 lib/mix/tasks/clone.ex create mode 100644 lib/umwelt/client.ex create mode 100644 lib/umwelt/client/agent.ex create mode 100644 lib/umwelt/client/application.ex create mode 100644 lib/umwelt/client/clone.ex create mode 100644 lib/umwelt/client/fetcher.ex create mode 100644 lib/umwelt/client/request.ex create mode 100644 lib/umwelt/client/writer.ex create mode 100644 test/mix/tasks/clone_test.exs create mode 100644 test/umwelt/client/agent_test.exs create mode 100644 test/umwelt/client/request_test.exs create mode 100644 test/umwelt/client/writer_test.exs diff --git a/.github/workflows/elixir.yml b/.github/workflows/elixir.yml index 8bbd132..64116c3 100644 --- a/.github/workflows/elixir.yml +++ b/.github/workflows/elixir.yml @@ -16,7 +16,7 @@ jobs: strategy: matrix: elixir: [1.17.2] - otp: [26.2.5.1, 27.0] + otp: [26.2.5.1, 27.0.1] steps: - name: Cancel Previous Runs uses: styfle/cancel-workflow-action@0.12.1 @@ -61,7 +61,7 @@ jobs: strategy: matrix: elixir: [1.17.2] - otp: [26.2.5.1, 27.0] + otp: [26.2.5.1, 27.0.1] steps: - name: Cancel Previous Runs uses: styfle/cancel-workflow-action@0.12.1 @@ -126,7 +126,7 @@ jobs: fail-fast: false matrix: elixir: [1.17.2] - otp: [26.2.5.1, 27.0] + otp: [26.2.5.1, 27.0.1] steps: - name: Cancel Previous Runs uses: styfle/cancel-workflow-action@0.12.1 diff --git a/.tool-versions b/.tool-versions index 94515c4..7879e57 100644 --- a/.tool-versions +++ b/.tool-versions @@ -1,2 +1,2 @@ elixir 1.17.2 -erlang 27.0 +erlang 27.0.1 diff --git a/README.md b/README.md index 3b7d7e0..8d14d53 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,6 @@ # Umwelt ![Umwelt CI](https://github.com/sovetnik/umwelt/actions/workflows/elixir.yml/badge.svg?event=push) [![wakatime](https://wakatime.com/badge/user/7542de1a-027f-4ed7-bc4b-c31d4cf9aa2a/project/018c9f92-bb93-4303-816f-bc0799a61194.svg)](https://wakatime.com/badge/user/7542de1a-027f-4ed7-bc4b-c31d4cf9aa2a/project/018c9f92-bb93-4303-816f-bc0799a61194) Client for [umwelt.dev](https://umwelt.dev) -## Implemented actions: - -### Dump - -Extracts Umwelt from Elixir project and dumps it into `root_name.bin` - ## Installation [available in Hex](https://hex.pm/packages/umwelt), the package can be installed @@ -22,24 +16,53 @@ end ## Usage -Right now it is a proof of concept, and in this version parser can parse some business-logic related code, via `mix dump`. +### Dump + +Extracts Umwelt from Elixir project and dumps it into `root_name.bin` In common case, when you want to parse your project and it's name from `Mix.Project.config()[:app]` matches root folder name `lib/root_name`, use: ```bash - mix dump + mix umwelt.dump ``` When you wanna parse another folder in lib, `lib/another_root_name`, use: ```bash - mix dump another_root_name + mix umwelt.dump another_root_name +``` + +### Clone + +Fetch and write all modules from specified phase. + +When your project is ready, you can get its code and specs. +Create a new elixir or phoenix app, add umwelt and pull the code. +```bash + mix new project_name + cd project_name ``` +add umwelt to deps in `mix.exs`. +You have to obtain a token on [profile page](https://umwelt.dev/auth/profile) +```bash + export UMWELT_TOKEN="your_token" + mix umwelt.clone phase_id + mix test --trace +``` +And now you will see all messages for failing tests and can start coding. + +If you want to reduce logger add this to your `config.exs` +``` + config :logger, + compile_time_purge_matching: [ + [application: :umwelt, level_lower_than: :warning] + ] +``` ## Planned Here is the list of planned features: ### Client functions -Set of push/pull/sync mix tasks to sync local code with remote representation on [umwelt.dev](https://umwelt.dev) +Set of pull/push/sync mix tasks to sync local code with remote representation on [umwelt.dev](https://umwelt.dev) ### Unparser Tools for update local code with changes made on web side. diff --git a/lib/mix/tasks/clone.ex b/lib/mix/tasks/clone.ex new file mode 100644 index 0000000..b5e9d17 --- /dev/null +++ b/lib/mix/tasks/clone.ex @@ -0,0 +1,79 @@ +defmodule Mix.Tasks.Umwelt.Clone do + @moduledoc "Clones phase modules and code" + @shortdoc "The code puller" + use Mix.Task + require Logger + + @impl Mix.Task + def run([phase_id]) do + case System.get_env("UMWELT_TOKEN", "no_token") do + "no_token" -> + """ + Token not found in env! + You can get it on umwelt.dev/auth/profile and do + + export UMWELT_TOKEN="token" + + or pass it directly in + + mix clone phase_id "token" + + """ + |> Logger.warning() + + token -> + run([phase_id, token]) + end + end + + @impl Mix.Task + def run([phase_id, token]) do + Umwelt.Client.Application.start(nil, nil) + + Umwelt.Client.Supervisor + |> Process.whereis() + |> Process.monitor() + + %{phase_id: phase_id, token: token} + |> assign_host() + |> assign_port() + |> Umwelt.Client.pull() + + receive do + {:DOWN, _, :process, _, _} -> + Logger.info("Done!") + + other -> + Logger.warning(inspect(other)) + end + end + + defp assign_host(params) do + host = + case Mix.env() do + :dev -> + System.get_env("UMWELT_HOST", "https://umwelt.dev") + + :test -> + "http://localhost" + end + + Map.put(params, :api_host, host) + end + + defp assign_port(params) do + port = + case Mix.env() do + :dev -> + case params.api_host do + "http://localhost" -> 4000 + "https://umwelt.dev" -> 443 + end + + :test -> + Application.get_env(:umwelt, :api_port) + end + + Map.put(params, :port, port) + end +end diff --git a/lib/mix/tasks/dump.ex b/lib/mix/tasks/dump.ex index 05e3fb5..54162dd 100644 --- a/lib/mix/tasks/dump.ex +++ b/lib/mix/tasks/dump.ex @@ -1,4 +1,4 @@ -defmodule Mix.Tasks.Dump do +defmodule Mix.Tasks.Umwelt.Dump do @moduledoc "This task for self-parse umwelt" @shortdoc "The lib parser" diff --git a/lib/umwelt/client.ex b/lib/umwelt/client.ex new file mode 100644 index 0000000..6d0514e --- /dev/null +++ b/lib/umwelt/client.ex @@ -0,0 +1,11 @@ +defmodule Umwelt.Client do + @moduledoc "Client for umwelt.dev" + + alias Umwelt.Client.Clone + + require Logger + + def pull(params) do + GenServer.cast(Clone, {:pull, params}) + end +end diff --git a/lib/umwelt/client/agent.ex b/lib/umwelt/client/agent.ex new file mode 100644 index 0000000..f975dbd --- /dev/null +++ b/lib/umwelt/client/agent.ex @@ -0,0 +1,93 @@ +defmodule Umwelt.Client.Agent do + @moduledoc "Keeps pulling metadata" + + use Agent + require Logger + + def start_link(_args) do + Agent.start_link( + fn -> + %{ + modules: %{}, + waiting: [], + fetching: [], + fetched: [], + writing: [], + written: [], + total: 0 + } + end, + name: __MODULE__ + ) + end + + def all_waiting, do: Agent.get(__MODULE__, fn state -> state.waiting end) + + def completed?, + do: Agent.get(__MODULE__, fn state -> state.total == Enum.count(state.written) end) + + def state, do: Agent.get(__MODULE__, fn state -> state end) + def total, do: Agent.get(__MODULE__, fn state -> state.total end) + def ready, do: Agent.get(__MODULE__, fn state -> Enum.count(state.written) end) + + def add_modules(modules) do + Agent.update(__MODULE__, fn state -> + %{ + state + | modules: modules, + waiting: Map.keys(modules), + fetching: [], + fetched: [], + writing: [], + written: [], + total: map_size(modules) + } + end) + end + + def next_waiting do + Agent.get_and_update(__MODULE__, fn state -> + case state.waiting do + [mod_name | modules] -> + { + %{id: state.modules[mod_name], name: mod_name}, + state + |> Map.put(:waiting, modules) + |> Map.put(:fetching, [mod_name | state.fetching]) + } + + [] -> + {nil, state} + end + end) + end + + def update_status(mod_name, :fetched) do + Agent.update(__MODULE__, fn state -> + state + |> Map.put(:fetching, List.delete(state.fetching, mod_name)) + |> Map.put(:fetched, [mod_name | state.fetched]) + end) + end + + def update_status(mod_name, :writing) do + Agent.update(__MODULE__, fn state -> + state + |> Map.put(:writing, [mod_name | state.writing]) + end) + end + + def update_status(mod_name, :written) do + Agent.update(__MODULE__, fn state -> + state + |> Map.put(:writing, List.delete(state.writing, mod_name)) + |> Map.put(:written, [mod_name | state.written]) + end) + + render_progress() + end + + def render_progress do + ProgressBar.render(ready(), total(), suffix: :count) + end +end diff --git a/lib/umwelt/client/application.ex b/lib/umwelt/client/application.ex new file mode 100644 index 0000000..624bcfd --- /dev/null +++ b/lib/umwelt/client/application.ex @@ -0,0 +1,15 @@ +defmodule Umwelt.Client.Application do + @moduledoc "Client app & Supervisor" + + use Application + + def start(_type, _args) do + children = [ + {Umwelt.Client.Agent, []}, + {Umwelt.Client.Clone, []} + ] + + opts = [strategy: :one_for_one, name: Umwelt.Client.Supervisor] + Supervisor.start_link(children, opts) + end +end diff --git a/lib/umwelt/client/clone.ex b/lib/umwelt/client/clone.ex new file mode 100644 index 0000000..e422dee --- /dev/null +++ b/lib/umwelt/client/clone.ex @@ -0,0 +1,95 @@ +defmodule Umwelt.Client.Clone do + @moduledoc "Clone main process" + + use GenServer + require Logger + + alias Umwelt.Client + + def start_link(_), + do: + GenServer.start_link( + __MODULE__, + %{phase_id: nil, port: nil}, + name: __MODULE__ + ) + + def init(state), do: {:ok, state} + + def handle_cast({:pull, params}, _state) do + send(self(), :start_pulling) + + {:noreply, params} + end + + def handle_info(:start_pulling, state) do + case Client.Request.fetch_modules(state) do + {:ok, modules} -> + Logger.info("Fetching modules: #{inspect(Map.keys(modules))}") + modules |> Client.Agent.add_modules() + + {:error, reason} -> + Logger.error("Failed to fetch modules: #{inspect(reason)}. Stopping...") + Supervisor.stop(Client.Supervisor) + end + + send(self(), :spawn_fetchers) + + {:noreply, state} + end + + def handle_info(:spawn_fetchers, state) do + total = Client.Agent.total() + Logger.debug("Spawning fetchers: #{total}") + + Client.Agent.all_waiting() + |> Enum.each(fn _ -> spawn_fetcher(state) end) + + {:noreply, state} + end + + def handle_info({:fetched, %{name: name, code: code}}, state) do + Client.Agent.update_status(name, :fetched) + spawn_writer(%{name: name, code: code}) + {:noreply, state} + end + + def handle_info({:fetch_failed, module}, state) do + Logger.warning("Respawning failed fetcher for module #{module.name}") + Client.Fetcher.start_link(module) + {:noreply, state} + end + + def handle_info({:written, mod_name}, state) do + Client.Agent.update_status(mod_name, :written) + send(self(), :maybe_stop) + {:noreply, state} + end + + def handle_info(:maybe_stop, state) do + if Client.Agent.completed?() do + Logger.debug("All modules processed. Stopping application.") + Supervisor.stop(Client.Supervisor) + end + + {:noreply, state} + end + + defp spawn_fetcher(state) do + case Client.Agent.next_waiting() do + nil -> + Logger.debug("No more modules to fetch") + :ok + + module -> + Logger.debug("Spawning fetcher for module #{inspect(module.name)}") + Client.Fetcher.start_link(Map.merge(module, state)) + end + end + + defp spawn_writer(%{name: name} = module) do + Logger.debug("Spawning writer for module #{inspect(name)}") + Client.Agent.update_status(name, :writing) + Client.Writer.start_link(module) + end +end diff --git a/lib/umwelt/client/fetcher.ex b/lib/umwelt/client/fetcher.ex new file mode 100644 index 0000000..bb294fc --- /dev/null +++ b/lib/umwelt/client/fetcher.ex @@ -0,0 +1,24 @@ +defmodule Umwelt.Client.Fetcher do + @moduledoc "Code fetcher task" + + use Task + require Logger + + alias Umwelt.Client + + def start_link(module) do + Task.start_link(__MODULE__, :run, [module]) + end + + def run(module) do + case Client.Request.fetch_code(module) do + {:ok, code} -> + Logger.debug("Success fetch #{module.name}") + send(Client.Clone, {:fetched, %{name: module.name, code: code}}) + + {:error, reason} -> + Logger.warning("Fail fetch #{module.name}: #{reason}") + send(Client.Clone, {:fetch_failed, module}) + end + end +end diff --git a/lib/umwelt/client/request.ex b/lib/umwelt/client/request.ex new file mode 100644 index 0000000..a2e5c4e --- /dev/null +++ b/lib/umwelt/client/request.ex @@ -0,0 +1,49 @@ +defmodule Umwelt.Client.Request do + @moduledoc "Umwelt client request" + + require Logger + + def fetch_modules(params) do + url = ~c"#{params.api_host}:#{params.port}/api/trees/#{params.phase_id}" + + case do_request(url, params.token) do + {:ok, {{~c"HTTP/1.1", 200, ~c"OK"}, _headers, body}} -> + {:ok, Jason.decode!(body)["data"]} + + {:ok, {{~c"HTTP/1.1", _status, _reason}, _headers, body}} -> + %{"error" => reason} = Jason.decode!(body) + + {:error, reason} + + {:error, reason} -> + {:error, reason} + end + end + + def fetch_code(params) do + url = ~c"#{params.api_host}:#{params.port}/api/code/#{params.phase_id}/#{params.id}" + + case do_request(url, params.token) do + {:ok, {{~c"HTTP/1.1", 200, ~c"OK"}, _headers, body}} -> + {:ok, Jason.decode!(body)["data"]} + + {:ok, {{~c"HTTP/1.1", _status, _reason}, _headers, body}} -> + %{"error" => reason} = Jason.decode!(body) + {:error, reason} + + {:error, reason} -> + {:error, reason} + end + end + + defp do_request(url, token) do + :httpc.request(:get, {url, headers(token)}, [], []) + end + + defp headers(token) do + [ + {~c"Authorization", ~c"Bearer #{token}"}, + {~c"Accept", ~c"application/json"} + ] + end +end diff --git a/lib/umwelt/client/writer.ex b/lib/umwelt/client/writer.ex new file mode 100644 index 0000000..8c6a9cf --- /dev/null +++ b/lib/umwelt/client/writer.ex @@ -0,0 +1,41 @@ +defmodule Umwelt.Client.Writer do + @moduledoc "File writer task" + + use Task + require Logger + + alias Umwelt.Client + + def start_link(module) do + Task.start_link(__MODULE__, :run, [module]) + end + + def run(module) do + module.code + |> Enum.each(fn {path, code} -> write_to_file(path, code) end) + + send(Client.Clone, {:written, module.name}) + end + + defp write_to_file(path, code) do + path = Path.expand(path, "umwelt_raw") + path |> Path.dirname() |> File.mkdir_p!() + + case File.read(path) do + {:ok, content} when content == code -> + Logger.debug("Write #{path}: identical") + {path, :identical} + + {:ok, content} -> + :ok = File.write!("#{path}_", content) + :ok = File.write!(path, code) + Logger.debug("Write #{path}: created_with_backup") + {path, :created_with_backup} + + {:error, _reason} -> + :ok = File.write!(path, code) + :ok = Logger.debug("Write #{path}: created") + {path, :created} + end + end +end diff --git a/mix.exs b/mix.exs index f0a2a84..f0dacf8 100644 --- a/mix.exs +++ b/mix.exs @@ -4,8 +4,8 @@ defmodule Umwelt.MixProject do def project do [ app: :umwelt, - version: "0.1.8", - elixir: "~> 1.15", + version: "0.2.0", + elixir: "~> 1.17", compilers: [:leex, :yecc] ++ Mix.compilers(), start_permanent: Mix.env() == :prod, deps: deps(), @@ -21,13 +21,18 @@ defmodule Umwelt.MixProject do # Run "mix help compile.app" to learn about applications. def application do [ - extra_applications: [:logger] + extra_applications: [:inets, :logger, :public_key, :ssl], + mod: {Umwelt.Client.Application, []} ] end # Run "mix help deps" to learn about dependencies. defp deps do [ + {:certifi, "~> 2.5"}, + {:jason, "~> 1.2"}, + {:progress_bar, "~> 3.0"}, + {:bypass, "~> 2.1", only: :test}, {:credo, "~> 1.6", only: [:dev, :test], runtime: false}, {:dialyxir, "~> 1.0", only: [:dev, :test], runtime: false}, {:ex_doc, "~> 0.29", only: :dev, runtime: false} diff --git a/mix.lock b/mix.lock index 0736b5c..9329361 100644 --- a/mix.lock +++ b/mix.lock @@ -1,6 +1,12 @@ %{ "bunt": {:hex, :bunt, "1.0.0", "081c2c665f086849e6d57900292b3a161727ab40431219529f13c4ddcf3e7a44", [:mix], [], "hexpm", "dc5f86aa08a5f6fa6b8096f0735c4e76d54ae5c9fa2c143e5a1fc7c1cd9bb6b5"}, + "bypass": {:hex, :bypass, "2.1.0", "909782781bf8e20ee86a9cabde36b259d44af8b9f38756173e8f5e2e1fabb9b1", [:mix], [{:plug, "~> 1.7", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 2.0", [hex: :plug_cowboy, repo: "hexpm", optional: false]}, {:ranch, "~> 1.3", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm", "d9b5df8fa5b7a6efa08384e9bbecfe4ce61c77d28a4282f79e02f1ef78d96b80"}, + "certifi": {:hex, :certifi, "2.13.0", "e52be248590050b2dd33b0bb274b56678f9068e67805dca8aa8b1ccdb016bbf6", [:rebar3], [], "hexpm", "8f3d9533a0f06070afdfd5d596b32e21c6580667a492891851b0e2737bc507a1"}, + "cowboy": {:hex, :cowboy, "2.12.0", "f276d521a1ff88b2b9b4c54d0e753da6c66dd7be6c9fca3d9418b561828a3731", [:make, :rebar3], [{:cowlib, "2.13.0", [hex: :cowlib, repo: "hexpm", optional: false]}, {:ranch, "1.8.0", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm", "8a7abe6d183372ceb21caa2709bec928ab2b72e18a3911aa1771639bef82651e"}, + "cowboy_telemetry": {:hex, :cowboy_telemetry, "0.4.0", "f239f68b588efa7707abce16a84d0d2acf3a0f50571f8bb7f56a15865aae820c", [:rebar3], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "7d98bac1ee4565d31b62d59f8823dfd8356a169e7fcbb83831b8a5397404c9de"}, + "cowlib": {:hex, :cowlib, "2.13.0", "db8f7505d8332d98ef50a3ef34b34c1afddec7506e4ee4dd4a3a266285d282ca", [:make, :rebar3], [], "hexpm", "e1e1284dc3fc030a64b1ad0d8382ae7e99da46c3246b815318a4b848873800a4"}, "credo": {:hex, :credo, "1.7.7", "771445037228f763f9b2afd612b6aa2fd8e28432a95dbbc60d8e03ce71ba4446", [:mix], [{:bunt, "~> 0.2.1 or ~> 1.0", [hex: :bunt, repo: "hexpm", optional: false]}, {:file_system, "~> 0.2 or ~> 1.0", [hex: :file_system, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "8bc87496c9aaacdc3f90f01b7b0582467b69b4bd2441fe8aae3109d843cc2f2e"}, + "decimal": {:hex, :decimal, "2.1.1", "5611dca5d4b2c3dd497dec8f68751f1f1a54755e8ed2a966c2633cf885973ad6", [:mix], [], "hexpm", "53cfe5f497ed0e7771ae1a475575603d77425099ba5faef9394932b35020ffcc"}, "dialyxir": {:hex, :dialyxir, "1.4.3", "edd0124f358f0b9e95bfe53a9fcf806d615d8f838e2202a9f430d59566b6b53b", [:mix], [{:erlex, ">= 0.2.6", [hex: :erlex, repo: "hexpm", optional: false]}], "hexpm", "bf2cfb75cd5c5006bec30141b131663299c661a864ec7fbbc72dfa557487a986"}, "earmark_parser": {:hex, :earmark_parser, "1.4.39", "424642f8335b05bb9eb611aa1564c148a8ee35c9c8a8bba6e129d51a3e3c6769", [:mix], [], "hexpm", "06553a88d1f1846da9ef066b87b57c6f605552cfbe40d20bd8d59cc6bde41944"}, "erlex": {:hex, :erlex, "0.2.6", "c7987d15e899c7a2f34f5420d2a2ea0d659682c06ac607572df55a43753aa12e", [:mix], [], "hexpm", "2ed2e25711feb44d52b17d2780eabf998452f6efda104877a3881c2f8c0c0c75"}, @@ -10,5 +16,12 @@ "makeup": {:hex, :makeup, "1.1.2", "9ba8837913bdf757787e71c1581c21f9d2455f4dd04cfca785c70bbfff1a76a3", [:mix], [{:nimble_parsec, "~> 1.2.2 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "cce1566b81fbcbd21eca8ffe808f33b221f9eee2cbc7a1706fc3da9ff18e6cac"}, "makeup_elixir": {:hex, :makeup_elixir, "0.16.2", "627e84b8e8bf22e60a2579dad15067c755531fea049ae26ef1020cad58fe9578", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.2.3 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "41193978704763f6bbe6cc2758b84909e62984c7752b3784bd3c218bb341706b"}, "makeup_erlang": {:hex, :makeup_erlang, "1.0.0", "6f0eff9c9c489f26b69b61440bf1b238d95badae49adac77973cbacae87e3c2e", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "ea7a9307de9d1548d2a72d299058d1fd2339e3d398560a0e46c27dab4891e4d2"}, + "mime": {:hex, :mime, "2.0.6", "8f18486773d9b15f95f4f4f1e39b710045fa1de891fada4516559967276e4dc2", [:mix], [], "hexpm", "c9945363a6b26d747389aac3643f8e0e09d30499a138ad64fe8fd1d13d9b153e"}, "nimble_parsec": {:hex, :nimble_parsec, "1.4.0", "51f9b613ea62cfa97b25ccc2c1b4216e81df970acd8e16e8d1bdc58fef21370d", [:mix], [], "hexpm", "9c565862810fb383e9838c1dd2d7d2c437b3d13b267414ba6af33e50d2d1cf28"}, + "plug": {:hex, :plug, "1.16.1", "40c74619c12f82736d2214557dedec2e9762029b2438d6d175c5074c933edc9d", [:mix], [{:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_crypto, "~> 1.1.1 or ~> 1.2 or ~> 2.0", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "a13ff6b9006b03d7e33874945b2755253841b238c34071ed85b0e86057f8cddc"}, + "plug_cowboy": {:hex, :plug_cowboy, "2.7.1", "87677ffe3b765bc96a89be7960f81703223fe2e21efa42c125fcd0127dd9d6b2", [:mix], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:cowboy_telemetry, "~> 0.3", [hex: :cowboy_telemetry, repo: "hexpm", optional: false]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "02dbd5f9ab571b864ae39418db7811618506256f6d13b4a45037e5fe78dc5de3"}, + "plug_crypto": {:hex, :plug_crypto, "2.1.0", "f44309c2b06d249c27c8d3f65cfe08158ade08418cf540fd4f72d4d6863abb7b", [:mix], [], "hexpm", "131216a4b030b8f8ce0f26038bc4421ae60e4bb95c5cf5395e1421437824c4fa"}, + "progress_bar": {:hex, :progress_bar, "3.0.0", "f54ff038c2ac540cfbb4c2bfe97c75e7116ead044f3c2b10c9f212452194b5cd", [:mix], [{:decimal, "~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}], "hexpm", "6981c2b25ab24aecc91a2dc46623658e1399c21a2ae24db986b90d678530f2b7"}, + "ranch": {:hex, :ranch, "1.8.0", "8c7a100a139fd57f17327b6413e4167ac559fbc04ca7448e9be9057311597a1d", [:make, :rebar3], [], "hexpm", "49fbcfd3682fab1f5d109351b61257676da1a2fdbe295904176d5e521a2ddfe5"}, + "telemetry": {:hex, :telemetry, "1.2.1", "68fdfe8d8f05a8428483a97d7aab2f268aaff24b49e0f599faa091f1d4e7f61c", [:rebar3], [], "hexpm", "dad9ce9d8effc621708f99eac538ef1cbe05d6a874dd741de2e689c47feafed5"}, } diff --git a/test/mix/tasks/clone_test.exs b/test/mix/tasks/clone_test.exs new file mode 100644 index 0000000..373f08a --- /dev/null +++ b/test/mix/tasks/clone_test.exs @@ -0,0 +1,93 @@ +defmodule Mix.Tasks.CloneTest do + use ExUnit.Case + import ExUnit.CaptureLog + + alias Bypass + alias Mix.Tasks.Umwelt.Clone + + setup do + bypass = Bypass.open() + Application.put_env(:umwelt, :api_port, bypass.port) + + :ok = Application.ensure_started(:umwelt) + + on_exit(fn -> File.rm_rf!("umwelt_raw/temp") end) + + {:ok, bypass: bypass} + end + + describe "clone" do + test "interface call" do + Application.put_env(:umwelt, :api_token, "no_token") + + assert capture_log([], fn -> assert :ok == Clone.run([42]) end) =~ "Token not found in env!" + end + + test "success", %{bypass: bypass} do + Bypass.expect_once(bypass, "GET", "/api/trees/23", fn conn -> + Plug.Conn.resp(conn, 200, ~s({"data": { + "Disco": "1", + "Disco.Chaos": "2", + "Disco.Discord": "3" + }})) + end) + + Bypass.expect_once(bypass, "GET", "/api/code/23/1", fn conn -> + Plug.Conn.resp( + conn, + 200, + ~s({"data": {"temp/lib/disco.ex": "defmodule Disco", + "temp/test/disco_test.ex": "defmodule DiscoTest"}}) + ) + end) + + Bypass.expect_once(bypass, "GET", "/api/code/23/2", fn conn -> + Plug.Conn.resp( + conn, + 200, + ~s({"data": {"temp/lib/disco/chaos.ex": "defmodule Disco.Chaos", + "temp/test/disco/chaos_test.ex": "defmodule Disco.ChaosTest"}}) + ) + end) + + # here we check that fetcher respawns after 401 + Bypass.expect_once(bypass, "GET", "/api/code/23/3", fn conn -> + Plug.Conn.resp(conn, 401, ~s({"error": "Unauthorized"})) + end) + + Bypass.expect_once(bypass, "GET", "/api/code/23/3", fn conn -> + Plug.Conn.resp( + conn, + 200, + ~s({"data": {"temp/lib/disco/discord.ex": "defmodule Disco.Discord", + "temp/test/disco/discord_test.ex": "defmodule Disco.DiscordTest"}}) + ) + end) + + assert capture_log([], fn -> assert :ok == Clone.run([23, "token"]) end) =~ "Done!" + + :timer.sleep(666) + assert "defmodule Disco" == File.read!("umwelt_raw/temp/lib/disco.ex") + assert "defmodule DiscoTest" == File.read!("umwelt_raw/temp/test/disco_test.ex") + assert "defmodule Disco.Chaos" == File.read!("umwelt_raw/temp/lib/disco/chaos.ex") + assert "defmodule Disco.ChaosTest" == File.read!("umwelt_raw/temp/test/disco/chaos_test.ex") + assert "defmodule Disco.Discord" == File.read!("umwelt_raw/temp/lib/disco/discord.ex") + + assert "defmodule Disco.DiscordTest" == + File.read!("umwelt_raw/temp/test/disco/discord_test.ex") + + assert {:ok, _} = File.rm_rf("umwelt_raw/temp") + end + + test "when fetch unsuccessful", %{bypass: bypass} do + Bypass.expect_once(bypass, "GET", "/api/trees/42", fn conn -> + Plug.Conn.resp(conn, 401, ~s({"error": "Unauthorized"})) + end) + + assert capture_log([], fn -> assert :ok == Clone.run([42, "bad_token"]) end) =~ + "Failed to fetch modules: \"Unauthorized\". Stopping..." + + Application.stop(:umwelt) + end + end +end diff --git a/test/mix/tasks/dump_test.exs b/test/mix/tasks/dump_test.exs index b6dd6f2..bf278ed 100644 --- a/test/mix/tasks/dump_test.exs +++ b/test/mix/tasks/dump_test.exs @@ -1,11 +1,14 @@ defmodule Mix.Tasks.DumpTest do use ExUnit.Case, async: true + import ExUnit.CaptureIO - alias Mix.Tasks.Dump + alias Mix.Tasks.Umwelt.Dump describe "parse([root_name])" do test "parse app from Mix.Project.config by default" do - assert :ok = Dump.run([]) + assert capture_io([], fn -> + assert :ok == Dump.run([]) + end) =~ "Parsing result saved into umwelt.bin" {:ok, bin} = File.read("umwelt.bin") @@ -15,7 +18,9 @@ defmodule Mix.Tasks.DumpTest do end test "parse app from given root_name" do - assert :ok = Dump.run(["umwelt"]) + assert capture_io([], fn -> + assert :ok == Dump.run(["umwelt"]) + end) =~ "Parsing result saved into umwelt.bin" {:ok, bin} = File.read("umwelt.bin") diff --git a/test/umwelt/client/agent_test.exs b/test/umwelt/client/agent_test.exs new file mode 100644 index 0000000..3f2f519 --- /dev/null +++ b/test/umwelt/client/agent_test.exs @@ -0,0 +1,165 @@ +defmodule Umwelt.Client.AgentTest do + use ExUnit.Case + + alias Umwelt.Client.Agent + + setup do + modules = %{ + "Discordian" => 248_169, + "Discordian.Aftermath" => 248_188, + "Discordian.Bureaucracy" => 248_187, + "Discordian.Chaos" => 248_184, + "Discordian.Confusion" => 248_186, + "Discordian.Discord" => 248_185 + } + + :ok = Application.ensure_started(:umwelt) + + {:ok, modules: modules} + end + + test "initial state is empty", _context do + assert %{ + modules: %{}, + waiting: [], + fetching: [], + fetched: [], + writing: [], + written: [], + total: 0 + } == Agent.state() + end + + test "add_modules/1 adds modules to state", %{modules: modules} do + Agent.add_modules(modules) + + assert %{ + modules: modules, + waiting: Map.keys(modules), + fetching: [], + fetched: [], + writing: [], + written: [], + total: 6 + } == Agent.state() + + Agent.add_modules(%{}) + end + + describe "next_waiting/0" do + test "retrieves and updates the state correctly", %{modules: modules} do + Agent.add_modules(modules) + + assert Agent.next_waiting() == %{id: 248_169, name: "Discordian"} + + assert %{ + modules: modules, + waiting: Map.keys(modules) -- ["Discordian"], + fetching: ["Discordian"], + fetched: [], + writing: [], + written: [], + total: 6 + } == Agent.state() + + Agent.add_modules(%{}) + end + + test "when waiting list is already empty" do + Agent.add_modules(%{"Discordian" => 248_169}) + + assert Agent.next_waiting() == %{id: 248_169, name: "Discordian"} + + assert %{ + modules: %{"Discordian" => 248_169}, + waiting: [], + fetching: ["Discordian"], + fetched: [], + writing: [], + written: [], + total: 1 + } == Agent.state() + + assert Agent.next_waiting() == nil + + assert %{ + modules: %{"Discordian" => 248_169}, + waiting: [], + fetching: ["Discordian"], + fetched: [], + writing: [], + written: [], + total: 1 + } == Agent.state() + + Agent.add_modules(%{}) + end + end + + describe "update_status/2" do + test "updates the module status to :fetched", %{modules: modules} do + Agent.add_modules(modules) + Agent.next_waiting() + + Agent.update_status("Discordian", :fetched) + state = Agent.state() + assert state.fetching == [] + assert state.fetched == ["Discordian"] + Agent.add_modules(%{}) + end + + test "updates the module status to :writing", %{modules: modules} do + Agent.add_modules(modules) + Agent.next_waiting() + Agent.update_status("Discordian", :fetched) + + Agent.update_status("Discordian", :writing) + state = Agent.state() + assert state.writing == ["Discordian"] + Agent.add_modules(%{}) + end + + test "updates the module status to :written", %{modules: modules} do + Agent.add_modules(modules) + Agent.next_waiting() + Agent.update_status("Discordian", :fetched) + Agent.update_status("Discordian", :writing) + + Agent.update_status("Discordian", :written) + state = Agent.state() + assert state.writing == [] + assert state.written == ["Discordian"] + Agent.add_modules(%{}) + end + end + + describe "completed?/0" do + test "returns true when all modules are written", %{modules: modules} do + Agent.add_modules(modules) + + for module_name <- Map.keys(modules) do + Agent.next_waiting() + Agent.update_status(module_name, :fetched) + Agent.update_status(module_name, :writing) + Agent.update_status(module_name, :written) + end + + assert Agent.completed?() + Agent.add_modules(%{}) + end + + test "returns false when not all modules are written", %{modules: modules} do + Agent.add_modules(modules) + + for module_name <- Enum.take(Map.keys(modules), 3) do + Agent.next_waiting() + Agent.update_status(module_name, :fetched) + Agent.update_status(module_name, :writing) + Agent.update_status(module_name, :written) + end + + refute Agent.completed?() + Agent.add_modules(%{}) + end + end +end diff --git a/test/umwelt/client/request_test.exs b/test/umwelt/client/request_test.exs new file mode 100644 index 0000000..619f03a --- /dev/null +++ b/test/umwelt/client/request_test.exs @@ -0,0 +1,51 @@ +defmodule Umwelt.Client.RequestTest do + use ExUnit.Case + + alias Bypass + alias Umwelt.Client.Request + + setup do + bypass = Bypass.open() + :ok = Application.ensure_started(:umwelt) + + {:ok, bypass: bypass} + end + + test "fetch_modules returns modules", %{bypass: bypass} do + Bypass.expect(bypass, fn conn -> + Plug.Conn.resp(conn, 200, ~s({"data": { + "Disco": "1", + "Disco.Chaos": "2", + "Disco.Discord": "3" + }})) + end) + + assert {:ok, + %{ + "Disco" => "1", + "Disco.Chaos" => "2", + "Disco.Discord" => "3" + }} == + Request.fetch_modules(%{ + api_host: "http://localhost", + phase_id: 1, + port: bypass.port, + token: "token" + }) + end + + test "fetch_code returns code files", %{bypass: bypass} do + Bypass.expect(bypass, fn conn -> + Plug.Conn.resp(conn, 200, ~s({"data": {"foo.ex": "code1", "foo_spec.exs": "code2"}})) + end) + + assert {:ok, %{"foo.ex" => "code1", "foo_spec.exs" => "code2"}} == + Request.fetch_code(%{ + api_host: "http://localhost", + id: 23, + phase_id: 5, + port: bypass.port, + token: "token" + }) + end +end diff --git a/test/umwelt/client/writer_test.exs b/test/umwelt/client/writer_test.exs new file mode 100644 index 0000000..6e5f441 --- /dev/null +++ b/test/umwelt/client/writer_test.exs @@ -0,0 +1,80 @@ +defmodule Umwelt.Client.WriterTest do + use ExUnit.Case + + alias Umwelt.Client.{Agent, Writer} + + setup do + module = %{ + code: %{ + "temp/lib/disco/chaos.ex" => "defmodule Disco.Chaos", + "temp/test/disco/chaos_test.exs" => "defmodule Disco.ChaosTest" + }, + name: "Disco.Chaos" + } + + :ok = Application.ensure_started(:umwelt) + + Agent.add_modules(%{"Disco.Chaos" => 23}) + Agent.update_status("Disco.Chaos", :fetched) + + on_exit(fn -> File.rm_rf!("umwelt_raw/temp") end) + {:ok, module: module} + end + + describe "run/1" do + test "writes new files correctly", %{module: module} do + lib_path = "umwelt_raw/temp/lib/disco/chaos.ex" + test_path = "umwelt_raw/temp/test/disco/chaos_test.exs" + + log = capture_log(fn -> Writer.run(module) end) + + :timer.sleep(666) + assert log =~ " #{Path.expand(lib_path)}: created" + assert log =~ " #{Path.expand(test_path)}: created" + + assert File.read!(lib_path) == "defmodule Disco.Chaos" + assert File.read!(test_path) == "defmodule Disco.ChaosTest" + end + + test "writes identical files correctly", %{module: module} do + lib_path = "umwelt_raw/temp/lib/disco/chaos.ex" + test_path = "umwelt_raw/temp/test/disco/chaos_test.exs" + + File.mkdir_p!(Path.dirname(lib_path)) + File.write!(lib_path, "defmodule Disco.Chaos") + File.mkdir_p!(Path.dirname(test_path)) + File.write!(test_path, "defmodule Disco.ChaosTest") + + log = capture_log(fn -> Writer.run(module) end) + + assert log =~ " #{Path.expand(lib_path)}: identical" + assert log =~ " #{Path.expand(test_path)}: identical" + end + + test "creates backup and writes new content", %{module: module} do + lib_path = "umwelt_raw/temp/lib/disco/chaos.ex" + test_path = "umwelt_raw/temp/test/disco/chaos_test.exs" + lib_backup_path = "#{lib_path}_" + test_backup_path = "#{test_path}_" + + File.mkdir_p!(Path.dirname(lib_path)) + File.write!(lib_path, "old lib content") + File.mkdir_p!(Path.dirname(test_path)) + File.write!(test_path, "old test content") + + log = capture_log(fn -> Writer.run(module) end) + + assert log =~ "#{Path.expand(lib_path)}: created_with_backup" + assert log =~ "#{Path.expand(test_path)}: created_with_backup" + + assert File.read!(lib_path) == "defmodule Disco.Chaos" + assert File.read!(test_path) == "defmodule Disco.ChaosTest" + assert File.read!(lib_backup_path) == "old lib content" + assert File.read!(test_backup_path) == "old test content" + end + end + + defp capture_log(fun) do + ExUnit.CaptureLog.capture_log(fn -> fun.() end) + end +end diff --git a/test/umwelt/parser_test.exs b/test/umwelt/parser_test.exs index a14c5c5..f459f80 100644 --- a/test/umwelt/parser_test.exs +++ b/test/umwelt/parser_test.exs @@ -225,6 +225,18 @@ defmodule Umwelt.ParserTest do } == Parser.parse_raw(code) end + + test "parsing error", %{} do + assert {:error, + {[ + opening_delimiter: :do, + expected_delimiter: :end, + line: 1, + column: 9, + end_line: 1, + end_column: 11 + ], "missing terminator: end", ""}} == Parser.parse_raw("def foo do") + end end describe "reading ast" do From 52dcbd6c8a3de90bd6bb05c39c43ec8d923ec3af Mon Sep 17 00:00:00 2001 From: Sovetnik <5460740+sovetnik@users.noreply.github.com> Date: Wed, 7 Aug 2024 19:41:21 +0300 Subject: [PATCH 2/6] Fix style --- lib/mix/tasks/clone.ex | 9 +++---- lib/umwelt/client/clone.ex | 17 +++----------- lib/umwelt/client/request.ex | 44 ++++++++++++++--------------------- lib/umwelt/client/writer.ex | 7 ++---- test/mix/tasks/clone_test.exs | 1 - 5 files changed, 26 insertions(+), 52 deletions(-) diff --git a/lib/mix/tasks/clone.ex b/lib/mix/tasks/clone.ex index b5e9d17..cb15b0e 100644 --- a/lib/mix/tasks/clone.ex +++ b/lib/mix/tasks/clone.ex @@ -28,7 +28,7 @@ defmodule Mix.Tasks.Umwelt.Clone do @impl Mix.Task def run([phase_id, token]) do - Umwelt.Client.Application.start(nil, nil) + Umwelt.Client.Application.start(:normal, []) Umwelt.Client.Supervisor |> Process.whereis() @@ -51,11 +51,8 @@ defmodule Mix.Tasks.Umwelt.Clone do defp assign_host(params) do host = case Mix.env() do - :dev -> - System.get_env("UMWELT_HOST", "https://umwelt.dev") - - :test -> - "http://localhost" + :test -> "http://localhost" + _ -> System.get_env("UMWELT_HOST", "https://umwelt.dev") end Map.put(params, :api_host, host) diff --git a/lib/umwelt/client/clone.ex b/lib/umwelt/client/clone.ex index e422dee..fb8e430 100644 --- a/lib/umwelt/client/clone.ex +++ b/lib/umwelt/client/clone.ex @@ -1,7 +1,7 @@ defmodule Umwelt.Client.Clone do @moduledoc "Clone main process" - use GenServer + use GenServer, restart: :transient, shutdown: 10_000 require Logger alias Umwelt.Client @@ -17,12 +17,10 @@ defmodule Umwelt.Client.Clone do def init(state), do: {:ok, state} def handle_cast({:pull, params}, _state) do - send(self(), :start_pulling) - - {:noreply, params} + {:noreply, params, {:continue, :start_pulling}} end - def handle_info(:start_pulling, state) do + def handle_continue(:start_pulling, state) do case Client.Request.fetch_modules(state) do {:ok, modules} -> Logger.info("Fetching modules: #{inspect(Map.keys(modules))}") @@ -33,15 +31,6 @@ defmodule Umwelt.Client.Clone do Supervisor.stop(Client.Supervisor) end - send(self(), :spawn_fetchers) - - {:noreply, state} - end - - def handle_info(:spawn_fetchers, state) do - total = Client.Agent.total() - Logger.debug("Spawning fetchers: #{total}") - Client.Agent.all_waiting() |> Enum.each(fn _ -> spawn_fetcher(state) end) diff --git a/lib/umwelt/client/request.ex b/lib/umwelt/client/request.ex index a2e5c4e..75429b9 100644 --- a/lib/umwelt/client/request.ex +++ b/lib/umwelt/client/request.ex @@ -4,40 +4,32 @@ defmodule Umwelt.Client.Request do require Logger def fetch_modules(params) do - url = ~c"#{params.api_host}:#{params.port}/api/trees/#{params.phase_id}" - - case do_request(url, params.token) do - {:ok, {{~c"HTTP/1.1", 200, ~c"OK"}, _headers, body}} -> - {:ok, Jason.decode!(body)["data"]} - - {:ok, {{~c"HTTP/1.1", _status, _reason}, _headers, body}} -> - %{"error" => reason} = Jason.decode!(body) - - {:error, reason} - - {:error, reason} -> - {:error, reason} - end + ~c"#{params.api_host}:#{params.port}/api/trees/#{params.phase_id}" + |> do_request(params.token) + |> handle_response() end def fetch_code(params) do - url = ~c"#{params.api_host}:#{params.port}/api/code/#{params.phase_id}/#{params.id}" + ~c"#{params.api_host}:#{params.port}/api/code/#{params.phase_id}/#{params.id}" + |> do_request(params.token) + |> handle_response() + end - case do_request(url, params.token) do - {:ok, {{~c"HTTP/1.1", 200, ~c"OK"}, _headers, body}} -> - {:ok, Jason.decode!(body)["data"]} + defp do_request(url, token) do + :httpc.request(:get, {url, headers(token)}, [], []) + end - {:ok, {{~c"HTTP/1.1", _status, _reason}, _headers, body}} -> - %{"error" => reason} = Jason.decode!(body) - {:error, reason} + defp handle_response({:ok, {{~c"HTTP/1.1", 200, ~c"OK"}, _headers, body}}) do + {:ok, Jason.decode!(body)["data"]} + end - {:error, reason} -> - {:error, reason} - end + defp handle_response({:ok, {{~c"HTTP/1.1", _status, _reason}, _headers, body}}) do + %{"error" => reason} = Jason.decode!(body) + {:error, reason} end - defp do_request(url, token) do - :httpc.request(:get, {url, headers(token)}, [], []) + defp handle_response({:error, reason}) do + {:error, reason} end defp headers(token) do diff --git a/lib/umwelt/client/writer.ex b/lib/umwelt/client/writer.ex index 8c6a9cf..da98c92 100644 --- a/lib/umwelt/client/writer.ex +++ b/lib/umwelt/client/writer.ex @@ -24,18 +24,15 @@ defmodule Umwelt.Client.Writer do case File.read(path) do {:ok, content} when content == code -> Logger.debug("Write #{path}: identical") - {path, :identical} {:ok, content} -> - :ok = File.write!("#{path}_", content) + :ok = File.write!(path <> "_", content) :ok = File.write!(path, code) Logger.debug("Write #{path}: created_with_backup") - {path, :created_with_backup} {:error, _reason} -> :ok = File.write!(path, code) - :ok = Logger.debug("Write #{path}: created") - {path, :created} + Logger.debug("Write #{path}: created") end end end diff --git a/test/mix/tasks/clone_test.exs b/test/mix/tasks/clone_test.exs index 373f08a..a168b3c 100644 --- a/test/mix/tasks/clone_test.exs +++ b/test/mix/tasks/clone_test.exs @@ -2,7 +2,6 @@ defmodule Mix.Tasks.CloneTest do use ExUnit.Case import ExUnit.CaptureLog - alias Bypass alias Mix.Tasks.Umwelt.Clone setup do From ee7ac9631ae2fa106098e5b03fb15d3b5d5d85a9 Mon Sep 17 00:00:00 2001 From: Sovetnik <5460740+sovetnik@users.noreply.github.com> Date: Thu, 8 Aug 2024 22:15:31 +0300 Subject: [PATCH 3/6] Add their own Supervisors for Fetchers and Writers --- lib/umwelt/client/application.ex | 2 ++ lib/umwelt/client/clone.ex | 8 ++++++-- lib/umwelt/client/fetcher.ex | 17 +++++++++-------- lib/umwelt/client/writer.ex | 11 ++++++----- test/umwelt/client/writer_test.exs | 1 + 5 files changed, 24 insertions(+), 15 deletions(-) diff --git a/lib/umwelt/client/application.ex b/lib/umwelt/client/application.ex index 624bcfd..350c4a0 100644 --- a/lib/umwelt/client/application.ex +++ b/lib/umwelt/client/application.ex @@ -6,6 +6,8 @@ defmodule Umwelt.Client.Application do def start(_type, _args) do children = [ {Umwelt.Client.Agent, []}, + {Task.Supervisor, name: Umwelt.Client.FetcherSupervisor}, + {Task.Supervisor, name: Umwelt.Client.WriterSupervisor}, {Umwelt.Client.Clone, []} ] diff --git a/lib/umwelt/client/clone.ex b/lib/umwelt/client/clone.ex index fb8e430..685af24 100644 --- a/lib/umwelt/client/clone.ex +++ b/lib/umwelt/client/clone.ex @@ -17,10 +17,10 @@ defmodule Umwelt.Client.Clone do def init(state), do: {:ok, state} def handle_cast({:pull, params}, _state) do - {:noreply, params, {:continue, :start_pulling}} + {:noreply, params, {:continue, :fetch_modules}} end - def handle_continue(:start_pulling, state) do + def handle_continue(:fetch_modules, state) do case Client.Request.fetch_modules(state) do {:ok, modules} -> Logger.info("Fetching modules: #{inspect(Map.keys(modules))}") @@ -31,6 +31,10 @@ defmodule Umwelt.Client.Clone do Supervisor.stop(Client.Supervisor) end + {:noreply, state, {:continue, :start_pulling}} + end + + def handle_continue(:start_pulling, state) do Client.Agent.all_waiting() |> Enum.each(fn _ -> spawn_fetcher(state) end) diff --git a/lib/umwelt/client/fetcher.ex b/lib/umwelt/client/fetcher.ex index bb294fc..4019af5 100644 --- a/lib/umwelt/client/fetcher.ex +++ b/lib/umwelt/client/fetcher.ex @@ -6,19 +6,20 @@ defmodule Umwelt.Client.Fetcher do alias Umwelt.Client - def start_link(module) do - Task.start_link(__MODULE__, :run, [module]) + def start_link(params) do + Client.FetcherSupervisor + |> Task.Supervisor.start_child(__MODULE__, :run, [params]) end - def run(module) do - case Client.Request.fetch_code(module) do + def run(params) do + case Client.Request.fetch_code(params) do {:ok, code} -> - Logger.debug("Success fetch #{module.name}") - send(Client.Clone, {:fetched, %{name: module.name, code: code}}) + Logger.debug("Success fetch #{params.name}") + send(Client.Clone, {:fetched, %{name: params.name, code: code}}) {:error, reason} -> - Logger.warning("Fail fetch #{module.name}: #{reason}") - send(Client.Clone, {:fetch_failed, module}) + Logger.warning("Fail fetch #{params.name}: #{reason}") + send(Client.Clone, {:fetch_failed, params}) end end end diff --git a/lib/umwelt/client/writer.ex b/lib/umwelt/client/writer.ex index da98c92..4d5ee2f 100644 --- a/lib/umwelt/client/writer.ex +++ b/lib/umwelt/client/writer.ex @@ -6,15 +6,16 @@ defmodule Umwelt.Client.Writer do alias Umwelt.Client - def start_link(module) do - Task.start_link(__MODULE__, :run, [module]) + def start_link(params) do + Client.WriterSupervisor + |> Task.Supervisor.start_child(__MODULE__, :run, [params]) end - def run(module) do - module.code + def run(params) do + params.code |> Enum.each(fn {path, code} -> write_to_file(path, code) end) - send(Client.Clone, {:written, module.name}) + send(Client.Clone, {:written, params.name}) end defp write_to_file(path, code) do diff --git a/test/umwelt/client/writer_test.exs b/test/umwelt/client/writer_test.exs index 6e5f441..c19c9c4 100644 --- a/test/umwelt/client/writer_test.exs +++ b/test/umwelt/client/writer_test.exs @@ -18,6 +18,7 @@ defmodule Umwelt.Client.WriterTest do Agent.update_status("Disco.Chaos", :fetched) on_exit(fn -> File.rm_rf!("umwelt_raw/temp") end) + {:ok, module: module} end From f61539e2ffbeef445e86294923d18766a2a8753c Mon Sep 17 00:00:00 2001 From: Sovetnik <5460740+sovetnik@users.noreply.github.com> Date: Sun, 18 Aug 2024 09:15:29 +0300 Subject: [PATCH 4/6] Write umwelt files directly in lib and test --- README.md | 1 + lib/umwelt/client/writer.ex | 2 +- test/mix/tasks/clone_test.exs | 18 +++++++----------- test/umwelt/client/writer_test.exs | 14 +++++++------- 4 files changed, 16 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index 8d14d53..58f03ba 100644 --- a/README.md +++ b/README.md @@ -33,6 +33,7 @@ When you wanna parse another folder in lib, `lib/another_root_name`, use: ### Clone Fetch and write all modules from specified phase. +Make sure that your project name and umwelt name didn't match to avoid file corruption. When your project is ready, you can get its code and specs. Create a new elixir or phoenix app, add umwelt and pull the code. diff --git a/lib/umwelt/client/writer.ex b/lib/umwelt/client/writer.ex index 4d5ee2f..666c79b 100644 --- a/lib/umwelt/client/writer.ex +++ b/lib/umwelt/client/writer.ex @@ -19,7 +19,7 @@ defmodule Umwelt.Client.Writer do end defp write_to_file(path, code) do - path = Path.expand(path, "umwelt_raw") + path = Path.expand(path) path |> Path.dirname() |> File.mkdir_p!() case File.read(path) do diff --git a/test/mix/tasks/clone_test.exs b/test/mix/tasks/clone_test.exs index a168b3c..fee7873 100644 --- a/test/mix/tasks/clone_test.exs +++ b/test/mix/tasks/clone_test.exs @@ -10,7 +10,7 @@ defmodule Mix.Tasks.CloneTest do :ok = Application.ensure_started(:umwelt) - on_exit(fn -> File.rm_rf!("umwelt_raw/temp") end) + on_exit(fn -> File.rm_rf!("temp") end) {:ok, bypass: bypass} end @@ -66,16 +66,12 @@ defmodule Mix.Tasks.CloneTest do assert capture_log([], fn -> assert :ok == Clone.run([23, "token"]) end) =~ "Done!" :timer.sleep(666) - assert "defmodule Disco" == File.read!("umwelt_raw/temp/lib/disco.ex") - assert "defmodule DiscoTest" == File.read!("umwelt_raw/temp/test/disco_test.ex") - assert "defmodule Disco.Chaos" == File.read!("umwelt_raw/temp/lib/disco/chaos.ex") - assert "defmodule Disco.ChaosTest" == File.read!("umwelt_raw/temp/test/disco/chaos_test.ex") - assert "defmodule Disco.Discord" == File.read!("umwelt_raw/temp/lib/disco/discord.ex") - - assert "defmodule Disco.DiscordTest" == - File.read!("umwelt_raw/temp/test/disco/discord_test.ex") - - assert {:ok, _} = File.rm_rf("umwelt_raw/temp") + assert "defmodule Disco" == File.read!("temp/lib/disco.ex") + assert "defmodule DiscoTest" == File.read!("temp/test/disco_test.ex") + assert "defmodule Disco.Chaos" == File.read!("temp/lib/disco/chaos.ex") + assert "defmodule Disco.ChaosTest" == File.read!("temp/test/disco/chaos_test.ex") + assert "defmodule Disco.Discord" == File.read!("temp/lib/disco/discord.ex") + assert "defmodule Disco.DiscordTest" == File.read!("temp/test/disco/discord_test.ex") end test "when fetch unsuccessful", %{bypass: bypass} do diff --git a/test/umwelt/client/writer_test.exs b/test/umwelt/client/writer_test.exs index c19c9c4..362f23c 100644 --- a/test/umwelt/client/writer_test.exs +++ b/test/umwelt/client/writer_test.exs @@ -17,15 +17,15 @@ defmodule Umwelt.Client.WriterTest do Agent.add_modules(%{"Disco.Chaos" => 23}) Agent.update_status("Disco.Chaos", :fetched) - on_exit(fn -> File.rm_rf!("umwelt_raw/temp") end) + on_exit(fn -> File.rm_rf!("temp") end) {:ok, module: module} end describe "run/1" do test "writes new files correctly", %{module: module} do - lib_path = "umwelt_raw/temp/lib/disco/chaos.ex" - test_path = "umwelt_raw/temp/test/disco/chaos_test.exs" + lib_path = "temp/lib/disco/chaos.ex" + test_path = "temp/test/disco/chaos_test.exs" log = capture_log(fn -> Writer.run(module) end) @@ -38,8 +38,8 @@ defmodule Umwelt.Client.WriterTest do end test "writes identical files correctly", %{module: module} do - lib_path = "umwelt_raw/temp/lib/disco/chaos.ex" - test_path = "umwelt_raw/temp/test/disco/chaos_test.exs" + lib_path = "temp/lib/disco/chaos.ex" + test_path = "temp/test/disco/chaos_test.exs" File.mkdir_p!(Path.dirname(lib_path)) File.write!(lib_path, "defmodule Disco.Chaos") @@ -53,8 +53,8 @@ defmodule Umwelt.Client.WriterTest do end test "creates backup and writes new content", %{module: module} do - lib_path = "umwelt_raw/temp/lib/disco/chaos.ex" - test_path = "umwelt_raw/temp/test/disco/chaos_test.exs" + lib_path = "temp/lib/disco/chaos.ex" + test_path = "temp/test/disco/chaos_test.exs" lib_backup_path = "#{lib_path}_" test_backup_path = "#{test_path}_" From d23154e7a9a30acc9c0d30960294347503912c6d Mon Sep 17 00:00:00 2001 From: Sovetnik <5460740+sovetnik@users.noreply.github.com> Date: Sun, 18 Aug 2024 09:20:17 +0300 Subject: [PATCH 5/6] Update deps --- mix.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/mix.lock b/mix.lock index 9329361..bab898f 100644 --- a/mix.lock +++ b/mix.lock @@ -8,14 +8,14 @@ "credo": {:hex, :credo, "1.7.7", "771445037228f763f9b2afd612b6aa2fd8e28432a95dbbc60d8e03ce71ba4446", [:mix], [{:bunt, "~> 0.2.1 or ~> 1.0", [hex: :bunt, repo: "hexpm", optional: false]}, {:file_system, "~> 0.2 or ~> 1.0", [hex: :file_system, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "8bc87496c9aaacdc3f90f01b7b0582467b69b4bd2441fe8aae3109d843cc2f2e"}, "decimal": {:hex, :decimal, "2.1.1", "5611dca5d4b2c3dd497dec8f68751f1f1a54755e8ed2a966c2633cf885973ad6", [:mix], [], "hexpm", "53cfe5f497ed0e7771ae1a475575603d77425099ba5faef9394932b35020ffcc"}, "dialyxir": {:hex, :dialyxir, "1.4.3", "edd0124f358f0b9e95bfe53a9fcf806d615d8f838e2202a9f430d59566b6b53b", [:mix], [{:erlex, ">= 0.2.6", [hex: :erlex, repo: "hexpm", optional: false]}], "hexpm", "bf2cfb75cd5c5006bec30141b131663299c661a864ec7fbbc72dfa557487a986"}, - "earmark_parser": {:hex, :earmark_parser, "1.4.39", "424642f8335b05bb9eb611aa1564c148a8ee35c9c8a8bba6e129d51a3e3c6769", [:mix], [], "hexpm", "06553a88d1f1846da9ef066b87b57c6f605552cfbe40d20bd8d59cc6bde41944"}, + "earmark_parser": {:hex, :earmark_parser, "1.4.41", "ab34711c9dc6212dda44fcd20ecb87ac3f3fce6f0ca2f28d4a00e4154f8cd599", [:mix], [], "hexpm", "a81a04c7e34b6617c2792e291b5a2e57ab316365c2644ddc553bb9ed863ebefa"}, "erlex": {:hex, :erlex, "0.2.6", "c7987d15e899c7a2f34f5420d2a2ea0d659682c06ac607572df55a43753aa12e", [:mix], [], "hexpm", "2ed2e25711feb44d52b17d2780eabf998452f6efda104877a3881c2f8c0c0c75"}, - "ex_doc": {:hex, :ex_doc, "0.34.1", "9751a0419bc15bc7580c73fde506b17b07f6402a1e5243be9e0f05a68c723368", [:mix], [{:earmark_parser, "~> 1.4.39", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_c, ">= 0.1.0", [hex: :makeup_c, repo: "hexpm", optional: true]}, {:makeup_elixir, "~> 0.14 or ~> 1.0", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1 or ~> 1.0", [hex: :makeup_erlang, repo: "hexpm", optional: false]}, {:makeup_html, ">= 0.1.0", [hex: :makeup_html, repo: "hexpm", optional: true]}], "hexpm", "d441f1a86a235f59088978eff870de2e815e290e44a8bd976fe5d64470a4c9d2"}, + "ex_doc": {:hex, :ex_doc, "0.34.2", "13eedf3844ccdce25cfd837b99bea9ad92c4e511233199440488d217c92571e8", [:mix], [{:earmark_parser, "~> 1.4.39", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_c, ">= 0.1.0", [hex: :makeup_c, repo: "hexpm", optional: true]}, {:makeup_elixir, "~> 0.14 or ~> 1.0", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1 or ~> 1.0", [hex: :makeup_erlang, repo: "hexpm", optional: false]}, {:makeup_html, ">= 0.1.0", [hex: :makeup_html, repo: "hexpm", optional: true]}], "hexpm", "5ce5f16b41208a50106afed3de6a2ed34f4acfd65715b82a0b84b49d995f95c1"}, "file_system": {:hex, :file_system, "1.0.0", "b689cc7dcee665f774de94b5a832e578bd7963c8e637ef940cd44327db7de2cd", [:mix], [], "hexpm", "6752092d66aec5a10e662aefeed8ddb9531d79db0bc145bb8c40325ca1d8536d"}, - "jason": {:hex, :jason, "1.4.1", "af1504e35f629ddcdd6addb3513c3853991f694921b1b9368b0bd32beb9f1b63", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "fbb01ecdfd565b56261302f7e1fcc27c4fb8f32d56eab74db621fc154604a7a1"}, + "jason": {:hex, :jason, "1.4.4", "b9226785a9aa77b6857ca22832cffa5d5011a667207eb2a0ad56adb5db443b8a", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "c5eb0cab91f094599f94d55bc63409236a8ec69a21a67814529e8d5f6cc90b3b"}, "makeup": {:hex, :makeup, "1.1.2", "9ba8837913bdf757787e71c1581c21f9d2455f4dd04cfca785c70bbfff1a76a3", [:mix], [{:nimble_parsec, "~> 1.2.2 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "cce1566b81fbcbd21eca8ffe808f33b221f9eee2cbc7a1706fc3da9ff18e6cac"}, "makeup_elixir": {:hex, :makeup_elixir, "0.16.2", "627e84b8e8bf22e60a2579dad15067c755531fea049ae26ef1020cad58fe9578", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.2.3 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "41193978704763f6bbe6cc2758b84909e62984c7752b3784bd3c218bb341706b"}, - "makeup_erlang": {:hex, :makeup_erlang, "1.0.0", "6f0eff9c9c489f26b69b61440bf1b238d95badae49adac77973cbacae87e3c2e", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "ea7a9307de9d1548d2a72d299058d1fd2339e3d398560a0e46c27dab4891e4d2"}, + "makeup_erlang": {:hex, :makeup_erlang, "1.0.1", "c7f58c120b2b5aa5fd80d540a89fdf866ed42f1f3994e4fe189abebeab610839", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "8a89a1eeccc2d798d6ea15496a6e4870b75e014d1af514b1b71fa33134f57814"}, "mime": {:hex, :mime, "2.0.6", "8f18486773d9b15f95f4f4f1e39b710045fa1de891fada4516559967276e4dc2", [:mix], [], "hexpm", "c9945363a6b26d747389aac3643f8e0e09d30499a138ad64fe8fd1d13d9b153e"}, "nimble_parsec": {:hex, :nimble_parsec, "1.4.0", "51f9b613ea62cfa97b25ccc2c1b4216e81df970acd8e16e8d1bdc58fef21370d", [:mix], [], "hexpm", "9c565862810fb383e9838c1dd2d7d2c437b3d13b267414ba6af33e50d2d1cf28"}, "plug": {:hex, :plug, "1.16.1", "40c74619c12f82736d2214557dedec2e9762029b2438d6d175c5074c933edc9d", [:mix], [{:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_crypto, "~> 1.1.1 or ~> 1.2 or ~> 2.0", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "a13ff6b9006b03d7e33874945b2755253841b238c34071ed85b0e86057f8cddc"}, From 26d1906983dd67f1c2c7b26915168a70014ff8a0 Mon Sep 17 00:00:00 2001 From: Sovetnik <5460740+sovetnik@users.noreply.github.com> Date: Sun, 18 Aug 2024 09:27:24 +0300 Subject: [PATCH 6/6] Possible fix blinking test --- lib/umwelt/client/clone.ex | 1 + test/mix/tasks/clone_test.exs | 6 ++++-- test/umwelt/client/writer_test.exs | 5 ++++- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/lib/umwelt/client/clone.ex b/lib/umwelt/client/clone.ex index 685af24..e2d6b5e 100644 --- a/lib/umwelt/client/clone.ex +++ b/lib/umwelt/client/clone.ex @@ -62,6 +62,7 @@ defmodule Umwelt.Client.Clone do def handle_info(:maybe_stop, state) do if Client.Agent.completed?() do Logger.debug("All modules processed. Stopping application.") + :timer.sleep(99) Supervisor.stop(Client.Supervisor) end diff --git a/test/mix/tasks/clone_test.exs b/test/mix/tasks/clone_test.exs index fee7873..921dcef 100644 --- a/test/mix/tasks/clone_test.exs +++ b/test/mix/tasks/clone_test.exs @@ -10,7 +10,10 @@ defmodule Mix.Tasks.CloneTest do :ok = Application.ensure_started(:umwelt) - on_exit(fn -> File.rm_rf!("temp") end) + on_exit(fn -> + :timer.sleep(99) + File.rm_rf!("temp") + end) {:ok, bypass: bypass} end @@ -65,7 +68,6 @@ defmodule Mix.Tasks.CloneTest do assert capture_log([], fn -> assert :ok == Clone.run([23, "token"]) end) =~ "Done!" - :timer.sleep(666) assert "defmodule Disco" == File.read!("temp/lib/disco.ex") assert "defmodule DiscoTest" == File.read!("temp/test/disco_test.ex") assert "defmodule Disco.Chaos" == File.read!("temp/lib/disco/chaos.ex") diff --git a/test/umwelt/client/writer_test.exs b/test/umwelt/client/writer_test.exs index 362f23c..f89ee2d 100644 --- a/test/umwelt/client/writer_test.exs +++ b/test/umwelt/client/writer_test.exs @@ -17,7 +17,10 @@ defmodule Umwelt.Client.WriterTest do Agent.add_modules(%{"Disco.Chaos" => 23}) Agent.update_status("Disco.Chaos", :fetched) - on_exit(fn -> File.rm_rf!("temp") end) + on_exit(fn -> + :timer.sleep(199) + File.rm_rf!("temp") + end) {:ok, module: module} end