Introduce Clone task

.tool-versions

@@ -1,2 +1,2 @@
 elixir 1.17.2
-erlang 27.0
+erlang 27.0.1

README.md

@@ -1,12 +1,6 @@
 # Umwelt ![Umwelt CI](https://github.com/sovetnik/umwelt/actions/workflows/elixir.yml/badge.svg?event=push) [![wakatime](https://wakatime.com/badge/user/7542de1a-027f-4ed7-bc4b-c31d4cf9aa2a/project/018c9f92-bb93-4303-816f-bc0799a61194.svg)](https://wakatime.com/badge/user/7542de1a-027f-4ed7-bc4b-c31d4cf9aa2a/project/018c9f92-bb93-4303-816f-bc0799a61194)
 Client for [umwelt.dev](https://umwelt.dev)
 
-## Implemented actions:
-
-### Dump
-
-Extracts Umwelt from Elixir project and dumps it into `root_name.bin`
-
 ## Installation
 
 [available in Hex](https://hex.pm/packages/umwelt), the package can be installed
@@ -22,24 +16,53 @@ end
 
 ## Usage
 
-Right now it is a proof of concept, and in this version parser can parse some business-logic related code, via `mix dump`.
+### Dump
+
+Extracts Umwelt from Elixir project and dumps it into `root_name.bin`
 
 In common case, when you want to parse your project and it's name from `Mix.Project.config()[:app]` matches root folder name `lib/root_name`, use:
 ```bash
-  mix dump
+  mix umwelt.dump
 ```
 
 When you wanna parse another folder in lib, `lib/another_root_name`, use:
 ```bash
-  mix dump another_root_name
+  mix umwelt.dump another_root_name
+```
+
+### Clone
+
+Fetch and write all modules from specified phase.
+
+When your project is ready, you can get its code and specs.
+Create a new elixir or phoenix app, add umwelt and pull the code.
+```bash
+  mix new project_name
+  cd project_name
 ```
+add umwelt to deps in `mix.exs`.
 
+You have to obtain a token on [profile page](https://umwelt.dev/auth/profile) 
+```bash
+  export UMWELT_TOKEN="your_token"
+  mix umwelt.clone phase_id
+  mix test --trace
+```
+And now you will see all messages for failing tests and can start coding.
+
+If you want to reduce logger add this to your `config.exs`
+```
+    config :logger,
+      compile_time_purge_matching: [
+        [application: :umwelt, level_lower_than: :warning]
+      ]
+```
 
 ## Planned
 Here is the list of planned features:
 
 ### Client functions
-Set of push/pull/sync mix tasks to sync local code with remote representation on [umwelt.dev](https://umwelt.dev)
+Set of pull/push/sync mix tasks to sync local code with remote representation on [umwelt.dev](https://umwelt.dev)
 
 ### Unparser
 Tools for update local code with changes made on web side.

lib/mix/tasks/clone.ex

@@ -0,0 +1,79 @@
+defmodule Mix.Tasks.Umwelt.Clone do
+  @moduledoc "Clones phase modules and code"
+  @shortdoc "The code puller"
+  use Mix.Task
+  require Logger
+
+  @impl Mix.Task
+  def run([phase_id]) do
+    case System.get_env("UMWELT_TOKEN", "no_token") do
+      "no_token" ->
+        """
+        Token not found in env!
+        You can get it on umwelt.dev/auth/profile and do 
+
+          export UMWELT_TOKEN="token"
+
+        or pass it directly in 
+
+          mix clone phase_id "token"
+
+        """
+        |> Logger.warning()
+
+      token ->
+        run([phase_id, token])
+    end
+  end
+
+  @impl Mix.Task
+  def run([phase_id, token]) do
+    Umwelt.Client.Application.start(nil, nil)
+
+    Umwelt.Client.Supervisor
+    |> Process.whereis()
+    |> Process.monitor()
+
+    %{phase_id: phase_id, token: token}
+    |> assign_host()
+    |> assign_port()
+    |> Umwelt.Client.pull()
+
+    receive do
+      {:DOWN, _, :process, _, _} ->
+        Logger.info("Done!")
+
+      other ->
+        Logger.warning(inspect(other))
+    end
+  end
+
+  defp assign_host(params) do
+    host =
+      case Mix.env() do
+        :dev ->
+          System.get_env("UMWELT_HOST", "https://umwelt.dev")
+
+        :test ->
+          "http://localhost"
+      end
+
+    Map.put(params, :api_host, host)
+  end
+
+  defp assign_port(params) do
+    port =
+      case Mix.env() do
+        :dev ->
+          case params.api_host do
+            "http://localhost" -> 4000
+            "https://umwelt.dev" -> 443
+          end
+
+        :test ->
+          Application.get_env(:umwelt, :api_port)
+      end
+
+    Map.put(params, :port, port)
+  end
+end

lib/mix/tasks/dump.ex

@@ -1,4 +1,4 @@
-defmodule Mix.Tasks.Dump do
+defmodule Mix.Tasks.Umwelt.Dump do
   @moduledoc "This task for self-parse umwelt"
   @shortdoc "The lib parser"
 

lib/umwelt/client.ex

@@ -0,0 +1,11 @@
+defmodule Umwelt.Client do
+  @moduledoc "Client for umwelt.dev"
+
+  alias Umwelt.Client.Clone
+
+  require Logger
+
+  def pull(params) do
+    GenServer.cast(Clone, {:pull, params})
+  end
+end

lib/umwelt/client/agent.ex

@@ -0,0 +1,93 @@
+defmodule Umwelt.Client.Agent do
+  @moduledoc "Keeps pulling metadata"
+
+  use Agent
+  require Logger
+
+  def start_link(_args) do
+    Agent.start_link(
+      fn ->
+        %{
+          modules: %{},
+          waiting: [],
+          fetching: [],
+          fetched: [],
+          writing: [],
+          written: [],
+          total: 0
+        }
+      end,
+      name: __MODULE__
+    )
+  end
+
+  def all_waiting, do: Agent.get(__MODULE__, fn state -> state.waiting end)
+
+  def completed?,
+    do: Agent.get(__MODULE__, fn state -> state.total == Enum.count(state.written) end)
+
+  def state, do: Agent.get(__MODULE__, fn state -> state end)
+  def total, do: Agent.get(__MODULE__, fn state -> state.total end)
+  def ready, do: Agent.get(__MODULE__, fn state -> Enum.count(state.written) end)
+
+  def add_modules(modules) do
+    Agent.update(__MODULE__, fn state ->
+      %{
+        state
+        | modules: modules,
+          waiting: Map.keys(modules),
+          fetching: [],
+          fetched: [],
+          writing: [],
+          written: [],
+          total: map_size(modules)
+      }
+    end)
+  end
+
+  def next_waiting do
+    Agent.get_and_update(__MODULE__, fn state ->
+      case state.waiting do
+        [mod_name | modules] ->
+          {
+            %{id: state.modules[mod_name], name: mod_name},
+            state
+            |> Map.put(:waiting, modules)
+            |> Map.put(:fetching, [mod_name | state.fetching])
+          }
+
+        [] ->
+          {nil, state}
+      end
+    end)
+  end
+
+  def update_status(mod_name, :fetched) do
+    Agent.update(__MODULE__, fn state ->
+      state
+      |> Map.put(:fetching, List.delete(state.fetching, mod_name))
+      |> Map.put(:fetched, [mod_name | state.fetched])
+    end)
+  end
+
+  def update_status(mod_name, :writing) do
+    Agent.update(__MODULE__, fn state ->
+      state
+      |> Map.put(:writing, [mod_name | state.writing])
+    end)
+  end
+
+  def update_status(mod_name, :written) do
+    Agent.update(__MODULE__, fn state ->
+      state
+      |> Map.put(:writing, List.delete(state.writing, mod_name))
+      |> Map.put(:written, [mod_name | state.written])
+    end)
+
+    render_progress()
+  end
+
+  def render_progress() do
+    ProgressBar.render(ready(), total(), suffix: :count)
+  end
+end

lib/umwelt/client/application.ex

@@ -0,0 +1,15 @@
+defmodule Umwelt.Client.Application do
+  @moduledoc "Client app & Supervisor"
+
+  use Application
+
+  def start(_type, _args) do
+    children = [
+      {Umwelt.Client.Agent, []},
+      {Umwelt.Client.Clone, []}
+    ]
+
+    opts = [strategy: :one_for_one, name: Umwelt.Client.Supervisor]
+    Supervisor.start_link(children, opts)
+  end
+end

lib/umwelt/client/clone.ex

@@ -0,0 +1,95 @@
+defmodule Umwelt.Client.Clone do
+  @moduledoc "Clone main process"
+
+  use GenServer
+  require Logger
+
+  alias Umwelt.Client
+
+  def start_link(_),
+    do:
+      GenServer.start_link(
+        __MODULE__,
+        %{phase_id: nil, port: nil},
+        name: __MODULE__
+      )
+
+  def init(state), do: {:ok, state}
+
+  def handle_cast({:pull, params}, _state) do
+    send(self(), :start_pulling)
+
+    {:noreply, params}
+  end
+
+  def handle_info(:start_pulling, state) do
+    case Client.Request.fetch_modules(state) do
+      {:ok, modules} ->
+        Logger.info("Fetching modules: #{inspect(Map.keys(modules))}")
+        modules |> Client.Agent.add_modules()
+
+      {:error, reason} ->
+        Logger.error("Failed to fetch modules: #{inspect(reason)}. Stopping...")
+        Supervisor.stop(Client.Supervisor)
+    end
+
+    send(self(), :spawn_fetchers)
+
+    {:noreply, state}
+  end
+
+  def handle_info(:spawn_fetchers, state) do
+    total = Client.Agent.total()
+    Logger.debug("Spawning fetchers: #{total}")
+
+    Client.Agent.all_waiting()
+    |> Enum.each(fn _ -> spawn_fetcher(state) end)
+
+    {:noreply, state}
+  end
+
+  def handle_info({:fetched, %{name: name, code: code}}, state) do
+    Client.Agent.update_status(name, :fetched)
+    spawn_writer(%{name: name, code: code})
+    {:noreply, state}
+  end
+
+  def handle_info({:fetch_failed, module}, state) do
+    Logger.warning("Respawning failed fetcher for module #{module.name}")
+    Client.Fetcher.start_link(module)
+    {:noreply, state}
+  end
+
+  def handle_info({:written, mod_name}, state) do
+    Client.Agent.update_status(mod_name, :written)
+    send(self(), :maybe_stop)
+    {:noreply, state}
+  end
+
+  def handle_info(:maybe_stop, state) do
+    if Client.Agent.completed?() do
+      Logger.debug("All modules processed. Stopping application.")
+      Supervisor.stop(Client.Supervisor)
+    end
+
+    {:noreply, state}
+  end
+
+  defp spawn_fetcher(state) do
+    case Client.Agent.next_waiting() do
+      nil ->
+        Logger.debug("No more modules to fetch")
+        :ok
+
+      module ->
+        Logger.debug("Spawning fetcher for module #{inspect(module.name)}")
+        Client.Fetcher.start_link(Map.merge(module, state))
+    end
+  end
+
+  defp spawn_writer(%{name: name} = module) do
+    Logger.debug("Spawning writer for module #{inspect(name)}")
+    Client.Agent.update_status(name, :writing)
+    Client.Writer.start_link(module)
+  end
+end