commit for comments

lib/github_webscraping/extract_file_infos.ex

@@ -0,0 +1,63 @@
+defmodule GithubWebscraping.ExtractFileInfos do
+  def fetch_file_name(html) do
+    file_name =
+      html
+      |> Floki.find("div.repository-content")
+      |> Floki.find("div.d-flex.flex-items-start.flex-shrink-0")
+      |> Floki.find("strong.final-path")
+      |> Floki.text()
+
+    file_name
+  end
+
+  def fetch_extension(html) do
+    file_name =
+      html
+      |> Floki.find("div.repository-content")
+      |> Floki.find("div.d-flex.flex-items-start.flex-shrink-0")
+      |> Floki.find("strong.final-path")
+      |> Floki.text()
+
+    Path.extname(file_name)
+  end
+
+  def fetch_line_numbers(html) do
+    line_numbers =
+      html
+      |> Floki.find("div.text-mono.f6.flex-auto.pr-3")
+      |> Floki.text()
+
+    if String.contains?(line_numbers, "sloc") do
+      kylo_bytes = List.first(List.first(Regex.scan(~r/(\d+) lines/, line_numbers)))
+      kylo_bytes = List.first(List.first(Regex.scan(~r/(\d+)/, kylo_bytes)))
+      {lines, _} = Integer.parse(kylo_bytes)
+      lines
+    else
+      0
+    end
+  end
+
+  def fetch_file_size_in_bytes(html) do
+    string_bytes =
+      html
+      |> Floki.find("div.text-mono.f6.flex-auto.pr-3")
+      |> Floki.text()
+
+    cond do
+      String.contains?(string_bytes, "KB") ->
+        k_bytes = String.trim(List.last(List.last(Regex.scan(~r/(\d+)?.?(\d+)/, string_bytes))))
+        {bytes, _} = Float.parse(k_bytes)
+        bytes * 1000.0
+
+      String.contains?(string_bytes, "MB") ->
+        m_bytes = String.trim(List.last(List.last(Regex.scan(~r/(\d+)?.?(\d+)/, string_bytes))))
+        {bytes, _} = Float.parse(m_bytes)
+        bytes * 1_000_000.0
+
+      String.contains?(string_bytes, "Bytes") ->
+        bytes = String.trim(List.last(List.last(Regex.scan(~r/(\d+)/, string_bytes))))
+        {bytes, _} = Float.parse(bytes)
+        bytes
+    end
+  end
+end

lib/github_webscraping/github_webscraping.ex

@@ -0,0 +1,8 @@
+defmodule Lib.GithubWebscraping do
+  @moduledoc """
+  GithubWebscraping API module
+  """
+  alias GithubWebscraping.MappingRepository
+
+  defdelegate get_repository_infos(url), to: MappingRepository, as: :process
+end

lib/github_webscraping/group_file_information.ex

@@ -0,0 +1,26 @@
+defmodule GithubWebscraping.GroupFileInformation do
+  @spec group_infos(list(%GithubWebscraping.Schemas.GithubFile{})) :: map()
+  def group_infos(files) do
+    all_lines = get_all_lines(files)
+    all_bytes = get_all_bytes(files)
+
+    repo_info = %{
+      all_lines: all_lines,
+      all_bytes: all_bytes,
+      files: files
+    }
+
+    repo_info
+  end
+
+  defp get_all_lines(files) do
+    all_lines = Enum.reduce(files, 0, fn file, acc -> file.file_lines + acc end)
+    all_lines
+  end
+
+  defp get_all_bytes(files) do
+    all_bytes = Enum.reduce(files, 0, fn file, acc -> file.file_bytes + acc end)
+
+    all_bytes
+  end
+end

lib/github_webscraping/mapping_repository.ex

@@ -0,0 +1,84 @@
+defmodule GithubWebscraping.MappingRepository do
+  alias GithubWebscraping.{ExtractFileInfos, GroupFileInformation}
+  alias GithubWebscraping.Schemas.GithubFile
+
+  @git_url_base "https://github.com"
+
+  @spec process(String.t()) :: map()
+  def process(url) do
+    files = mapping_repository_by_url(url)
+    files_with_all_infos = GroupFileInformation.group_infos(files)
+    files_with_all_infos
+  end
+
+  defp mapping_repository_by_url(url) do
+    urls = get_urls(is_first_url(url))
+    files_url = get_files_url(urls)
+    pastes_url = get_pastes_url(urls)
+
+    files =
+      Enum.map(files_url, fn file_url ->
+        build_file(@git_url_base <> file_url)
+      end)
+
+    IO.puts("\n----------Returned files----------\n")
+    IO.inspect(files)
+    IO.puts("\n----------Returned urls-----------\n")
+    IO.inspect(pastes_url)
+
+    other_files =
+      if Enum.count(pastes_url) > 0 do
+        Enum.map(pastes_url, fn url ->
+          mapping_repository_by_url(url)
+        end)
+      else
+        []
+      end
+
+    if Enum.count(other_files) > 0 do
+      Enum.concat(files, Enum.reduce(other_files, fn elem, acc -> elem ++ acc end))
+    else
+      files
+    end
+  end
+
+  defp build_file(url) do
+    html = download_string_url(url) |> Floki.parse_document!()
+    name = ExtractFileInfos.fetch_file_name(html)
+    lines = ExtractFileInfos.fetch_line_numbers(html)
+    extension = ExtractFileInfos.fetch_extension(html)
+    bytes = ExtractFileInfos.fetch_file_size_in_bytes(html)
+
+    GithubFile.build(url, name, extension, bytes, lines)
+  end
+
+  defp get_urls(url) do
+    url
+    |> download_string_url()
+    |> Floki.parse_document!()
+    |> Floki.find("div.js-details-container.Details")
+    |> Floki.find("div.js-navigation-item")
+    |> Floki.find("a.js-navigation-open.Link--primary")
+    |> Floki.attribute("href")
+  end
+
+  defp download_string_url(url) do
+    HTTPoison.get!(url).body
+  end
+
+  defp get_files_url(urls) do
+    Enum.filter(urls, fn url -> url =~ "blob" end)
+  end
+
+  defp get_pastes_url(urls) do
+    Enum.filter(urls, fn url -> url =~ "tree" end)
+  end
+
+  defp is_first_url(url) do
+    if url =~ "https://github.com/" do
+      url
+    else
+      "https://github.com" <> url
+    end
+  end
+end

lib/github_webscraping/schemas/github_file.ex

@@ -0,0 +1,16 @@
+defmodule GithubWebscraping.Schemas.GithubFile do
+  alias GithubWebscraping.Schemas.GithubFile
+
+  @derive {Jason.Encoder, only: [:file_url, :file_name, :extension, :file_bytes, :file_lines]}
+  defstruct [:file_url, :file_name, :extension, :file_bytes, :file_lines]
+
+  def build(file_url, file_name, extension, file_bytes, file_lines) do
+    %GithubFile{
+      file_url: file_url,
+      file_name: file_name,
+      extension: extension,
+      file_bytes: file_bytes,
+      file_lines: file_lines
+    }
+  end
+end

lib/github_webscraping_web/controllers/github_webscraping_controller.ex

@@ -0,0 +1,10 @@
+defmodule GithubWebscrapingWeb.GithubWebscrapingController do
+  use GithubWebscrapingWeb, :controller
+
+  alias Lib.GithubWebscraping
+
+  def index(conn, %{"github_url" => github_url}) do
+    files = GithubWebscraping.get_repository_infos(github_url)
+    json(conn, files)
+  end
+end

lib/github_webscraping_web/router.ex

@@ -14,9 +14,9 @@ defmodule GithubWebscrapingWeb.Router do
   end
 
   scope "/", GithubWebscrapingWeb do
-    pipe_through :browser
+    pipe_through :api
 
-    get "/", PageController, :index
+    post "/get_repository_info", GithubWebscrapingController, :index
   end
 
   # Other scopes may use custom stacks.

mix.exs

@@ -44,7 +44,9 @@ defmodule GithubWebscraping.MixProject do
       {:telemetry_poller, "~> 0.4"},
       {:gettext, "~> 0.11"},
       {:jason, "~> 1.0"},
-      {:plug_cowboy, "~> 2.0"}
+      {:plug_cowboy, "~> 2.0"},
+      {:httpoison, "~> 1.8"},
+      {:floki, "~> 0.30.0"}
     ]
   end
 

mix.lock

@@ -1,4 +1,5 @@
 %{
+  "certifi": {:hex, :certifi, "2.6.1", "dbab8e5e155a0763eea978c913ca280a6b544bfa115633fa20249c3d396d9493", [:rebar3], [], "hexpm", "524c97b4991b3849dd5c17a631223896272c6b0af446778ba4675a1dff53bb7e"},
   "connection": {:hex, :connection, "1.1.0", "ff2a49c4b75b6fb3e674bfc5536451607270aac754ffd1bdfe175abe4a6d7a68", [:mix], [], "hexpm", "722c1eb0a418fbe91ba7bd59a47e28008a189d47e37e0e7bb85585a016b2869c"},
   "cowboy": {:hex, :cowboy, "2.8.0", "f3dc62e35797ecd9ac1b50db74611193c29815401e53bac9a5c0577bd7bc667d", [:rebar3], [{:cowlib, "~> 2.9.1", [hex: :cowlib, repo: "hexpm", optional: false]}, {:ranch, "~> 1.7.1", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm", "4643e4fba74ac96d4d152c75803de6fad0b3fa5df354c71afdd6cbeeb15fac8a"},
   "cowboy_telemetry": {:hex, :cowboy_telemetry, "0.3.1", "ebd1a1d7aff97f27c66654e78ece187abdc646992714164380d8a041eda16754", [:rebar3], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "3a6efd3366130eab84ca372cbd4a7d3c3a97bdfcfb4911233b035d117063f0af"},
@@ -8,9 +9,18 @@
   "ecto": {:hex, :ecto, "3.5.8", "8ebf12be6016cb99313348ba7bb4612f4114b9a506d6da79a2adc7ef449340bc", [:mix], [{:decimal, "~> 1.6 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "ea0be182ea8922eb7742e3ae8e71b67ee00ae177de1bf76210299a5f16ba4c77"},
   "ecto_sql": {:hex, :ecto_sql, "3.5.4", "a9e292c40bd79fff88885f95f1ecd7b2516e09aa99c7dd0201aa84c54d2358e4", [:mix], [{:db_connection, "~> 2.2", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.5.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:myxql, "~> 0.3.0 or ~> 0.4.0", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.15.0 or ~> 1.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:tds, "~> 2.1.1", [hex: :tds, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "1fff1a28a898d7bbef263f1f3ea425b04ba9f33816d843238c84eff883347343"},
   "file_system": {:hex, :file_system, "0.2.10", "fb082005a9cd1711c05b5248710f8826b02d7d1784e7c3451f9c1231d4fc162d", [:mix], [], "hexpm", "41195edbfb562a593726eda3b3e8b103a309b733ad25f3d642ba49696bf715dc"},
+  "floki": {:hex, :floki, "0.30.0", "22ebbe681a5d3777cdd830ca091b1b806d33c3449c26312eadca7f7be685c0c8", [:mix], [{:html_entities, "~> 0.5.0", [hex: :html_entities, repo: "hexpm", optional: false]}], "hexpm", "a9e128a4ca9bb71f11affa315b6768a9ad326d5996ff1e92acf1d7a01a10076a"},
   "gettext": {:hex, :gettext, "0.18.2", "7df3ea191bb56c0309c00a783334b288d08a879f53a7014341284635850a6e55", [:mix], [], "hexpm", "f9f537b13d4fdd30f3039d33cb80144c3aa1f8d9698e47d7bcbcc8df93b1f5c5"},
+  "hackney": {:hex, :hackney, "1.17.4", "99da4674592504d3fb0cfef0db84c3ba02b4508bae2dff8c0108baa0d6e0977c", [:rebar3], [{:certifi, "~>2.6.1", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "~>6.1.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "~>1.0.0", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~>1.1", [hex: :mimerl, repo: "hexpm", optional: false]}, {:parse_trans, "3.3.1", [hex: :parse_trans, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "~>1.1.0", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}, {:unicode_util_compat, "~>0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "de16ff4996556c8548d512f4dbe22dd58a587bf3332e7fd362430a7ef3986b16"},
+  "hound": {:hex, :hound, "1.1.1", "d3afce4cf0f446331d9d00427e9eb74fa135c296b1d3745d4bbe2096ce259087", [:mix], [{:hackney, "~> 1.5", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "8c6342b49f53bb0e5c51d5ecca18a8ce872c44da05a8ce6f828385ebd744fe2a"},
+  "html_entities": {:hex, :html_entities, "0.5.2", "9e47e70598da7de2a9ff6af8758399251db6dbb7eebe2b013f2bbd2515895c3c", [:mix], [], "hexpm", "c53ba390403485615623b9531e97696f076ed415e8d8058b1dbaa28181f4fdcc"},
+  "httpoison": {:hex, :httpoison, "1.8.0", "6b85dea15820b7804ef607ff78406ab449dd78bed923a49c7160e1886e987a3d", [:mix], [{:hackney, "~> 1.17", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm", "28089eaa98cf90c66265b6b5ad87c59a3729bea2e74e9d08f9b51eb9729b3c3a"},
+  "idna": {:hex, :idna, "6.1.1", "8a63070e9f7d0c62eb9d9fcb360a7de382448200fbbd1b106cc96d3d8099df8d", [:rebar3], [{:unicode_util_compat, "~>0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "92376eb7894412ed19ac475e4a86f7b413c1b9fbb5bd16dccd57934157944cea"},
   "jason": {:hex, :jason, "1.2.2", "ba43e3f2709fd1aa1dce90aaabfd039d000469c05c56f0b8e31978e03fa39052", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "18a228f5f0058ee183f29f9eae0805c6e59d61c3b006760668d8d18ff0d12179"},
+  "metrics": {:hex, :metrics, "1.0.1", "25f094dea2cda98213cecc3aeff09e940299d950904393b2a29d191c346a8486", [:rebar3], [], "hexpm", "69b09adddc4f74a40716ae54d140f93beb0fb8978d8636eaded0c31b6f099f16"},
   "mime": {:hex, :mime, "1.5.0", "203ef35ef3389aae6d361918bf3f952fa17a09e8e43b5aa592b93eba05d0fb8d", [:mix], [], "hexpm", "55a94c0f552249fc1a3dd9cd2d3ab9de9d3c89b559c2bd01121f824834f24746"},
+  "mimerl": {:hex, :mimerl, "1.2.0", "67e2d3f571088d5cfd3e550c383094b47159f3eee8ffa08e64106cdf5e981be3", [:rebar3], [], "hexpm", "f278585650aa581986264638ebf698f8bb19df297f66ad91b18910dfc6e19323"},
+  "parse_trans": {:hex, :parse_trans, "3.3.1", "16328ab840cc09919bd10dab29e431da3af9e9e7e7e6f0089dd5a2d2820011d8", [:rebar3], [], "hexpm", "07cd9577885f56362d414e8c4c4e6bdf10d43a8767abb92d24cbe8b24c54888b"},
   "phoenix": {:hex, :phoenix, "1.5.8", "71cfa7a9bb9a37af4df98939790642f210e35f696b935ca6d9d9c55a884621a4", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix_html, "~> 2.13", [hex: :phoenix_html, repo: "hexpm", optional: true]}, {:phoenix_pubsub, "~> 2.0", [hex: :phoenix_pubsub, repo: "hexpm", optional: false]}, {:plug, "~> 1.10", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 1.0 or ~> 2.2", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:plug_crypto, "~> 1.1.2 or ~> 1.2", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "35ded0a32f4836168c7ab6c33b88822eccd201bcd9492125a9bea4c54332d955"},
   "phoenix_ecto": {:hex, :phoenix_ecto, "4.2.1", "13f124cf0a3ce0f1948cf24654c7b9f2347169ff75c1123f44674afee6af3b03", [:mix], [{:ecto, "~> 3.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:phoenix_html, "~> 2.14.2 or ~> 2.15", [hex: :phoenix_html, repo: "hexpm", optional: true]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "478a1bae899cac0a6e02be1deec7e2944b7754c04e7d4107fc5a517f877743c0"},
   "phoenix_html": {:hex, :phoenix_html, "2.14.3", "51f720d0d543e4e157ff06b65de38e13303d5778a7919bcc696599e5934271b8", [:mix], [{:plug, "~> 1.5", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "efd697a7fff35a13eeeb6b43db884705cba353a1a41d127d118fda5f90c8e80f"},
@@ -21,9 +31,12 @@
   "plug": {:hex, :plug, "1.11.1", "f2992bac66fdae679453c9e86134a4201f6f43a687d8ff1cd1b2862d53c80259", [:mix], [{:mime, "~> 1.0", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_crypto, "~> 1.1.1 or ~> 1.2", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "23524e4fefbb587c11f0833b3910bfb414bf2e2534d61928e920f54e3a1b881f"},
   "plug_cowboy": {:hex, :plug_cowboy, "2.4.1", "779ba386c0915027f22e14a48919a9545714f849505fa15af2631a0d298abf0f", [:mix], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:cowboy_telemetry, "~> 0.3", [hex: :cowboy_telemetry, repo: "hexpm", optional: false]}, {:plug, "~> 1.7", [hex: :plug, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "d72113b6dff7b37a7d9b2a5b68892808e3a9a752f2bf7e503240945385b70507"},
   "plug_crypto": {:hex, :plug_crypto, "1.2.1", "5c854427528bf61d159855cedddffc0625e2228b5f30eff76d5a4de42d896ef4", [:mix], [], "hexpm", "6961c0e17febd9d0bfa89632d391d2545d2e0eb73768f5f50305a23961d8782c"},
+  "poison": {:hex, :poison, "3.1.0", "d9eb636610e096f86f25d9a46f35a9facac35609a7591b3be3326e99a0484665", [:mix], [], "hexpm", "fec8660eb7733ee4117b85f55799fd3833eb769a6df71ccf8903e8dc5447cfce"},
   "postgrex": {:hex, :postgrex, "0.15.8", "f5e782bbe5e8fa178d5e3cd1999c857dc48eda95f0a4d7f7bd92a50e84a0d491", [:mix], [{:connection, "~> 1.0", [hex: :connection, repo: "hexpm", optional: false]}, {:db_connection, "~> 2.1", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm", "698fbfacea34c4cf22c8281abeb5cf68d99628d541874f085520ab3b53d356fe"},
   "ranch": {:hex, :ranch, "1.7.1", "6b1fab51b49196860b733a49c07604465a47bdb78aa10c1c16a3d199f7f8c881", [:rebar3], [], "hexpm", "451d8527787df716d99dc36162fca05934915db0b6141bbdac2ea8d3c7afc7d7"},
+  "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.6", "cf344f5692c82d2cd7554f5ec8fd961548d4fd09e7d22f5b62482e5aeaebd4b0", [:make, :mix, :rebar3], [], "hexpm", "bdb0d2471f453c88ff3908e7686f86f9be327d065cc1ec16fa4540197ea04680"},
   "telemetry": {:hex, :telemetry, "0.4.2", "2808c992455e08d6177322f14d3bdb6b625fbcfd233a73505870d8738a2f4599", [:rebar3], [], "hexpm", "2d1419bd9dda6a206d7b5852179511722e2b18812310d304620c7bd92a13fcef"},
   "telemetry_metrics": {:hex, :telemetry_metrics, "0.6.0", "da9d49ee7e6bb1c259d36ce6539cd45ae14d81247a2b0c90edf55e2b50507f7b", [:mix], [{:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "5cfe67ad464b243835512aa44321cee91faed6ea868d7fb761d7016e02915c3d"},
   "telemetry_poller": {:hex, :telemetry_poller, "0.5.1", "21071cc2e536810bac5628b935521ff3e28f0303e770951158c73eaaa01e962a", [:rebar3], [{:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "4cab72069210bc6e7a080cec9afffad1b33370149ed5d379b81c7c5f0c663fd4"},
+  "unicode_util_compat": {:hex, :unicode_util_compat, "0.7.0", "bc84380c9ab48177092f43ac89e4dfa2c6d62b40b8bd132b1059ecc7232f9a78", [:rebar3], [], "hexpm", "25eee6d67df61960cf6a794239566599b09e17e668d3700247bc498638152521"},
 }