defmodule DediboxPrices.OffersCrawler do require Logger alias DediboxPrices.Entities.Offer def get_fresh_data do case HTTPoison.get!("https://console.online.net/fr/order/server") do %HTTPoison.Response{status_code: 200, body: body} -> parse_body(body) %HTTPoison.Error{reason: reason} -> {:error, reason} end end defp parse_body(body) do {:ok, document} = Floki.parse_document(body) [_header | lines] = Floki.find(document, ".server-availability tr") {:ok, lines |> Enum.map(&parse_one_item/1) |> Enum.reject(&is_nil/1)} end defp parse_one_item({"tr", [{"id", _number}], _rest} = html_tag) do [name, cpu, memory, disk, _, bandwith, rpn, dispo, price, _] = html_tag |> Floki.find("td") |> Enum.map(fn text -> text |> Floki.text() |> String.trim() end) [memory, _] = String.split(memory, " ") [id] = Floki.attribute(html_tag, "id") with {:ok, price} <- parse_price(price), {:ok, {disk_amount, disk_size, disk_properties}} <- parse_disk(disk), {:ok, disponibility} <- parse_disponibility(dispo) do %Offer{ name: "#{name} (#{id})", bandwith: parse_bandwith(bandwith), cpu: cpu, disk_amount: disk_amount, disk_properties: disk_properties, disk_size: disk_size, disponibility: disponibility, memory: String.to_integer(memory), price: price, rpn: String.replace(rpn, ~r/\W/, ""), tag: Floki.raw_html(html_tag) } else error -> Logger.error([ "[parse_one_item] with ", inspect(html_tag), " and error : ", inspect(error) ]) nil end end defp parse_one_item(_), do: nil defp parse_disponibility("victime de son succès"), do: {:ok, 0} defp parse_disponibility(disponibility) do case Integer.parse(disponibility) do {value, ""} -> value {_value, _rest} -> :error :error -> :error end end defp parse_disk(disk) do case Regex.scan(~r"^(\d+) ?x ?(\d+) ?(\w+)( (\w+))?", disk) do [[_, amount, memory, "TB"]] -> {:ok, {String.to_integer(amount), String.to_integer(memory) * 1000, nil}} [[_, amount, memory, "GB"]] -> {:ok, {String.to_integer(amount), String.to_integer(memory) * 1, nil}} [[_, amount, memory, "TB", _, properties]] -> {:ok, {String.to_integer(amount), String.to_integer(memory) * 1000, properties}} [[_, amount, memory, "GB", _, properties]] -> {:ok, {String.to_integer(amount), String.to_integer(memory) * 1, properties}} error -> {:error, error} end end defp parse_bandwith(bandwith) do case Regex.scan(~r"^(\d+) ?(\w+)/sec", bandwith) do [[_, value, "Gbit"]] -> String.to_integer(value) * 1000 [[_, value, "Mbit"]] -> String.to_integer(value) * 1 end end defp parse_price(price) do case String.replace(price, " ", "") |> Float.parse() do {value, _rest} -> {:ok, value} _ -> {:error, "no price found"} end end end