From c0706bfe9b354abfadaa2d38f698d3309db3428b Mon Sep 17 00:00:00 2001 From: Ben Busby Date: Tue, 11 Jul 2023 12:30:13 -0600 Subject: [PATCH] Allow separation of instance and API URLs Instance URLs and API URLs are not always the same, and require different testing strategies. This allows the two types of URLs for an instance, as well as the testing path, to be separated by a "|" character. When Farside runs the instance checker, it will split instance strings by that character and perform separate tests. Instances without a separate API URL will only be tested once against their primary instance URL. Currently the only service using this paradigm is Piped, but allows for others using the same approach to be added/tested in Farside in the future. Closes #115 --- lib/farside/instances.ex | 50 ++++++++++++++++++++++++++++++++-------- mix.lock | 2 +- services-full.json | 43 ++++++++++++++++------------------ services.json | 29 +++++++++++------------ 4 files changed, 75 insertions(+), 49 deletions(-) diff --git a/lib/farside/instances.ex b/lib/farside/instances.ex index 279d48a..d02a082 100644 --- a/lib/farside/instances.ex +++ b/lib/farside/instances.ex @@ -20,6 +20,8 @@ defmodule Farside.Instances do end def request(url) do + IO.puts("#{@debug_spacer}#{url}") + cond do System.get_env("FARSIDE_TEST") -> :good @@ -57,19 +59,26 @@ defmodule Farside.Instances do result = cond do Enum.member?(@skip_service_updates, service.type) -> - service.instances + get_service_vals(service.instances) true -> Enum.filter(service.instances, fn instance_url -> - request_url = - instance_url <> - EEx.eval_string( - service.test_url, - query: Enum.random(@queries) - ) + test_url = get_test_val(instance_url) + test_path = get_test_val(service.test_url) + test_request_url = gen_validation_url(test_url, test_path) - IO.puts("#{@debug_spacer}#{request_url}") + service_url = get_service_val(instance_url) + service_path = get_service_val(service.test_url) + service_request_url = gen_validation_url(service_url, service_path) - request(request_url) == :good + cond do + service_url != test_url -> + service_up = request(service_request_url) + test_up = request(test_request_url) + + service_up == :good && test_up == :good + true -> + request(test_request_url) == :good + end end) end @@ -79,6 +88,9 @@ defmodule Farside.Instances do end def add_to_db(service, instances) do + # Ensure only service URLs are inserted, not test URLs (separated by "|") + instances = get_service_vals(instances) + # Remove previous list of instances CubDB.delete(CubDB, "#{@service_prefix}#{service.type}") @@ -99,4 +111,24 @@ defmodule Farside.Instances do IO.write(file, "#{service_name}: #{inspect(results)}\n") File.close(file) end + + def gen_validation_url(url, path) do + url <> EEx.eval_string(path, query: Enum.random(@queries)) + end + + def get_service_vals(services) do + Enum.map(services, fn x -> get_service_val(x) end) + end + + def get_service_val(service) do + String.split(service, "|") |> List.first + end + + def get_test_vals(services) do + Enum.map(services, fn x -> get_test_val(x) end) + end + + def get_test_val(service) do + String.split(service, "|") |> List.last + end end diff --git a/mix.lock b/mix.lock index 87a9cad..a51f790 100644 --- a/mix.lock +++ b/mix.lock @@ -22,7 +22,7 @@ "plug_crypto": {:hex, :plug_crypto, "1.2.3", "8f77d13aeb32bfd9e654cb68f0af517b371fb34c56c9f2b58fe3df1235c1251a", [:mix], [], "hexpm", "b5672099c6ad5c202c45f5a403f21a3411247f164e4a8fab056e5cd8a290f4a2"}, "quantum": {:hex, :quantum, "3.5.0", "8d2c5ba68c55991e8975aca368e3ab844ba01f4b87c4185a7403280e2c99cf34", [:mix], [{:crontab, "~> 1.1", [hex: :crontab, repo: "hexpm", optional: false]}, {:gen_stage, "~> 0.14 or ~> 1.0", [hex: :gen_stage, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:telemetry_registry, "~> 0.2", [hex: :telemetry_registry, repo: "hexpm", optional: false]}], "hexpm", "cab737d1d9779f43cb1d701f46dd05ea58146fd96238d91c9e0da662c1982bb6"}, "ranch": {:hex, :ranch, "1.8.0", "8c7a100a139fd57f17327b6413e4167ac559fbc04ca7448e9be9057311597a1d", [:make, :rebar3], [], "hexpm", "49fbcfd3682fab1f5d109351b61257676da1a2fdbe295904176d5e521a2ddfe5"}, - "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.6", "cf344f5692c82d2cd7554f5ec8fd961548d4fd09e7d22f5b62482e5aeaebd4b0", [:make, :mix, :rebar3], [], "hexpm", "bdb0d2471f453c88ff3908e7686f86f9be327d065cc1ec16fa4540197ea04680"}, + "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.7", "354c321cf377240c7b8716899e182ce4890c5938111a1296add3ec74cf1715df", [:make, :mix, :rebar3], [], "hexpm", "fe4c190e8f37401d30167c8c405eda19469f34577987c76dde613e838bbc67f8"}, "telemetry": {:hex, :telemetry, "1.1.0", "a589817034a27eab11144ad24d5c0f9fab1f58173274b1e9bae7074af9cbee51", [:rebar3], [], "hexpm", "b727b2a1f75614774cff2d7565b64d0dfa5bd52ba517f16543e6fc7efcc0df48"}, "telemetry_registry": {:hex, :telemetry_registry, "0.3.0", "6768f151ea53fc0fbca70dbff5b20a8d663ee4e0c0b2ae589590e08658e76f1e", [:mix, :rebar3], [{:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "492e2adbc609f3e79ece7f29fec363a97a2c484ac78a83098535d6564781e917"}, "unicode_util_compat": {:hex, :unicode_util_compat, "0.7.0", "bc84380c9ab48177092f43ac89e4dfa2c6d62b40b8bd132b1059ecc7232f9a78", [:rebar3], [], "hexpm", "25eee6d67df61960cf6a794239566599b09e17e668d3700247bc498638152521"}, diff --git a/services-full.json b/services-full.json index 0f2590f..470e679 100644 --- a/services-full.json +++ b/services-full.json @@ -133,31 +133,28 @@ }, { "type": "piped", - "test_url": "/watch?v=eBGIQ7ZuuiU", + "test_url": "/watch?v=dQw4w9WgXcQ|/streams/dQw4w9WgXcQ", "fallback": "https://piped.video", "instances": [ - "https://piped.video", - "https://piped.tokhmi.xyz", - "https://piped.moomoo.me", - "https://piped.mha.fi", - "https://watch.whatever.social", - "https://piped.garudalinux.org", - "https://efy.piped.pages.dev", - "https://watch.leptons.xyz", - "https://piped.lunar.icu", - "https://yt.jae.fi", - "https://il.ax", - "https://piped.esmailelbob.xyz", - "https://piped.projectsegfau.lt", - "https://piped.privacydev.net", - "https://piped.palveluntarjoaja.eu", - "https://piped.smnz.de", - "https://piped.adminforge.de", - "https://piped.qdi.fi", - "https://piped.hostux.net", - "https://pd.vern.cc", - "https://piped.chauvet.pro", - "https://piped.colinslegacy.com" + "https://piped.video|https://pipedapi.kavin.rocks", + "https://piped.tokhmi.xyz|https://pipedapi.tokhmi.xyz", + "https://piped.moomoo.me|https://pipedapi.moomoo.me", + "https://piped.mha.fi|https://api-piped.mha.fi", + "https://efy.piped.pages.dev|https://pipedapi.aeong.one", + "https://watch.whatever.social|https://watchapi.whatever.social", + "https://piped.garudalinux.org|https://piped-api.garudalinux.org", + "https://watch.leptons.xyz|https://pipedapi.leptons.xyz", + "https://piped.lunar.icu|https://piped-api.lunar.icu", + "https://il.ax|https://pa.il.ax", + "https://piped.projectsegfau.lt|https://api.piped.projectsegfau.lt", + "https://piped.privacydev.net|https://api.piped.privacydev.net", + "https://piped.palveluntarjoaja.eu|https://pipedapi.palveluntarjoaja.eu", + "https://piped.smnz.de|https://pipedapi.smnz.de", + "https://piped.adminforge.de|https://pipedapi.adminforge.de", + "https://piped.qdi.fi|https://pipedapi.qdi.fi", + "https://piped.hostux.net|https://piped-api.hostux.net", + "https://pd.vern.cc|https://pdapi.vern.cc", + "https://piped.colinslegacy.com|https://pipedapi.colinslegacy.com" ] }, { diff --git a/services.json b/services.json index 34d43e4..e3afc5f 100644 --- a/services.json +++ b/services.json @@ -115,24 +115,21 @@ }, { "type": "piped", - "test_url": "/watch?v=eBGIQ7ZuuiU", + "test_url": "/watch?v=dQw4w9WgXcQ|/streams/dQw4w9WgXcQ", "fallback": "https://piped.video", "instances": [ - "https://watch.whatever.social", - "https://piped.lunar.icu", - "https://yt.jae.fi", - "https://il.ax", - "https://piped.esmailelbob.xyz", - "https://piped.projectsegfau.lt", - "https://piped.privacydev.net", - "https://piped.palveluntarjoaja.eu", - "https://piped.smnz.de", - "https://piped.adminforge.de", - "https://piped.qdi.fi", - "https://piped.hostux.net", - "https://pd.vern.cc", - "https://piped.chauvet.pro", - "https://piped.colinslegacy.com" + "https://watch.whatever.social|https://watchapi.whatever.social", + "https://piped.lunar.icu|https://piped-api.lunar.icu", + "https://il.ax|https://pa.il.ax", + "https://piped.projectsegfau.lt|https://api.piped.projectsegfau.lt", + "https://piped.privacydev.net|https://api.piped.privacydev.net", + "https://piped.palveluntarjoaja.eu|https://pipedapi.palveluntarjoaja.eu", + "https://piped.smnz.de|https://pipedapi.smnz.de", + "https://piped.adminforge.de|https://pipedapi.adminforge.de", + "https://piped.qdi.fi|https://pipedapi.qdi.fi", + "https://piped.hostux.net|https://piped-api.hostux.net", + "https://pd.vern.cc|https://pdapi.vern.cc", + "https://piped.colinslegacy.com|https://pipedapi.colinslegacy.com" ] }, {