beenull/farside
1
0
Fork 0
Code Issues Pull requests Packages Projects Releases Wiki Activity

Remove cloudflare filter from update workflow

Browse source 
The cloudflare filter, when performed as part of the github action
workflow, doesn't seem to work nearly as reliably when run on an actual
machine.

The farside server will instead run the un-cloudflare script whenever it
pulls in new changes to services-full.json, which should be a much more
reliable approach to filtering out cloudflare instances.
This commit is contained in:
Ben Busby 2022-08-04 13:16:58 -06:00
parent e16d4bb330
commit e1e46dcd92
No known key found for this signature in database
GPG key ID: B9B7231E01D924A1
3 changed files with 14 additions and 95 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

32
.github/workflows/update-instances.yml vendored
View file

@ -27,33 +27,6 @@ jobs:
sed -i 's/\/"/"/g' services-full.json
}
function remove_cf_instances() {
rm -f out.json
file="services-full.json"
while read -r line; do
if [[ "$line" == "\"https://"* ]]; then
domain=$(echo "$line" | sed -e "s/^\"https:\/\///" -e "s/\",//" -e "s/\"//")
ns=$(dig ns "$domain" || true)
if [[ "$ns" == *"cloudflare"* ]]; then
echo "\"$domain\" using cloudflare, skipping..."
elif [[ "$ns" != *"NOERROR"* ]]; then
echo "Unable to verify records for \"$domain\", skipping..."
else
echo "$line" >> out.json
fi
else
echo "$line" >> out.json
fi
done <$file
# Remove any trailing commas from new instance lists
sed -i -e ':begin' -e '$!N' -e 's/,\n]/\n]/g' -e 'tbegin' -e 'P' -e 'D' out.json
cat out.json | jq --indent 2 . > services.json
rm -f out.json
}
# ==============================================================
# Git config
# ==============================================================
@ -216,11 +189,6 @@ jobs:
# TODO: Update instances for other services
# ==============================================================
# ==============================================================
# Remove cloudflare instances
# ==============================================================
remove_cf_instances
# ==============================================================
# Push changes
# ==============================================================

73
services.json
View file

@ -4,29 +4,19 @@
"test_url": "/r/popular",
"fallback": "https://libredd.it",
"instances": [
"https://libreddit.albony.xyz",
"https://libreddit.spike.codes",
"https://libreddit.dothq.co",
"https://libreddit.kavin.rocks",
"https://libreddit.40two.app",
"https://reddit.invak.id",
"https://lr.riverside.rocks",
"https://libreddit.silkky.cloud",
"https://libreddit.database.red",
"https://libreddit.exonip.de",
"https://libreddit.domain.glass",
"https://libreddit.trevorthalacker.com",
"https://reddit.phii.me",
"https://libreddit.sugoma.tk",
"https://reddit.artemislena.eu",
"https://libreddit.some-things.org",
"https://reddit.stuehieyr.com",
"https://lr.mint.lgbt",
"https://libreddit.igna.rocks",
"https://libreddit.autarkic.org",
"https://libreddit.flux.industries",
"https://libreddit.drivet.xyz",
"https://libreddit.de",
"https://libreddit.bus-hit.me",
"https://de.leddit.xyz"
"https://libreddit.bus-hit.me"
]
},
{
@ -37,18 +27,15 @@
"https://incogsnoo.com",
"https://rdt.trom.tf",
"https://reddit.lol",
"https://snoo.ioens.is",
"https://teddit.adminforge.de",
"https://teddit.bus-hit.me",
"https://teddit.domain.glass",
"https://teddit.encrypted-data.xyz",
"https://teddit.froth.zone",
"https://teddit.garudalinux.org",
"https://teddit.ggc-project.de",
"https://teddit.httpjames.space",
"https://teddit.namazso.eu",
"https://teddit.nautolan.racing",
"https://teddit.net",
"https://teddit.pussthecat.org",
"https://teddit.sethforprivacy.com",
"https://teddit.tinfoil-hat.net",
"https://teddit.totaldarkness.net",
"https://teddit.zaggy.nl"
]
@ -58,7 +45,6 @@
"test_url": "/u/officialrickastley",
"fallback": "https://bibliogram.art",
"instances": [
"https://bib.actionsack.com",
"https://bibliogram.1d4.us",
"https://bibliogram.art",
"https://bibliogram.froth.zone",
@ -67,8 +53,6 @@
"https://bibliogram.pussthecat.org",
"https://bibliogram.snopyta.org",
"https://ig.beparanoid.de",
"https://ig.funami.tech",
"https://ig.tokhmi.xyz",
"https://insta.trom.tf",
"https://insta.tromdienste.de"
]
@ -79,13 +63,9 @@
"fallback": "https://invidious.snopyta.org",
"instances": [
"https://inv.bp.projectsegfau.lt",
"https://inv.privacy.com.de",
"https://inv.riverside.rocks",
"https://inv.vern.cc",
"https://invidious.esmailelbob.xyz",
"https://invidious.flokinet.to",
"https://invidious.kavin.rocks",
"https://invidious.namazso.eu",
"https://invidious.nerdvpn.de",
"https://invidious.osi.kr",
"https://invidious.projectsegfau.lt",
@ -106,10 +86,6 @@
"test_url": "/watch?v=eBGIQ7ZuuiU",
"fallback": "https://piped.kavin.rocks",
"instances": [
"https://piped.kavin.rocks",
"https://piped.silkky.cloud",
"https://piped.tokhmi.xyz",
"https://piped.moomoo.me",
"https://piped.mint.lgbt"
]
},
@ -124,29 +100,21 @@
"https://nitter.nixnet.services",
"https://nitter.fdn.fr",
"https://nitter.1d4.us",
"https://nitter.kavin.rocks",
"https://nitter.vxempire.xyz",
"https://nitter.unixfox.eu",
"https://nitter.eu",
"https://nitter.namazso.eu",
"https://nitter.mailstation.de",
"https://n.actionsack.com",
"https://nitter.cattube.org",
"https://nitter.exonip.de",
"https://twitr.gq",
"https://nitter.moomoo.me",
"https://bird.trom.tf",
"https://nitter.it",
"https://twitter.censors.us",
"https://nitter.grimneko.de",
"https://nitter.koyu.space",
"https://n.0x0.st",
"https://n.hyperborea.cloud",
"https://nitter.ca",
"https://twitter.076.ne.jp",
"https://nitter.sethforprivacy.com",
"https://nitter.bus-hit.me",
"https://de.nttr.stream",
"https://n.l5.ca",
"https://unofficialbird.com"
]
@ -177,10 +145,9 @@
"https://tl.vern.cc",
"https://translate.beparanoid.de",
"https://translate.bus-hit.me",
"https://translate.namazso.eu",
"https://translate.josias.dev",
"https://translate.northboot.xyz",
"https://translate.priv.pw",
"https://translate.riverside.rocks",
"https://translate.slipfox.xyz",
"https://translate.tiekoetter.com"
]
@ -202,7 +169,6 @@
"instances": [
"https://i.bcow.xyz",
"https://rimgo.pussthecat.org",
"https://img.riverside.rocks",
"https://rimgo.totaldarkness.net",
"https://rimgo.bus-hit.me"
]
@ -213,15 +179,10 @@
"fallback": "https://whoogle.fossho.st",
"instances": [
"https://gowogle.voring.me",
"https://s.tokhmi.xyz",
"https://search.albony.xyz",
"https://search.dr460nf1r3.org",
"https://search.garudalinux.org",
"https://search.sethforprivacy.com",
"https://search.wef.lol",
"https://whoogle.dcs0.hu",
"https://whoogle.esmailelbob.xyz",
"https://whoogle.fossho.st",
"https://whoogle.lunar.icu",
"https://whoogle.privacydev.net",
"https://www.whooglesearch.ml"
]
@ -235,7 +196,6 @@
"https://dynabyte.ca",
"https://search.neet.works",
"https://searx.divided-by-zero.eu",
"https://searx.gnu.style",
"https://searx.ru",
"https://searx.sp-codes.de",
"https://searx.stuehieyr.com",
@ -249,10 +209,10 @@
"test_url": "/search?q=<%=query%>",
"fallback": "https://searx.be",
"instances": [
"https://darmarit.org/searx",
"https://de.xcxc.ml",
"https://jackgoss.xyz",
"https://priv.au",
"https://s.frlt.one",
"https://s.zhaocloud.net",
"https://saber.tk",
"https://search.0relay.com",
@ -263,12 +223,8 @@
"https://search.mdosch.de",
"https://search.neet.works",
"https://search.ononoki.org",
"https://search.privacyguides.net",
"https://search.projectsegfau.lt",
"https://search.rabbit-company.com",
"https://search.sapti.me",
"https://search.teamriverbubbles.com",
"https://search.unlocked.link",
"https://search.zzls.xyz",
"https://searx.be",
"https://searx.becomesovran.com",
@ -278,14 +234,12 @@
"https://searx.fmac.xyz",
"https://searx.mha.fi",
"https://searx.mistli.net",
"https://searx.namejeff.xyz",
"https://searx.priv.pw",
"https://searx.prvcy.eu",
"https://searx.sev.monster",
"https://searx.slipfox.xyz/searx",
"https://searx.tiekoetter.com",
"https://searx.zcyph.cc",
"https://searxng.tordenskjold.de",
"https://searxng.zackptg5.com",
"https://serx.ml",
"https://swag.pw",
"https://xcxc.ml"
@ -296,11 +250,10 @@
"test_url": "/wiki/Wikipedia?lang=en",
"fallback": "https://wikiless.org",
"instances": [
"https://wiki.604kph.xyz",
"https://wiki.froth.zone",
"https://wiki.slipfox.xyz",
"https://wikiless.esmailelbob.xyz",
"https://wikiless.funami.tech",
"https://wikiless.lunar.icu",
"https://wikiless.northboot.xyz",
"https://wikiless.org",
"https://wikiless.sethforprivacy.com",
@ -336,9 +289,7 @@
"fallback": "https://librex.beparanoid.de",
"instances": [
"https://librex.beparanoid.de",
"https://search.davidovski.xyz",
"https://search.funami.tech",
"https://librex.catalyst.sx"
"https://search.davidovski.xyz"
]
},
{

4
tools/un-cloudflare.sh
View file

@ -10,7 +10,7 @@ while read -r line; do
ns=$(dig ns "$domain" || true)
if [[ "$ns" == *"cloudflare"* ]]; then
echo "\"$domain\" using cloudflare, skipping..."
elif [ ${#ns} -eq 0 ]; then
elif [[ "$ns" != *"NOERROR"* ]]; then
echo "Unable to verify records for \"$domain\", skipping..."
else
echo "$line" >> out.json
@ -21,7 +21,7 @@ while read -r line; do
done <$file
# Remove any trailing commas from new instance lists
sed -i '' -e ':begin' -e '$!N' -e 's/,\n]/\n]/g' -e 'tbegin' -e 'P' -e 'D' out.json
sed -i -e ':begin' -e '$!N' -e 's/,\n]/\n]/g' -e 'tbegin' -e 'P' -e 'D' out.json
cat out.json | jq --indent 2 . > services.json
rm -f out.json