inital
This commit is contained in:
commit
c13cf29f28
33
Makefile
Normal file
33
Makefile
Normal file
|
@ -0,0 +1,33 @@
|
||||||
|
all: clean multi-tor start_router haproxy check_install rc-local polipo-config
|
||||||
|
|
||||||
|
clean:
|
||||||
|
rm -rvf build
|
||||||
|
|
||||||
|
multi-tor:
|
||||||
|
mkdir -pv build/usr/local/bin
|
||||||
|
php multi-tor.sh.php > build/usr/local/bin/multi-tor.sh
|
||||||
|
chmod +x build/usr/local/bin/multi-tor.sh
|
||||||
|
|
||||||
|
start_router:
|
||||||
|
mkdir -pv build/usr/local/bin
|
||||||
|
php start_router.sh.php > build/usr/local/bin/start_router.sh
|
||||||
|
chmod +x build/usr/local/bin/start_router.sh
|
||||||
|
|
||||||
|
haproxy:
|
||||||
|
mkdir -pv build/etc/haproxy
|
||||||
|
php haproxy.cfg.php > build/etc/haproxy/haproxy.cfg
|
||||||
|
|
||||||
|
check_install:
|
||||||
|
mkdir -pv build/usr/local/bin
|
||||||
|
php check_install.sh.php > build/usr/local/bin/check_install.sh
|
||||||
|
chmod +x build/usr/local/bin/check_install.sh
|
||||||
|
|
||||||
|
rc-local:
|
||||||
|
mkdir -pv build/etc
|
||||||
|
php rc.local.php > build/etc/rc.local
|
||||||
|
chmod +x build/etc/rc.local
|
||||||
|
|
||||||
|
polipo-config:
|
||||||
|
mkdir -pv build/etc/polipo
|
||||||
|
php polipo-config.php > build/etc/polipo/config
|
||||||
|
chmod +x build/etc/polipo/config
|
12
check_install.sh.php
Normal file
12
check_install.sh.php
Normal file
|
@ -0,0 +1,12 @@
|
||||||
|
<?php require('./config.php'); ?>
|
||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
export INSTALL=""
|
||||||
|
<?php foreach (explode(',', CHECK_INSTALL) as $install) { ?>
|
||||||
|
if [ ! -e /usr/bin/polipo ]; then
|
||||||
|
export INSTALL="$INSTALL <?php echo $install; ?>"
|
||||||
|
update-rc.d -f <?php echo $install; ?> remove
|
||||||
|
fi
|
||||||
|
<?php } ?>
|
||||||
|
|
||||||
|
apt-get install -y $INSTALL
|
12
config.php
Normal file
12
config.php
Normal file
|
@ -0,0 +1,12 @@
|
||||||
|
<?php
|
||||||
|
define("BASE_PORT", "9051");
|
||||||
|
define("CONTROL_PORT", "15000");
|
||||||
|
define("TOR_DATA_DIR", "/data");
|
||||||
|
define("INSTANCES", "10");
|
||||||
|
define("BALANCE_MODE", "roundrobin");
|
||||||
|
define("SOCKS_PORT", "9050");
|
||||||
|
define("CHECK_INSTALL", "polipo,haproxy,tor");
|
||||||
|
define("CACHE_DIR", "");
|
||||||
|
define("PROXY_USERNAME", "da1b91ca-b548-48ab-8c42-a556dd95d2f0");
|
||||||
|
define("PROXY_PASSWORD", "1cdf3c32-7699-4ed2-ad88-7e9897c8f7fc");
|
||||||
|
?>
|
31
haproxy.cfg.php
Normal file
31
haproxy.cfg.php
Normal file
|
@ -0,0 +1,31 @@
|
||||||
|
<?php require('./config.php'); ?>
|
||||||
|
global
|
||||||
|
log /dev/log local0
|
||||||
|
log /dev/log local1 notice
|
||||||
|
chroot /var/lib/haproxy
|
||||||
|
user haproxy
|
||||||
|
group haproxy
|
||||||
|
daemon
|
||||||
|
|
||||||
|
defaults
|
||||||
|
log global
|
||||||
|
mode http
|
||||||
|
option dontlognull
|
||||||
|
contimeout 5000
|
||||||
|
clitimeout 50000
|
||||||
|
srvtimeout 50000
|
||||||
|
errorfile 400 /etc/haproxy/errors/400.http
|
||||||
|
errorfile 403 /etc/haproxy/errors/403.http
|
||||||
|
errorfile 408 /etc/haproxy/errors/408.http
|
||||||
|
errorfile 500 /etc/haproxy/errors/500.http
|
||||||
|
errorfile 502 /etc/haproxy/errors/502.http
|
||||||
|
errorfile 503 /etc/haproxy/errors/503.http
|
||||||
|
errorfile 504 /etc/haproxy/errors/504.http
|
||||||
|
|
||||||
|
listen socks :<?php echo SOCKS_PORT."\n"; ?>
|
||||||
|
mode tcp
|
||||||
|
balance <?php echo BALANCE_MODE."\n"; ?>
|
||||||
|
|
||||||
|
<?php $i = 1; $port = BASE_PORT; while ($i < INSTANCES) { ?>
|
||||||
|
server tor<?php echo $i++; ?> 127.0.0.1:<?php echo $port++; ?> check
|
||||||
|
<?php } ?>
|
37
multi-tor.sh.php
Normal file
37
multi-tor.sh.php
Normal file
|
@ -0,0 +1,37 @@
|
||||||
|
<?php require('./config.php'); ?>
|
||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
# Original script from
|
||||||
|
# http://blog.databigbang.com/distributed-scraping-with-multiple-tor-circuits/
|
||||||
|
|
||||||
|
base_socks_port=<?php echo BASE_PORT."\n"; ?>
|
||||||
|
base_control_port=<?php echo CONTROL_PORT."\n"; ?>
|
||||||
|
|
||||||
|
# Create data directory if it doesn't exist
|
||||||
|
if [ ! -d "<?php echo TOR_DATA_DIR; ?>" ]; then
|
||||||
|
mkdir "<?php echo TOR_DATA_DIR; ?>"
|
||||||
|
fi
|
||||||
|
|
||||||
|
TOR_INSTANCES="$1"
|
||||||
|
|
||||||
|
if [ ! $TOR_INSTANCES ] || [ $TOR_INSTANCES -lt 1 ]; then
|
||||||
|
echo "Please supply an instance count"
|
||||||
|
echo "Example: ./multi-tor.sh 5"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
for i in $(seq $TOR_INSTANCES)
|
||||||
|
do
|
||||||
|
j=$((i+1))
|
||||||
|
socks_port=$((base_socks_port+i))
|
||||||
|
control_port=$((base_control_port+i))
|
||||||
|
if [ ! -d "<?php echo TOR_DATA_DIR; ?>/tor$i" ]; then
|
||||||
|
echo "Creating directory data/tor$i"
|
||||||
|
mkdir "<?php echo TOR_DATA_DIR; ?>/tor$i"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Take into account that authentication for the control port is disabled. Must be used in secure and controlled environments
|
||||||
|
echo "Running: tor --MaxCircuitDirtiness 60 --RunAsDaemon 1 --CookieAuthentication 0 --HashedControlPassword \"\" --ControlPort $control_port --PidFile tor$i.pid --SocksPort $socks_port --DataDirectory <?php echo TOR_DATA_DIR; ?>/tor$i"
|
||||||
|
|
||||||
|
tor --MaxCircuitDirtiness 60 --RunAsDaemon 1 --CookieAuthentication 0 --HashedControlPassword "" --ControlPort $control_port --PidFile tor$i.pid --SocksPort $socks_port --DataDirectory <?php echo TOR_DATA_DIR; ?>/tor$i
|
||||||
|
done
|
154
polipo-config.php
Normal file
154
polipo-config.php
Normal file
|
@ -0,0 +1,154 @@
|
||||||
|
<?php require('./config.php'); ?>
|
||||||
|
# Sample configuration file for Polipo. -*-sh-*-
|
||||||
|
|
||||||
|
# You should not need to edit this configuration file; all configuration
|
||||||
|
# variables have reasonable defaults.
|
||||||
|
|
||||||
|
# This file only contains some of the configuration variables; see the
|
||||||
|
# list given by ``polipo -v'' and the manual for more.
|
||||||
|
|
||||||
|
|
||||||
|
### Basic configuration
|
||||||
|
### *******************
|
||||||
|
|
||||||
|
# Uncomment one of these if you want to allow remote clients to
|
||||||
|
# connect:
|
||||||
|
|
||||||
|
proxyAddress = "::0" # both IPv4 and IPv6
|
||||||
|
# proxyAddress = "0.0.0.0" # IPv4 only
|
||||||
|
|
||||||
|
# If you are enabling 'proxyAddress' above, then you want to enable the
|
||||||
|
# 'allowedClients' variable to the address of your network, e.g.
|
||||||
|
# allowedClients = 127.0.0.1, 192.168.42.0/24
|
||||||
|
|
||||||
|
# allowedClients = 127.0.0.1
|
||||||
|
|
||||||
|
# Uncomment this if you want your Polipo to identify itself by
|
||||||
|
# something else than the host name:
|
||||||
|
|
||||||
|
proxyName = "<?php echo gethostname(); ?>"
|
||||||
|
|
||||||
|
# Uncomment this if there's only one user using this instance of Polipo:
|
||||||
|
|
||||||
|
# cacheIsShared = false
|
||||||
|
|
||||||
|
# Uncomment this if you want to use a parent proxy:
|
||||||
|
|
||||||
|
# parentProxy = "squid.example.org:3128"
|
||||||
|
|
||||||
|
# Uncomment this if you want to use a parent SOCKS proxy:
|
||||||
|
|
||||||
|
socksParentProxy = "localhost:9050"
|
||||||
|
socksProxyType = socks5
|
||||||
|
|
||||||
|
|
||||||
|
### Memory
|
||||||
|
### ******
|
||||||
|
|
||||||
|
# Uncomment this if you want Polipo to use a ridiculously small amount
|
||||||
|
# of memory (a hundred C-64 worth or so):
|
||||||
|
|
||||||
|
# chunkHighMark = 819200
|
||||||
|
# objectHighMark = 128
|
||||||
|
|
||||||
|
# Uncomment this if you've got plenty of memory:
|
||||||
|
|
||||||
|
# chunkHighMark = 50331648
|
||||||
|
# objectHighMark = 16384
|
||||||
|
|
||||||
|
|
||||||
|
### On-disk data
|
||||||
|
### ************
|
||||||
|
|
||||||
|
# Uncomment this if you want to disable the on-disk cache:
|
||||||
|
|
||||||
|
diskCacheRoot = "<?php echo CACHE_DIR;?>"
|
||||||
|
|
||||||
|
# Uncomment this if you want to put the on-disk cache in a
|
||||||
|
# non-standard location:
|
||||||
|
|
||||||
|
# diskCacheRoot = "~/.polipo-cache/"
|
||||||
|
|
||||||
|
# Uncomment this if you want to disable the local web server:
|
||||||
|
|
||||||
|
# localDocumentRoot = ""
|
||||||
|
|
||||||
|
# Uncomment this if you want to enable the pages under /polipo/index?
|
||||||
|
# and /polipo/servers?. This is a serious privacy leak if your proxy
|
||||||
|
# is shared.
|
||||||
|
|
||||||
|
# disableIndexing = false
|
||||||
|
# disableServersList = false
|
||||||
|
|
||||||
|
|
||||||
|
### Domain Name System
|
||||||
|
### ******************
|
||||||
|
|
||||||
|
# Uncomment this if you want to contact IPv4 hosts only (and make DNS
|
||||||
|
# queries somewhat faster):
|
||||||
|
|
||||||
|
# dnsQueryIPv6 = no
|
||||||
|
|
||||||
|
# Uncomment this if you want Polipo to prefer IPv4 to IPv6 for
|
||||||
|
# double-stack hosts:
|
||||||
|
|
||||||
|
# dnsQueryIPv6 = reluctantly
|
||||||
|
|
||||||
|
# Uncomment this to disable Polipo's DNS resolver and use the system's
|
||||||
|
# default resolver instead. If you do that, Polipo will freeze during
|
||||||
|
# every DNS query:
|
||||||
|
|
||||||
|
# dnsUseGethostbyname = yes
|
||||||
|
|
||||||
|
|
||||||
|
### HTTP
|
||||||
|
### ****
|
||||||
|
|
||||||
|
# Uncomment this if you want to enable detection of proxy loops.
|
||||||
|
# This will cause your hostname (or whatever you put into proxyName
|
||||||
|
# above) to be included in every request:
|
||||||
|
|
||||||
|
# disableVia=false
|
||||||
|
|
||||||
|
# Uncomment this if you want to slightly reduce the amount of
|
||||||
|
# information that you leak about yourself:
|
||||||
|
|
||||||
|
# censoredHeaders = from, accept-language
|
||||||
|
# censorReferer = maybe
|
||||||
|
|
||||||
|
# Uncomment this if you're paranoid. This will break a lot of sites,
|
||||||
|
# though:
|
||||||
|
|
||||||
|
# censoredHeaders = set-cookie, cookie, cookie2, from, accept-language
|
||||||
|
# censorReferer = true
|
||||||
|
|
||||||
|
# Uncomment this if you want to use Poor Man's Multiplexing; increase
|
||||||
|
# the sizes if you're on a fast line. They should each amount to a few
|
||||||
|
# seconds' worth of transfer; if pmmSize is small, you'll want
|
||||||
|
# pmmFirstSize to be larger.
|
||||||
|
|
||||||
|
# Note that PMM is somewhat unreliable.
|
||||||
|
|
||||||
|
# pmmFirstSize = 16384
|
||||||
|
# pmmSize = 8192
|
||||||
|
|
||||||
|
# Uncomment this if your user-agent does something reasonable with
|
||||||
|
# Warning headers (most don't):
|
||||||
|
|
||||||
|
# relaxTransparency = maybe
|
||||||
|
|
||||||
|
# Uncomment this if you never want to revalidate instances for which
|
||||||
|
# data is available (this is not a good idea):
|
||||||
|
|
||||||
|
# relaxTransparency = yes
|
||||||
|
|
||||||
|
# Uncomment this if you have no network:
|
||||||
|
|
||||||
|
# proxyOffline = yes
|
||||||
|
|
||||||
|
# Uncomment this if you want to avoid revalidating instances with a
|
||||||
|
# Vary header (this is not a good idea):
|
||||||
|
|
||||||
|
# mindlesslyCacheVary = true
|
||||||
|
|
||||||
|
authCredentials = <?php echo PROXY_USERNAME; ?>:<?php echo PROXY_PASSWORD."\n"; ?>
|
3
rc.local.php
Normal file
3
rc.local.php
Normal file
|
@ -0,0 +1,3 @@
|
||||||
|
<?php require('./config.php'); ?>
|
||||||
|
/usr/local/bin/check_install.sh
|
||||||
|
/usr/local/bin/start_router.sh
|
12
start_router.sh.php
Normal file
12
start_router.sh.php
Normal file
|
@ -0,0 +1,12 @@
|
||||||
|
<?php require('./config.php'); ?>
|
||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
/etc/init.d/tor stop
|
||||||
|
/etc/init.d/polipo stop
|
||||||
|
/etc/init.d/haproxy stop
|
||||||
|
killall haproxy
|
||||||
|
killall tor
|
||||||
|
bash -c "/usr/local/bin/multi-tor.sh <?php echo INSTANCES; ?>"
|
||||||
|
bash -c "haproxy -f /etc/haproxy/haproxy.cfg -D"
|
||||||
|
sleep 5
|
||||||
|
/etc/init.d/polipo start
|
Loading…
Reference in a new issue