inital

Makefile

@@ -0,0 +1,33 @@
+all: clean multi-tor start_router haproxy check_install rc-local polipo-config
+
+clean:
+	rm -rvf build
+
+multi-tor:
+	mkdir -pv build/usr/local/bin
+	php multi-tor.sh.php > build/usr/local/bin/multi-tor.sh
+	chmod +x build/usr/local/bin/multi-tor.sh
+
+start_router: 
+	mkdir -pv build/usr/local/bin
+	php start_router.sh.php > build/usr/local/bin/start_router.sh
+	chmod +x build/usr/local/bin/start_router.sh
+
+haproxy:
+	mkdir -pv build/etc/haproxy
+	php haproxy.cfg.php > build/etc/haproxy/haproxy.cfg
+
+check_install: 
+	mkdir -pv build/usr/local/bin
+	php check_install.sh.php > build/usr/local/bin/check_install.sh
+	chmod +x build/usr/local/bin/check_install.sh
+
+rc-local: 
+	mkdir -pv build/etc
+	php rc.local.php > build/etc/rc.local
+	chmod +x build/etc/rc.local
+
+polipo-config: 
+	mkdir -pv build/etc/polipo
+	php polipo-config.php > build/etc/polipo/config
+	chmod +x build/etc/polipo/config

check_install.sh.php

@@ -0,0 +1,12 @@
+<?php require('./config.php'); ?>
+#!/bin/bash
+
+export INSTALL=""
+<?php foreach (explode(',', CHECK_INSTALL) as $install) { ?>
+if [ ! -e /usr/bin/polipo ]; then
+	export INSTALL="$INSTALL <?php echo $install; ?>"
+	update-rc.d -f <?php echo $install; ?> remove
+fi
+<?php } ?>
+
+apt-get install -y $INSTALL

config.php

@@ -0,0 +1,12 @@
+<?php 
+	define("BASE_PORT", "9051");
+	define("CONTROL_PORT", "15000");
+	define("TOR_DATA_DIR", "/data");
+	define("INSTANCES", "10");
+	define("BALANCE_MODE", "roundrobin");
+	define("SOCKS_PORT", "9050");
+	define("CHECK_INSTALL", "polipo,haproxy,tor");
+	define("CACHE_DIR", "");
+	define("PROXY_USERNAME", "da1b91ca-b548-48ab-8c42-a556dd95d2f0");
+	define("PROXY_PASSWORD", "1cdf3c32-7699-4ed2-ad88-7e9897c8f7fc");
+?>

haproxy.cfg.php

@@ -0,0 +1,31 @@
+<?php require('./config.php'); ?>
+global
+        log /dev/log    local0
+        log /dev/log    local1 notice
+        chroot /var/lib/haproxy
+        user haproxy
+        group haproxy
+        daemon
+
+defaults
+        log     global
+        mode    http
+        option  dontlognull
+        contimeout 5000
+        clitimeout 50000
+        srvtimeout 50000
+        errorfile 400 /etc/haproxy/errors/400.http
+        errorfile 403 /etc/haproxy/errors/403.http
+        errorfile 408 /etc/haproxy/errors/408.http
+        errorfile 500 /etc/haproxy/errors/500.http
+        errorfile 502 /etc/haproxy/errors/502.http
+        errorfile 503 /etc/haproxy/errors/503.http
+        errorfile 504 /etc/haproxy/errors/504.http
+
+listen socks :<?php echo SOCKS_PORT."\n"; ?>
+        mode tcp
+        balance <?php echo BALANCE_MODE."\n"; ?>
+       
+        <?php $i = 1; $port = BASE_PORT; while ($i < INSTANCES) { ?>
+server tor<?php echo $i++; ?> 127.0.0.1:<?php echo $port++; ?> check
+        <?php } ?>

multi-tor.sh.php

@@ -0,0 +1,37 @@
+<?php require('./config.php'); ?>
+#!/bin/bash
+
+# Original script from
+# http://blog.databigbang.com/distributed-scraping-with-multiple-tor-circuits/
+
+base_socks_port=<?php echo BASE_PORT."\n"; ?>
+base_control_port=<?php echo CONTROL_PORT."\n"; ?>
+
+# Create data directory if it doesn't exist
+if [ ! -d "<?php echo TOR_DATA_DIR; ?>" ]; then
+	mkdir "<?php echo TOR_DATA_DIR; ?>"
+fi
+
+TOR_INSTANCES="$1"
+
+if [ ! $TOR_INSTANCES ] || [ $TOR_INSTANCES -lt 1 ]; then
+    echo "Please supply an instance count"
+    echo "Example: ./multi-tor.sh 5"
+    exit 1
+fi
+
+for i in $(seq $TOR_INSTANCES)
+do
+	j=$((i+1))
+	socks_port=$((base_socks_port+i))
+	control_port=$((base_control_port+i))
+	if [ ! -d "<?php echo TOR_DATA_DIR; ?>/tor$i" ]; then
+		echo "Creating directory data/tor$i"
+		mkdir "<?php echo TOR_DATA_DIR; ?>/tor$i"
+	fi
+
+	# Take into account that authentication for the control port is disabled. Must be used in secure and controlled environments
+	echo "Running: tor --MaxCircuitDirtiness 60 --RunAsDaemon 1 --CookieAuthentication 0 --HashedControlPassword \"\" --ControlPort $control_port --PidFile tor$i.pid --SocksPort $socks_port --DataDirectory <?php echo TOR_DATA_DIR; ?>/tor$i"
+
+	tor --MaxCircuitDirtiness 60 --RunAsDaemon 1 --CookieAuthentication 0 --HashedControlPassword "" --ControlPort $control_port --PidFile tor$i.pid --SocksPort $socks_port --DataDirectory <?php echo TOR_DATA_DIR; ?>/tor$i
+done

polipo-config.php

@@ -0,0 +1,154 @@
+<?php require('./config.php'); ?>
+# Sample configuration file for Polipo. -*-sh-*-
+
+# You should not need to edit this configuration file; all configuration
+# variables have reasonable defaults.
+
+# This file only contains some of the configuration variables; see the
+# list given by ``polipo -v'' and the manual for more.
+
+
+### Basic configuration
+### *******************
+
+# Uncomment one of these if you want to allow remote clients to
+# connect:
+
+proxyAddress = "::0"        # both IPv4 and IPv6
+# proxyAddress = "0.0.0.0"    # IPv4 only
+
+# If you are enabling 'proxyAddress' above, then you want to enable the
+# 'allowedClients' variable to the address of your network, e.g.
+# allowedClients = 127.0.0.1, 192.168.42.0/24
+                                            
+# allowedClients = 127.0.0.1              
+
+# Uncomment this if you want your Polipo to identify itself by
+# something else than the host name:
+
+proxyName = "<?php echo gethostname(); ?>"
+
+# Uncomment this if there's only one user using this instance of Polipo:
+
+# cacheIsShared = false
+
+# Uncomment this if you want to use a parent proxy:
+
+# parentProxy = "squid.example.org:3128"
+
+# Uncomment this if you want to use a parent SOCKS proxy:
+
+socksParentProxy = "localhost:9050"
+socksProxyType = socks5
+
+
+### Memory
+### ******
+
+# Uncomment this if you want Polipo to use a ridiculously small amount
+# of memory (a hundred C-64 worth or so):
+
+# chunkHighMark = 819200
+# objectHighMark = 128
+
+# Uncomment this if you've got plenty of memory:
+
+# chunkHighMark = 50331648
+# objectHighMark = 16384
+
+
+### On-disk data
+### ************
+
+# Uncomment this if you want to disable the on-disk cache:
+
+diskCacheRoot = "<?php echo CACHE_DIR;?>"
+
+# Uncomment this if you want to put the on-disk cache in a
+# non-standard location:
+
+# diskCacheRoot = "~/.polipo-cache/"
+
+# Uncomment this if you want to disable the local web server:
+
+# localDocumentRoot = ""
+
+# Uncomment this if you want to enable the pages under /polipo/index?
+# and /polipo/servers?.  This is a serious privacy leak if your proxy
+# is shared.
+
+# disableIndexing = false
+# disableServersList = false
+
+
+### Domain Name System
+### ******************
+
+# Uncomment this if you want to contact IPv4 hosts only (and make DNS
+# queries somewhat faster):
+
+# dnsQueryIPv6 = no
+
+# Uncomment this if you want Polipo to prefer IPv4 to IPv6 for
+# double-stack hosts:
+
+# dnsQueryIPv6 = reluctantly
+
+# Uncomment this to disable Polipo's DNS resolver and use the system's
+# default resolver instead.  If you do that, Polipo will freeze during
+# every DNS query:
+
+# dnsUseGethostbyname = yes
+
+
+### HTTP
+### ****
+
+# Uncomment this if you want to enable detection of proxy loops.
+# This will cause your hostname (or whatever you put into proxyName
+# above) to be included in every request:
+
+# disableVia=false
+
+# Uncomment this if you want to slightly reduce the amount of
+# information that you leak about yourself:
+
+# censoredHeaders = from, accept-language
+# censorReferer = maybe
+
+# Uncomment this if you're paranoid.  This will break a lot of sites,
+# though:
+
+# censoredHeaders = set-cookie, cookie, cookie2, from, accept-language
+# censorReferer = true
+
+# Uncomment this if you want to use Poor Man's Multiplexing; increase
+# the sizes if you're on a fast line.  They should each amount to a few
+# seconds' worth of transfer; if pmmSize is small, you'll want
+# pmmFirstSize to be larger.
+
+# Note that PMM is somewhat unreliable.
+
+# pmmFirstSize = 16384
+# pmmSize = 8192
+
+# Uncomment this if your user-agent does something reasonable with
+# Warning headers (most don't):
+
+# relaxTransparency = maybe
+
+# Uncomment this if you never want to revalidate instances for which
+# data is available (this is not a good idea):
+
+# relaxTransparency = yes
+
+# Uncomment this if you have no network:
+
+# proxyOffline = yes
+
+# Uncomment this if you want to avoid revalidating instances with a
+# Vary header (this is not a good idea):
+
+# mindlesslyCacheVary = true
+
+authCredentials = <?php echo PROXY_USERNAME; ?>:<?php echo PROXY_PASSWORD."\n"; ?>

rc.local.php

@@ -0,0 +1,3 @@
+<?php require('./config.php'); ?>
+/usr/local/bin/check_install.sh
+/usr/local/bin/start_router.sh

start_router.sh.php

@@ -0,0 +1,12 @@
+<?php require('./config.php'); ?>
+#!/bin/bash
+
+/etc/init.d/tor stop
+/etc/init.d/polipo stop
+/etc/init.d/haproxy stop
+killall haproxy
+killall tor
+bash -c "/usr/local/bin/multi-tor.sh <?php echo INSTANCES; ?>"
+bash -c "haproxy -f /etc/haproxy/haproxy.cfg -D"
+sleep 5
+/etc/init.d/polipo start