commit c13cf29f2831756304bf11998b9e15d3fbb5ac2c Author: Zachary Robert Thomas Boyd Date: Sat Mar 8 22:34:23 2014 -0500 inital diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..eaeb771 --- /dev/null +++ b/Makefile @@ -0,0 +1,33 @@ +all: clean multi-tor start_router haproxy check_install rc-local polipo-config + +clean: + rm -rvf build + +multi-tor: + mkdir -pv build/usr/local/bin + php multi-tor.sh.php > build/usr/local/bin/multi-tor.sh + chmod +x build/usr/local/bin/multi-tor.sh + +start_router: + mkdir -pv build/usr/local/bin + php start_router.sh.php > build/usr/local/bin/start_router.sh + chmod +x build/usr/local/bin/start_router.sh + +haproxy: + mkdir -pv build/etc/haproxy + php haproxy.cfg.php > build/etc/haproxy/haproxy.cfg + +check_install: + mkdir -pv build/usr/local/bin + php check_install.sh.php > build/usr/local/bin/check_install.sh + chmod +x build/usr/local/bin/check_install.sh + +rc-local: + mkdir -pv build/etc + php rc.local.php > build/etc/rc.local + chmod +x build/etc/rc.local + +polipo-config: + mkdir -pv build/etc/polipo + php polipo-config.php > build/etc/polipo/config + chmod +x build/etc/polipo/config \ No newline at end of file diff --git a/check_install.sh.php b/check_install.sh.php new file mode 100644 index 0000000..c936203 --- /dev/null +++ b/check_install.sh.php @@ -0,0 +1,12 @@ + +#!/bin/bash + +export INSTALL="" + +if [ ! -e /usr/bin/polipo ]; then + export INSTALL="$INSTALL " + update-rc.d -f remove +fi + + +apt-get install -y $INSTALL \ No newline at end of file diff --git a/config.php b/config.php new file mode 100644 index 0000000..e89c189 --- /dev/null +++ b/config.php @@ -0,0 +1,12 @@ + \ No newline at end of file diff --git a/haproxy.cfg.php b/haproxy.cfg.php new file mode 100644 index 0000000..fdf65c6 --- /dev/null +++ b/haproxy.cfg.php @@ -0,0 +1,31 @@ + +global + log /dev/log local0 + log /dev/log local1 notice + chroot /var/lib/haproxy + user haproxy + group haproxy + daemon + +defaults + log global + mode http + option dontlognull + contimeout 5000 + clitimeout 50000 + srvtimeout 50000 + errorfile 400 /etc/haproxy/errors/400.http + errorfile 403 /etc/haproxy/errors/403.http + errorfile 408 /etc/haproxy/errors/408.http + errorfile 500 /etc/haproxy/errors/500.http + errorfile 502 /etc/haproxy/errors/502.http + errorfile 503 /etc/haproxy/errors/503.http + errorfile 504 /etc/haproxy/errors/504.http + +listen socks : + mode tcp + balance + + +server tor 127.0.0.1: check + \ No newline at end of file diff --git a/multi-tor.sh.php b/multi-tor.sh.php new file mode 100644 index 0000000..b4ab575 --- /dev/null +++ b/multi-tor.sh.php @@ -0,0 +1,37 @@ + +#!/bin/bash + +# Original script from +# http://blog.databigbang.com/distributed-scraping-with-multiple-tor-circuits/ + +base_socks_port= +base_control_port= + +# Create data directory if it doesn't exist +if [ ! -d "" ]; then + mkdir "" +fi + +TOR_INSTANCES="$1" + +if [ ! $TOR_INSTANCES ] || [ $TOR_INSTANCES -lt 1 ]; then + echo "Please supply an instance count" + echo "Example: ./multi-tor.sh 5" + exit 1 +fi + +for i in $(seq $TOR_INSTANCES) +do + j=$((i+1)) + socks_port=$((base_socks_port+i)) + control_port=$((base_control_port+i)) + if [ ! -d "/tor$i" ]; then + echo "Creating directory data/tor$i" + mkdir "/tor$i" + fi + + # Take into account that authentication for the control port is disabled. Must be used in secure and controlled environments + echo "Running: tor --MaxCircuitDirtiness 60 --RunAsDaemon 1 --CookieAuthentication 0 --HashedControlPassword \"\" --ControlPort $control_port --PidFile tor$i.pid --SocksPort $socks_port --DataDirectory /tor$i" + + tor --MaxCircuitDirtiness 60 --RunAsDaemon 1 --CookieAuthentication 0 --HashedControlPassword "" --ControlPort $control_port --PidFile tor$i.pid --SocksPort $socks_port --DataDirectory /tor$i +done \ No newline at end of file diff --git a/polipo-config.php b/polipo-config.php new file mode 100644 index 0000000..bf2d96c --- /dev/null +++ b/polipo-config.php @@ -0,0 +1,154 @@ + +# Sample configuration file for Polipo. -*-sh-*- + +# You should not need to edit this configuration file; all configuration +# variables have reasonable defaults. + +# This file only contains some of the configuration variables; see the +# list given by ``polipo -v'' and the manual for more. + + +### Basic configuration +### ******************* + +# Uncomment one of these if you want to allow remote clients to +# connect: + +proxyAddress = "::0" # both IPv4 and IPv6 +# proxyAddress = "0.0.0.0" # IPv4 only + +# If you are enabling 'proxyAddress' above, then you want to enable the +# 'allowedClients' variable to the address of your network, e.g. +# allowedClients = 127.0.0.1, 192.168.42.0/24 + +# allowedClients = 127.0.0.1 + +# Uncomment this if you want your Polipo to identify itself by +# something else than the host name: + +proxyName = "" + +# Uncomment this if there's only one user using this instance of Polipo: + +# cacheIsShared = false + +# Uncomment this if you want to use a parent proxy: + +# parentProxy = "squid.example.org:3128" + +# Uncomment this if you want to use a parent SOCKS proxy: + +socksParentProxy = "localhost:9050" +socksProxyType = socks5 + + +### Memory +### ****** + +# Uncomment this if you want Polipo to use a ridiculously small amount +# of memory (a hundred C-64 worth or so): + +# chunkHighMark = 819200 +# objectHighMark = 128 + +# Uncomment this if you've got plenty of memory: + +# chunkHighMark = 50331648 +# objectHighMark = 16384 + + +### On-disk data +### ************ + +# Uncomment this if you want to disable the on-disk cache: + +diskCacheRoot = "" + +# Uncomment this if you want to put the on-disk cache in a +# non-standard location: + +# diskCacheRoot = "~/.polipo-cache/" + +# Uncomment this if you want to disable the local web server: + +# localDocumentRoot = "" + +# Uncomment this if you want to enable the pages under /polipo/index? +# and /polipo/servers?. This is a serious privacy leak if your proxy +# is shared. + +# disableIndexing = false +# disableServersList = false + + +### Domain Name System +### ****************** + +# Uncomment this if you want to contact IPv4 hosts only (and make DNS +# queries somewhat faster): + +# dnsQueryIPv6 = no + +# Uncomment this if you want Polipo to prefer IPv4 to IPv6 for +# double-stack hosts: + +# dnsQueryIPv6 = reluctantly + +# Uncomment this to disable Polipo's DNS resolver and use the system's +# default resolver instead. If you do that, Polipo will freeze during +# every DNS query: + +# dnsUseGethostbyname = yes + + +### HTTP +### **** + +# Uncomment this if you want to enable detection of proxy loops. +# This will cause your hostname (or whatever you put into proxyName +# above) to be included in every request: + +# disableVia=false + +# Uncomment this if you want to slightly reduce the amount of +# information that you leak about yourself: + +# censoredHeaders = from, accept-language +# censorReferer = maybe + +# Uncomment this if you're paranoid. This will break a lot of sites, +# though: + +# censoredHeaders = set-cookie, cookie, cookie2, from, accept-language +# censorReferer = true + +# Uncomment this if you want to use Poor Man's Multiplexing; increase +# the sizes if you're on a fast line. They should each amount to a few +# seconds' worth of transfer; if pmmSize is small, you'll want +# pmmFirstSize to be larger. + +# Note that PMM is somewhat unreliable. + +# pmmFirstSize = 16384 +# pmmSize = 8192 + +# Uncomment this if your user-agent does something reasonable with +# Warning headers (most don't): + +# relaxTransparency = maybe + +# Uncomment this if you never want to revalidate instances for which +# data is available (this is not a good idea): + +# relaxTransparency = yes + +# Uncomment this if you have no network: + +# proxyOffline = yes + +# Uncomment this if you want to avoid revalidating instances with a +# Vary header (this is not a good idea): + +# mindlesslyCacheVary = true + +authCredentials = : diff --git a/rc.local.php b/rc.local.php new file mode 100644 index 0000000..2fc31b5 --- /dev/null +++ b/rc.local.php @@ -0,0 +1,3 @@ + +/usr/local/bin/check_install.sh +/usr/local/bin/start_router.sh \ No newline at end of file diff --git a/start_router.sh.php b/start_router.sh.php new file mode 100644 index 0000000..1bda8ab --- /dev/null +++ b/start_router.sh.php @@ -0,0 +1,12 @@ + +#!/bin/bash + +/etc/init.d/tor stop +/etc/init.d/polipo stop +/etc/init.d/haproxy stop +killall haproxy +killall tor +bash -c "/usr/local/bin/multi-tor.sh " +bash -c "haproxy -f /etc/haproxy/haproxy.cfg -D" +sleep 5 +/etc/init.d/polipo start \ No newline at end of file