Block user agents and domains using user config

interfacer/src/browsh/config_sample.go

@@ -50,6 +50,14 @@ jpeg_compression = 0.9
 # Rate limit. For syntax, see: https://github.com/ulule/limiter
 rate-limit = "10-M"
 
+# Blocking is useful if the HTTP server is made public. All values are evaluated as
+# regular expressions.
+blocked-domains = [
+]
+
+blocked-user-agents = [
+]
+
 # HTML snippets to show at top and bottom of final page.
 header = ""
 footer = ""

interfacer/src/browsh/raw_text_server.go

@@ -97,10 +97,16 @@ func handleHTTPServerRequest(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 	w.Header().Set("Cache-Control", "public, max-age=600")
-	if isDisallowedURL(urlForBrowsh) {
+	if isDisallowedDomain(urlForBrowsh) {
 		http.Redirect(w, r, "/", 301)
 		return
 	}
+	if isDisallowedUserAgent(r.Header.Get("User-Agent")) {
+		if urlForBrowsh != "" {
+			http.Redirect(w, r, "/", 403)
+			return
+		}
+	}
 	if strings.TrimSpace(urlForBrowsh) == "" {
 		if strings.Contains(r.Host, "text.") {
 			message = "Welcome to the Browsh plain text client.\n" +
@@ -137,9 +143,24 @@ func deRecurseURL(urlForBrowsh string) (string, bool) {
 	return deRecurseURL(strings.TrimPrefix(nestedURL.RequestURI(), "/"))
 }
 
-func isDisallowedURL(urlForBrowsh string) bool {
-	r, _ := regexp.Compile("[mail|accounts].google.com")
-	return r.MatchString(urlForBrowsh)
+func isDisallowedDomain(urlForBrowsh string) bool {
+	for _, domainish := range viper.GetStringSlice("http-server.blocked-domains") {
+		r, _ := regexp.Compile(domainish)
+		if r.MatchString(urlForBrowsh) {
+			return true
+		}
+	}
+	return false
+}
+
+func isDisallowedUserAgent(userAgent string) bool {
+	for _, agentish := range viper.GetStringSlice("http-server.blocked-user-agents") {
+		r, _ := regexp.Compile(agentish)
+		if r.MatchString(userAgent) {
+			return true
+		}
+	}
+	return false
 }
 
 func isProductionHTTP(r *http.Request) bool {

interfacer/test/http-server/server_test.go

@@ -1,10 +1,13 @@
 package test
 
 import (
+	"io/ioutil"
+	"net/http"
 	"testing"
 
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
+	"github.com/spf13/viper"
 )
 
 func TestHTTPServer(t *testing.T) {
@@ -30,4 +33,43 @@ var _ = Describe("HTTP Server", func() {
 		response := getPath("/smorgasbord", "html")
 		Expect(response).To(ContainSubstring("background-image: url(data:image/jpeg"))
 	})
+
+	It("should block specified domains", func() {
+		viper.Set(
+			"http-server.blocked-domains",
+			[]string{"[mail|accounts].google.com", "other"},
+		)
+		url := getBrowshServiceBase() + "/mail.google.com"
+		client := &http.Client{}
+		request, _ := http.NewRequest("GET", url, nil)
+		response, _ := client.Do(request)
+		contents, _ := ioutil.ReadAll(response.Body)
+		Expect(string(contents)).To(ContainSubstring("Welcome to the Browsh HTML"))
+	})
+
+	It("should block specified user agents", func() {
+		viper.Set(
+			"http-server.blocked-user-agents",
+			[]string{"MJ12bot", "other"},
+		)
+		url := getBrowshServiceBase() + "/example.com"
+		client := &http.Client{}
+		request, _ := http.NewRequest("GET", url, nil)
+		request.Header.Add("User-Agent", "Blah blah MJ12bot etc")
+		response, _ := client.Do(request)
+		Expect(response.StatusCode).To(Equal(403))
+	})
+
+	It("should allow a blocked user agent to see the home page", func() {
+		viper.Set(
+			"http-server.blocked-user-agents",
+			[]string{"MJ12bot", "other"},
+		)
+		url := getBrowshServiceBase()
+		client := &http.Client{}
+		request, _ := http.NewRequest("GET", url, nil)
+		request.Header.Add("User-Agent", "Blah blah MJ12bot etc")
+		response, _ := client.Do(request)
+		Expect(response.StatusCode).To(Equal(200))
+	})
 })

interfacer/test/http-server/setup.go

@@ -28,8 +28,12 @@ func startBrowsh() {
 	browsh.HTTPServerStart()
 }
 
+func getBrowshServiceBase() string {
+	return "http://localhost:" + viper.GetString("http-server.port")
+}
+
 func getPath(path string, mode string) string {
-	browshServiceBase := "http://localhost:" + viper.GetString("http-server.port")
+	browshServiceBase := getBrowshServiceBase()
 	staticFileServerBase := "http://localhost:" + staticFileServerPort
 	fullBase := browshServiceBase + "/" + staticFileServerBase
 	client := &http.Client{}