diff --git a/.gitignore b/.gitignore index 4cd81aa..c340c84 100644 --- a/.gitignore +++ b/.gitignore @@ -10,4 +10,5 @@ config_dev.old.json tests todo.txt LICENCE -tokens.json \ No newline at end of file +tokens.json +.vscode \ No newline at end of file diff --git a/docker.sh b/docker.sh index d141d80..151ef91 100644 --- a/docker.sh +++ b/docker.sh @@ -1,28 +1,37 @@ VERSION=$(npm pkg get version | tr -d \") +LATEST="latest" if [ -n "$ARCHI" ]; then VERSION="$ARCHI-$VERSION" fi -echo "Pushing azukaar/cosmos-server:$VERSION to docker hub" +# if branch is unstable in git for circle ci +if [ -n "$CIRCLE_BRANCH" ]; then + if [ "$CIRCLE_BRANCH" != "master" ]; then + VERSION="$VERSION-$CIRCLE_BRANCH" + LATEST="$LATEST-$CIRCLE_BRANCH" + fi +fi + +echo "Pushing azukaar/cosmos-server:$VERSION and azukaar/cosmos-server:$LATEST" sh build.sh docker build \ -t azukaar/cosmos-server:$VERSION \ - -t azukaar/cosmos-server:latest \ + -t azukaar/cosmos-server:$LATEST \ . sh build arm64.sh docker build \ -t azukaar/cosmos-server:$VERSION-arm64 \ - -t azukaar/cosmos-server:latest-arm64 \ + -t azukaar/cosmos-server:$LATEST-arm64 \ -f dockerfile.arm64 \ --platform linux/arm64 \ . docker push azukaar/cosmos-server:$VERSION -docker push azukaar/cosmos-server:latest +docker push azukaar/cosmos-server:$LATEST docker push azukaar/cosmos-server:$VERSION-arm64 -docker push azukaar/cosmos-server:latest-arm64 \ No newline at end of file +docker push azukaar/cosmos-server:$LATEST-arm64 \ No newline at end of file diff --git a/go.mod b/go.mod index aabd9de..dfd70db 100644 --- a/go.mod +++ b/go.mod @@ -42,6 +42,7 @@ require ( github.com/go-chi/chi v4.0.2+incompatible // indirect github.com/go-chi/httprate v0.7.1 // indirect github.com/go-errors/errors v1.1.1 // indirect + github.com/go-ole/go-ole v1.2.6 // indirect github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect github.com/go-playground/validator/v10 v10.12.0 // indirect @@ -82,6 +83,7 @@ require ( github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.1 // indirect github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe // indirect + github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect github.com/namedotcom/go v0.0.0-20180403034216-08470befbe04 // indirect github.com/nrdcg/auroradns v1.0.1 // indirect github.com/nrdcg/desec v0.5.0 // indirect @@ -95,19 +97,24 @@ require ( github.com/patrickmn/go-cache v2.1.0+incompatible // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect github.com/pquerna/ffjson v0.0.0-20190930134022-aa0246cd15f7 // indirect github.com/pquerna/otp v1.3.0 // indirect github.com/roberthodgen/spa-server v0.0.0-20171007154335-bb87b4ff3253 // indirect github.com/sacloud/libsacloud v1.36.2 // indirect + github.com/shirou/gopsutil/v3 v3.23.3 // indirect github.com/sirupsen/logrus v1.7.0 // indirect github.com/stretchr/objx v0.5.0 // indirect github.com/stretchr/testify v1.8.2 // indirect + github.com/tklauser/go-sysconf v0.3.11 // indirect + github.com/tklauser/numcpus v0.6.0 // indirect github.com/transip/gotransip/v6 v6.5.0 // indirect github.com/vultr/govultr v1.1.1 // indirect github.com/xdg-go/pbkdf2 v1.0.0 // indirect github.com/xdg-go/scram v1.1.1 // indirect github.com/xdg-go/stringprep v1.0.3 // indirect github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect + github.com/yusufpapurcu/wmi v1.2.2 // indirect go.mongodb.org/mongo-driver v1.11.3 // indirect go.opencensus.io v0.22.5 // indirect go.uber.org/ratelimit v0.1.0 // indirect diff --git a/go.sum b/go.sum index c26df79..89ff29a 100644 --- a/go.sum +++ b/go.sum @@ -181,6 +181,8 @@ github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= +github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= @@ -246,6 +248,8 @@ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -378,6 +382,8 @@ github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3Rllmb github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe h1:iruDEfMl2E6fbMZ9s0scYfZQ84/6SPL6zC8ACM2oIL0= github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f h1:y5//uYreIhSUg3J1GEMiLbxo1LJaP8RfCpH6pymGZus= +github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= github.com/namedotcom/go v0.0.0-20180403034216-08470befbe04 h1:o6uBwrhM5C8Ll3MAAxrQxRHEu7FkapwTuI2WmL1rw4g= github.com/namedotcom/go v0.0.0-20180403034216-08470befbe04/go.mod h1:5sN+Lt1CaY4wsPvgQH/jsuJi4XO2ssZbdsIizr4CVC8= github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32/go.mod h1:9wM+0iRr9ahx58uYLpLIr5fm8diHn0JbqRycJi6w0Ms= @@ -416,6 +422,8 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw= +github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/pquerna/ffjson v0.0.0-20190930134022-aa0246cd15f7 h1:xoIK0ctDddBMnc74udxJYBqlo9Ylnsp1waqjLsnef20= github.com/pquerna/ffjson v0.0.0-20190930134022-aa0246cd15f7/go.mod h1:YARuvh7BUWHNhzDq2OM5tzR2RiCcN2D7sapiKyCel/M= github.com/pquerna/otp v1.2.0/go.mod h1:dkJfzwRKNiegxyNb54X/3fLwhCynbMspSyWKnvi1AEg= @@ -446,6 +454,10 @@ github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQD github.com/rwtodd/Go.Sed v0.0.0-20210816025313-55464686f9ef/go.mod h1:8AEUvGVi2uQ5b24BIhcr0GCcpd/RNAFWaN2CJFrWIIQ= github.com/sacloud/libsacloud v1.36.2 h1:aosI7clbQ9IU0Hj+3rpk3SKJop5nLPpLThnWCivPqjI= github.com/sacloud/libsacloud v1.36.2/go.mod h1:P7YAOVmnIn3DKHqCZcUKYUXmSwGBm3yS7IBEjKVSrjg= +github.com/shirou/gopsutil/v3 v3.23.3 h1:Syt5vVZXUDXPEXpIBt5ziWsJ4LdSAAxF4l/xZeQgSEE= +github.com/shirou/gopsutil/v3 v3.23.3/go.mod h1:lSBNN6t3+D6W5e5nXTxc8KIMMVxAcS+6IJlffjRRlMU= +github.com/shoenig/go-m1cpu v0.1.4/go.mod h1:Wwvst4LR89UxjeFtLRMrpgRiyY4xPsejnVZym39dbAQ= +github.com/shoenig/test v0.6.3/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnjqq0k= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= @@ -472,6 +484,10 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= +github.com/tklauser/go-sysconf v0.3.11 h1:89WgdJhk5SNwJfu+GKyYveZ4IaJ7xAkecBo+KdJV0CM= +github.com/tklauser/go-sysconf v0.3.11/go.mod h1:GqXfhXY3kiPa0nAXPDIQIWzJbMCB7AmcWpGR8lSZfqI= +github.com/tklauser/numcpus v0.6.0 h1:kebhY2Qt+3U6RNK7UqpYNA+tJ23IBEGKkB7JQBfDYms= +github.com/tklauser/numcpus v0.6.0/go.mod h1:FEZLMke0lhOUG6w2JadTzp0a+Nl8PF/GFkQ5UVIcaL4= github.com/transip/gotransip/v6 v6.2.0/go.mod h1:pQZ36hWWRahCUXkFWlx9Hs711gLd8J4qdgLdRzmtY+g= github.com/transip/gotransip/v6 v6.5.0 h1:mMybbSvyJSA/SzoNHa4ioudmjDpYcVrZhFhxIeeHRx0= github.com/transip/gotransip/v6 v6.5.0/go.mod h1:pQZ36hWWRahCUXkFWlx9Hs711gLd8J4qdgLdRzmtY+g= @@ -499,6 +515,8 @@ github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg= +github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= go.mongodb.org/mongo-driver v1.11.3 h1:Ql6K6qYHEzB6xvu4+AU0BoRoqf9vFPcc4o7MUIdPW8Y= go.mongodb.org/mongo-driver v1.11.3/go.mod h1:PTSz5yu21bkT/wXpkS7WR5f0ddqw5quethTUn9WM+2g= go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= @@ -649,6 +667,7 @@ golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -674,9 +693,11 @@ golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= diff --git a/package-lock.json b/package-lock.json index 9b3d3b5..9a686dd 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "cosmos-server", - "version": "0.1.15", + "version": "0.1.16", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "cosmos-server", - "version": "0.1.15", + "version": "0.1.16", "dependencies": { "@ant-design/colors": "^6.0.0", "@ant-design/icons": "^4.7.0", diff --git a/package.json b/package.json index 0a82aee..4a84b0d 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "cosmos-server", - "version": "0.1.17", + "version": "0.1.17-unstable", "description": "", "main": "test-server.js", "bugs": { diff --git a/src/httpServer.go b/src/httpServer.go index 252b35a..985070e 100644 --- a/src/httpServer.go +++ b/src/httpServer.go @@ -118,6 +118,7 @@ func startHTTPSServer(router *mux.Router, tlsCert string, tlsKey string) { func tokenMiddleware(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + //Header.Del r.Header.Set("x-cosmos-user", "") r.Header.Set("x-cosmos-role", "") @@ -170,7 +171,9 @@ func StartServer() { router := mux.NewRouter().StrictSlash(true) - router.Use(middleware.Recoverer) + // need rewrite bc it catches too many things and prevent + // client to be notified of the error + // router.Use(middleware.Recoverer) router.Use(middleware.Logger) router.Use(utils.SetSecurityHeaders) diff --git a/src/index.go b/src/index.go index e133961..31fd3c6 100644 --- a/src/index.go +++ b/src/index.go @@ -1,26 +1,28 @@ package main import ( - "github.com/azukaar/cosmos-server/src/utils" - "time" - "github.com/azukaar/cosmos-server/src/docker" "math/rand" + "time" + + "github.com/azukaar/cosmos-server/src/docker" + "github.com/azukaar/cosmos-server/src/utils" ) func main() { - utils.Log("Starting...") + utils.Log("Starting...") + // utils.Log("Smart Shield estimates the capacity at " + strconv.Itoa((int)(proxy.MaxUsers)) + " concurrent users") - rand.Seed(time.Now().UnixNano()) + rand.Seed(time.Now().UnixNano()) - LoadConfig() - - go CRON() + LoadConfig() - docker.Test() + go CRON() - docker.DockerListenEvents() + docker.Test() - docker.BootstrapAllContainersFromTags() - - StartServer() -} \ No newline at end of file + docker.DockerListenEvents() + + docker.BootstrapAllContainersFromTags() + + StartServer() +} diff --git a/src/proxy/SmartResponseWriter.go b/src/proxy/SmartResponseWriter.go new file mode 100644 index 0000000..5e11387 --- /dev/null +++ b/src/proxy/SmartResponseWriter.go @@ -0,0 +1,90 @@ +package proxy + +import ( + "github.com/azukaar/cosmos-server/src/utils" + "bufio" + "net" + "net/http" + "time" + "fmt" + "errors" +) + +type SmartResponseWriterWrapper struct { + http.ResponseWriter + ClientID string + Status int + Bytes int64 + ThrottleNext int + TimeStarted time.Time + TimeEnded time.Time + RequestCost int + Method string + shield smartShieldState + policy utils.SmartShieldPolicy + isOver bool +} + +func (w *SmartResponseWriterWrapper) IsOver() bool { + return w.isOver +} + +func (w *SmartResponseWriterWrapper) IsOld() bool { + if !w.IsOver() { + return false + } + oneHourAgo := time.Now().Add(-time.Hour) + if w.TimeEnded.Before(oneHourAgo) { + return true + } + return false +} + +func (w *SmartResponseWriterWrapper) WriteHeader(status int) { + w.Status = status + w.RequestCost = 1 + if w.Method != "GET" { + w.RequestCost = 5 + } + if w.Status >= 400 { + w.RequestCost *= 30 + } + w.ResponseWriter.WriteHeader(status) +} + +func (w *SmartResponseWriterWrapper) Write(p []byte) (int, error) { + userConsumed := shield.GetUserUsedBudgets(w.ClientID) + if !shield.isAllowedToReqest(w.policy, userConsumed) { + utils.Log(fmt.Sprintf("SmartShield: %s is banned", w.ClientID)) + w.isOver = true + w.TimeEnded = time.Now() + w.ResponseWriter.WriteHeader(http.StatusServiceUnavailable) + w.ResponseWriter.(http.Flusher).Flush() + return 0, errors.New("Pending request cancelled due to SmartShield") + } + thro := shield.computeThrottle(w.policy, userConsumed) + utils.Debug(fmt.Sprintf("Throttle: %d", thro)) + w.ThrottleNext = 0 + if thro > 0 { + time.Sleep(time.Duration(thro) * time.Millisecond) + } + n, err := w.ResponseWriter.Write(p) + w.Bytes += int64(n) + return n, err +} + +func (w *SmartResponseWriterWrapper) Hijack() (net.Conn, *bufio.ReadWriter, error) { + hijacker, ok := w.ResponseWriter.(http.Hijacker) + if !ok { + return nil, nil, http.ErrNotSupported + } + return hijacker.Hijack() +} + +func (w *SmartResponseWriterWrapper) Flush() { + flusher, ok := w.ResponseWriter.(http.Flusher) + if ok { + flusher.Flush() + } +} + diff --git a/src/proxy/routeTo.go b/src/proxy/routeTo.go index 106d1db..e12b0b0 100644 --- a/src/proxy/routeTo.go +++ b/src/proxy/routeTo.go @@ -2,14 +2,10 @@ package proxy import ( "net/http" - "net/http/httputil" + "net/http/httputil" "net/url" spa "github.com/roberthodgen/spa-server" "github.com/azukaar/cosmos-server/src/utils" - // "io/ioutil" - // "io" - // "os" - // "golang.org/x/crypto/bcrypt" ) // NewProxy takes target host and creates a reverse proxy @@ -24,6 +20,7 @@ func NewProxy(targetHost string) (*httputil.ReverseProxy, error) { proxy.ModifyResponse = func(resp *http.Response) error { utils.Debug("Response from backend: " + resp.Status) utils.Debug("URL was " + resp.Request.URL.String()) + return nil } @@ -31,7 +28,7 @@ func NewProxy(targetHost string) (*httputil.ReverseProxy, error) { } -func RouteTo(route utils.ProxyRouteConfig) http.Handler /*func(http.ResponseWriter, *http.Request)*/ { +func RouteTo(route utils.ProxyRouteConfig) http.Handler { // initialize a reverse proxy and pass the actual backend server url here destination := route.Target diff --git a/src/proxy/routerGen.go b/src/proxy/routerGen.go index ad469fa..25f6e61 100644 --- a/src/proxy/routerGen.go +++ b/src/proxy/routerGen.go @@ -2,13 +2,14 @@ package proxy import ( "net/http" - "github.com/gorilla/mux" - "time" - "github.com/azukaar/cosmos-server/src/utils" - "github.com/azukaar/cosmos-server/src/user" - "strconv" - "github.com/go-chi/httprate" "regexp" + "strconv" + "time" + + "github.com/azukaar/cosmos-server/src/user" + "github.com/azukaar/cosmos-server/src/utils" + "github.com/go-chi/httprate" + "github.com/gorilla/mux" ) func tokenMiddleware(enabled bool) func(next http.Handler) http.Handler { @@ -16,16 +17,16 @@ func tokenMiddleware(enabled bool) func(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { r.Header.Set("x-cosmos-user", "") r.Header.Set("x-cosmos-role", "") - + u, err := user.RefreshUserToken(w, r) - + if err != nil { return } - + r.Header.Set("x-cosmos-user", u.Nickname) r.Header.Set("x-cosmos-role", strconv.Itoa((int)(u.Role))) - + ogcookies := r.Header.Get("Cookie") cookieRemoveRegex := regexp.MustCompile(`jwttoken=[^;]*;`) cookies := cookieRemoveRegex.ReplaceAllString(ogcookies, "") @@ -34,8 +35,8 @@ func tokenMiddleware(enabled bool) func(next http.Handler) http.Handler { // Replace the token with a application speicfic one r.Header.Set("x-cosmos-token", "1234567890") - if(enabled) { - utils.LoggedInOnlyWithRedirect(w, r); + if enabled { + utils.LoggedInOnlyWithRedirect(w, r) } next.ServeHTTP(w, r) @@ -46,24 +47,26 @@ func tokenMiddleware(enabled bool) func(next http.Handler) http.Handler { func RouterGen(route utils.ProxyRouteConfig, router *mux.Router, destination http.Handler) *mux.Route { origin := router.Methods("GET", "POST", "PUT", "DELETE", "PATCH", "OPTIONS", "HEAD") - if(route.UseHost) { + if route.UseHost { origin = origin.Host(route.Host) } - if(route.UsePathPrefix) { - if(route.PathPrefix != "" && route.PathPrefix[0] != '/') { + if route.UsePathPrefix { + if route.PathPrefix != "" && route.PathPrefix[0] != '/' { utils.Error("PathPrefix must start with a /", nil) } origin = origin.PathPrefix(route.PathPrefix) } - - if(route.UsePathPrefix && route.StripPathPrefix) { - if(route.PathPrefix != "" && route.PathPrefix[0] != '/') { + + if route.UsePathPrefix && route.StripPathPrefix { + if route.PathPrefix != "" && route.PathPrefix[0] != '/' { utils.Error("PathPrefix must start with a /", nil) } destination = http.StripPrefix(route.PathPrefix, destination) } + destination = SmartShieldMiddleware(route.SmartShield)(destination) + originCORS := route.CORSOrigin if originCORS == "" { @@ -74,34 +77,34 @@ func RouterGen(route utils.ProxyRouteConfig, router *mux.Router, destination htt } } - if(route.UsePathPrefix && !route.StripPathPrefix && (route.Mode == "STATIC" || route.Mode == "SPA")) { + if route.UsePathPrefix && !route.StripPathPrefix && (route.Mode == "STATIC" || route.Mode == "SPA") { utils.Warn("PathPrefix is used, but StripPathPrefix is false. The route mode is " + (string)(route.Mode) + ". This will likely cause issues with the route. Ignore this warning if you know what you are doing.") } - + timeout := route.Timeout - - if(timeout > 0) { + + if timeout > 0 { destination = utils.MiddlewareTimeout(timeout * time.Millisecond)(destination) } throttlePerMinute := route.ThrottlePerMinute - if(throttlePerMinute > 0) { + if throttlePerMinute > 0 { throtthleTime := time.Minute - destination = httprate.Limit(throttlePerMinute, throtthleTime, + destination = httprate.Limit(throttlePerMinute, throtthleTime, httprate.WithKeyFuncs(httprate.KeyByIP), httprate.WithLimitHandler(func(w http.ResponseWriter, r *http.Request) { utils.Error("Too many requests. Throttling", nil) - utils.HTTPError(w, "Too many requests", + utils.HTTPError(w, "Too many requests", http.StatusTooManyRequests, "HTTP003") - return + return }), )(destination) } origin.Handler(tokenMiddleware(route.AuthEnabled)(utils.CORSHeader(originCORS)((destination)))) - utils.Log("Added route: ["+ (string)(route.Mode) + "] " + route.Host + route.PathPrefix + " to " + route.Target + "") + utils.Log("Added route: [" + (string)(route.Mode) + "] " + route.Host + route.PathPrefix + " to " + route.Target + "") return origin -} \ No newline at end of file +} diff --git a/src/proxy/shield.go b/src/proxy/shield.go new file mode 100644 index 0000000..5e9985d --- /dev/null +++ b/src/proxy/shield.go @@ -0,0 +1,235 @@ +package proxy + +import ( + "github.com/azukaar/cosmos-server/src/utils" + "sync" + "time" + "net/http" + "fmt" + "net" +) + +/* + TODO : + - Recalculate throttle every gb for writer wrapper? +*/ + +const ( + STRIKE = 0 + TEMP = 1 + PERM = 2 +) +type userBan struct { + ClientID string + banType int + time time.Time +} + +type smartShieldState struct { + sync.Mutex + requests []*SmartResponseWriterWrapper + bans []*userBan +} + +type userUsedBudget struct { + ClientID string + Time float64 + Requests int + Bytes int64 +} + +var shield smartShieldState + +func (shield *smartShieldState) GetUserUsedBudgets(ClientID string) userUsedBudget { + shield.Lock() + defer shield.Unlock() + + userConsumed := userUsedBudget{ + ClientID: ClientID, + Time: 0, + Requests: 0, + Bytes: 0, + } + + // Check for recent requests + for i := len(shield.requests) - 1; i >= 0; i-- { + request := shield.requests[i] + if(request.IsOld()) { + return userConsumed + } + if request.ClientID == ClientID && !request.IsOld() { + if(request.IsOver()) { + userConsumed.Time += request.TimeEnded.Sub(request.TimeStarted).Seconds() + } else { + userConsumed.Time += time.Now().Sub(request.TimeStarted).Seconds() + } + userConsumed.Requests += request.RequestCost + userConsumed.Bytes += request.Bytes + } + } + + return userConsumed +} + +func (shield *smartShieldState) isAllowedToReqest(policy utils.SmartShieldPolicy, userConsumed userUsedBudget) bool { + shield.Lock() + defer shield.Unlock() + + ClientID := userConsumed.ClientID + + nbTempBans := 0 + nbStrikes := 0 + + // Check for bans + for i := len(shield.bans) - 1; i >= 0; i-- { + ban := shield.bans[i] + if ban.banType == PERM { + return false + } else if ban.banType == TEMP { + if(ban.time.Add(4 * 3600 * time.Second).Before(time.Now())) { + return false + } else if (ban.time.Add(72 * 3600 * time.Second).Before(time.Now())) { + nbTempBans++ + } + } else if ban.banType == STRIKE { + return false + if(ban.time.Add(3600 * time.Second).Before(time.Now())) { + return false + } else if (ban.time.Add(24 * 3600 * time.Second).Before(time.Now())) { + nbStrikes++ + } + } + } + + // Check for new bans + if nbTempBans >= 3 { + // perm ban + shield.bans = append(shield.bans, &userBan{ + ClientID: ClientID, + banType: PERM, + time: time.Now(), + }) + return false + } else if nbStrikes >= 3 { + // temp ban + shield.bans = append(shield.bans, &userBan{ + ClientID: ClientID, + banType: TEMP, + time: time.Now(), + }) + return false + } + + // Check for new strikes + if (userConsumed.Time > (policy.PerUserTimeBudget * float64(policy.PolicyStrictness))) || + (userConsumed.Requests > (policy.PerUserRequestLimit * policy.PolicyStrictness)) || + (userConsumed.Bytes > (policy.PerUserByteLimit * int64(policy.PolicyStrictness))) { + shield.bans = append(shield.bans, &userBan{ + ClientID: ClientID, + banType: STRIKE, + time: time.Now(), + }) + return false + } + + return true +} + +func (shield *smartShieldState) computeThrottle(policy utils.SmartShieldPolicy, userConsumed userUsedBudget) int { + shield.Lock() + defer shield.Unlock() + + throttle := 0 + + overReq := policy.PerUserRequestLimit - userConsumed.Requests + overReqRatio := float64(overReq) / float64(policy.PerUserRequestLimit) + if overReq < 0 { + newThrottle := int(float64(2500) * -overReqRatio) + if newThrottle > throttle { + throttle = newThrottle + } + } + + overByte := policy.PerUserByteLimit - userConsumed.Bytes + overByteRatio := float64(overByte) / float64(policy.PerUserByteLimit) + if overByte < 0 { + newThrottle := int(float64(150) * -overByteRatio) + if newThrottle > throttle { + throttle = newThrottle + } + } + + if throttle > 0 { + utils.Debug(fmt.Sprintf("User Time: %f, Requests: %d, Bytes: %d", userConsumed.Time, userConsumed.Requests, userConsumed.Bytes)) + utils.Debug(fmt.Sprintf("Policy Time: %f, Requests: %d, Bytes: %d", policy.PerUserTimeBudget, policy.PerUserRequestLimit, policy.PerUserByteLimit)) + utils.Debug(fmt.Sprintf("Throttling: %d", throttle)) + } + + return throttle +} + +func GetClientID(r *http.Request) string { + ip, _, _ := net.SplitHostPort(r.RemoteAddr) + return ip +} + +func SmartShieldMiddleware(policy utils.SmartShieldPolicy) func(http.Handler) http.Handler { + if policy.Enabled == false { + return func(next http.Handler) http.Handler { + return next + } + } else { + if(policy.PerUserTimeBudget == 0) { + policy.PerUserTimeBudget = 2 * 60 * 60 * 1000 // 2 hours + } + if(policy.PerUserRequestLimit == 0) { + policy.PerUserRequestLimit = 6000 // 100 requests per minute + } + if(policy.PerUserByteLimit == 0) { + policy.PerUserByteLimit = 3 * 60 * 1024 * 1024 * 1024 // 180GB + } + if(policy.PolicyStrictness == 0) { + policy.PolicyStrictness = 2 // NORMAL + } + } + return func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + utils.Log("SmartShield: Request received") + clientID := GetClientID(r) + userConsumed := shield.GetUserUsedBudgets(clientID) + + if !shield.isAllowedToReqest(policy, userConsumed) { + utils.Log("SmartShield: User is banned") + http.Error(w, "Too many requests", http.StatusTooManyRequests) + return + } else { + utils.Debug("SmartShield: Creating request") + throttle := shield.computeThrottle(policy, userConsumed) + wrapper := &SmartResponseWriterWrapper { + ResponseWriter: w, + ThrottleNext: throttle, + TimeStarted: time.Now(), + ClientID: clientID, + RequestCost: 1, + Method: r.Method, + shield: shield, + policy: policy, + } + + utils.Debug("SmartShield: Adding request") + shield.Lock() + shield.requests = append(shield.requests, wrapper) + shield.Unlock() + + utils.Debug("SmartShield: Processing request") + next.ServeHTTP(wrapper, r) + + shield.Lock() + wrapper.TimeEnded = time.Now() + wrapper.isOver = true + shield.Unlock() + utils.Debug("SmartShield: Request finished") + } + }) + } +} \ No newline at end of file diff --git a/src/utils/middleware.go b/src/utils/middleware.go index ac3e614..6ffb15f 100644 --- a/src/utils/middleware.go +++ b/src/utils/middleware.go @@ -4,6 +4,7 @@ import ( "context" "net/http" "time" + "github.com/mxk/go-flowrate/flowrate" ) // https://github.com/go-chi/chi/blob/master/middleware/timeout.go @@ -31,6 +32,28 @@ func MiddlewareTimeout(timeout time.Duration) func(next http.Handler) http.Handl } } +type responseWriter struct { + http.ResponseWriter + *flowrate.Writer +} + +func (w *responseWriter) Write(b []byte) (int, error) { + return w.Writer.Write(b) +} + +func BandwithLimiterMiddleware(max int64) func(next http.Handler) http.Handler { + return func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if(max > 0) { + fw := flowrate.NewWriter(w, max) + w = &responseWriter{w, fw} + } + + next.ServeHTTP(w, r) + }) + } +} + func SetSecurityHeaders(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if(IsHTTPS) { diff --git a/src/utils/types.go b/src/utils/types.go index 481c726..0b1aa7b 100644 --- a/src/utils/types.go +++ b/src/utils/types.go @@ -93,6 +93,19 @@ type HTTPConfig struct { SSLEmail string `validate:"omitempty,email"` } +const ( + STRICT = 1 + NORMAL = 2 + LENIENT = 3 +) +type SmartShieldPolicy struct { + Enabled bool + PolicyStrictness int + PerUserTimeBudget float64 + PerUserRequestLimit int + PerUserByteLimit int64 +} + type DockerConfig struct { SkipPruneNetwork bool } @@ -114,5 +127,6 @@ type ProxyRouteConfig struct { StripPathPrefix bool AuthEnabled bool Target string `validate:"required"` + SmartShield SmartShieldPolicy Mode ProxyMode } diff --git a/src/utils/utils.go b/src/utils/utils.go index 181fa6b..69a95fa 100644 --- a/src/utils/utils.go +++ b/src/utils/utils.go @@ -1,13 +1,15 @@ package utils import ( - "os" - "net/http" "encoding/json" - "strconv" - "strings" - "math/rand" "errors" + "math/rand" + "net/http" + "os" + "strconv" + "strings" + + "github.com/shirou/gopsutil/v3/mem" ) var BaseMainConfig Config @@ -16,13 +18,13 @@ var IsHTTPS = false var DefaultConfig = Config{ LoggingLevel: "INFO", - NewInstall: true, + NewInstall: true, HTTPConfig: HTTPConfig{ - HTTPSCertificateMode: "DISABLED", + HTTPSCertificateMode: "DISABLED", GenerateMissingAuthCert: true, - HTTPPort: "80", - HTTPSPort: "443", - Hostname: "localhost", + HTTPPort: "80", + HTTPSPort: "443", + Hostname: "localhost", ProxyConfig: ProxyConfig{ Routes: []ProxyRouteConfig{}, }, @@ -30,7 +32,7 @@ var DefaultConfig = Config{ } func FileExists(path string) bool { - _, err := os.Stat(path) + _, err := os.Stat(path) if err == nil { return true } @@ -51,36 +53,37 @@ func GetPublicAuthKey() string { } var AlphaNumRunes = []rune("0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ") + func GenerateRandomString(n int) string { b := make([]rune, n) for i := range b { - b[i] = AlphaNumRunes[rand.Intn(len(AlphaNumRunes))] + b[i] = AlphaNumRunes[rand.Intn(len(AlphaNumRunes))] } return string(b) } type HTTPErrorResult struct { - Status string `json:"status"` + Status string `json:"status"` Message string `json:"message"` - Code string `json:"code"` + Code string `json:"code"` } func HTTPError(w http.ResponseWriter, message string, code int, userCode string) { w.WriteHeader(code) json.NewEncoder(w).Encode(HTTPErrorResult{ - Status: "error", + Status: "error", Message: message, - Code: userCode, + Code: userCode, }) - Error("HTTP Request returned Error " + strconv.Itoa(code) + " : " + message, nil) + Error("HTTP Request returned Error "+strconv.Itoa(code)+" : "+message, nil) } -func SetBaseMainConfig(config Config){ +func SetBaseMainConfig(config Config) { LoadBaseMainConfig(config) SaveConfigTofile(config) } -func LoadBaseMainConfig(config Config){ +func LoadBaseMainConfig(config Config) { BaseMainConfig = config MainConfig = config @@ -135,12 +138,11 @@ func Sanitize(s string) string { func GetConfigFileName() string { configFile := os.Getenv("CONFIG_FILE") - + if configFile == "" { configFile = "/config/cosmos.config.json" } - return configFile } @@ -190,7 +192,7 @@ func SaveConfigTofile(config Config) { Fatal("Writing Config File", err) } - Log("Config file saved."); + Log("Config file saved.") } func RestartServer() { @@ -205,9 +207,9 @@ func LoggedInOnlyWithRedirect(w http.ResponseWriter, req *http.Request) error { if !isUserLoggedIn || userNickname == "" { Error("LoggedInOnlyWithRedirect: User is not logged in", nil) - http.Redirect(w, req, "/ui/login?notlogged=1&redirect=" + req.URL.Path, http.StatusFound) + http.Redirect(w, req, "/ui/login?notlogged=1&redirect="+req.URL.Path, http.StatusFound) } - + return nil } @@ -222,7 +224,7 @@ func LoggedInOnly(w http.ResponseWriter, req *http.Request) error { HTTPError(w, "User not logged in", http.StatusUnauthorized, "HTTP004") return errors.New("User not logged in") } - + return nil } @@ -260,7 +262,7 @@ func AdminOrItselfOnly(w http.ResponseWriter, req *http.Request, nickname string return errors.New("User not logged in") } - if nickname != userNickname && !isUserAdmin { + if nickname != userNickname && !isUserAdmin { Error("AdminOrItselfOnly: User is not admin", nil) HTTPError(w, "User unauthorized", http.StatusUnauthorized, "HTTP005") return errors.New("User not Admin") @@ -275,7 +277,7 @@ func GetAllHostnames() []string { } proxies := GetMainConfig().HTTPConfig.ProxyConfig.Routes for _, proxy := range proxies { - if (proxy.UseHost && proxy.Host != "" && strings.Contains(proxy.Host, ".") && !strings.Contains(proxy.Host, ",") && !strings.Contains(proxy.Host, " ")){ + if proxy.UseHost && proxy.Host != "" && strings.Contains(proxy.Host, ".") && !strings.Contains(proxy.Host, ",") && !strings.Contains(proxy.Host, " ") { hostnames = append(hostnames, proxy.Host) } } @@ -290,4 +292,14 @@ func GetAllHostnames() []string { } Debug("Hostnames are " + strings.Join(uniqueHostnames, ", ")) return uniqueHostnames -} \ No newline at end of file +} + +func GetAvailableRAM() uint64 { + vmStat, err := mem.VirtualMemory() + if err != nil { + panic(err) + } + + // Use total available memory as an approximation + return vmStat.Available +}