diff --git a/client/src/api/downloadButton.jsx b/client/src/api/downloadButton.jsx index 1468aba..3372739 100644 --- a/client/src/api/downloadButton.jsx +++ b/client/src/api/downloadButton.jsx @@ -1,4 +1,6 @@ +import { ArrowDownOutlined } from "@ant-design/icons"; import { Button } from "@mui/material"; +import ResponsiveButton from "../components/responseiveButton"; export const DownloadFile = ({ filename, content, contentGetter, label }) => { const downloadFile = async () => { @@ -34,8 +36,13 @@ export const DownloadFile = ({ filename, content, contentGetter, label }) => { } return ( - + ); } \ No newline at end of file diff --git a/client/src/pages/config/users/configman.jsx b/client/src/pages/config/users/configman.jsx index 0a333f3..bfdd1b9 100644 --- a/client/src/pages/config/users/configman.jsx +++ b/client/src/pages/config/users/configman.jsx @@ -78,13 +78,6 @@ const ConfigManagement = () => { }} label={'Purge Metrics Dashboard'} content={'Are you sure you want to purge all the metrics data from the dashboards?'} /> - - - {config && <> diff --git a/client/src/pages/dashboard/proxyDashboard.jsx b/client/src/pages/dashboard/proxyDashboard.jsx index ffeaa74..81be379 100644 --- a/client/src/pages/dashboard/proxyDashboard.jsx +++ b/client/src/pages/dashboard/proxyDashboard.jsx @@ -11,18 +11,35 @@ const ProxyDashboard = ({ xAxis, zoom, setZoom, slot, metrics }) => { return (<> - - + + + + + - key.startsWith("cosmos.proxy.route.")).map((key) => metrics[key]) } /> + + + + + + key.startsWith("cosmos.proxy.blocked.")).map((key) => metrics[key]) + } /> ) } diff --git a/client/src/pages/servapps/servapps.jsx b/client/src/pages/servapps/servapps.jsx index 8f37fa4..20259fa 100644 --- a/client/src/pages/servapps/servapps.jsx +++ b/client/src/pages/servapps/servapps.jsx @@ -21,6 +21,7 @@ import DockerComposeImport from './containers/docker-compose'; import { ContainerNetworkWarning } from '../../components/containers'; import { ServAppIcon } from '../../utils/servapp-icon'; import MiniPlotComponent from '../dashboard/components/mini-plot'; +import { DownloadFile } from '../../api/downloadButton'; const Item = styled(Paper)(({ theme }) => ({ backgroundColor: theme.palette.mode === 'dark' ? '#1A2027' : '#fff', @@ -152,6 +153,11 @@ const ServApps = () => { >Start ServApp + diff --git a/package.json b/package.json index 58a5fd4..89429cc 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "cosmos-server", - "version": "0.12.0-unstable31", + "version": "0.12.0-unstable32", "description": "", "main": "test-server.js", "bugs": { diff --git a/src/httpServer.go b/src/httpServer.go index 5d10fc5..28df340 100644 --- a/src/httpServer.go +++ b/src/httpServer.go @@ -153,10 +153,13 @@ func SecureAPI(userRouter *mux.Router, public bool) { } userRouter.Use(proxy.SmartShieldMiddleware( "__COSMOS", - utils.SmartShieldPolicy{ - Enabled: true, - PolicyStrictness: 1, - PerUserRequestLimit: 5000, + utils.ProxyRouteConfig{ + Name: "_Cosmos", + SmartShield: utils.SmartShieldPolicy{ + Enabled: true, + PolicyStrictness: 1, + PerUserRequestLimit: 5000, + }, }, )) userRouter.Use(utils.MiddlewareTimeout(45 * time.Second)) diff --git a/src/index.go b/src/index.go index 936ccbb..5561b7e 100644 --- a/src/index.go +++ b/src/index.go @@ -17,6 +17,7 @@ func main() { utils.Log("Starting...") utils.ReBootstrapContainer = docker.BootstrapContainerFromTags + utils.PushShieldMetrics = metrics.PushShieldMetrics rand.Seed(time.Now().UnixNano()) diff --git a/src/metrics/middleware.go b/src/metrics/middleware.go index edb3105..9b22fa5 100644 --- a/src/metrics/middleware.go +++ b/src/metrics/middleware.go @@ -1,93 +1,109 @@ package metrics import ( - "net/http" - "fmt" "time" "github.com/azukaar/cosmos-server/src/utils" ) -// responseWriter wraps the original http.ResponseWriter to capture the status code. -// type responseWriter struct { -// http.ResponseWriter -// status int -// } -// func (rw *responseWriter) WriteHeader(status int) { -// rw.status = status -// rw.ResponseWriter.WriteHeader(status) -// } +func PushRequestMetrics(route utils.ProxyRouteConfig, statusCode int, TimeStarted time.Time, size int64) error { + responseTime := time.Since(TimeStarted) -func MetricsMiddleware(route utils.ProxyRouteConfig) func(next http.Handler) http.Handler { - return func(next http.Handler) http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - startTime := time.Now() - - // Call the next handler (which can be another middleware or the final handler). - // wrappedWriter := &responseWriter{ResponseWriter: w} - - next.ServeHTTP(w, r) - - // Calculate and log the response time. - responseTime := time.Since(startTime) - - utils.Debug(fmt.Sprintf("[%s] %s %s %v", r.Method, r.RequestURI, r.RemoteAddr, responseTime)) - - if !utils.GetMainConfig().MonitoringDisabled { - go func() { - // if wrappedWriter.status >= 400 { - // PushSetMetric("proxy.all.error", 1, DataDef{ - // Max: 0, - // Period: time.Second * 30, - // Label: "Global Request Errors", - // AggloType: "sum", - // SetOperation: "sum", - // }) - // PushSetMetric("proxy.route.error."+route.Name, 1, DataDef{ - // Max: 0, - // Period: time.Second * 30, - // Label: "Request Errors " + route.Name, - // AggloType: "sum", - // SetOperation: "sum", - // }) - // } else { - // PushSetMetric("proxy.all.success", 1, DataDef{ - // Max: 0, - // Period: time.Second * 30, - // Label: "Global Request Success", - // AggloType: "sum", - // SetOperation: "sum", - // }) - // PushSetMetric("proxy.route.success."+route.Name, 1, DataDef{ - // Max: 0, - // Period: time.Second * 30, - // Label: "Request Success " + route.Name, - // AggloType: "sum", - // SetOperation: "sum", - // }) - // } - - PushSetMetric("proxy.all.time", int(responseTime.Milliseconds()), DataDef{ - Max: 0, - Period: time.Second * 30, - Label: "Global Response Time", - AggloType: "avg", - SetOperation: "max", - Unit: "ms", - }) - - PushSetMetric("proxy.route.time."+route.Name, int(responseTime.Milliseconds()), DataDef{ - Max: 0, - Period: time.Second * 30, - Label: "Response Time " + route.Name, - AggloType: "avg", - SetOperation: "max", - Unit: "ms", - }) - }() + if !utils.GetMainConfig().MonitoringDisabled { + if statusCode >= 400 { + PushSetMetric("proxy.all.error", 1, DataDef{ + Max: 0, + Period: time.Second * 30, + Label: "Global Request Errors", + AggloType: "sum", + SetOperation: "sum", + }) + PushSetMetric("proxy.route.error."+route.Name, 1, DataDef{ + Max: 0, + Period: time.Second * 30, + Label: "Request Errors " + route.Name, + AggloType: "sum", + SetOperation: "sum", + }) + } else { + PushSetMetric("proxy.all.success", 1, DataDef{ + Max: 0, + Period: time.Second * 30, + Label: "Global Request Success", + AggloType: "sum", + SetOperation: "sum", + }) + PushSetMetric("proxy.route.success."+route.Name, 1, DataDef{ + Max: 0, + Period: time.Second * 30, + Label: "Request Success " + route.Name, + AggloType: "sum", + SetOperation: "sum", + }) } - }) + PushSetMetric("proxy.all.time", int(responseTime.Milliseconds()), DataDef{ + Max: 0, + Period: time.Second * 30, + Label: "Global Response Time", + AggloType: "avg", + SetOperation: "max", + Unit: "ms", + }) + + PushSetMetric("proxy.route.time."+route.Name, int(responseTime.Milliseconds()), DataDef{ + Max: 0, + Period: time.Second * 30, + Label: "Response Time " + route.Name, + AggloType: "avg", + SetOperation: "max", + Unit: "ms", + }) + + PushSetMetric("proxy.all.bytes", int(size), DataDef{ + Max: 0, + Period: time.Second * 30, + Label: "Global Transfered Bytes", + AggloType: "sum", + Unit: "B", + }) + + PushSetMetric("proxy.route.bytes."+route.Name, int(size), DataDef{ + Max: 0, + Period: time.Second * 30, + Label: "Transfered Bytes " + route.Name, + AggloType: "sum", + Unit: "B", + }) + } + + return nil } + +func PushShieldMetrics(reason string) { + reasonStr := map[string]string{ + "bots": "Bots", + "geo": "By Geolocation", + "referer": "By Referer", + "hostname": "By Hostname", + "ip-whitelists": "By IP Whitelists", + "smart-shield": "Smart Shield", + } + + PushSetMetric("proxy.blocked."+reason, 1, DataDef{ + Max: 0, + Period: time.Second * 30, + Label: "Blocked " + reasonStr[reason], + AggloType: "sum", + SetOperation: "sum", + }) + + PushSetMetric("proxy.all.blocked", 1, DataDef{ + Max: 0, + Period: time.Second * 30, + Label: "Global Blocked Requests", + AggloType: "sum", + SetOperation: "sum", + }) } \ No newline at end of file diff --git a/src/proxy/botblock.go b/src/proxy/botblock.go index 4ce539f..40687b0 100644 --- a/src/proxy/botblock.go +++ b/src/proxy/botblock.go @@ -2,6 +2,8 @@ package proxy import ( "net/http" + + "github.com/azukaar/cosmos-server/src/metrics" ) var botUserAgents = []string{ @@ -28,12 +30,14 @@ func BotDetectionMiddleware(next http.Handler) http.Handler { userAgent := r.UserAgent() if userAgent == "" { + go metrics.PushShieldMetrics("bots") http.Error(w, "Access denied: Bots are not allowed.", http.StatusForbidden) return } for _, botUserAgent := range botUserAgents { if userAgent == botUserAgent { + go metrics.PushShieldMetrics("bots") http.Error(w, "Access denied: Bots are not allowed.", http.StatusForbidden) return } diff --git a/src/proxy/routerGen.go b/src/proxy/routerGen.go index d1a7609..f7748ea 100644 --- a/src/proxy/routerGen.go +++ b/src/proxy/routerGen.go @@ -8,7 +8,6 @@ import ( "github.com/azukaar/cosmos-server/src/user" "github.com/azukaar/cosmos-server/src/utils" - "github.com/azukaar/cosmos-server/src/metrics" "github.com/go-chi/httprate" "github.com/gorilla/mux" ) @@ -88,7 +87,7 @@ func RouterGen(route utils.ProxyRouteConfig, router *mux.Router, destination htt destination = utils.Restrictions(route.RestrictToConstellation, route.WhitelistInboundIPs)(destination) - destination = SmartShieldMiddleware(route.Name, route.SmartShield)(destination) + destination = SmartShieldMiddleware(route.Name, route)(destination) originCORS := route.CORSOrigin @@ -146,8 +145,6 @@ func RouterGen(route utils.ProxyRouteConfig, router *mux.Router, destination htt destination = utils.SetSecurityHeaders(destination) } - destination = metrics.MetricsMiddleware(route)(destination) - destination = tokenMiddleware(route.AuthEnabled, route.AdminOnly)(utils.CORSHeader(originCORS)((destination))) origin.Handler(destination) diff --git a/src/proxy/shield.go b/src/proxy/shield.go index aff0bc4..c5905b7 100644 --- a/src/proxy/shield.go +++ b/src/proxy/shield.go @@ -1,7 +1,6 @@ package proxy import ( - "github.com/azukaar/cosmos-server/src/utils" "sync" "time" "net/http" @@ -9,6 +8,9 @@ import ( "net" "math" "strconv" + + "github.com/azukaar/cosmos-server/src/utils" + "github.com/azukaar/cosmos-server/src/metrics" ) /* @@ -267,12 +269,10 @@ func isPrivileged(req *http.Request, policy utils.SmartShieldPolicy) bool { return role >= policy.PrivilegedGroups } -func SmartShieldMiddleware(shieldID string, policy utils.SmartShieldPolicy) func(http.Handler) http.Handler { - if policy.Enabled == false { - return func(next http.Handler) http.Handler { - return next - } - } else { +func SmartShieldMiddleware(shieldID string, route utils.ProxyRouteConfig) func(http.Handler) http.Handler { + policy := route.SmartShield + + if policy.Enabled { if(policy.PerUserTimeBudget == 0) { policy.PerUserTimeBudget = 2 * 60 * 60 * 1000 // 2 hours } @@ -298,7 +298,31 @@ func SmartShieldMiddleware(shieldID string, policy utils.SmartShieldPolicy) func return func(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - utils.Log("SmartShield: Request received") + clientID := GetClientID(r) + + wrapper := &SmartResponseWriterWrapper { + ResponseWriter: w, + ThrottleNext: 0, + TimeStarted: time.Now(), + ClientID: clientID, + RequestCost: 1, + Method: r.Method, + shield: shield, + shieldID: shieldID, + policy: policy, + isPrivileged: isPrivileged(r, policy), + } + + if !policy.Enabled { + next.ServeHTTP(wrapper, r) + wrapper.TimeEnded = time.Now() + wrapper.isOver = true + + go metrics.PushRequestMetrics(route, wrapper.Status, wrapper.TimeStarted, wrapper.Bytes) + + return + } + currentGlobalRequests := shield.GetServerNbReq(shieldID) + 1 utils.Debug(fmt.Sprintf("SmartShield: Current global requests: %d", currentGlobalRequests)) @@ -307,6 +331,7 @@ func SmartShieldMiddleware(shieldID string, policy utils.SmartShieldPolicy) func wayTooManyReq := currentGlobalRequests > policy.MaxGlobalSimultaneous * 10 retries := 50 if wayTooManyReq { + go metrics.PushShieldMetrics("smart-shield") utils.Log("SmartShield: WAYYYY Too many users on the server. Aborting right away.") http.Error(w, "Too many requests", http.StatusTooManyRequests) return @@ -317,6 +342,7 @@ func SmartShieldMiddleware(shieldID string, policy utils.SmartShieldPolicy) func tooManyReq = currentGlobalRequests > policy.MaxGlobalSimultaneous retries-- if retries <= 0 { + go metrics.PushShieldMetrics("smart-shield") utils.Log("SmartShield: Too many users on the server") http.Error(w, "Too many requests", http.StatusTooManyRequests) return @@ -324,11 +350,11 @@ func SmartShieldMiddleware(shieldID string, policy utils.SmartShieldPolicy) func } } - clientID := GetClientID(r) userConsumed := shield.GetUserUsedBudgets(shieldID, clientID) if !isPrivileged(r, policy) && !shield.isAllowedToReqest(shieldID, policy, userConsumed) { lastBan := shield.GetLastBan(policy, userConsumed) + go metrics.PushShieldMetrics("smart-shield") utils.Log("SmartShield: User is blocked due to abuse: " + fmt.Sprintf("%+v", lastBan)) http.Error(w, "Too many requests", http.StatusTooManyRequests) return @@ -337,6 +363,7 @@ func SmartShieldMiddleware(shieldID string, policy utils.SmartShieldPolicy) func if(!isPrivileged(r, policy)) { throttle = shield.computeThrottle(policy, userConsumed) } + wrapper := &SmartResponseWriterWrapper { ResponseWriter: w, ThrottleNext: throttle, @@ -371,6 +398,7 @@ func SmartShieldMiddleware(shieldID string, policy utils.SmartShieldPolicy) func shield.Lock() wrapper.TimeEnded = time.Now() wrapper.isOver = true + go metrics.PushRequestMetrics(route, wrapper.Status, wrapper.TimeStarted, wrapper.Bytes) shield.Unlock() })() diff --git a/src/utils/middleware.go b/src/utils/middleware.go index d31e533..11cf288 100644 --- a/src/utils/middleware.go +++ b/src/utils/middleware.go @@ -14,6 +14,8 @@ import ( // https://github.com/go-chi/chi/blob/master/middleware/timeout.go +var PushShieldMetrics func(string) + func MiddlewareTimeout(timeout time.Duration) func(next http.Handler) http.Handler { return func(next http.Handler) http.Handler { fn := func(w http.ResponseWriter, r *http.Request) { @@ -148,6 +150,7 @@ func BlockByCountryMiddleware(blockedCountries []string, CountryBlacklistIsWhite } if blocked { + PushShieldMetrics("geo") http.Error(w, "Access denied", http.StatusForbidden) return } @@ -177,6 +180,7 @@ func BlockPostWithoutReferer(next http.Handler) http.Handler { if r.Method == "POST" || r.Method == "PUT" || r.Method == "PATCH" || r.Method == "DELETE" { referer := r.Header.Get("Referer") if referer == "" { + PushShieldMetrics("referer") Error("Blocked POST request without Referer header", nil) http.Error(w, "Bad Request: Invalid request.", http.StatusBadRequest) return @@ -213,6 +217,7 @@ func EnsureHostname(next http.Handler) http.Handler { } if !isOk { + PushShieldMetrics("hostname") Error("Invalid Hostname " + r.Host + " for request. Expecting one of " + fmt.Sprintf("%v", hostnames), nil) w.WriteHeader(http.StatusBadRequest) http.Error(w, "Bad Request: Invalid hostname. Use your domain instead of your IP to access your server. Check logs if more details are needed.", http.StatusBadRequest) @@ -306,11 +311,13 @@ func Restrictions(RestrictToConstellation bool, WhitelistInboundIPs []string) fu if(RestrictToConstellation) { if(!isInConstellation) { if(!isUsingWhiteList) { + PushShieldMetrics("ip-whitelists") Error("Request from " + ip + " is blocked because of restrictions", nil) Debug("Blocked by RestrictToConstellation isInConstellation isUsingWhiteList") http.Error(w, "Access denied", http.StatusForbidden) return } else if (!isInWhitelist) { + PushShieldMetrics("ip-whitelists") Error("Request from " + ip + " is blocked because of restrictions", nil) Debug("Blocked by RestrictToConstellation isInConstellation isInWhitelist") http.Error(w, "Access denied", http.StatusForbidden) @@ -318,6 +325,7 @@ func Restrictions(RestrictToConstellation bool, WhitelistInboundIPs []string) fu } } } else if(isUsingWhiteList && !isInWhitelist) { + PushShieldMetrics("ip-whitelists") Error("Request from " + ip + " is blocked because of restrictions", nil) Debug("Blocked by RestrictToConstellation isInConstellation isUsingWhiteList isInWhitelist") http.Error(w, "Access denied", http.StatusForbidden)