[release] v0.12.0-unstable32

client/src/api/downloadButton.jsx

@@ -1,4 +1,6 @@
+import { ArrowDownOutlined } from "@ant-design/icons";
 import { Button } from "@mui/material";
+import ResponsiveButton from "../components/responseiveButton";
 
 export const DownloadFile = ({ filename, content, contentGetter, label }) => {
     const downloadFile = async () => {
@@ -34,8 +36,13 @@ export const DownloadFile = ({ filename, content, contentGetter, label }) => {
     }
 
     return (
-        <Button onClick={downloadFile}>
+        <ResponsiveButton
+            color="primary"
+            onClick={downloadFile}
+            variant={"outlined"}
+            startIcon={<ArrowDownOutlined />}
+        >
             {label}
-        </Button>
+        </ResponsiveButton>
     );
 }

client/src/pages/config/users/configman.jsx

@@ -78,13 +78,6 @@ const ConfigManagement = () => {
       }}
       label={'Purge Metrics Dashboard'} 
       content={'Are you sure you want to purge all the metrics data from the dashboards?'} />
-
-      <DownloadFile
-        filename={'backup.cosmos-compose.json'}
-        label={'Download Docker Backup'}
-        contentGetter={API.config.getBackup}
-      />
-
     </Stack>
     
     {config && <>

client/src/pages/dashboard/proxyDashboard.jsx

@@ -11,18 +11,35 @@ const ProxyDashboard = ({ xAxis, zoom, setZoom, slot, metrics }) => {
   return (<>
 
     <Grid container rowSpacing={4.5} columnSpacing={2.75} >
-      <Grid item xs={12} md={7} lg={8}>
+      <Grid item xs={12} md={6} lg={6}>
-        <PlotComponent xAxis={xAxis} zoom={zoom} setZoom={setZoom} slot={slot} title={'Requests'} data={[
+        <PlotComponent xAxis={xAxis} zoom={zoom} setZoom={setZoom} slot={slot} title={'Requests Resources'} data={[
           metrics["cosmos.proxy.all.time"],
+          metrics["cosmos.proxy.all.bytes"],
+        ]} />
+      </Grid>
+
+      <Grid item xs={12} md={6} lg={6}>
+        <PlotComponent xAxis={xAxis} zoom={zoom} setZoom={setZoom} slot={slot} title={'Requests Responses'} data={[
           metrics["cosmos.proxy.all.success"],
           metrics["cosmos.proxy.all.error"],
         ]} />
       </Grid>
 
-      <TableComponent xAxis={xAxis} zoom={zoom} setZoom={setZoom} slot={slot} title="Containers - Resources" data={
+
+      <TableComponent xAxis={xAxis} zoom={zoom} setZoom={setZoom} slot={slot} title="Requests Per URLs" data={
         Object.keys(metrics).filter((key) => key.startsWith("cosmos.proxy.route.")).map((key) => metrics[key])
       } />
 
+      <Grid item xs={12} md={4} lg={4}>
+        <PlotComponent xAxis={xAxis} zoom={zoom} setZoom={setZoom} slot={slot} title={'Blocked Requests'} data={[
+          metrics["cosmos.proxy.all.blocked"],
+        ]} />
+      </Grid>
+
+
+      <TableComponent xAxis={xAxis} zoom={zoom} setZoom={setZoom} slot={slot} title="Reasons For Blocked Requests" data={
+        Object.keys(metrics).filter((key) => key.startsWith("cosmos.proxy.blocked.")).map((key) => metrics[key])
+      } />
     </Grid>
   </>)
 }

client/src/pages/servapps/servapps.jsx

@@ -21,6 +21,7 @@ import DockerComposeImport from './containers/docker-compose';
 import { ContainerNetworkWarning } from '../../components/containers';
 import { ServAppIcon } from '../../utils/servapp-icon';
 import MiniPlotComponent from '../dashboard/components/mini-plot';
+import { DownloadFile } from '../../api/downloadButton';
 
 const Item = styled(Paper)(({ theme }) => ({
   backgroundColor: theme.palette.mode === 'dark' ? '#1A2027' : '#fff',
@@ -152,6 +153,11 @@ const ServApps = () => {
             >Start ServApp</ResponsiveButton>
         </Link>
         <DockerComposeImport refresh={refreshServApps}/>
+        <DownloadFile
+          filename={'backup.cosmos-compose.json'}
+          label={'Export Docker Backup'}
+          contentGetter={API.config.getBackup}
+        />
       </Stack>
       
       <Grid2 container spacing={{xs: 1, sm: 1, md: 2 }}>

package.json

@@ -1,6 +1,6 @@
 {
   "name": "cosmos-server",
-  "version": "0.12.0-unstable31",
+  "version": "0.12.0-unstable32",
   "description": "",
   "main": "test-server.js",
   "bugs": {

src/httpServer.go

@@ -153,10 +153,13 @@ func SecureAPI(userRouter *mux.Router, public bool) {
 	}
 	userRouter.Use(proxy.SmartShieldMiddleware(
 		"__COSMOS",
-		utils.SmartShieldPolicy{
+		utils.ProxyRouteConfig{
-			Enabled: true,
+			Name: "_Cosmos",
-			PolicyStrictness: 1,
+			SmartShield: utils.SmartShieldPolicy{
-			PerUserRequestLimit: 5000,
+				Enabled: true,
+				PolicyStrictness: 1,
+				PerUserRequestLimit: 5000,
+			},
 		},
 	))
 	userRouter.Use(utils.MiddlewareTimeout(45 * time.Second))

src/index.go

@@ -17,6 +17,7 @@ func main() {
 	utils.Log("Starting...")
 	
 	utils.ReBootstrapContainer = docker.BootstrapContainerFromTags
+	utils.PushShieldMetrics = metrics.PushShieldMetrics
 
 	rand.Seed(time.Now().UnixNano())
 

src/metrics/middleware.go

@@ -1,93 +1,109 @@
 package metrics
 
 import (
-	"net/http"
-	"fmt"
 	"time"
 
 	"github.com/azukaar/cosmos-server/src/utils"
 )
 
-// responseWriter wraps the original http.ResponseWriter to capture the status code.
-// type responseWriter struct {
-// 	http.ResponseWriter
-// 	status int
-// }
 
-// func (rw *responseWriter) WriteHeader(status int) {
+func PushRequestMetrics(route utils.ProxyRouteConfig, statusCode int, TimeStarted time.Time, size int64) error {
-// 	rw.status = status
+	responseTime := time.Since(TimeStarted)
-// 	rw.ResponseWriter.WriteHeader(status)
-// }
 
-func MetricsMiddleware(route utils.ProxyRouteConfig) func(next http.Handler) http.Handler {
+	if !utils.GetMainConfig().MonitoringDisabled {
-	return func(next http.Handler) http.Handler {
+		if statusCode >= 400 {
-		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			PushSetMetric("proxy.all.error", 1, DataDef{
-		startTime := time.Now()
+				Max: 0,
-
+				Period: time.Second * 30,
-		// Call the next handler (which can be another middleware or the final handler).
+				Label: "Global Request Errors",
-		// wrappedWriter := &responseWriter{ResponseWriter: w}
+				AggloType: "sum",
-
+				SetOperation: "sum",
-		next.ServeHTTP(w, r)
+			})
-
+			PushSetMetric("proxy.route.error."+route.Name, 1, DataDef{
-		// Calculate and log the response time.
+				Max: 0,
-		responseTime := time.Since(startTime)
+				Period: time.Second * 30,
-
+				Label: "Request Errors " + route.Name,
-		utils.Debug(fmt.Sprintf("[%s] %s %s %v", r.Method, r.RequestURI, r.RemoteAddr, responseTime))
+				AggloType: "sum",
-
+				SetOperation: "sum",
-		if !utils.GetMainConfig().MonitoringDisabled {
+			})
-			go func() {
+		} else {
-				// if wrappedWriter.status >= 400 {
+			PushSetMetric("proxy.all.success", 1, DataDef{
-				// 	PushSetMetric("proxy.all.error", 1, DataDef{
+				Max: 0,
-				// 		Max: 0,
+				Period: time.Second * 30,
-				// 		Period: time.Second * 30,
+				Label: "Global Request Success",
-				// 		Label: "Global Request Errors",
+				AggloType: "sum",
-				// 		AggloType: "sum",
+				SetOperation: "sum",
-				// 		SetOperation: "sum",
+			})
-				// 	})
+			PushSetMetric("proxy.route.success."+route.Name, 1, DataDef{
-				// 	PushSetMetric("proxy.route.error."+route.Name, 1, DataDef{
+				Max: 0,
-				// 		Max: 0,
+				Period: time.Second * 30,
-				// 		Period: time.Second * 30,
+				Label: "Request Success " + route.Name,
-				// 		Label: "Request Errors " + route.Name,
+				AggloType: "sum",
-				// 		AggloType: "sum",
+				SetOperation: "sum",
-				// 		SetOperation: "sum",
+			})
-				// 	})
-				// } else {
-				// 	PushSetMetric("proxy.all.success", 1, DataDef{
-				// 		Max: 0,
-				// 		Period: time.Second * 30,
-				// 		Label: "Global Request Success",
-				// 		AggloType: "sum",
-				// 		SetOperation: "sum",
-				// 	})
-				// 	PushSetMetric("proxy.route.success."+route.Name, 1, DataDef{
-				// 		Max: 0,
-				// 		Period: time.Second * 30,
-				// 		Label: "Request Success " + route.Name,
-				// 		AggloType: "sum",
-				// 		SetOperation: "sum",
-				// 	})
-				// }
-
-				PushSetMetric("proxy.all.time", int(responseTime.Milliseconds()), DataDef{
-					Max: 0,
-					Period: time.Second * 30,
-					Label: "Global Response Time",
-					AggloType: "avg",
-					SetOperation: "max",
-					Unit: "ms",
-				})
-
-				PushSetMetric("proxy.route.time."+route.Name, int(responseTime.Milliseconds()), DataDef{
-					Max: 0,
-					Period: time.Second * 30,
-					Label: "Response Time " + route.Name,
-					AggloType: "avg",
-					SetOperation: "max",
-					Unit: "ms",
-				})
-			}()
 		}
 
-	})
+		PushSetMetric("proxy.all.time", int(responseTime.Milliseconds()), DataDef{
+			Max: 0,
+			Period: time.Second * 30,
+			Label: "Global Response Time",
+			AggloType: "avg",
+			SetOperation: "max",
+			Unit: "ms",
+		})
+
+		PushSetMetric("proxy.route.time."+route.Name, int(responseTime.Milliseconds()), DataDef{
+			Max: 0,
+			Period: time.Second * 30,
+			Label: "Response Time " + route.Name,
+			AggloType: "avg",
+			SetOperation: "max",
+			Unit: "ms",
+		})
+
+		PushSetMetric("proxy.all.bytes", int(size), DataDef{
+			Max: 0,
+			Period: time.Second * 30,
+			Label: "Global Transfered Bytes",
+			AggloType: "sum",
+			Unit: "B",
+		})
+
+		PushSetMetric("proxy.route.bytes."+route.Name, int(size), DataDef{
+			Max: 0,
+			Period: time.Second * 30,
+			Label: "Transfered Bytes " + route.Name,
+			AggloType: "sum",
+			Unit: "B",
+		})
+	}
+
+	return nil
 }
+
+func PushShieldMetrics(reason string) {
+	reasonStr := map[string]string{
+		"bots": "Bots",
+		"geo": "By Geolocation",
+		"referer": "By Referer",
+		"hostname": "By Hostname",
+		"ip-whitelists": "By IP Whitelists",
+		"smart-shield": "Smart Shield",
+	}
+
+	PushSetMetric("proxy.blocked."+reason, 1, DataDef{
+		Max: 0,
+		Period: time.Second * 30,
+		Label: "Blocked " + reasonStr[reason],
+		AggloType: "sum",
+		SetOperation: "sum",
+	})
+	
+	PushSetMetric("proxy.all.blocked", 1, DataDef{
+		Max: 0,
+		Period: time.Second * 30,
+		Label: "Global Blocked Requests",
+		AggloType: "sum",
+		SetOperation: "sum",
+	})
 }

src/proxy/botblock.go

@@ -2,6 +2,8 @@ package proxy
 
 import (
 	"net/http"
+
+	"github.com/azukaar/cosmos-server/src/metrics"
 )
 
 var botUserAgents = []string{
@@ -28,12 +30,14 @@ func BotDetectionMiddleware(next http.Handler) http.Handler {
 		userAgent := r.UserAgent()
 		
 		if userAgent == "" {
+			go metrics.PushShieldMetrics("bots")
 			http.Error(w, "Access denied: Bots are not allowed.", http.StatusForbidden)
 			return
 		}
 
 		for _, botUserAgent := range botUserAgents {
 			if userAgent == botUserAgent {
+			go metrics.PushShieldMetrics("bots")
 				http.Error(w, "Access denied: Bots are not allowed.", http.StatusForbidden)
 				return
 			}

src/proxy/routerGen.go

@@ -8,7 +8,6 @@ import (
 
 	"github.com/azukaar/cosmos-server/src/user"
 	"github.com/azukaar/cosmos-server/src/utils"
-	"github.com/azukaar/cosmos-server/src/metrics"
 	"github.com/go-chi/httprate"
 	"github.com/gorilla/mux"
 )
@@ -88,7 +87,7 @@ func RouterGen(route utils.ProxyRouteConfig, router *mux.Router, destination htt
 	
 	destination = utils.Restrictions(route.RestrictToConstellation, route.WhitelistInboundIPs)(destination)
 	
-	destination = SmartShieldMiddleware(route.Name, route.SmartShield)(destination)
+	destination = SmartShieldMiddleware(route.Name, route)(destination)
 
 	originCORS := route.CORSOrigin
 
@@ -146,8 +145,6 @@ func RouterGen(route utils.ProxyRouteConfig, router *mux.Router, destination htt
 		destination = utils.SetSecurityHeaders(destination)
 	}
 
-	destination = metrics.MetricsMiddleware(route)(destination)
-
 	destination = tokenMiddleware(route.AuthEnabled, route.AdminOnly)(utils.CORSHeader(originCORS)((destination)))
 
 	origin.Handler(destination)

src/proxy/shield.go

@@ -1,7 +1,6 @@
 package proxy
 
 import (
-	"github.com/azukaar/cosmos-server/src/utils"
 	"sync"
 	"time"
 	"net/http"
@@ -9,6 +8,9 @@ import (
 	"net"
 	"math"
 	"strconv"
+
+	"github.com/azukaar/cosmos-server/src/utils"
+	"github.com/azukaar/cosmos-server/src/metrics"
 )
 
 /*
@@ -267,12 +269,10 @@ func isPrivileged(req *http.Request, policy utils.SmartShieldPolicy) bool {
 	return role >= policy.PrivilegedGroups
 }
 
-func SmartShieldMiddleware(shieldID string, policy utils.SmartShieldPolicy) func(http.Handler) http.Handler {
+func SmartShieldMiddleware(shieldID string, route utils.ProxyRouteConfig) func(http.Handler) http.Handler {
-	if policy.Enabled == false {
+	policy := route.SmartShield
-		return func(next http.Handler) http.Handler {
+
-			return next
+	if policy.Enabled {
-		}
-	} else {
 		if(policy.PerUserTimeBudget == 0) {
 			policy.PerUserTimeBudget = 2 * 60 * 60 * 1000 // 2 hours
 		}
@@ -298,7 +298,31 @@ func SmartShieldMiddleware(shieldID string, policy utils.SmartShieldPolicy) func
 
 	return func(next http.Handler) http.Handler {
 		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-			utils.Log("SmartShield: Request received")
+			clientID := GetClientID(r)
+
+			wrapper := &SmartResponseWriterWrapper {
+				ResponseWriter: w,
+				ThrottleNext:   0,
+				TimeStarted:    time.Now(),
+				ClientID:       clientID,
+				RequestCost:    1,
+				Method: 				r.Method,
+				shield: shield,
+				shieldID: shieldID,
+				policy: policy,
+				isPrivileged: isPrivileged(r, policy),
+			}
+
+			if !policy.Enabled {
+				next.ServeHTTP(wrapper, r)
+				wrapper.TimeEnded = time.Now()
+				wrapper.isOver = true
+
+				go metrics.PushRequestMetrics(route, wrapper.Status, wrapper.TimeStarted, wrapper.Bytes)
+
+				return
+			}
+
 			currentGlobalRequests := shield.GetServerNbReq(shieldID) + 1
 			utils.Debug(fmt.Sprintf("SmartShield: Current global requests: %d", currentGlobalRequests))
 
@@ -307,6 +331,7 @@ func SmartShieldMiddleware(shieldID string, policy utils.SmartShieldPolicy) func
 				wayTooManyReq := currentGlobalRequests > policy.MaxGlobalSimultaneous * 10
 				retries := 50
 				if wayTooManyReq {
+					go metrics.PushShieldMetrics("smart-shield")
 					utils.Log("SmartShield: WAYYYY Too many users on the server. Aborting right away.")
 					http.Error(w, "Too many requests", http.StatusTooManyRequests)
 					return
@@ -317,6 +342,7 @@ func SmartShieldMiddleware(shieldID string, policy utils.SmartShieldPolicy) func
 					tooManyReq = currentGlobalRequests > policy.MaxGlobalSimultaneous
 					retries--
 					if retries <= 0 {
+						go metrics.PushShieldMetrics("smart-shield")
 						utils.Log("SmartShield: Too many users on the server")
 						http.Error(w, "Too many requests", http.StatusTooManyRequests)
 						return
@@ -324,11 +350,11 @@ func SmartShieldMiddleware(shieldID string, policy utils.SmartShieldPolicy) func
 				}
 			}
 
-			clientID := GetClientID(r)
 			userConsumed := shield.GetUserUsedBudgets(shieldID, clientID)
 
 			if !isPrivileged(r, policy) && !shield.isAllowedToReqest(shieldID, policy, userConsumed) {
 				lastBan := shield.GetLastBan(policy, userConsumed)
+				go metrics.PushShieldMetrics("smart-shield")
 				utils.Log("SmartShield: User is blocked due to abuse: " + fmt.Sprintf("%+v", lastBan))
 				http.Error(w, "Too many requests", http.StatusTooManyRequests)
 				return
@@ -337,6 +363,7 @@ func SmartShieldMiddleware(shieldID string, policy utils.SmartShieldPolicy) func
 				if(!isPrivileged(r, policy)) {
 					throttle = shield.computeThrottle(policy, userConsumed)
 				}
+
 				wrapper := &SmartResponseWriterWrapper {
 					ResponseWriter: w,
 					ThrottleNext:   throttle,
@@ -371,6 +398,7 @@ func SmartShieldMiddleware(shieldID string, policy utils.SmartShieldPolicy) func
 					shield.Lock()
 					wrapper.TimeEnded = time.Now()
 					wrapper.isOver = true
+					go metrics.PushRequestMetrics(route, wrapper.Status, wrapper.TimeStarted, wrapper.Bytes)
 					shield.Unlock()
 				})()
 

src/utils/middleware.go

@@ -14,6 +14,8 @@ import (
 
 // https://github.com/go-chi/chi/blob/master/middleware/timeout.go
 
+var PushShieldMetrics func(string)
+
 func MiddlewareTimeout(timeout time.Duration) func(next http.Handler) http.Handler {
 	return func(next http.Handler) http.Handler {
 		fn := func(w http.ResponseWriter, r *http.Request) {
@@ -148,6 +150,7 @@ func BlockByCountryMiddleware(blockedCountries []string, CountryBlacklistIsWhite
 						}
 
 						if blocked {
+							PushShieldMetrics("geo")
 							http.Error(w, "Access denied", http.StatusForbidden)
 							return
 						}
@@ -177,6 +180,7 @@ func BlockPostWithoutReferer(next http.Handler) http.Handler {
 		if r.Method == "POST" || r.Method == "PUT" || r.Method == "PATCH" || r.Method == "DELETE" {
 			referer := r.Header.Get("Referer")
 			if referer == "" {
+				PushShieldMetrics("referer")
 				Error("Blocked POST request without Referer header", nil)
 				http.Error(w, "Bad Request: Invalid request.", http.StatusBadRequest)
 				return
@@ -213,6 +217,7 @@ func EnsureHostname(next http.Handler) http.Handler {
 		}
 
 		if !isOk {
+			PushShieldMetrics("hostname")
 			Error("Invalid Hostname " + r.Host + " for request. Expecting one of " + fmt.Sprintf("%v", hostnames), nil)
 			w.WriteHeader(http.StatusBadRequest)
 			http.Error(w, "Bad Request: Invalid hostname. Use your domain instead of your IP to access your server. Check logs if more details are needed.", http.StatusBadRequest)
@@ -306,11 +311,13 @@ func Restrictions(RestrictToConstellation bool, WhitelistInboundIPs []string) fu
 		if(RestrictToConstellation) {
 			if(!isInConstellation) {
 				if(!isUsingWhiteList) {
+					PushShieldMetrics("ip-whitelists")
 					Error("Request from " + ip + " is blocked because of restrictions", nil)
 					Debug("Blocked by RestrictToConstellation isInConstellation isUsingWhiteList")
 					http.Error(w, "Access denied", http.StatusForbidden)
 					return
 				} else if (!isInWhitelist) {
+					PushShieldMetrics("ip-whitelists")
 					Error("Request from " + ip + " is blocked because of restrictions", nil)
 					Debug("Blocked by RestrictToConstellation isInConstellation isInWhitelist")
 					http.Error(w, "Access denied", http.StatusForbidden)
@@ -318,6 +325,7 @@ func Restrictions(RestrictToConstellation bool, WhitelistInboundIPs []string) fu
 				}
 			}
 		} else if(isUsingWhiteList && !isInWhitelist) {
+			PushShieldMetrics("ip-whitelists")
 			Error("Request from " + ip + " is blocked because of restrictions", nil)
 			Debug("Blocked by RestrictToConstellation isInConstellation isUsingWhiteList isInWhitelist")
 			http.Error(w, "Access denied", http.StatusForbidden)