beenull/Cosmos-Server
1
0
Fork 0
Code Issues Pull requests Packages Projects Releases Wiki Activity

[release] v0.12.0-unstable32

Browse source
This commit is contained in:
Yann Stepienik 2023-11-02 10:39:47 +00:00
parent bdfcf0a2bf
commit 6a5f9b74fe
12 changed files with 189 additions and 109 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
client/src/api/downloadButton.jsx
View file

@ -1,4 +1,6 @@
import { ArrowDownOutlined } from "@ant-design/icons";
import { Button } from "@mui/material";
import ResponsiveButton from "../components/responseiveButton";
export const DownloadFile = ({ filename, content, contentGetter, label }) => {
const downloadFile = async () => {
@ -34,8 +36,13 @@ export const DownloadFile = ({ filename, content, contentGetter, label }) => {
}
return (
<Button onClick={downloadFile}>
<ResponsiveButton
color="primary"
onClick={downloadFile}
variant={"outlined"}
startIcon={<ArrowDownOutlined />}
>
{label}
</Button>
</ResponsiveButton>
);
}

7
client/src/pages/config/users/configman.jsx
View file

@ -78,13 +78,6 @@ const ConfigManagement = () => {
}}
label={'Purge Metrics Dashboard'}
content={'Are you sure you want to purge all the metrics data from the dashboards?'} />
<DownloadFile
filename={'backup.cosmos-compose.json'}
label={'Download Docker Backup'}
contentGetter={API.config.getBackup}
/>
</Stack>
{config && <>

23
client/src/pages/dashboard/proxyDashboard.jsx
View file

@ -11,18 +11,35 @@ const ProxyDashboard = ({ xAxis, zoom, setZoom, slot, metrics }) => {
return (<>
<Grid container rowSpacing={4.5} columnSpacing={2.75} >
<Grid item xs={12} md={7} lg={8}>
<PlotComponent xAxis={xAxis} zoom={zoom} setZoom={setZoom} slot={slot} title={'Requests'} data={[
<Grid item xs={12} md={6} lg={6}>
<PlotComponent xAxis={xAxis} zoom={zoom} setZoom={setZoom} slot={slot} title={'Requests Resources'} data={[
metrics["cosmos.proxy.all.time"],
metrics["cosmos.proxy.all.bytes"],
]} />
</Grid>
<Grid item xs={12} md={6} lg={6}>
<PlotComponent xAxis={xAxis} zoom={zoom} setZoom={setZoom} slot={slot} title={'Requests Responses'} data={[
metrics["cosmos.proxy.all.success"],
metrics["cosmos.proxy.all.error"],
]} />
</Grid>
<TableComponent xAxis={xAxis} zoom={zoom} setZoom={setZoom} slot={slot} title="Containers - Resources" data={
<TableComponent xAxis={xAxis} zoom={zoom} setZoom={setZoom} slot={slot} title="Requests Per URLs" data={
Object.keys(metrics).filter((key) => key.startsWith("cosmos.proxy.route.")).map((key) => metrics[key])
} />
<Grid item xs={12} md={4} lg={4}>
<PlotComponent xAxis={xAxis} zoom={zoom} setZoom={setZoom} slot={slot} title={'Blocked Requests'} data={[
metrics["cosmos.proxy.all.blocked"],
]} />
</Grid>
<TableComponent xAxis={xAxis} zoom={zoom} setZoom={setZoom} slot={slot} title="Reasons For Blocked Requests" data={
Object.keys(metrics).filter((key) => key.startsWith("cosmos.proxy.blocked.")).map((key) => metrics[key])
} />
</Grid>
</>)
}

6
client/src/pages/servapps/servapps.jsx
View file

@ -21,6 +21,7 @@ import DockerComposeImport from './containers/docker-compose';
import { ContainerNetworkWarning } from '../../components/containers';
import { ServAppIcon } from '../../utils/servapp-icon';
import MiniPlotComponent from '../dashboard/components/mini-plot';
import { DownloadFile } from '../../api/downloadButton';
const Item = styled(Paper)(({ theme }) => ({
backgroundColor: theme.palette.mode === 'dark' ? '#1A2027' : '#fff',
@ -152,6 +153,11 @@ const ServApps = () => {
>Start ServApp</ResponsiveButton>
</Link>
<DockerComposeImport refresh={refreshServApps}/>
<DownloadFile
filename={'backup.cosmos-compose.json'}
label={'Export Docker Backup'}
contentGetter={API.config.getBackup}
/>
</Stack>
<Grid2 container spacing={{xs: 1, sm: 1, md: 2 }}>

2
package.json
View file

@ -1,6 +1,6 @@
{
"name": "cosmos-server",
"version": "0.12.0-unstable31",
"version": "0.12.0-unstable32",
"description": "",
"main": "test-server.js",
"bugs": {

5
src/httpServer.go
View file

@ -153,11 +153,14 @@ func SecureAPI(userRouter *mux.Router, public bool) {
}
userRouter.Use(proxy.SmartShieldMiddleware(
"__COSMOS",
utils.SmartShieldPolicy{
utils.ProxyRouteConfig{
Name: "_Cosmos",
SmartShield: utils.SmartShieldPolicy{
Enabled: true,
PolicyStrictness: 1,
PerUserRequestLimit: 5000,
},
},
))
userRouter.Use(utils.MiddlewareTimeout(45 * time.Second))
userRouter.Use(proxy.BotDetectionMiddleware)

1
src/index.go
View file

@ -17,6 +17,7 @@ func main() {
utils.Log("Starting...")
utils.ReBootstrapContainer = docker.BootstrapContainerFromTags
utils.PushShieldMetrics = metrics.PushShieldMetrics
rand.Seed(time.Now().UnixNano())

136
src/metrics/middleware.go
View file

@ -1,72 +1,47 @@
package metrics
import (
"net/http"
"fmt"
"time"
"github.com/azukaar/cosmos-server/src/utils"
)
// responseWriter wraps the original http.ResponseWriter to capture the status code.
// type responseWriter struct {
// http.ResponseWriter
// status int
// }
// func (rw *responseWriter) WriteHeader(status int) {
// rw.status = status
// rw.ResponseWriter.WriteHeader(status)
// }
func MetricsMiddleware(route utils.ProxyRouteConfig) func(next http.Handler) http.Handler {
return func(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
startTime := time.Now()
// Call the next handler (which can be another middleware or the final handler).
// wrappedWriter := &responseWriter{ResponseWriter: w}
next.ServeHTTP(w, r)
// Calculate and log the response time.
responseTime := time.Since(startTime)
utils.Debug(fmt.Sprintf("[%s] %s %s %v", r.Method, r.RequestURI, r.RemoteAddr, responseTime))
func PushRequestMetrics(route utils.ProxyRouteConfig, statusCode int, TimeStarted time.Time, size int64) error {
responseTime := time.Since(TimeStarted)
if !utils.GetMainConfig().MonitoringDisabled {
go func() {
// if wrappedWriter.status >= 400 {
// PushSetMetric("proxy.all.error", 1, DataDef{
// Max: 0,
// Period: time.Second * 30,
// Label: "Global Request Errors",
// AggloType: "sum",
// SetOperation: "sum",
// })
// PushSetMetric("proxy.route.error."+route.Name, 1, DataDef{
// Max: 0,
// Period: time.Second * 30,
// Label: "Request Errors " + route.Name,
// AggloType: "sum",
// SetOperation: "sum",
// })
// } else {
// PushSetMetric("proxy.all.success", 1, DataDef{
// Max: 0,
// Period: time.Second * 30,
// Label: "Global Request Success",
// AggloType: "sum",
// SetOperation: "sum",
// })
// PushSetMetric("proxy.route.success."+route.Name, 1, DataDef{
// Max: 0,
// Period: time.Second * 30,
// Label: "Request Success " + route.Name,
// AggloType: "sum",
// SetOperation: "sum",
// })
// }
if statusCode >= 400 {
PushSetMetric("proxy.all.error", 1, DataDef{
Max: 0,
Period: time.Second * 30,
Label: "Global Request Errors",
AggloType: "sum",
SetOperation: "sum",
})
PushSetMetric("proxy.route.error."+route.Name, 1, DataDef{
Max: 0,
Period: time.Second * 30,
Label: "Request Errors " + route.Name,
AggloType: "sum",
SetOperation: "sum",
})
} else {
PushSetMetric("proxy.all.success", 1, DataDef{
Max: 0,
Period: time.Second * 30,
Label: "Global Request Success",
AggloType: "sum",
SetOperation: "sum",
})
PushSetMetric("proxy.route.success."+route.Name, 1, DataDef{
Max: 0,
Period: time.Second * 30,
Label: "Request Success " + route.Name,
AggloType: "sum",
SetOperation: "sum",
})
}
PushSetMetric("proxy.all.time", int(responseTime.Milliseconds()), DataDef{
Max: 0,
@ -85,9 +60,50 @@ func MetricsMiddleware(route utils.ProxyRouteConfig) func(next http.Handler) htt
SetOperation: "max",
Unit: "ms",
})
}()
PushSetMetric("proxy.all.bytes", int(size), DataDef{
Max: 0,
Period: time.Second * 30,
Label: "Global Transfered Bytes",
AggloType: "sum",
Unit: "B",
})
PushSetMetric("proxy.route.bytes."+route.Name, int(size), DataDef{
Max: 0,
Period: time.Second * 30,
Label: "Transfered Bytes " + route.Name,
AggloType: "sum",
Unit: "B",
})
}
return nil
}
func PushShieldMetrics(reason string) {
reasonStr := map[string]string{
"bots": "Bots",
"geo": "By Geolocation",
"referer": "By Referer",
"hostname": "By Hostname",
"ip-whitelists": "By IP Whitelists",
"smart-shield": "Smart Shield",
}
PushSetMetric("proxy.blocked."+reason, 1, DataDef{
Max: 0,
Period: time.Second * 30,
Label: "Blocked " + reasonStr[reason],
AggloType: "sum",
SetOperation: "sum",
})
PushSetMetric("proxy.all.blocked", 1, DataDef{
Max: 0,
Period: time.Second * 30,
Label: "Global Blocked Requests",
AggloType: "sum",
SetOperation: "sum",
})
}
}

4
src/proxy/botblock.go
View file

@ -2,6 +2,8 @@ package proxy
import (
"net/http"
"github.com/azukaar/cosmos-server/src/metrics"
)
var botUserAgents = []string{
@ -28,12 +30,14 @@ func BotDetectionMiddleware(next http.Handler) http.Handler {
userAgent := r.UserAgent()
if userAgent == "" {
go metrics.PushShieldMetrics("bots")
http.Error(w, "Access denied: Bots are not allowed.", http.StatusForbidden)
return
}
for _, botUserAgent := range botUserAgents {
if userAgent == botUserAgent {
go metrics.PushShieldMetrics("bots")
http.Error(w, "Access denied: Bots are not allowed.", http.StatusForbidden)
return
}

5
src/proxy/routerGen.go
View file

@ -8,7 +8,6 @@ import (
"github.com/azukaar/cosmos-server/src/user"
"github.com/azukaar/cosmos-server/src/utils"
"github.com/azukaar/cosmos-server/src/metrics"
"github.com/go-chi/httprate"
"github.com/gorilla/mux"
)
@ -88,7 +87,7 @@ func RouterGen(route utils.ProxyRouteConfig, router *mux.Router, destination htt
destination = utils.Restrictions(route.RestrictToConstellation, route.WhitelistInboundIPs)(destination)
destination = SmartShieldMiddleware(route.Name, route.SmartShield)(destination)
destination = SmartShieldMiddleware(route.Name, route)(destination)
originCORS := route.CORSOrigin
@ -146,8 +145,6 @@ func RouterGen(route utils.ProxyRouteConfig, router *mux.Router, destination htt
destination = utils.SetSecurityHeaders(destination)
}
destination = metrics.MetricsMiddleware(route)(destination)
destination = tokenMiddleware(route.AuthEnabled, route.AdminOnly)(utils.CORSHeader(originCORS)((destination)))
origin.Handler(destination)

46
src/proxy/shield.go
View file

@ -1,7 +1,6 @@
package proxy
import (
"github.com/azukaar/cosmos-server/src/utils"
"sync"
"time"
"net/http"
@ -9,6 +8,9 @@ import (
"net"
"math"
"strconv"
"github.com/azukaar/cosmos-server/src/utils"
"github.com/azukaar/cosmos-server/src/metrics"
)
/*
@ -267,12 +269,10 @@ func isPrivileged(req *http.Request, policy utils.SmartShieldPolicy) bool {
return role >= policy.PrivilegedGroups
}
func SmartShieldMiddleware(shieldID string, policy utils.SmartShieldPolicy) func(http.Handler) http.Handler {
if policy.Enabled == false {
return func(next http.Handler) http.Handler {
return next
}
} else {
func SmartShieldMiddleware(shieldID string, route utils.ProxyRouteConfig) func(http.Handler) http.Handler {
policy := route.SmartShield
if policy.Enabled {
if(policy.PerUserTimeBudget == 0) {
policy.PerUserTimeBudget = 2 * 60 * 60 * 1000 // 2 hours
}
@ -298,7 +298,31 @@ func SmartShieldMiddleware(shieldID string, policy utils.SmartShieldPolicy) func
return func(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
utils.Log("SmartShield: Request received")
clientID := GetClientID(r)
wrapper := &SmartResponseWriterWrapper {
ResponseWriter: w,
ThrottleNext: 0,
TimeStarted: time.Now(),
ClientID: clientID,
RequestCost: 1,
Method: r.Method,
shield: shield,
shieldID: shieldID,
policy: policy,
isPrivileged: isPrivileged(r, policy),
}
if !policy.Enabled {
next.ServeHTTP(wrapper, r)
wrapper.TimeEnded = time.Now()
wrapper.isOver = true
go metrics.PushRequestMetrics(route, wrapper.Status, wrapper.TimeStarted, wrapper.Bytes)
return
}
currentGlobalRequests := shield.GetServerNbReq(shieldID) + 1
utils.Debug(fmt.Sprintf("SmartShield: Current global requests: %d", currentGlobalRequests))
@ -307,6 +331,7 @@ func SmartShieldMiddleware(shieldID string, policy utils.SmartShieldPolicy) func
wayTooManyReq := currentGlobalRequests > policy.MaxGlobalSimultaneous * 10
retries := 50
if wayTooManyReq {
go metrics.PushShieldMetrics("smart-shield")
utils.Log("SmartShield: WAYYYY Too many users on the server. Aborting right away.")
http.Error(w, "Too many requests", http.StatusTooManyRequests)
return
@ -317,6 +342,7 @@ func SmartShieldMiddleware(shieldID string, policy utils.SmartShieldPolicy) func
tooManyReq = currentGlobalRequests > policy.MaxGlobalSimultaneous
retries--
if retries <= 0 {
go metrics.PushShieldMetrics("smart-shield")
utils.Log("SmartShield: Too many users on the server")
http.Error(w, "Too many requests", http.StatusTooManyRequests)
return
@ -324,11 +350,11 @@ func SmartShieldMiddleware(shieldID string, policy utils.SmartShieldPolicy) func
}
}
clientID := GetClientID(r)
userConsumed := shield.GetUserUsedBudgets(shieldID, clientID)
if !isPrivileged(r, policy) && !shield.isAllowedToReqest(shieldID, policy, userConsumed) {
lastBan := shield.GetLastBan(policy, userConsumed)
go metrics.PushShieldMetrics("smart-shield")
utils.Log("SmartShield: User is blocked due to abuse: " + fmt.Sprintf("%+v", lastBan))
http.Error(w, "Too many requests", http.StatusTooManyRequests)
return
@ -337,6 +363,7 @@ func SmartShieldMiddleware(shieldID string, policy utils.SmartShieldPolicy) func
if(!isPrivileged(r, policy)) {
throttle = shield.computeThrottle(policy, userConsumed)
}
wrapper := &SmartResponseWriterWrapper {
ResponseWriter: w,
ThrottleNext: throttle,
@ -371,6 +398,7 @@ func SmartShieldMiddleware(shieldID string, policy utils.SmartShieldPolicy) func
shield.Lock()
wrapper.TimeEnded = time.Now()
wrapper.isOver = true
go metrics.PushRequestMetrics(route, wrapper.Status, wrapper.TimeStarted, wrapper.Bytes)
shield.Unlock()
})()

8
src/utils/middleware.go
View file

@ -14,6 +14,8 @@ import (
// https://github.com/go-chi/chi/blob/master/middleware/timeout.go
var PushShieldMetrics func(string)
func MiddlewareTimeout(timeout time.Duration) func(next http.Handler) http.Handler {
return func(next http.Handler) http.Handler {
fn := func(w http.ResponseWriter, r *http.Request) {
@ -148,6 +150,7 @@ func BlockByCountryMiddleware(blockedCountries []string, CountryBlacklistIsWhite
}
if blocked {
PushShieldMetrics("geo")
http.Error(w, "Access denied", http.StatusForbidden)
return
}
@ -177,6 +180,7 @@ func BlockPostWithoutReferer(next http.Handler) http.Handler {
if r.Method == "POST" || r.Method == "PUT" || r.Method == "PATCH" || r.Method == "DELETE" {
referer := r.Header.Get("Referer")
if referer == "" {
PushShieldMetrics("referer")
Error("Blocked POST request without Referer header", nil)
http.Error(w, "Bad Request: Invalid request.", http.StatusBadRequest)
return
@ -213,6 +217,7 @@ func EnsureHostname(next http.Handler) http.Handler {
}
if !isOk {
PushShieldMetrics("hostname")
Error("Invalid Hostname " + r.Host + " for request. Expecting one of " + fmt.Sprintf("%v", hostnames), nil)
w.WriteHeader(http.StatusBadRequest)
http.Error(w, "Bad Request: Invalid hostname. Use your domain instead of your IP to access your server. Check logs if more details are needed.", http.StatusBadRequest)
@ -306,11 +311,13 @@ func Restrictions(RestrictToConstellation bool, WhitelistInboundIPs []string) fu
if(RestrictToConstellation) {
if(!isInConstellation) {
if(!isUsingWhiteList) {
PushShieldMetrics("ip-whitelists")
Error("Request from " + ip + " is blocked because of restrictions", nil)
Debug("Blocked by RestrictToConstellation isInConstellation isUsingWhiteList")
http.Error(w, "Access denied", http.StatusForbidden)
return
} else if (!isInWhitelist) {
PushShieldMetrics("ip-whitelists")
Error("Request from " + ip + " is blocked because of restrictions", nil)
Debug("Blocked by RestrictToConstellation isInConstellation isInWhitelist")
http.Error(w, "Access denied", http.StatusForbidden)
@ -318,6 +325,7 @@ func Restrictions(RestrictToConstellation bool, WhitelistInboundIPs []string) fu
}
}
} else if(isUsingWhiteList && !isInWhitelist) {
PushShieldMetrics("ip-whitelists")
Error("Request from " + ip + " is blocked because of restrictions", nil)
Debug("Blocked by RestrictToConstellation isInConstellation isUsingWhiteList isInWhitelist")
http.Error(w, "Access denied", http.StatusForbidden)