From 9ec6784b268e2b9df5e3eb9112a2aa907d86d0b2 Mon Sep 17 00:00:00 2001 From: Yann Stepienik Date: Tue, 7 Nov 2023 16:26:51 +0000 Subject: [PATCH] [release] v0.12.0-unstable47 --- changelog.md | 26 +-- .../src/pages/config/users/formShortcuts.jsx | 4 +- client/src/pages/market/details.jsx | 0 client/src/pages/market/listing.jsx | 83 ++++++-- client/src/pages/market/sources.jsx | 183 ++++++++++++++++++ package.json | 2 +- readme.md | 7 +- src/CRON.go | 4 + src/configapi/patch.go | 8 + src/configapi/set.go | 8 + src/constellation/api_devices_create.go | 12 ++ src/docker/api_blueprint.go | 14 ++ src/docker/api_secureContainer.go | 10 + src/docker/docker.go | 9 + src/docker/events.go | 70 +++---- src/httpServer.go | 20 +- src/market/index.go | 32 ++- src/market/init.go | 52 ++++- src/market/update.go | 15 ++ src/proxy/shield.go | 24 ++- src/user/create.go | 9 + src/user/password_reset.go | 9 + src/user/register.go | 9 + src/utils/cleanup.go | 40 ++++ src/utils/emails.go | 10 + src/utils/log.go | 10 + src/utils/middleware.go | 74 +++++++ 27 files changed, 653 insertions(+), 91 deletions(-) delete mode 100644 client/src/pages/market/details.jsx create mode 100644 client/src/pages/market/sources.jsx create mode 100644 src/utils/cleanup.go diff --git a/changelog.md b/changelog.md index d40a656..af27223 100644 --- a/changelog.md +++ b/changelog.md @@ -1,27 +1,31 @@ ## Version 0.12.0 - - New Dashboard - - New metrics gathering system - - New alerts system - - New notification center - - New events manager - - Integrated a new docker-less mode of functioning for networking - - Added Button to force reset HTTPS cert in settings - - New color slider with reset buttons + - New real time persisting andd optimized metrics monitoring system (RAM, CPU, Network, disk, requests, errors, etc...) + - New Dashboard with graphs for metrics, including graphs in many screens such as home, routes and servapps + - New customizable alerts system based on metrics in real time, with included preset for anti-crypto mining and anti memory leak + - New events manager (improved logs with requests and advanced search) + - New notification system + - Added Marketplace UI to edit sources, with new display of 3rd party sources - Added a notification when updating a container, renewing certs, etc... + - Certificates now renew sooner to avoid Let's Encrypt sending emails about expiring certificates + - Added option to disable routes without deleting them - Improved icon loading speed, and added proper placeholder - - Added lazyloading to URL and Servapp pages images + - Marketplace now fetch faster (removed the domain indirection to directly fetch from github) + - Integrated a new docker-less mode of functioning for networking - Added a dangerous IP detector that stops sending HTTP response to IPs that are abusing various shields features - Added a button in the servapp page to easily download the docker backup + - Added Button to force reset HTTPS cert in settings + - Added lazyloading to URL and Servapp pages images + - Fixed annoying marketplace screenshot bug (you know what I'm talking about!) + - New color slider with reset buttons - Redirect static folder to host if possible - New Homescreen look - - Added option to disable routes without deleting them - Fixed blinking modals issues - Improve display or icons [fixes #121] - Refactored Mongo connection code [fixes #111] - Forward simultaneously TCP and UDP [fixes #122] ## Version 0.11.3 - - Fix missing even subscriber on export + - Fix missing event subscriber on export ## Version 0.11.2 - Improve Docker exports logs diff --git a/client/src/pages/config/users/formShortcuts.jsx b/client/src/pages/config/users/formShortcuts.jsx index fce7fc8..76005b8 100644 --- a/client/src/pages/config/users/formShortcuts.jsx +++ b/client/src/pages/config/users/formShortcuts.jsx @@ -47,7 +47,7 @@ export const CosmosInputText = ({ name, style, value, errors, multiline, type, p { @@ -101,7 +101,7 @@ export const CosmosInputPassword = ({ name, noStrength, type, placeholder, autoC 1 ? ( { - screenshots.map((item, i) => ) + screenshots.map((item, i) =>
+ +
) } ) - : + :
+ +
} function Showcases({ showcase, isDark, isAdmin }) { @@ -122,18 +146,44 @@ const MarketPage = () => { // borderTop: '1px solid rgb(220,220,220)' }; - useEffect(() => { + const refresh = () => { API.market.list().then((res) => { setApps(res.data.all); setShowcase(res.data.showcase); }); + }; + + useEffect(() => { + refresh(); }, []); let openedApp = null; if (appName && Object.keys(apps).length > 0) { openedApp = apps[appStore].find((app) => app.name === appName); + openedApp.appstore = appStore; } + let appList = apps && Object.keys(apps).reduce((acc, appstore) => { + const a = apps[appstore].map((app) => { + app.appstore = appstore; + return app; + }); + + return acc.concat(a); + }, []); + + appList.sort((a, b) => { + if (a.name > b.name) { + return 1; + } + + if (a.name < b.name) { + return -1; + } + + return 0; + }); + return <> @@ -186,7 +236,7 @@ const MarketPage = () => { -

{openedApp.name}

+

{openedApp.name} {openedApp.appstore != 'cosmos-cloud' ? (' @ '+openedApp.appstore) : ''}
@@ -197,6 +247,14 @@ const MarketPage = () => { {openedApp.supported_architectures && openedApp.supported_architectures.slice(0, 8).map((tag) => )}
+ {openedApp.appstore != 'cosmos-cloud' &&
+
+ + + source: {openedApp.appstore} +
+
} +
repository: {openedApp.repository}
image: {openedApp.image}
@@ -259,6 +317,7 @@ const MarketPage = () => { >Start ServApp { }} /> + {(!apps || !Object.keys(apps).length) && { } {apps && Object.keys(apps).length > 0 && - {Object.keys(apps).map(appstore => apps[appstore] - .filter((app) => { + {appList.filter((app) => { if (!search || search.length <= 2) { return true; } @@ -284,17 +342,18 @@ const MarketPage = () => { app.tags.join(' ').toLowerCase().includes(search.toLowerCase()); }) .map((app) => { - return
-
{app.name}
+
{app.name}{app.appstore != 'cosmos-cloud' ? (' @ '+app.appstore) : ''}
{ - }))} + })} } diff --git a/client/src/pages/market/sources.jsx b/client/src/pages/market/sources.jsx new file mode 100644 index 0000000..de789de --- /dev/null +++ b/client/src/pages/market/sources.jsx @@ -0,0 +1,183 @@ +import * as React from 'react'; +import IsLoggedIn from '../../isLoggedIn'; +import * as API from '../../api'; +import MainCard from '../../components/MainCard'; +import { Formik, Field, useFormik, FormikProvider } from 'formik'; +import * as Yup from 'yup'; +import { + Alert, + Button, + Checkbox, + FormControlLabel, + Grid, + InputLabel, + OutlinedInput, + Stack, + FormHelperText, + TextField, + MenuItem, + Skeleton, + CircularProgress, + Dialog, + DialogTitle, + DialogContent, + DialogContentText, + DialogActions, + Box, +} from '@mui/material'; +import { ContainerOutlined, ExclamationCircleOutlined, InfoCircleOutlined, PlusCircleOutlined, SyncOutlined, WarningOutlined } from '@ant-design/icons'; +import PrettyTableView from '../../components/tableView/prettyTableView'; +import { DeleteButton } from '../../components/delete'; +import { CosmosCheckbox, CosmosFormDivider, CosmosInputText, CosmosSelect } from '../config/users/formShortcuts'; +import ResponsiveButton from '../../components/responseiveButton'; + +const AlertValidationSchema = Yup.object().shape({ + name: Yup.string().required('Name is required'), + trackingMetric: Yup.string().required('Tracking metric is required'), + conditionOperator: Yup.string().required('Condition operator is required'), + conditionValue: Yup.number().required('Condition value is required'), + period: Yup.string().required('Period is required'), +}); + +const EditSourcesModal = ({ onSave }) => { + const [config, setConfig] = React.useState(null); + const [open, setOpen] = React.useState(false); + + function getConfig() { + API.config.get().then((res) => { + setConfig(res.data); + }); + } + + React.useEffect(() => { + getConfig(); + }, []); + + const formik = useFormik({ + initialValues: { + sources: config ? config.MarketConfig.Sources : [], + }, + enableReinitialize: true, // This will reinitialize the form when `config` changes + // validationSchema: AlertValidationSchema, + onSubmit: (values) => { + values.sources = values.sources.filter((a) => !a.removed); + + // setIsLoading(true); + + let toSave = { + ...config, + MarketConfig: { + ...config.MarketConfig, + Sources: values.sources, + } + }; + + setOpen(false); + + return API.config.set(toSave).then(() => { + onSave(); + }); + }, + validate: (values) => { + const errors = {}; + + values.sources.forEach((source, index) => { + if (source.Name === '') { + errors[`sources.${index}.Name`] = 'Name is required'; + } + if (source.Url === '') { + errors[`sources.${index}.Url`] = 'URL is required'; + } + + if (source.Name === 'cosmos-cloud' || values.sources.filter((s) => s.Name === source.Name).length > 1) { + errors[`sources.${index}.Name`] = 'Name must be unique'; + } + }); + + return errors; + } + }); + + return (<> + setOpen(false)} maxWidth="sm" fullWidth> + Edit Sources + {config && +
+ + + {formik.values.sources + .map((action, index) => { + return !action.removed && <> + + + +
+ +
+ + + { + formik.setFieldValue(`sources.${index}.removed`, true); + }} + /> + + +
+ {formik.errors[`sources.${index}.Name`]} +
+
+ {formik.errors[`sources.${index}.Url`]} +
+ + + })} + + + + + + + + +
+ } + + + + } + onClick={() => setOpen(true)} + >Sources + + ); +}; + +export default EditSourcesModal; \ No newline at end of file diff --git a/package.json b/package.json index 646c124..e0e7ee7 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "cosmos-server", - "version": "0.12.0-unstable46", + "version": "0.12.0-unstable47", "description": "", "main": "test-server.js", "bugs": { diff --git a/readme.md b/readme.md index f9d429a..653d612 100644 --- a/readme.md +++ b/readme.md @@ -16,8 +16,7 @@ [![DiscordLink](https://img.shields.io/discord/1083875833824944188?label=Discord&logo=Discord&style=flat-square)](https://discord.gg/PwMWwsrwHA) ![CircleCI](https://img.shields.io/circleci/build/github/azukaar/Cosmos-Server?token=6efd010d0f82f97175f04a6acf2dae2bbcc4063c&style=flat-square) -Cosmos is a self-hosted platform for running server applications securely and with built-in privacy features. It acts as a secure gateway to your application, as well as a server manager. It aims to solve the increasingly worrying problem of vulnerable self-hosted applications and personal servers. - +☁️ Cosmos is the most secure and easy way to selfhost a Home Server. It acts as a secure gateway to your application, as well as a server manager. It aims to solve the increasingly worrying problem of vulnerable self-hosted applications and personal servers.


@@ -37,7 +36,7 @@ Cosmos is a self-hosted platform for running server applications securely and wi ![screenshot1](./screenshot1.png) -Whether you have a **server**, a **NAS**, or a **Raspberry Pi** with applications such as **Plex**, **HomeAssistant** or even a blog, Cosmos is the perfect solution to secure them all. Simply install Cosmos on your server and connect to your applications through it to enjoy built-in security and robustness for all your services, right out of the box. +Whether you have a **server**, a **NAS**, or a **Raspberry Pi** with applications such as **Plex**, **HomeAssistant** or even a blog, Cosmos is the perfect solution torun and secure them all. Simply install Cosmos on your server and connect to your applications through it to enjoy built-in security and robustness for all your services, right out of the box. Cosmos is a: @@ -155,7 +154,7 @@ in this command, `-v /:/mnt/host` is optional and allow to manage folders from C `--privileged` is also optional, but it is required if you use hardening software like AppArmor or SELinux, as they restrict access to the docker socket. It is also required for Constellation to work. If you don't want to use it, you can add the following capabilities: NET_ADMIN for Constellation. -Once installed, simply go to `http://your-server-ip` and follow the instructions of the setup wizard. +Once installed, simply go to `http://your-server-ip` and follow the instructions of the setup wizard. **always start the install with the browser in incognito mode** to avoid issues with your browser cache. Port 4242 is a UDP port used for the Constellation VPN. diff --git a/src/CRON.go b/src/CRON.go index 9d66655..087cdcd 100644 --- a/src/CRON.go +++ b/src/CRON.go @@ -123,6 +123,10 @@ func CRON() { s.Every(1).Day().At("01:00").Do(checkCerts) s.Every(6).Hours().Do(checkUpdatesAvailable) s.Every(1).Hours().Do(utils.CleanBannedIPs) + s.Every(1).Day().At("00:00").Do(func() { + utils.CleanupByDate("notifications") + utils.CleanupByDate("events") + }) s.Start() }() } \ No newline at end of file diff --git a/src/configapi/patch.go b/src/configapi/patch.go index 56221f7..cf1c1e0 100644 --- a/src/configapi/patch.go +++ b/src/configapi/patch.go @@ -95,6 +95,14 @@ func ConfigApiPatch(w http.ResponseWriter, req *http.Request) { config.HTTPConfig.ProxyConfig.Routes = routes utils.SetBaseMainConfig(config) + + utils.TriggerEvent( + "cosmos.settings", + "Settings updated", + "success", + "", + map[string]interface{}{ + }) utils.RestartHTTPServer() diff --git a/src/configapi/set.go b/src/configapi/set.go index a2aad63..6d95cec 100644 --- a/src/configapi/set.go +++ b/src/configapi/set.go @@ -40,6 +40,14 @@ func ConfigApiSet(w http.ResponseWriter, req *http.Request) { request.NewInstall = config.NewInstall utils.SetBaseMainConfig(request) + + utils.TriggerEvent( + "cosmos.settings", + "Settings updated", + "success", + "", + map[string]interface{}{ + }) utils.DisconnectDB() authorizationserver.Init() diff --git a/src/constellation/api_devices_create.go b/src/constellation/api_devices_create.go index 995669b..4bb4436 100644 --- a/src/constellation/api_devices_create.go +++ b/src/constellation/api_devices_create.go @@ -147,6 +147,18 @@ func DeviceCreate(w http.ResponseWriter, req *http.Request) { return } + utils.TriggerEvent( + "cosmos.constellation.device.create", + "Device created", + "success", + "", + map[string]interface{}{ + "deviceName": deviceName, + "nickname": nickname, + "publicKey": key, + "ip": request.IP, + }) + json.NewEncoder(w).Encode(map[string]interface{}{ "status": "OK", "data": map[string]interface{}{ diff --git a/src/docker/api_blueprint.go b/src/docker/api_blueprint.go index 0785769..d669e10 100644 --- a/src/docker/api_blueprint.go +++ b/src/docker/api_blueprint.go @@ -966,6 +966,20 @@ func CreateService(serviceRequest DockerServiceCreateRequest, OnLog func(string) OnLog("\n") OnLog(utils.DoSuccess("[OPERATION SUCCEEDED]. SERVICE STARTED\n")) + servicesNames := []string{} + for _, service := range serviceRequest.Services { + servicesNames = append(servicesNames, service.Name) + } + + utils.TriggerEvent( + "cosmos.docker.compose.create", + "Service created", + "success", + "", + map[string]interface{}{ + "services": servicesNames, + }) + return nil } diff --git a/src/docker/api_secureContainer.go b/src/docker/api_secureContainer.go index 2d7478e..b889d29 100644 --- a/src/docker/api_secureContainer.go +++ b/src/docker/api_secureContainer.go @@ -49,6 +49,16 @@ func SecureContainerRoute(w http.ResponseWriter, req *http.Request) { return } + utils.TriggerEvent( + "cosmos.docker.isolate", + "Container network isolation changed", + "success", + "container@"+containerName, + map[string]interface{}{ + "container": containerName, + "status": status, + }) + json.NewEncoder(w).Encode(map[string]interface{}{ "status": "OK", }) diff --git a/src/docker/docker.go b/src/docker/docker.go index 2e972a7..b1af97f 100644 --- a/src/docker/docker.go +++ b/src/docker/docker.go @@ -101,6 +101,15 @@ func RecreateContainer(containerID string, containerConfig types.ContainerJSON) } else { return EditContainer(containerID, containerConfig, false) } + + utils.TriggerEvent( + "cosmos.docker.recreate", + "Cosmos Container Recreate", + "success", + "container@" + containerID, + map[string]interface{}{ + "container": containerID, + }) return "", nil } diff --git a/src/docker/events.go b/src/docker/events.go index 7b80b32..d569daa 100644 --- a/src/docker/events.go +++ b/src/docker/events.go @@ -60,41 +60,43 @@ func DockerListenEvents() error { onNetworkConnect(msg.Actor.ID) } - level := "info" - if msg.Type == "image" { - level = "debug" + if msg.Action != "exec_create" && msg.Action != "exec_start" && msg.Action != "exec_die" { + level := "info" + if msg.Type == "image" { + level = "debug" + } + if msg.Action == "destroy" || msg.Action == "delete" || msg.Action == "kill" || msg.Action == "die" { + level = "warning" + } + if msg.Action == "create" || msg.Action == "start" { + level = "success" + } + + object := "" + if msg.Type == "container" { + object = "container@" + msg.Actor.Attributes["name"] + } else if msg.Type == "network" { + object = "network@" + msg.Actor.Attributes["name"] + } else if msg.Type == "image" { + object = "image@" + msg.Actor.Attributes["name"] + } else if msg.Type == "volume" && msg.Actor.Attributes["name"] != "" { + object = "volume@" + msg.Actor.Attributes["name"] + } + + utils.TriggerEvent( + "cosmos.docker.event." + msg.Type + "." + msg.Action, + "Docker Event " + msg.Type + " " + msg.Action, + level, + object, + map[string]interface{}{ + "type": msg.Type, + "action": msg.Action, + "actor": msg.Actor, + "status": msg.Status, + "from": msg.From, + "scope": msg.Scope, + }) } - if msg.Action == "destroy" || msg.Action == "delete" || msg.Action == "kill" || msg.Action == "die" { - level = "warning" - } - if msg.Action == "create" || msg.Action == "start" { - level = "success" - } - - object := "" - if msg.Type == "container" { - object = "container@" + msg.Actor.Attributes["name"] - } else if msg.Type == "network" { - object = "network@" + msg.Actor.Attributes["name"] - } else if msg.Type == "image" { - object = "image@" + msg.Actor.Attributes["name"] - } else if msg.Type == "volume" && msg.Actor.Attributes["name"] != "" { - object = "volume@" + msg.Actor.Attributes["name"] - } - - utils.TriggerEvent( - "cosmos.docker.event." + msg.Type + "." + msg.Action, - "Docker Event " + msg.Type + " " + msg.Action, - level, - object, - map[string]interface{}{ - "type": msg.Type, - "action": msg.Action, - "actor": msg.Actor, - "status": msg.Status, - "from": msg.From, - "scope": msg.Scope, - }) } } }() diff --git a/src/httpServer.go b/src/httpServer.go index b9dffb4..ac39d8a 100644 --- a/src/httpServer.go +++ b/src/httpServer.go @@ -147,7 +147,7 @@ func tokenMiddleware(next http.Handler) http.Handler { }) } -func SecureAPI(userRouter *mux.Router, public bool) { +func SecureAPI(userRouter *mux.Router, public bool, publicCors bool) { if(!public) { userRouter.Use(tokenMiddleware) } @@ -162,9 +162,13 @@ func SecureAPI(userRouter *mux.Router, public bool) { }, }, )) + + if(publicCors || public) { + userRouter.Use(utils.PublicCORS) + } + userRouter.Use(utils.MiddlewareTimeout(45 * time.Second)) - userRouter.Use(proxy.BotDetectionMiddleware) - userRouter.Use(httprate.Limit(120, 1*time.Minute, + userRouter.Use(httprate.Limit(180, 1*time.Minute, httprate.WithKeyFuncs(httprate.KeyByIP), httprate.WithLimitHandler(func(w http.ResponseWriter, r *http.Request) { utils.Error("Too many requests. Throttling", nil) @@ -334,7 +338,7 @@ func InitServer() *mux.Router { } logoAPI := router.PathPrefix("/logo").Subrouter() - SecureAPI(logoAPI, true) + SecureAPI(logoAPI, true, true) logoAPI.HandleFunc("/", SendLogo) @@ -413,7 +417,7 @@ func InitServer() *mux.Router { srapi.Use(utils.EnsureHostname) } - SecureAPI(srapi, false) + SecureAPI(srapi, false, false) pwd,_ := os.Getwd() utils.Log("Starting in " + pwd) @@ -437,13 +441,13 @@ func InitServer() *mux.Router { })) userRouter := router.PathPrefix("/oauth2").Subrouter() - SecureAPI(userRouter, false) + SecureAPI(userRouter, false, true) serverRouter := router.PathPrefix("/oauth2").Subrouter() - SecureAPI(serverRouter, true) + SecureAPI(serverRouter, true, true) wellKnownRouter := router.PathPrefix("/").Subrouter() - SecureAPI(wellKnownRouter, true) + SecureAPI(wellKnownRouter, true, true) authorizationserver.RegisterHandlers(wellKnownRouter, userRouter, serverRouter) diff --git a/src/market/index.go b/src/market/index.go index ee5d4af..0af350b 100644 --- a/src/market/index.go +++ b/src/market/index.go @@ -3,6 +3,7 @@ package market import ( "net/http" "encoding/json" + "fmt" "github.com/azukaar/cosmos-server/src/utils" ) @@ -17,8 +18,23 @@ func MarketGet(w http.ResponseWriter, req *http.Request) { } if(req.Method == "GET") { + config := utils.GetMainConfig() + configSourcesList := config.MarketConfig.Sources + configSources := map[string]bool{ + "cosmos-cloud": true, + } + for _, source := range configSourcesList { + configSources[source.Name] = true + } + + utils.Debug(fmt.Sprintf("MarketGet: Config sources: %v", configSources)) + + Init() + err := updateCache(w, req) if err != nil { + utils.Error("MarketGet: Error while updating cache", err) + utils.HTTPError(w, "Error while updating cache", http.StatusInternalServerError, "MK002") return } @@ -28,19 +44,23 @@ func MarketGet(w http.ResponseWriter, req *http.Request) { } for _, market := range currentMarketcache { + if !configSources[market.Name] { + continue + } + utils.Debug(fmt.Sprintf("MarketGet: Adding market %v", market.Name)) results := []appDefinition{} for _, app := range market.Results.All { - // if i < 10 { - results = append(results, app) - // } else { - // break - // } + results = append(results, app) } marketGetResult.All[market.Name] = results } if len(currentMarketcache) > 0 { - marketGetResult.Showcase = currentMarketcache[0].Results.Showcase + for _, market := range currentMarketcache { + if market.Name == "cosmos-cloud" { + marketGetResult.Showcase = market.Results.Showcase + } + } } json.NewEncoder(w).Encode(map[string]interface{}{ diff --git a/src/market/init.go b/src/market/init.go index 3e6f75b..b203317 100644 --- a/src/market/init.go +++ b/src/market/init.go @@ -6,23 +6,59 @@ import ( func Init() { config := utils.GetMainConfig() - currentMarketcache = []marketCacheObject{} sources := config.MarketConfig.Sources + inConfig := map[string]bool{ + "cosmos-cloud": true, + } + for _, source := range sources { + inConfig[source.Name] = true + } + + if currentMarketcache == nil { + currentMarketcache = []marketCacheObject{} + } + + inCache := map[string]bool{} + toRemove := []string{} + for _, cachedMarket := range currentMarketcache { + inCache[cachedMarket.Name] = true + + if !inConfig[cachedMarket.Name] { + utils.Log("MarketInit: Removing market " + cachedMarket.Name) + toRemove = append(toRemove, cachedMarket.Name) + } + } + + // remove markets that are not in config + for _, name := range toRemove { + for index, cachedMarket := range currentMarketcache { + if cachedMarket.Name == name { + currentMarketcache = append(currentMarketcache[:index], currentMarketcache[index+1:]...) + break + } + } + } + // prepend the default market defaultMarket := utils.MarketSource{ - Url: "https://cosmos-cloud.io/repository", + Url: "https://azukaar.github.io/cosmos-servapps-official/index.json", Name: "cosmos-cloud", } + sources = append([]utils.MarketSource{defaultMarket}, sources...) for _, marketDef := range sources { - market := marketCacheObject{ - Url: marketDef.Url, - Name: marketDef.Name, - } - currentMarketcache = append(currentMarketcache, market) + // add markets that are in config but not in cache + if !inCache[marketDef.Name] { + market := marketCacheObject{ + Url: marketDef.Url, + Name: marketDef.Name, + } - utils.Log("MarketInit: Added market " + market.Name) + currentMarketcache = append(currentMarketcache, market) + + utils.Log("MarketInit: Added market " + market.Name) + } } } \ No newline at end of file diff --git a/src/market/update.go b/src/market/update.go index e8038e6..78b57ce 100644 --- a/src/market/update.go +++ b/src/market/update.go @@ -61,9 +61,24 @@ func updateCache(w http.ResponseWriter, req *http.Request) error { continue } + result.Source = cachedMarket.Url + if cachedMarket.Name != "cosmos-cloud" { + result.Showcase = []appDefinition{} + } + cachedMarket.Results = result cachedMarket.LastUpdate = time.Now() + utils.TriggerEvent( + "cosmos.market.update", + "Market updated", + "success", + "", + map[string]interface{}{ + "market": cachedMarket.Name, + "numberOfApps": len(result.All), + }) + utils.Log("MarketUpdate: Updated market " + result.Source + " with " + string(len(result.All)) + " results") // save to cache diff --git a/src/proxy/shield.go b/src/proxy/shield.go index 4db141c..9611f9c 100644 --- a/src/proxy/shield.go +++ b/src/proxy/shield.go @@ -38,11 +38,11 @@ type smartShieldState struct { } type userUsedBudget struct { - ClientID string - Time float64 - Requests int - Bytes int64 - Simultaneous int + ClientID string `json:"clientID"` + Time float64 `json:"time"` + Requests int `json:"requests"` + Bytes int64 `json:"bytes"` + Simultaneous int `json:"simultaneous"` } var shield smartShieldState @@ -379,6 +379,20 @@ func SmartShieldMiddleware(shieldID string, route utils.ProxyRouteConfig) func(h lastBan := shield.GetLastBan(policy, userConsumed) go metrics.PushShieldMetrics("smart-shield") utils.IncrementIPAbuseCounter(clientID) + + utils.TriggerEvent( + "cosmos.proxy.shield.abuse." + route.Name, + "Proxy Shield " + route.Name + " Abuse by " + clientID, + "warning", + "route@" + route.Name, + map[string]interface{}{ + "route": route.Name, + "consumed": userConsumed, + "lastBan": lastBan, + "clientID": clientID, + "url": r.URL, + }) + utils.Log("SmartShield: User is blocked due to abuse: " + fmt.Sprintf("%+v", lastBan)) http.Error(w, "Too many requests", http.StatusTooManyRequests) return diff --git a/src/user/create.go b/src/user/create.go index b2e2f1c..9edef5e 100644 --- a/src/user/create.go +++ b/src/user/create.go @@ -77,6 +77,15 @@ func UserCreate(w http.ResponseWriter, req *http.Request) { return } + utils.TriggerEvent( + "cosmos.user.create", + "User created", + "success", + "", + map[string]interface{}{ + "nickname": nickname, + }) + json.NewEncoder(w).Encode(map[string]interface{}{ "status": "OK", "data": map[string]interface{}{ diff --git a/src/user/password_reset.go b/src/user/password_reset.go index 602c701..7b6e5e9 100644 --- a/src/user/password_reset.go +++ b/src/user/password_reset.go @@ -86,6 +86,15 @@ func ResetPassword(w http.ResponseWriter, req *http.Request) { return } + utils.TriggerEvent( + "cosmos.user.passwordreset", + "Password reset sent", + "success", + "", + map[string]interface{}{ + "nickname": user.Nickname, + }) + json.NewEncoder(w).Encode(map[string]interface{}{ "status": "OK", }) diff --git a/src/user/register.go b/src/user/register.go index e5e7ed9..0762def 100644 --- a/src/user/register.go +++ b/src/user/register.go @@ -102,6 +102,15 @@ func UserRegister(w http.ResponseWriter, req *http.Request) { } } + utils.TriggerEvent( + "cosmos.user.register", + "User registered", + "success", + "", + map[string]interface{}{ + "nickname": nickname, + }) + json.NewEncoder(w).Encode(map[string]interface{}{ "status": "OK", }) diff --git a/src/utils/cleanup.go b/src/utils/cleanup.go new file mode 100644 index 0000000..348870f --- /dev/null +++ b/src/utils/cleanup.go @@ -0,0 +1,40 @@ +package utils + +import ( + "time" + "strconv" + "context" + + "go.mongodb.org/mongo-driver/bson" +) + +type CleanupObject struct { + Date time.Time +} + +func CleanupByDate(collectionName string) { + c, errCo := GetCollection(GetRootAppId(), collectionName) + if errCo != nil { + MajorError("Database Cleanup", errCo) + return + } + + del, err := c.DeleteMany(context.Background(), bson.M{"Date": bson.M{"$lt": time.Now().AddDate(0, -1, 0)}}) + + if err != nil { + MajorError("Database Cleanup", err) + return + } + + Log("Cleanup: " + collectionName + " " + strconv.Itoa(int(del.DeletedCount)) + " objects deleted") + + TriggerEvent( + "cosmos.database.cleanup", + "Database Cleanup of " + collectionName, + "success", + "", + map[string]interface{}{ + "collection": collectionName, + "deleted": del.DeletedCount, + }) +} diff --git a/src/utils/emails.go b/src/utils/emails.go index c20f81b..f1a1eee 100644 --- a/src/utils/emails.go +++ b/src/utils/emails.go @@ -147,5 +147,15 @@ func SendEmail(recipients []string, subject string, body string) error { ServerURL, )) + TriggerEvent( + "cosmos.email.send", + "Email sent", + "success", + "", + map[string]interface{}{ + "recipients": recipients, + "subject": subject, + }) + return send(hostPort, auth, config.EmailConfig.From, recipients, msg) } \ No newline at end of file diff --git a/src/utils/log.go b/src/utils/log.go index 7e6ed8f..62b5597 100644 --- a/src/utils/log.go +++ b/src/utils/log.go @@ -57,6 +57,16 @@ func MajorError(message string, err error) { log.Println(Red + "[ERROR] " + message + " : " + errStr + Reset) } + TriggerEvent( + "cosmos.error", + "Critical Error", + "error", + "", + map[string]interface{}{ + "message": message, + "error": errStr, + }) + WriteNotification(Notification{ Recipient: "admin", Title: "Server Error", diff --git a/src/utils/middleware.go b/src/utils/middleware.go index 0ba426f..816b40d 100644 --- a/src/utils/middleware.go +++ b/src/utils/middleware.go @@ -163,6 +163,15 @@ func CORSHeader(origin string) func(next http.Handler) http.Handler { } } +func PublicCORS(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Access-Control-Allow-Origin", "*") + w.Header().Set("Access-Control-Allow-Credentials", "true") + + next.ServeHTTP(w, r) + }) +} + func AcceptHeader(accept string) func(next http.Handler) http.Handler { return func(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -218,6 +227,18 @@ func BlockByCountryMiddleware(blockedCountries []string, CountryBlacklistIsWhite if blocked { PushShieldMetrics("geo") IncrementIPAbuseCounter(ip) + + TriggerEvent( + "cosmos.proxy.shield.geo", + "Proxy Shield Geo blocked", + "warning", + "", + map[string]interface{}{ + "clientID": ip, + "country": countryCode, + "url": r.URL.String(), + }) + http.Error(w, "Access denied", http.StatusForbidden) return } @@ -253,6 +274,16 @@ func BlockPostWithoutReferer(next http.Handler) http.Handler { ip, _, _ := net.SplitHostPort(r.RemoteAddr) if ip != "" { + TriggerEvent( + "cosmos.proxy.shield.referer", + "Proxy Shield Referer blocked", + "warning", + "", + map[string]interface{}{ + "clientID": ip, + "url": r.URL.String(), + }) + IncrementIPAbuseCounter(ip) } @@ -295,6 +326,16 @@ func EnsureHostname(next http.Handler) http.Handler { ip, _, _ := net.SplitHostPort(r.RemoteAddr) if ip != "" { + TriggerEvent( + "cosmos.proxy.shield.hostname", + "Proxy Shield hostname blocked", + "warning", + "", + map[string]interface{}{ + "clientID": ip, + "hostname": r.Host, + "url": r.URL.String(), + }) IncrementIPAbuseCounter(ip) } @@ -389,6 +430,17 @@ func Restrictions(RestrictToConstellation bool, WhitelistInboundIPs []string) fu if(!isInConstellation) { if(!isUsingWhiteList) { PushShieldMetrics("ip-whitelists") + + TriggerEvent( + "cosmos.proxy.shield.whitelist", + "Proxy Shield IP blocked by whitelist", + "warning", + "", + map[string]interface{}{ + "clientID": ip, + "url": r.URL.String(), + }) + IncrementIPAbuseCounter(ip) Error("Request from " + ip + " is blocked because of restrictions", nil) Debug("Blocked by RestrictToConstellation isInConstellation isUsingWhiteList") @@ -396,6 +448,17 @@ func Restrictions(RestrictToConstellation bool, WhitelistInboundIPs []string) fu return } else if (!isInWhitelist) { PushShieldMetrics("ip-whitelists") + + TriggerEvent( + "cosmos.proxy.shield.whitelist", + "Proxy Shield IP blocked by whitelist", + "warning", + "", + map[string]interface{}{ + "clientID": ip, + "url": r.URL.String(), + }) + IncrementIPAbuseCounter(ip) Error("Request from " + ip + " is blocked because of restrictions", nil) Debug("Blocked by RestrictToConstellation isInConstellation isInWhitelist") @@ -405,6 +468,17 @@ func Restrictions(RestrictToConstellation bool, WhitelistInboundIPs []string) fu } } else if(isUsingWhiteList && !isInWhitelist) { PushShieldMetrics("ip-whitelists") + + TriggerEvent( + "cosmos.proxy.shield.whitelist", + "Proxy Shield IP blocked by whitelist", + "warning", + "", + map[string]interface{}{ + "clientID": ip, + "url": r.URL.String(), + }) + IncrementIPAbuseCounter(ip) Error("Request from " + ip + " is blocked because of restrictions", nil) Debug("Blocked by RestrictToConstellation isInConstellation isUsingWhiteList isInWhitelist")