diff --git a/changelog.md b/changelog.md index aca582a..8379b6d 100644 --- a/changelog.md +++ b/changelog.md @@ -1,3 +1,9 @@ +## Version 0.14.0 + - Cosmos is now fully dockerless + - Improved network IP resolution for containers, including supporting any network mode + - Integrated MongoDB as container + - Removed all sort of container bootstrapping (much faster boot) + ## Version 0.13.0 - Display container stacks as a group in the UI - New Delete modal to delete services entirely diff --git a/client/src/api/docker.jsx b/client/src/api/docker.jsx index e0418c9..1100bce 100644 --- a/client/src/api/docker.jsx +++ b/client/src/api/docker.jsx @@ -1,4 +1,5 @@ import wrap from './wrap'; +import yaml from 'js-yaml'; function list() { return wrap(fetch('/cosmos/api/servapps', { @@ -177,9 +178,9 @@ function createService(serviceData, onProgress) { const requestOptions = { method: 'POST', headers: { - 'Content-Type': 'application/json' + 'Content-Type': 'application/yaml' }, - body: JSON.stringify(serviceData) + body: serviceData }; return fetch('/cosmos/api/docker-service', requestOptions) diff --git a/client/src/pages/servapps/containers/docker-compose.jsx b/client/src/pages/servapps/containers/docker-compose.jsx index 4b1770e..5695995 100644 --- a/client/src/pages/servapps/containers/docker-compose.jsx +++ b/client/src/pages/servapps/containers/docker-compose.jsx @@ -176,196 +176,196 @@ const DockerComposeImport = ({ refresh, dockerComposeInit, installerInit, defaul try { doc = yaml.load(dockerCompose); - if (typeof doc === 'object' && doc !== null && Object.keys(doc).length > 0 && - !doc.services && !doc.networks && !doc.volumes) { - doc = { - services: Object.assign({}, doc) - } - } + // if (typeof doc === 'object' && doc !== null && Object.keys(doc).length > 0 && + // !doc.services && !doc.networks && !doc.volumes) { + // doc = { + // services: Object.assign({}, doc) + // } + // } - // convert to the proper format - if (doc.services) { - Object.keys(doc.services).forEach((key) => { - // convert volumes - if (doc.services[key].volumes) { - if (Array.isArray(doc.services[key].volumes)) { - let volumes = []; - doc.services[key].volumes.forEach((volume) => { - if (typeof volume === 'object') { - volumes.push(volume); - } else { - let volumeSplit = volume.split(':'); - let volumeObj = { - source: volumeSplit[0], - target: volumeSplit[1], - type: (volume[0] === '/' || volume[0] === '.') ? 'bind' : 'volume', - }; - volumes.push(volumeObj); - } - }); - doc.services[key].volumes = volumes; - } - } + // // convert to the proper format + // if (doc.services) { + // Object.keys(doc.services).forEach((key) => { + // // convert volumes + // if (doc.services[key].volumes) { + // if (Array.isArray(doc.services[key].volumes)) { + // let volumes = []; + // doc.services[key].volumes.forEach((volume) => { + // if (typeof volume === 'object') { + // volumes.push(volume); + // } else { + // let volumeSplit = volume.split(':'); + // let volumeObj = { + // source: volumeSplit[0], + // target: volumeSplit[1], + // type: (volume[0] === '/' || volume[0] === '.') ? 'bind' : 'volume', + // }; + // volumes.push(volumeObj); + // } + // }); + // doc.services[key].volumes = volumes; + // } + // } - if(doc.services[key].volumes) - Object.values(doc.services[key].volumes).forEach((volume) => { - if (volume.source && volume.source[0] === '.') { - let defaultPath = (config && config.DockerConfig && config.DockerConfig.DefaultDataPath) || "/usr" - volume.source = defaultPath + volume.source.replace('.', ''); - } - }); + // if(doc.services[key].volumes) + // Object.values(doc.services[key].volumes).forEach((volume) => { + // if (volume.source && volume.source[0] === '.') { + // let defaultPath = (config && config.DockerConfig && config.DockerConfig.DefaultDataPath) || "/usr" + // volume.source = defaultPath + volume.source.replace('.', ''); + // } + // }); - // convert expose - if (doc.services[key].expose) { - doc.services[key].expose = doc.services[key].expose.map((port) => { - return '' + port; - }) - } + // // convert expose + // if (doc.services[key].expose) { + // doc.services[key].expose = doc.services[key].expose.map((port) => { + // return '' + port; + // }) + // } - //convert user - if (doc.services[key].user) { - doc.services[key].user = '' + doc.services[key].user; - } + // //convert user + // if (doc.services[key].user) { + // doc.services[key].user = '' + doc.services[key].user; + // } - // convert labels: - if (doc.services[key].labels) { - if (Array.isArray(doc.services[key].labels)) { - let labels = {}; - doc.services[key].labels.forEach((label) => { - const [key, value] = label.split(/=(.*)/s); - labels['' + key] = '' + value; - }); - doc.services[key].labels = labels; - } - if (typeof doc.services[key].labels == 'object') { - let labels = {}; - Object.keys(doc.services[key].labels).forEach((keylabel) => { - labels['' + keylabel] = '' + doc.services[key].labels[keylabel]; - }); - doc.services[key].labels = labels; - } - } + // // convert labels: + // if (doc.services[key].labels) { + // if (Array.isArray(doc.services[key].labels)) { + // let labels = {}; + // doc.services[key].labels.forEach((label) => { + // const [key, value] = label.split(/=(.*)/s); + // labels['' + key] = '' + value; + // }); + // doc.services[key].labels = labels; + // } + // if (typeof doc.services[key].labels == 'object') { + // let labels = {}; + // Object.keys(doc.services[key].labels).forEach((keylabel) => { + // labels['' + keylabel] = '' + doc.services[key].labels[keylabel]; + // }); + // doc.services[key].labels = labels; + // } + // } - // convert environment - if (doc.services[key].environment) { - if (!Array.isArray(doc.services[key].environment)) { - let environment = []; - Object.keys(doc.services[key].environment).forEach((keyenv) => { - environment.push(keyenv + '=' + doc.services[key].environment[keyenv]); - }); - doc.services[key].environment = environment; - } - } + // // convert environment + // if (doc.services[key].environment) { + // if (!Array.isArray(doc.services[key].environment)) { + // let environment = []; + // Object.keys(doc.services[key].environment).forEach((keyenv) => { + // environment.push(keyenv + '=' + doc.services[key].environment[keyenv]); + // }); + // doc.services[key].environment = environment; + // } + // } - // convert network - if (doc.services[key].networks) { - if (Array.isArray(doc.services[key].networks)) { - let networks = {}; - doc.services[key].networks.forEach((network) => { - if (typeof network === 'object') { - networks['' + network.name] = network; - } - else - networks['' + network] = {}; - }); - doc.services[key].networks = networks; - } - } + // // convert network + // if (doc.services[key].networks) { + // if (Array.isArray(doc.services[key].networks)) { + // let networks = {}; + // doc.services[key].networks.forEach((network) => { + // if (typeof network === 'object') { + // networks['' + network.name] = network; + // } + // else + // networks['' + network] = {}; + // }); + // doc.services[key].networks = networks; + // } + // } - // convert devices - if (doc.services[key].devices) { - console.log(1) - if (Array.isArray(doc.services[key].devices)) { - console.log(2) - let devices = []; - doc.services[key].devices.forEach((device) => { - if(device.indexOf(':') === -1) { - devices.push(device + ':' + device); - } else { - devices.push(device); - } - }); - doc.services[key].devices = devices; - } - } + // // convert devices + // if (doc.services[key].devices) { + // console.log(1) + // if (Array.isArray(doc.services[key].devices)) { + // console.log(2) + // let devices = []; + // doc.services[key].devices.forEach((device) => { + // if(device.indexOf(':') === -1) { + // devices.push(device + ':' + device); + // } else { + // devices.push(device); + // } + // }); + // doc.services[key].devices = devices; + // } + // } - // convert command - if (doc.services[key].command) { - if (typeof doc.services[key].command !== 'string') { - doc.services[key].command = doc.services[key].command.join(' '); - } - } + // // convert command + // if (doc.services[key].command) { + // if (typeof doc.services[key].command !== 'string') { + // doc.services[key].command = doc.services[key].command.join(' '); + // } + // } - // ensure container_name - if (!doc.services[key].container_name) { - doc.services[key].container_name = key; - } + // // ensure container_name + // if (!doc.services[key].container_name) { + // doc.services[key].container_name = key; + // } - // convert healthcheck - if (doc.services[key].healthcheck) { - const toConvert = ["timeout", "interval", "start_period"]; - toConvert.forEach((valT) => { - if(typeof doc.services[key].healthcheck[valT] === 'string') { - let original = doc.services[key].healthcheck[valT]; - let value = parseInt(original); - if (original.endsWith('m')) { - value = value * 60; - } else if (original.endsWith('h')) { - value = value * 60 * 60; - } else if (original.endsWith('d')) { - value = value * 60 * 60 * 24; - } - doc.services[key].healthcheck[valT] = value; - } - }); - } - }); - } + // // convert healthcheck + // if (doc.services[key].healthcheck) { + // const toConvert = ["timeout", "interval", "start_period"]; + // toConvert.forEach((valT) => { + // if(typeof doc.services[key].healthcheck[valT] === 'string') { + // let original = doc.services[key].healthcheck[valT]; + // let value = parseInt(original); + // if (original.endsWith('m')) { + // value = value * 60; + // } else if (original.endsWith('h')) { + // value = value * 60 * 60; + // } else if (original.endsWith('d')) { + // value = value * 60 * 60 * 24; + // } + // doc.services[key].healthcheck[valT] = value; + // } + // }); + // } + // }); + // } - // convert networks - if (doc.networks) { - if (Array.isArray(doc.networks)) { - let networks = {}; - doc.networks.forEach((network) => { - if (typeof network === 'object') { - networks['' + network.name] = network; - } - else - networks['' + network] = {}; - }); - doc.networks = networks; - } else { - let networks = {}; - Object.keys(doc.networks).forEach((key) => { - networks['' + key] = doc.networks[key] || {}; - }); - doc.networks = networks; - } - } + // // convert networks + // if (doc.networks) { + // if (Array.isArray(doc.networks)) { + // let networks = {}; + // doc.networks.forEach((network) => { + // if (typeof network === 'object') { + // networks['' + network.name] = network; + // } + // else + // networks['' + network] = {}; + // }); + // doc.networks = networks; + // } else { + // let networks = {}; + // Object.keys(doc.networks).forEach((key) => { + // networks['' + key] = doc.networks[key] || {}; + // }); + // doc.networks = networks; + // } + // } - // convert volumes - if (doc.volumes) { - if (Array.isArray(doc.volumes)) { - let volumes = {}; - doc.volumes.forEach((volume) => { - if (!volume) { - volume = {}; - } - if (typeof volume === 'object') { - volumes['' + volume.name] = volume; - } - else - volumes['' + volume] = {}; - }); - doc.volumes = volumes; - } else { - let volumes = {}; - Object.keys(doc.volumes).forEach((key) => { - volumes['' + key] = doc.volumes[key] || {}; - }); - doc.volumes = volumes; - } - } + // // convert volumes + // if (doc.volumes) { + // if (Array.isArray(doc.volumes)) { + // let volumes = {}; + // doc.volumes.forEach((volume) => { + // if (!volume) { + // volume = {}; + // } + // if (typeof volume === 'object') { + // volumes['' + volume.name] = volume; + // } + // else + // volumes['' + volume] = {}; + // }); + // doc.volumes = volumes; + // } else { + // let volumes = {}; + // Object.keys(doc.volumes).forEach((key) => { + // volumes['' + key] = doc.volumes[key] || {}; + // }); + // doc.volumes = volumes; + // } + // } } catch (e) { setYmlError(e.message); diff --git a/client/src/pages/servapps/containers/newService.jsx b/client/src/pages/servapps/containers/newService.jsx index 021ed92..024eb67 100644 --- a/client/src/pages/servapps/containers/newService.jsx +++ b/client/src/pages/servapps/containers/newService.jsx @@ -21,6 +21,7 @@ import { Link } from 'react-router-dom'; import { smartDockerLogConcat, tryParseProgressLog } from '../../../utils/docker'; import { LoadingButton } from '@mui/lab'; import LogLine from '../../../components/logLine'; +import yaml from 'js-yaml'; const preStyle = { backgroundColor: '#000', @@ -84,7 +85,7 @@ const NewDockerService = ({service, refresh}) => { setLog([ 'Creating Service... ', ]) - API.docker.createService(service, (newlog) => { + API.docker.createService(yaml.dump(service), (newlog) => { setLog((old) => smartDockerLogConcat(old, newlog)); preRef.current.scrollTop = preRef.current.scrollHeight; if (newlog.includes('[OPERATION SUCCEEDED]')) { @@ -118,8 +119,7 @@ const NewDockerService = ({service, refresh}) => {
         {!log.length && `
 # You are about to create the following service(s):
-
-${JSON.stringify(service, false ,2)}`
+${yaml.dump(service)}`
         }
         {log.map((l) => {
           return 
diff --git a/readme.md b/readme.md
index 3d3dd22..97737fa 100644
--- a/readme.md
+++ b/readme.md
@@ -12,6 +12,7 @@
 Owen
 null
 null
+null
 Daniel C
 

diff --git a/src/configapi/patch.go b/src/configapi/patch.go index 91303df..9e487b4 100644 --- a/src/configapi/patch.go +++ b/src/configapi/patch.go @@ -115,7 +115,8 @@ func ConfigApiPatch(w http.ResponseWriter, req *http.Request) { utils.Log("RouteSettingsUpdate: Service needs update: "+name) - utils.ReBootstrapContainer(name) + // TODO CACHE BURST IN IP RESOLUTION + // utils.ReBootstrapContainer(name) } json.NewEncoder(w).Encode(map[string]interface{}{ diff --git a/src/docker/api_blueprint.go b/src/docker/api_blueprint.go index e7ec40f..db57119 100644 --- a/src/docker/api_blueprint.go +++ b/src/docker/api_blueprint.go @@ -1,7 +1,7 @@ package docker import ( - "encoding/json" + // "encoding/json" "fmt" "net/http" "strings" @@ -12,6 +12,7 @@ import ( "io/ioutil" "os/user" "errors" + // "gopkg.in/yaml.v2" "github.com/docker/go-connections/nat" "github.com/docker/docker/api/types/mount" "github.com/docker/docker/api/types/network" @@ -229,23 +230,76 @@ func CreateServiceRoute(w http.ResponseWriter, req *http.Request) { return } - decoder := json.NewDecoder(req.Body) - var serviceRequest DockerServiceCreateRequest - err := decoder.Decode(&serviceRequest) - if err != nil { - utils.Error("CreateService - decode - ", err) - fmt.Fprintf(w, "[OPERATION FAILED] Bad request: "+err.Error(), http.StatusBadRequest, "DS003") - flusher.Flush() - utils.HTTPError(w, "Bad request: " + err.Error(), http.StatusBadRequest, "DS003") - return - } + // decoder := yaml.NewDecoder(req.Body) + // var serviceRequest DockerServiceCreateRequest + // err := decoder.Decode(&serviceRequest) + // if err != nil { + // utils.Error("CreateService - decode - ", err) + // fmt.Fprintf(w, "[OPERATION FAILED] Bad request: "+err.Error(), http.StatusBadRequest, "DS003") + // flusher.Flush() + // utils.HTTPError(w, "Bad request: " + err.Error(), http.StatusBadRequest, "DS003") + // return + // } - CreateService(serviceRequest, + /*CreateService(serviceRequest, func (msg string) { fmt.Fprintf(w, msg) flusher.Flush() }, - ) + )*/ + + ServiceName := "test-wesh" + + filePath := utils.CONFIGFOLDER + "compose/" + ServiceName + + // Create the folder if does not exist, and output docker-compose.yml + if _, err := os.Stat(utils.CONFIGFOLDER + "compose/"); os.IsNotExist(err) { + os.MkdirAll(utils.CONFIGFOLDER + "compose/", 0750) + } + if _, err := os.Stat(filePath); os.IsNotExist(err) { + os.MkdirAll(filePath, 0750) + } + + // create or truncate the file + file, err := os.Create(filePath + "/docker-compose.yml") + if err != nil { + utils.Error("CreateService - create - ", err) + fmt.Fprintf(w, "[OPERATION FAILED] Internal server error: "+err.Error(), http.StatusInternalServerError, "DS004") + flusher.Flush() + return + } + defer file.Close() + + // write to file + ymlBody := req.Body + writer := bufio.NewWriter(file) + _, err = writer.ReadFrom(ymlBody) + if err != nil { + utils.Error("CreateService - write - ", err) + fmt.Fprintf(w, "[OPERATION FAILED] Internal server error: "+err.Error(), http.StatusInternalServerError, "DS005") + flusher.Flush() + return + } + + writer.Flush() + + // Compose up + err = ComposeUp(filePath, func(message string, outputType int) { + fmt.Fprintf(w, "%s\n", message) + flusher.Flush() + }) + + if err != nil { + utils.Error("CreateService - composeup - ", err) + fmt.Fprintf(w, "[OPERATION FAILED] Internal server error: "+err.Error(), http.StatusInternalServerError, "DS006") + flusher.Flush() + return + } + + + // Write a response to the client + fmt.Fprintf(w, "[OPERATION SUCCESSFUL] Service created successfully", http.StatusOK, "DS007") + flusher.Flush() } else { utils.Error("CreateService: Method not allowed" + req.Method, nil) utils.HTTPError(w, "Method not allowed", http.StatusMethodNotAllowed, "HTTP001") @@ -286,6 +340,21 @@ func CreateService(serviceRequest DockerServiceCreateRequest, OnLog func(string) var rollbackActions []DockerServiceCreateRollback var err error + serviceName := "" + for serviceName = range serviceRequest.Services { + break + } + + // check if serviceRequest.Networks contains service-default, if not, create it + if _, ok := serviceRequest.Networks[serviceName + "-default"]; !ok { + serviceRequest.Networks[serviceName + "-default"] = ContainerCreateRequestNetwork{ + Name: serviceName + "-default", + Driver: "bridge", + Attachable: true, + Internal: false, + } + } + // Create networks for networkToCreateName, networkToCreate := range serviceRequest.Networks { utils.Log(fmt.Sprintf("Creating network %s...", networkToCreateName)) @@ -296,7 +365,7 @@ func CreateService(serviceRequest DockerServiceCreateRequest, OnLog func(string) if err == nil { if networkToCreate.Driver == "" { - networkToCreate.Driver = "bridge" + networkToCreate.Driver = serviceName + "-default" } if (exNetworkDef.Driver != networkToCreate.Driver) { @@ -422,7 +491,8 @@ func CreateService(serviceRequest DockerServiceCreateRequest, OnLog func(string) OnLog(fmt.Sprintf("Checking service %s...\n", serviceName)) // If container request a Cosmos network, create and attach it - if (container.Labels["cosmos-force-network-secured"] == "true" || strings.ToLower(container.Labels["cosmos-network-name"]) == "auto") && + + /*if (container.Labels["cosmos-force-network-secured"] == "true" || strings.ToLower(container.Labels["cosmos-network-name"]) == "auto") && container.Labels["cosmos-network-name"] == "" { utils.Log(fmt.Sprintf("Forcing secure %s...", serviceName)) OnLog(fmt.Sprintf("Forcing secure %s...\n", serviceName)) @@ -465,7 +535,7 @@ func CreateService(serviceRequest DockerServiceCreateRequest, OnLog func(string) AttachNetworkToCosmos(container.Labels["cosmos-network-name"]) } - } + }*/ utils.Log(fmt.Sprintf("Creating container %s...", container.Name)) OnLog(fmt.Sprintf("Creating container %s...\n", container.Name)) @@ -819,6 +889,8 @@ func CreateService(serviceRequest DockerServiceCreateRequest, OnLog func(string) // connect to networks for netName, netConfig := range container.Networks { utils.Log("CreateService: Connecting to network: " + netName) + + //TODO: THIS IS WRONG https://pkg.go.dev/github.com/docker/docker@v24.0.7+incompatible/api/types/network#EndpointSettings err = DockerClient.NetworkConnect(DockerContext, netName, container.Name, &network.EndpointSettings{ Aliases: netConfig.Aliases, IPAddress: netConfig.IPV4Address, diff --git a/src/docker/bootstrap.go b/src/docker/bootstrap.go index 0880977..713c3a4 100644 --- a/src/docker/bootstrap.go +++ b/src/docker/bootstrap.go @@ -1,150 +1,150 @@ package docker -import ( - "github.com/azukaar/cosmos-server/src/utils" - "github.com/docker/docker/api/types" - "os" - "fmt" - "regexp" -) +// import ( +// "github.com/azukaar/cosmos-server/src/utils" +// "github.com/docker/docker/api/types" +// "os" +// "fmt" +// "regexp" +// ) -func BootstrapAllContainersFromTags() []error { - errD := Connect() - if errD != nil { - return []error{errD} - } +// func BootstrapAllContainersFromTags() []error { +// errD := Connect() +// if errD != nil { +// return []error{errD} +// } - errors := []error{} +// errors := []error{} - containers, err := DockerClient.ContainerList(DockerContext, types.ContainerListOptions{}) - if err != nil { - utils.Error("Docker Container List", err) - return []error{err} - } +// containers, err := DockerClient.ContainerList(DockerContext, types.ContainerListOptions{}) +// if err != nil { +// utils.Error("Docker Container List", err) +// return []error{err} +// } - for _, container := range containers { - errB := BootstrapContainerFromTags(container.ID) - if errB != nil { - utils.Error("Bootstrap Container From Tags", errB) - errors = append(errors, errB) - } - } +// for _, container := range containers { +// errB := BootstrapContainerFromTags(container.ID) +// if errB != nil { +// utils.Error("Bootstrap Container From Tags", errB) +// errors = append(errors, errB) +// } +// } - return errors -} +// return errors +// } -func UnsecureContainer(container types.ContainerJSON) (string, error) { - RemoveLabels(container, []string{ - "cosmos-force-network-secured", - }); - return EditContainer(container.ID, container, false) -} +// func UnsecureContainer(container types.ContainerJSON) (string, error) { +// RemoveLabels(container, []string{ +// "cosmos-force-network-secured", +// }); +// return EditContainer(container.ID, container, false) +// } -func BootstrapContainerFromTags(containerID string) error { - errD := Connect() - if errD != nil { - return errD - } +// func BootstrapContainerFromTags(containerID string) error { +// errD := Connect() +// if errD != nil { +// return errD +// } - selfContainer := types.ContainerJSON{} - if os.Getenv("HOSTNAME") != "" { - var errS error - selfContainer, errS = DockerClient.ContainerInspect(DockerContext, os.Getenv("HOSTNAME")) - if errS != nil { - utils.Error("DockerContainerBootstrapSelfInspect", errS) - return errS - } - } +// selfContainer := types.ContainerJSON{} +// if os.Getenv("HOSTNAME") != "" { +// var errS error +// selfContainer, errS = DockerClient.ContainerInspect(DockerContext, os.Getenv("HOSTNAME")) +// if errS != nil { +// utils.Error("DockerContainerBootstrapSelfInspect", errS) +// return errS +// } +// } - utils.Log("Bootstrap Container From Tags: " + containerID) +// utils.Log("Bootstrap Container From Tags: " + containerID) - container, err := DockerClient.ContainerInspect(DockerContext, containerID) - if err != nil { - utils.Error("DockerContainerBootstrapInspect", err) - return err - } +// container, err := DockerClient.ContainerInspect(DockerContext, containerID) +// if err != nil { +// utils.Error("DockerContainerBootstrapInspect", err) +// return err +// } - // check if any route has been added to the container - config := utils.GetMainConfig() - if(!HasLabel(container, "cosmos-network-name")) { - for _, route := range config.HTTPConfig.ProxyConfig.Routes { - utils.Debug("No cosmos-network-name label on container "+container.Name) - pattern := fmt.Sprintf(`(?i)^(([a-z]+):\/\/)?%s(:?[0-9]+)?$`, container.Name[1:]) - match, _ := regexp.MatchString(pattern, route.Target) - if route.Mode == "SERVAPP" && match { - utils.Log("Adding cosmos-network-name label to container "+container.Name) - AddLabels(container, map[string]string{ - "cosmos-network-name": "auto", - }) - } - } - } +// // check if any route has been added to the container +// config := utils.GetMainConfig() +// if(!HasLabel(container, "cosmos-network-name")) { +// for _, route := range config.HTTPConfig.ProxyConfig.Routes { +// utils.Debug("No cosmos-network-name label on container "+container.Name) +// pattern := fmt.Sprintf(`(?i)^(([a-z]+):\/\/)?%s(:?[0-9]+)?$`, container.Name[1:]) +// match, _ := regexp.MatchString(pattern, route.Target) +// if route.Mode == "SERVAPP" && match { +// utils.Log("Adding cosmos-network-name label to container "+container.Name) +// AddLabels(container, map[string]string{ +// "cosmos-network-name": "auto", +// }) +// } +// } +// } - // Check cosmos-network-name tag - isCosmosCon, _, needsUpdate := IsConnectedToASecureCosmosNetwork(selfContainer, container) +// // Check cosmos-network-name tag +// isCosmosCon, _, needsUpdate := IsConnectedToASecureCosmosNetwork(selfContainer, container) - if(IsLabel(container, "cosmos-force-network-secured")) { - utils.Log(container.Name+": Checking Force network secured") +// if(IsLabel(container, "cosmos-force-network-secured")) { +// utils.Log(container.Name+": Checking Force network secured") - // check if connected to bridge and to a cosmos network - isCon := IsConnectedToNetwork(container, "bridge") +// // check if connected to bridge and to a cosmos network +// isCon := IsConnectedToNetwork(container, "bridge") - if isCon || !isCosmosCon { - utils.Log(container.Name+": Needs isolating on a secured network") - needsRestart := false - var errCT error - if !isCosmosCon { - utils.Debug(container.Name+": Not connected to a cosmos network") - needsRestart, errCT = ConnectToSecureNetwork(container) - if errCT != nil { - utils.Warn("DockerContainerBootstrapConnectToSecureNetwork -- Cannot connect to network, removing force secure") - _, errUn := UnsecureContainer(container) - if errUn != nil { - utils.Fatal("DockerContainerBootstrapUnsecureContainer -- A broken container state is preventing Cosmos from functionning. Please remove the cosmos-force-secure label from the container "+container.Name+" manually", errUn) - return errCT - } - return errCT - } - if needsRestart { - utils.Log(container.Name+": Will restart to apply changes") - needsUpdate = true - } else { - utils.Log(container.Name+": Connected to new network") - } - } - if !needsRestart && isCon { - utils.Log(container.Name+": Disconnecting from bridge network") - errDisc := DockerClient.NetworkDisconnect(DockerContext, "bridge", containerID, true) - if errDisc != nil { - utils.Warn("DockerContainerBootstrapDisconnectFromBridge -- Cannot disconnect from Bridge, removing force secure") - _, errUn := UnsecureContainer(container) - if errUn != nil { - utils.Fatal("DockerContainerBootstrapUnsecureContainer -- A broken container state is preventing Cosmos from functionning. Please remove the cosmos-force-secure label from the container "+container.Name+" manually", errUn) - return errDisc - } - return errDisc - } - } - } +// if isCon || !isCosmosCon { +// utils.Log(container.Name+": Needs isolating on a secured network") +// needsRestart := false +// var errCT error +// if !isCosmosCon { +// utils.Debug(container.Name+": Not connected to a cosmos network") +// needsRestart, errCT = ConnectToSecureNetwork(container) +// if errCT != nil { +// utils.Warn("DockerContainerBootstrapConnectToSecureNetwork -- Cannot connect to network, removing force secure") +// _, errUn := UnsecureContainer(container) +// if errUn != nil { +// utils.Fatal("DockerContainerBootstrapUnsecureContainer -- A broken container state is preventing Cosmos from functionning. Please remove the cosmos-force-secure label from the container "+container.Name+" manually", errUn) +// return errCT +// } +// return errCT +// } +// if needsRestart { +// utils.Log(container.Name+": Will restart to apply changes") +// needsUpdate = true +// } else { +// utils.Log(container.Name+": Connected to new network") +// } +// } +// if !needsRestart && isCon { +// utils.Log(container.Name+": Disconnecting from bridge network") +// errDisc := DockerClient.NetworkDisconnect(DockerContext, "bridge", containerID, true) +// if errDisc != nil { +// utils.Warn("DockerContainerBootstrapDisconnectFromBridge -- Cannot disconnect from Bridge, removing force secure") +// _, errUn := UnsecureContainer(container) +// if errUn != nil { +// utils.Fatal("DockerContainerBootstrapUnsecureContainer -- A broken container state is preventing Cosmos from functionning. Please remove the cosmos-force-secure label from the container "+container.Name+" manually", errUn) +// return errDisc +// } +// return errDisc +// } +// } +// } - if(len(GetAllPorts(container)) > 0) { - utils.Log("Removing unsecure ports bindings from "+container.Name) - // remove all ports - UnexposeAllPorts(&container) - needsUpdate = true - } - } +// if(len(GetAllPorts(container)) > 0) { +// utils.Log("Removing unsecure ports bindings from "+container.Name) +// // remove all ports +// UnexposeAllPorts(&container) +// needsUpdate = true +// } +// } - if(needsUpdate) { - _, errEdit := EditContainer(containerID, container, false) - if errEdit != nil { - utils.Error("Docker Boostrap, couldn't update container: ", errEdit) - return errEdit - } - utils.Debug("Done updating Container From Tags after Bootstrapping: " + container.Name) - } +// if(needsUpdate) { +// _, errEdit := EditContainer(containerID, container, false) +// if errEdit != nil { +// utils.Error("Docker Boostrap, couldn't update container: ", errEdit) +// return errEdit +// } +// utils.Debug("Done updating Container From Tags after Bootstrapping: " + container.Name) +// } - utils.Log("Done bootstrapping Container From Tags: " + container.Name) +// utils.Log("Done bootstrapping Container From Tags: " + container.Name) - return nil -} +// return nil +// } diff --git a/src/docker/compose.go b/src/docker/compose.go new file mode 100644 index 0000000..892510b --- /dev/null +++ b/src/docker/compose.go @@ -0,0 +1,58 @@ +package docker + +import ( + "bufio" + "io" + "os/exec" + + "github.com/azukaar/cosmos-server/src/utils" +) + +const ( + Stdout = iota + Stderr +) + + +// readOutput reads from the given reader and uses the callback function to handle the output. +func readOutput(r io.Reader, callback func(message string, outputType int), outputType int) { + scanner := bufio.NewScanner(r) + for scanner.Scan() { + callback(scanner.Text(), outputType) + } +} + +// ExecCommand runs the specified command and uses a callback function to handle the output. +func ExecCommand(callback func(message string, outputType int), args ...string) error { + cmd := exec.Command(args[0], args[1:]...) + utils.Debug("Running command: " + cmd.String()) + + callback("Running command: " + cmd.String(), Stdout) + + // Create pipes for stdout and stderr + stdoutPipe, err := cmd.StdoutPipe() + if err != nil { + return err + } + + stderrPipe, err := cmd.StderrPipe() + if err != nil { + return err + } + + // Start the command + if err := cmd.Start(); err != nil { + return err + } + + // Read from stdout and stderr in separate goroutines + go readOutput(stdoutPipe, callback, Stdout) + go readOutput(stderrPipe, callback, Stderr) + + // Wait for the command to finish + return cmd.Wait() +} + +func ComposeUp(filePath string, callback func(message string, outputType int)) error { + return ExecCommand(callback, "docker", "compose", "--project-directory", filePath, "up", "--remove-orphans", "-d") +} \ No newline at end of file diff --git a/src/docker/events.go b/src/docker/events.go index e62e306..047ea7d 100644 --- a/src/docker/events.go +++ b/src/docker/events.go @@ -140,7 +140,7 @@ func DebouncedExportDocker() { func onDockerStarted(containerID string) { utils.Debug("onDockerStarted: " + containerID) - BootstrapContainerFromTags(containerID) + // BootstrapContainerFromTags(containerID) DebouncedExportDocker() } diff --git a/src/docker/ip.go b/src/docker/ip.go index c9ca337..8a2e7dc 100644 --- a/src/docker/ip.go +++ b/src/docker/ip.go @@ -4,8 +4,10 @@ import ( "fmt" "sync" "time" + "strings" "github.com/azukaar/cosmos-server/src/utils" + "github.com/docker/docker/api/types" ) type Cache struct { @@ -49,22 +51,68 @@ func (c *Cache) Set(key string, value string, duration time.Duration) { } func _getContainerIPByName(containerName string) (string, error) { + errD := Connect() + if errD != nil { + return "", errD + } + container, err := DockerClient.ContainerInspect(DockerContext, containerName) if err != nil { return "", err } // Prioritize "host" - if net, ok := container.NetworkSettings.Networks["host"]; ok && net.IPAddress != "" { - return net.IPAddress, nil - } + // if net, ok := container.NetworkSettings.Networks["host"]; ok && net.IPAddress != "" { + // return "localhost", nil + // } // Next, prioritize "bridge" - if net, ok := container.NetworkSettings.Networks["bridge"]; ok && net.IPAddress != "" { - return net.IPAddress, nil + // if net, ok := container.NetworkSettings.Networks["bridge"]; ok && net.IPAddress != "" { + // return net.IPAddress, nil + // } + + // if container is in network mode host + if container.HostConfig.NetworkMode == "host" { + return "localhost", nil + } else if container.HostConfig.NetworkMode == "bridge" || container.HostConfig.NetworkMode == "default" || container.HostConfig.NetworkMode == "" { + // if container is in network mode bridge or default or not set + + for _, net := range container.NetworkSettings.Networks { + // if this network is using the bridge driver + if net.IPAddress != "" { + return net.IPAddress, nil + } + } + } else if strings.HasPrefix(string(container.HostConfig.NetworkMode), "container:") { + // if container is in network mode container: + + // get the container name + otherContainerName := strings.TrimPrefix(string(container.HostConfig.NetworkMode), "container:") + // get the ip of the other container + ip, err := _getContainerIPByName(otherContainerName) + if err != nil { + return "", err + } + return ip, nil + } else { + // if container is in network mode + + // get the network name + networkName := string(container.HostConfig.NetworkMode) + // get the network + network, err := DockerClient.NetworkInspect(DockerContext, networkName, types.NetworkInspectOptions{}) + if err != nil { + return "", err + } + // get the ip of the container in the network + for _, containerEP := range network.Containers { + if containerEP.Name == container.Name { + return network.IPAM.Config[0].Gateway, nil + } + } } - // Finally, return the IP of the first network we find + // Finally, if nothing, return the IP of the first network we find for _, net := range container.NetworkSettings.Networks { if net.IPAddress != "" { return net.IPAddress, nil diff --git a/src/httpServer.go b/src/httpServer.go index 12a9244..db94ce1 100644 --- a/src/httpServer.go +++ b/src/httpServer.go @@ -41,7 +41,11 @@ func startHTTPServer(router *mux.Router) error { DisableGeneralOptionsHandler: true, } - docker.CheckPorts() + if os.Getenv("HOSTNAME") != "" { + docker.CheckPorts() + } else { + proxy.InitInternalTCPProxy() + } utils.Log("Listening to HTTP on : 0.0.0.0:" + serverPortHTTP) @@ -119,7 +123,11 @@ func startHTTPSServer(router *mux.Router) error { } // Redirect ports - docker.CheckPorts() + if os.Getenv("HOSTNAME") != "" { + docker.CheckPorts() + } else { + proxy.InitInternalTCPProxy() + } utils.Log("Now listening to HTTPS on :" + serverPortHTTPS) diff --git a/src/index.go b/src/index.go index 385bb2a..02f07cf 100644 --- a/src/index.go +++ b/src/index.go @@ -16,8 +16,9 @@ import ( func main() { utils.Log("Starting...") - utils.ReBootstrapContainer = docker.BootstrapContainerFromTags + // utils.ReBootstrapContainer = docker.BootstrapContainerFromTags utils.PushShieldMetrics = metrics.PushShieldMetrics + utils.GetContainerIPByName = docker.GetContainerIPByName rand.Seed(time.Now().UnixNano()) @@ -31,7 +32,7 @@ func main() { docker.DockerListenEvents() - docker.BootstrapAllContainersFromTags() + // docker.BootstrapAllContainersFromTags() docker.RemoveSelfUpdater() diff --git a/src/proxy/TCPProxy.go b/src/proxy/TCPProxy.go new file mode 100644 index 0000000..fc9d927 --- /dev/null +++ b/src/proxy/TCPProxy.go @@ -0,0 +1,142 @@ +package proxy + +import ( + "io" + "net" + "sync" + "strings" + + "github.com/azukaar/cosmos-server/src/utils" +) + +var ( + activeProxies map[string]chan bool + proxiesLock sync.Mutex +) + +func handleClient(client net.Conn, server net.Conn, stop chan bool) { + defer client.Close() + defer server.Close() + + // Forward data between client and server, and watch for stop signal + done := make(chan struct{}) + go func() { + io.Copy(server, client) + done <- struct{}{} + }() + go func() { + io.Copy(client, server) + done <- struct{}{} + }() + + select { + case <-stop: + return + case <-done: + return + } +} + +func startProxy(listenAddr string, target string, stop chan bool) { + listener, err := net.Listen("tcp", listenAddr) + if err != nil { + utils.Error("Failed to listen on " + listenAddr, err) + } + defer listener.Close() + + utils.Log("Proxy listening on "+listenAddr+", forwarding to " + target) + + for { + select { + case <-stop: + return + default: + client, err := listener.Accept() + if err != nil { + utils.Error("Failed to accept connection: %v", err) + continue + } + + server, err := net.Dial("tcp", target) + if err != nil { + utils.Error("Failed to connect to server: %v", err) + client.Close() + continue + } + + go handleClient(client, server, stop) + } + } +} + +func initInternalPortProxy(ports []string, destination string) { + proxiesLock.Lock() + defer proxiesLock.Unlock() + + // Initialize activeProxies map if it's nil + if activeProxies == nil { + activeProxies = make(map[string]chan bool) + } + + // Stop any existing proxies that are not in the new list + for port, stop := range activeProxies { + if !contains(ports, port) { + close(stop) + delete(activeProxies, port) + } + } + + // Start new proxies for ports in the list that aren't already running + for _, port := range ports { + if _, exists := activeProxies[port]; !exists { + utils.Log("Network Starting internal proxy for port " + port) + stop := make(chan bool) + activeProxies[port] = stop + go startProxy(":"+port, destination, stop) + } + } +} + +// Helper function to check if a slice contains a string +func contains(slice []string, item string) bool { + for _, a := range slice { + if a == item { + return true + } + } + return false +} + +func InitInternalTCPProxy() { + utils.Log("Network: Initializing internal TCP proxy") + + config := utils.GetMainConfig() + expectedPorts := []string{} + isHTTPS := utils.IsHTTPS + HTTPPort := config.HTTPConfig.HTTPPort + HTTPSPort := config.HTTPConfig.HTTPSPort + routes := config.HTTPConfig.ProxyConfig.Routes + targetPort := HTTPPort + if isHTTPS { + targetPort = HTTPSPort + } + + for _, route := range routes { + if route.UseHost && strings.Contains(route.Host, ":") { + hostname := route.Host + port := strings.Split(hostname, ":")[1] + expectedPorts = append(expectedPorts, port) + } + } + + // append hostname port + hostname := config.HTTPConfig.Hostname + if strings.Contains(hostname, ":") { + hostnameport := strings.Split(hostname, ":")[1] + if hostnameport != targetPort { + expectedPorts = append(expectedPorts, hostnameport) + } + } + + initInternalPortProxy(expectedPorts, "localhost:"+targetPort) +} \ No newline at end of file diff --git a/src/utils/db.go b/src/utils/db.go index 1f60d35..5479cd9 100644 --- a/src/utils/db.go +++ b/src/utils/db.go @@ -16,6 +16,7 @@ import ( var client *mongo.Client +var IsDBaContainer bool func DB() error { if(GetMainConfig().DisableUserManagement) { @@ -37,6 +38,19 @@ func DB() error { var err error opts := options.Client().ApplyURI(mongoURL).SetRetryWrites(true).SetWriteConcern(writeconcern.New(writeconcern.WMajority())) + + IsDBaContainer = false + + if os.Getenv("HOSTNAME") == "" { + hostname := opts.Hosts[0] + Log("Getting Mongo DB IP from name : " + hostname) + ip, _ := GetContainerIPByName(hostname) + if ip != "" { + IsDBaContainer = true + opts.SetHosts([]string{ip + ":27017"}) + Log("Mongo DB IP : " + ip) + } + } client, err = mongo.Connect(context.TODO(), opts) diff --git a/src/utils/utils.go b/src/utils/utils.go index 2d0c850..829457f 100644 --- a/src/utils/utils.go +++ b/src/utils/utils.go @@ -33,7 +33,8 @@ var NeedsRestart = false var UpdateAvailable = map[string]bool{} var RestartHTTPServer func() -var ReBootstrapContainer func(string) error +// var ReBootstrapContainer func(string) error +var GetContainerIPByName func(string) (string, error) var LetsEncryptErrors = []string{}