Merge pull request #22 from Moonlight-Panel/CleanupFix

Fixed cleanup
This commit is contained in:
Marcel Baumgartner 2023-04-03 02:27:56 +02:00 committed by GitHub
commit 0d7c29bc8e
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
5 changed files with 253 additions and 478 deletions

View file

@ -13,353 +13,204 @@ namespace Moonlight.App.Services;
public class CleanupService
{
#region Stats
public DateTime StartedAt { get; private set; }
public DateTime CompletedAt { get; private set; }
public int ServersCleaned { get; private set; }
public int CleanupsPerformed { get; private set; }
public int ServersRunning { get; private set; }
public bool IsRunning { get; private set; }
public bool Activated { get; set; }
public int PercentProgress { get; private set; } = 100;
public string Status { get; private set; } = "N/A";
private Task PerformTask;
private readonly ConfigService ConfigService;
private readonly IServiceScopeFactory ServiceScopeFactory;
private int RequiredCpu;
private long RequiredMemory;
private int WaitTime;
public EventHandler OnUpdated;
#endregion
public CleanupService(ConfigService configService, IServiceScopeFactory serviceScopeFactory)
private readonly ConfigService ConfigService;
private readonly MessageService MessageService;
private readonly IServiceScopeFactory ServiceScopeFactory;
private readonly PeriodicTimer Timer;
public CleanupService(
ConfigService configService,
IServiceScopeFactory serviceScopeFactory,
MessageService messageService)
{
ServiceScopeFactory = serviceScopeFactory;
MessageService = messageService;
ConfigService = configService;
var config = configService.GetSection("Moonlight").GetSection("Cleanup");
RequiredCpu = config.GetValue<int>("Cpu");
RequiredMemory = config.GetValue<long>("Memory");
WaitTime = config.GetValue<int>("Wait");
if (!ConfigService.DebugMode)
Task.Run(Start);
}
private void Start()
{
StartedAt = DateTime.Now;
CompletedAt = DateTime.Now;
IsRunning = false;
Activated = true;
var config = ConfigService.GetSection("Moonlight").GetSection("Cleanup");
DoWaiting();
}
private async void DoWaiting()
{
while (true)
if (!config.GetValue<bool>("Enable") || ConfigService.DebugMode)
{
if (Activated)
await Perform();
try
{
await Task.Delay((int) TimeSpan.FromMinutes(WaitTime).TotalMilliseconds);
}
catch (Exception ex)
{
}
Logger.Info("Disabling cleanup service");
return;
}
}
public async Task TriggerPerform()
{
if (IsRunning)
return;
Timer = new(TimeSpan.FromMinutes(config.GetValue<int>("Wait")));
PerformTask = new Task(async () => await Perform());
PerformTask.Start();
Task.Run(Run);
}
private async Task Perform()
private async Task Run()
{
if (IsRunning)
return;
IsRunning = true;
StartedAt = DateTime.Now;
ServersRunning = 0;
OnUpdated?.Invoke(this, null);
using (var scope = ServiceScopeFactory.CreateScope())
while (await Timer.WaitForNextTickAsync())
{
// Setup time measure
var watch = new Stopwatch();
watch.Start();
IsRunning = true;
// Get repos from dependency injection
var serverRepository = scope.ServiceProvider.GetRequiredService<ServerRepository>();
using var scope = ServiceScopeFactory.CreateScope();
var config = ConfigService.GetSection("Moonlight").GetSection("Cleanup");
var maxCpu = config.GetValue<int>("Cpu");
var minMemory = config.GetValue<int>("Memory");
var maxUptime = config.GetValue<int>("Uptime");
var minUptime = config.GetValue<int>("MinUptime");
var nodeRepository = scope.ServiceProvider.GetRequiredService<NodeRepository>();
var nodeService = scope.ServiceProvider.GetRequiredService<NodeService>();
var imageRepo = scope.ServiceProvider.GetRequiredService<ImageRepository>();
var serverService = scope.ServiceProvider.GetRequiredService<ServerService>();
// Fetching data from mysql
var servers = serverRepository.Get()
.Include(x => x.Image)
var nodes = nodeRepository
.Get()
.ToArray();
var nodes = nodeRepository.Get().ToArray();
var exceptions = servers.Where(x => x.IsCleanupException);
var images = imageRepo.Get().ToArray();
var nodeCount = nodes.Count();
// We use this counter for the foreach loops
int counter = 0;
PercentProgress = 0;
// Fetching data from nodes so we know what nodes to scan
var nodeContainers = new Dictionary<Node, ContainerStats.Container[]>();
Status = "Checking Nodes";
counter = 0;
PercentProgress = 0;
OnUpdated?.Invoke(this, null);
foreach (var node in nodes)
{
try
{
var cpu = await nodeService.GetCpuStats(node);
var freeMemory = await nodeService.GetMemoryStats(node);
var cpuStats = await nodeService.GetCpuStats(node);
var memoryStats = await nodeService.GetMemoryStats(node);
if (cpu.Usage > RequiredCpu || freeMemory.Free < RequiredMemory)
if (cpuStats.Usage > maxCpu || memoryStats.Free < minMemory)
{
var c = await nodeService.GetContainerStats(node);
var containers = c.Containers;
nodeContainers.Add(node, containers.ToArray());
}
}
catch (Exception e)
{
Logger.Error($"Error fetching cleanup data from node {node.Id}");
Logger.Error(e);
}
var containerStats = await nodeService.GetContainerStats(node);
counter++;
CalculateAndUpdateProgress(counter, nodeCount);
OnUpdated?.Invoke(this, null);
}
var serverRepository = scope.ServiceProvider.GetRequiredService<ServerRepository>();
var imageRepository = scope.ServiceProvider.GetRequiredService<ImageRepository>();
// Searching for servers we can actually stop because they have the cleanup tag
// and determine which servers we have to check for an illegal mc server
var serversToCheck = new List<Server>();
var serversToCheckForMc = new List<Server>();
Status = "Checking found servers";
counter = 0;
PercentProgress = 0;
OnUpdated?.Invoke(this, null);
// Count every container for progress calculation
var allContainers = 0;
foreach (var array in nodeContainers)
{
allContainers += array.Value.Length;
}
foreach (var nodeContainer in nodeContainers)
{
try
{
foreach (var container in nodeContainer.Value)
{
var server = servers.First(x => x.Uuid.ToString() == container.Name);
var tagsJson = imageRepo
var images = imageRepository
.Get()
.First(x => x.Id == server.Image.Id).TagsJson;
.ToArray();
var tags = JsonConvert.DeserializeObject<string[]>(tagsJson) ?? Array.Empty<string>();
var imagesWithFlag = images
.Where(x =>
(JsonConvert.DeserializeObject<string[]>(x.TagsJson) ?? Array.Empty<string>()).Contains("cleanup")
)
.ToArray();
var containerMappedToServers = new Dictionary<ContainerStats.Container, Server>();
if (tags.FirstOrDefault(x => x == "cleanup") != null)
foreach (var container in containerStats.Containers)
{
serversToCheck.Add(server);
}
if (tags.FirstOrDefault(x => x == "illegalmc") != null)
{
serversToCheckForMc.Add(server);
}
}
}
catch (Exception e)
{
Logger.Error($"Error processing cleanup data from node {nodeContainer.Key.Id}");
Logger.Error(e);
}
counter++;
CalculateAndUpdateProgress(counter, allContainers);
OnUpdated?.Invoke(this, null);
}
// Now we gonna scan every tagged server
Status = "Scanning servers";
counter = 0;
PercentProgress = 0;
OnUpdated?.Invoke(this, null);
foreach (var server in serversToCheck)
{
try
{
var serverData = serverRepository
.Get()
.Include(x => x.MainAllocation)
.Include(x => x.Node)
.Include(x => x.Variables)
.First(x => x.Id == server.Id);
var players = GetPlayers(serverData.Node, serverData.MainAllocation);
var stats = await serverService.GetDetails(server);
var exception = exceptions.FirstOrDefault(x => x.Id == server.Id) != null;
if (stats != null)
{
if (exception)
{
if (players == 0 && stats.Utilization.Uptime > TimeSpan.FromHours(6).TotalMilliseconds)
if (Guid.TryParse(container.Name, out Guid uuid))
{
await serverService.SetPowerState(server, PowerSignal.Restart);
ServersCleaned++;
OnUpdated?.Invoke(this, null);
}
else
{
ServersRunning++;
OnUpdated?.Invoke(this, null);
}
}
else
{
if (players == 0 && stats.Utilization.Uptime > TimeSpan.FromMinutes(10).TotalMilliseconds)
{
var cleanupVar = serverData.Variables.FirstOrDefault(x => x.Key == "J2S");
var server = serverRepository
.Get()
.Include(x => x.Image)
.Include(x => x.MainAllocation)
.Include(x => x.Variables)
.FirstOrDefault(x => x.Uuid == uuid);
if (cleanupVar == null)
if (server != null && imagesWithFlag.Any(y => y.Id == server.Image.Id))
{
await serverService.SetPowerState(server, PowerSignal.Stop);
ServersCleaned++;
OnUpdated?.Invoke(this, null);
containerMappedToServers.Add(container, server);
}
}
}
var serverService = scope.ServiceProvider.GetRequiredService<ServerService>();
foreach (var containerMapped in containerMappedToServers)
{
var server = containerMapped.Value;
try
{
var stats = await serverService.GetDetails(server);
if (server.IsCleanupException)
{
if (stats.Utilization.Uptime > TimeSpan.FromHours(maxUptime).TotalMilliseconds)
{
var players = GetPlayers(node, server.MainAllocation);
if (players == 0)
{
await serverService.SetPowerState(server, PowerSignal.Restart);
ServersCleaned++;
}
else
{
ServersRunning++;
}
await MessageService.Emit("cleanup.updated", null);
}
}
else
{
if (cleanupVar.Value == "1")
if (stats.Utilization.Uptime > TimeSpan.FromMinutes(minUptime).TotalMilliseconds)
{
await serverService.SetPowerState(server, PowerSignal.Restart);
ServersCleaned++;
OnUpdated?.Invoke(this, null);
}
else
{
await serverService.SetPowerState(server, PowerSignal.Stop);
ServersCleaned++;
OnUpdated?.Invoke(this, null);
var players = GetPlayers(node, server.MainAllocation);
if (players < 1)
{
var j2SVar = server.Variables.FirstOrDefault(x => x.Key == "J2S");
var handleJ2S = j2SVar != null && j2SVar.Value == "1";
if (handleJ2S)
{
await serverService.SetPowerState(server, PowerSignal.Restart);
}
else
{
await serverService.SetPowerState(server, PowerSignal.Stop);
}
ServersCleaned++;
}
else
{
ServersRunning++;
}
await MessageService.Emit("cleanup.updated", null);
}
}
}
else
catch (Exception e)
{
ServersRunning++;
OnUpdated?.Invoke(this, null);
Logger.Warn($"Error checking server {server.Name} ({server.Id})");
Logger.Warn(e);
}
}
}
}
catch (Exception e)
{
Logger.Error($"Error scanning {server.Name}");
Logger.Error($"Error performing cleanup on node {node.Name} ({node.Id})");
Logger.Error(e);
}
counter++;
CalculateAndUpdateProgress(counter, serversToCheck.Count);
OnUpdated?.Invoke(this, null);
}
// Finally we have to check all code container allocations
// for illegal hosted mc servers
Status = "Scanning code containers";
counter = 0;
PercentProgress = 0;
OnUpdated?.Invoke(this, null);
foreach (var server in serversToCheckForMc)
{
try
{
var serverData = serverRepository
.Get()
.Include(x => x.Allocations)
.Include(x => x.Node)
.First(x => x.Id == server.Id);
foreach (var allocation in serverData.Allocations)
{
if (GetPlayers(server.Node, allocation) != -1)
{
// TODO: Suspend server
Logger.Warn("Found CC running mc: https://moonlight.endelon-hosting.de/server/" +
server.Uuid + "/");
}
}
}
catch (Exception e)
{
Logger.Error($"Error scanning (cc) {server.Name}");
Logger.Error(e);
}
counter++;
CalculateAndUpdateProgress(counter, serversToCheckForMc.Count);
OnUpdated?.Invoke(this, null);
}
watch.Stop();
Status = $"Cleanup finished. Duration: {Math.Round(TimeSpan.FromMilliseconds(watch.ElapsedMilliseconds).TotalMinutes, 2)} Minutes";
PercentProgress = 100;
OnUpdated?.Invoke(this, null);
IsRunning = false;
CleanupsPerformed++;
await MessageService.Emit("cleanup.updated", null);
}
IsRunning = false;
CompletedAt = DateTime.Now;
CleanupsPerformed++;
OnUpdated?.Invoke(this, null);
}
private int GetPlayers(Node node, NodeAllocation allocation)
{
var ms = new MineStat(node.Fqdn, (ushort)allocation.Port);
//TODO: Add fake player check
if (ms.ServerUp)
{
return ms.CurrentPlayersInt;
}
else
{
return -1;
}
}
private void CalculateAndUpdateProgress(int now, int all)
{
PercentProgress = (int)Math.Round((now / (double)all) * 100);
return -1;
}
}

View file

@ -1,4 +1,5 @@
using System.Text;
using Logging.Net;
using Microsoft.Extensions.Primitives;
using Moonlight.App.Helpers;
@ -21,6 +22,9 @@ public class ConfigService : IConfiguration
if (debugVar != null)
DebugMode = bool.Parse(debugVar);
if(DebugMode)
Logger.Debug("Debug mode enabled");
}
public IEnumerable<IConfigurationSection> GetChildren()

View file

@ -0,0 +1,103 @@
@page "/admin/servers/cleanup"
@using Moonlight.App.Services
@using Moonlight.App.Models.Misc
@using Moonlight.App.Services.LogServices
@inject CleanupService CleanupService
@inject AuditLogService AuditLogService
@inject MessageService MessageService
@implements IDisposable
<OnlyAdmin>
<div class="row g-5 g-xl-10 mb-5 mb-xl-10">
<div class="col-xl-3">
<div class="card card-flush bgi-no-repeat bgi-size-contain bgi-position-x-end h-xl-100" style="background-color: #170049;">
<div class="card-header pt-5 mb-3">
<div class="d-flex flex-center rounded-circle h-80px w-80px" style="border: 1px rgba(255, 255, 255, 0.4);background-color: #7239EA">
<i class="text-white bx bxs-skull bx-lg"></i>
</div>
</div>
<div class="card-body d-flex align-items-end mb-3">
<div class="d-flex align-items-center">
<span class="fs-4hx text-white fw-bold me-6">@(CleanupService.ServersCleaned)</span>
<div class="fw-bold fs-6 text-white">
<span class="d-block">
<TL>Servers</TL>
</span>
<span>
<TL>stopped</TL>
</span>
</div>
</div>
</div>
</div>
</div>
<div class="col-xl-3">
<div class="card card-flush bgi-no-repeat bgi-size-contain bgi-position-x-end h-xl-100" style="background-color: #170049;">
<div class="card-header pt-5 mb-3">
<div class="d-flex flex-center rounded-circle h-80px w-80px" style="border: 1px rgba(255, 255, 255, 0.4);background-color: #7239EA">
<i class="text-white bx bx-transfer bx-lg"></i>
</div>
</div>
<div class="card-body d-flex align-items-end mb-3">
<div class="d-flex align-items-center">
<span class="fs-4hx text-white fw-bold me-6">@(CleanupService.CleanupsPerformed)</span>
<div class="fw-bold fs-6 text-white">
<span class="d-block">
<TL>Cleanups</TL>
</span>
<span>
<TL>executed</TL>
</span>
</div>
</div>
</div>
</div>
</div>
<div class="col-xl-3">
<div class="card card-flush bgi-no-repeat bgi-size-contain bgi-position-x-end h-xl-100" style="background-color: #170049;">
<div class="card-header pt-5 mb-3">
<div class="d-flex flex-center rounded-circle h-80px w-80px" style="border: 1px rgba(255, 255, 255, 0.4);background-color: #7239EA">
<i class="text-white bx bx-rocket bx-lg"></i>
</div>
</div>
<div class="card-body d-flex align-items-end mb-3">
<div class="d-flex align-items-center">
<span class="fs-4hx text-white fw-bold me-6">@(CleanupService.ServersRunning)</span>
<div class="fw-bold fs-6 text-white">
<span class="d-block">
<TL>Running cleanup</TL>
</span>
<span>
<TL>servers</TL>
</span>
</div>
</div>
</div>
</div>
</div>
</div>
</OnlyAdmin>
@code
{
protected override void OnInitialized()
{
MessageService.Subscribe<Cleanup, Object>("cleanup.updated", this, _ =>
{
Task.Run(async () =>
{
await InvokeAsync(StateHasChanged);
});
return Task.CompletedTask;
});
}
public void Dispose()
{
MessageService.Unsubscribe("cleanup.updated", this);
}
}

View file

@ -1,200 +0,0 @@
@page "/admin/servers/cleanup"
@using Moonlight.App.Services
@using Logging.Net
@using Moonlight.App.Models.Misc
@using Moonlight.App.Services.LogServices
@inject CleanupService CleanupService
@inject AuditLogService AuditLogService
@implements IDisposable
<PageTitle>
<TL>Cleanup</TL>
</PageTitle>
<IsAdmin>
<div class="row g-5 g-xl-10 mb-5 mb-xl-10">
<div class="col-xl-3">
<div class="card card-flush bgi-no-repeat bgi-size-contain bgi-position-x-end h-xl-100" style="background-color: #170049;">
<div class="card-header pt-5 mb-3">
<div class="d-flex flex-center rounded-circle h-80px w-80px" style="border: 1px rgba(255, 255, 255, 0.4);background-color: #7239EA">
<i class="text-white bx bxs-skull bx-lg"></i>
</div>
</div>
<div class="card-body d-flex align-items-end mb-3">
<div class="d-flex align-items-center">
<span class="fs-4hx text-white fw-bold me-6">@(CleanupService.ServersCleaned)</span>
<div class="fw-bold fs-6 text-white">
<span class="d-block">
<TL>Servers</TL>
</span>
<span>
<TL>stopped</TL>
</span>
</div>
</div>
</div>
</div>
</div>
<div class="col-xl-3">
<div class="card card-flush bgi-no-repeat bgi-size-contain bgi-position-x-end h-xl-100" style="background-color: #170049;">
<div class="card-header pt-5 mb-3">
<div class="d-flex flex-center rounded-circle h-80px w-80px" style="border: 1px rgba(255, 255, 255, 0.4);background-color: #7239EA">
<i class="text-white bx bx-transfer bx-lg"></i>
</div>
</div>
<div class="card-body d-flex align-items-end mb-3">
<div class="d-flex align-items-center">
<span class="fs-4hx text-white fw-bold me-6">@(CleanupService.CleanupsPerformed)</span>
<div class="fw-bold fs-6 text-white">
<span class="d-block">
<TL>Cleanups</TL>
</span>
<span>
<TL>executed</TL>
</span>
</div>
</div>
</div>
</div>
</div>
<div class="col-xl-3">
<div class="card card-flush bgi-no-repeat bgi-size-contain bgi-position-x-end h-xl-100" style="background-color: #170049;">
<div class="card-header pt-5 mb-3">
<div class="d-flex flex-center rounded-circle h-80px w-80px" style="border: 1px rgba(255, 255, 255, 0.4);background-color: #7239EA">
<i class="text-white bx bx-rocket bx-lg"></i>
</div>
</div>
<div class="card-body d-flex align-items-end mb-3">
<div class="d-flex align-items-center">
<span class="fs-4hx text-white fw-bold me-6">@(CleanupService.ServersRunning)</span>
<div class="fw-bold fs-6 text-white">
<span class="d-block">
<TL>Used clanup</TL>
</span>
<span>
<TL>Servers</TL>
</span>
</div>
</div>
</div>
</div>
</div>
<div class="col-xl-3">
<div class="card card-flush bgi-no-repeat bgi-size-contain bgi-position-x-end h-xl-100" style="background-color: #170049;">
<div class="card-body d-flex align-items-end mb-3">
<div class="d-flex align-items-center">
<div class="mb-3">
<label class="form-label"><TL>Status</TL>: @(CleanupService.Status)</label>
<div class="progress">
<div class="progress-bar progress-bar-striped progress-bar-animated bg-primary" role="progressbar" aria-valuenow="@(CleanupService.PercentProgress)" aria-valuemin="0" aria-valuemax="100" style="width: @(CleanupService.PercentProgress)%"></div>
</div>
</div>
</div>
</div>
</div>
</div>
<div class="col-xl-3">
<div class="card card-flush bgi-no-repeat bgi-size-contain bgi-position-x-end">
<div class="card-body mb-3">
<div class="align-items-top">
<div class="fw-bold fs-6 text-white">
<div class="row mb-3">
@if (CleanupService.IsRunning)
{
<button class="btn btn-primary disabled" disabled="">
<TL>Start</TL>
</button>
}
else
{
<button @onclick="Trigger" class="btn btn-primary">
<TL>Start</TL>
</button>
}
</div>
<div class="row mb-3">
@if (CleanupService.Activated)
{
<button class="btn btn-success disabled" disabled="">
<TL>Enable</TL>
</button>
}
else
{
<button @onclick="Enable" class="btn btn-success">
<TL>Enable</TL>
</button>
}
</div>
<div class="row">
@if (CleanupService.Activated)
{
<button @onclick="Disabled" class="btn btn-danger">
<TL>Disable</TL>
</button>
}
else
{
<button class="btn btn-danger disabled" disabled="">
<TL>Disable</TL>
</button>
}
</div>
</div>
</div>
</div>
</div>
</div>
</div>
</IsAdmin>
<NonAdmin>
<UnauthorizedAlert></UnauthorizedAlert>
</NonAdmin>
@code
{
protected async override Task OnAfterRenderAsync(bool firstRender)
{
if (firstRender)
{
CleanupService.OnUpdated += OnUpdated;
}
}
private void OnUpdated(object? sender, EventArgs e)
{
Logger.Debug(CleanupService.PercentProgress);
InvokeAsync(StateHasChanged);
}
public void Dispose()
{
CleanupService.OnUpdated -= OnUpdated;
}
private void Enable()
{
CleanupService.Activated = true;
AuditLogService.Log(AuditLogType.CleanupEnabled, "The automatic cleanup has been activated");
InvokeAsync(StateHasChanged);
}
private void Disabled()
{
CleanupService.Activated = false;
AuditLogService.Log(AuditLogType.CleanupDisabled, "The automatic cleanup has been disabled");
InvokeAsync(StateHasChanged);
}
private async void Trigger()
{
await CleanupService.TriggerPerform();
await AuditLogService.Log(AuditLogType.CleanupTriggered, "The automatic cleanup has been manually triggered");
await InvokeAsync(StateHasChanged);
}
}

View file

@ -406,4 +406,21 @@ The City field is required.;The City field is required.
The State field is required.;The State field is required.
The Country field is required.;The Country field is required.
Street and house number requered;Street and house number requered
Max lenght reached;Max lenght reached
Max lenght reached;Max lenght reached
Server;Server
stopped;stopped
Cleanups;Cleanups
executed;executed
Used clanup;Used clanup
Enable;Enable
Disabble;Disabble
Disable;Disable
Addons;Addons
Javascript version;Javascript version
Javascript file;Javascript file
Select javascript file to execute on start;Select javascript file to execute on start
Submit;Submit
Processing;Processing
Go up;Go up
Running cleanup;Running cleanup
servers;servers