beenull/Moonlight
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

cleanup service

Browse source
This commit is contained in:
Daniel Balk 2023-04-02 22:51:27 +02:00
parent 146678cebf
commit dfda444956
8 changed files with 1711 additions and 6 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
Moonlight/App/Database/DataContext.cs
View file

@ -2,6 +2,7 @@
using Moonlight.App.Database.Entities;
using Moonlight.App.Database.Entities.LogsEntries;
using Moonlight.App.Database.Entities.Notification;
using Moonlight.App.Models.Misc;
using Moonlight.App.Services;
namespace Moonlight.App.Database;
@ -42,6 +43,7 @@ public class DataContext : DbContext
public DbSet<AaPanel> AaPanels { get; set; }
public DbSet<Website> Websites { get; set; }
public DbSet<DdosAttack> DdosAttacks { get; set; }
public DbSet<CleanupException> CleanupExceptions { get; set; }
protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder)
{

1082
Moonlight/App/Database/Migrations/20230402204329_AddCleanupExceptionsTable.Designer.cs generated Normal file
View file

File diff suppressed because it is too large Load diff

170
Moonlight/App/Database/Migrations/20230402204329_AddCleanupExceptionsTable.cs Normal file
View file

@ -0,0 +1,170 @@
using Microsoft.EntityFrameworkCore.Metadata;
using Microsoft.EntityFrameworkCore.Migrations;
#nullable disable
namespace Moonlight.App.Database.Migrations
{
/// <inheritdoc />
public partial class AddCleanupExceptionsTable : Migration
{
/// <inheritdoc />
protected override void Up(MigrationBuilder migrationBuilder)
{
migrationBuilder.AlterColumn<string>(
name: "State",
table: "Users",
type: "varchar(64)",
maxLength: 64,
nullable: false,
oldClrType: typeof(string),
oldType: "longtext")
.Annotation("MySql:CharSet", "utf8mb4")
.OldAnnotation("MySql:CharSet", "utf8mb4");
migrationBuilder.AlterColumn<string>(
name: "LastName",
table: "Users",
type: "varchar(64)",
maxLength: 64,
nullable: false,
oldClrType: typeof(string),
oldType: "longtext")
.Annotation("MySql:CharSet", "utf8mb4")
.OldAnnotation("MySql:CharSet", "utf8mb4");
migrationBuilder.AlterColumn<string>(
name: "FirstName",
table: "Users",
type: "varchar(64)",
maxLength: 64,
nullable: false,
oldClrType: typeof(string),
oldType: "longtext")
.Annotation("MySql:CharSet", "utf8mb4")
.OldAnnotation("MySql:CharSet", "utf8mb4");
migrationBuilder.AlterColumn<string>(
name: "Country",
table: "Users",
type: "varchar(64)",
maxLength: 64,
nullable: false,
oldClrType: typeof(string),
oldType: "longtext")
.Annotation("MySql:CharSet", "utf8mb4")
.OldAnnotation("MySql:CharSet", "utf8mb4");
migrationBuilder.AlterColumn<string>(
name: "City",
table: "Users",
type: "varchar(128)",
maxLength: 128,
nullable: false,
oldClrType: typeof(string),
oldType: "longtext")
.Annotation("MySql:CharSet", "utf8mb4")
.OldAnnotation("MySql:CharSet", "utf8mb4");
migrationBuilder.AlterColumn<string>(
name: "Address",
table: "Users",
type: "varchar(128)",
maxLength: 128,
nullable: false,
oldClrType: typeof(string),
oldType: "longtext")
.Annotation("MySql:CharSet", "utf8mb4")
.OldAnnotation("MySql:CharSet", "utf8mb4");
migrationBuilder.CreateTable(
name: "CleanupExceptions",
columns: table => new
{
Id = table.Column<int>(type: "int", nullable: false)
.Annotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn),
ServerId = table.Column<int>(type: "int", nullable: false),
Note = table.Column<string>(type: "longtext", nullable: false)
.Annotation("MySql:CharSet", "utf8mb4")
},
constraints: table =>
{
table.PrimaryKey("PK_CleanupExceptions", x => x.Id);
})
.Annotation("MySql:CharSet", "utf8mb4");
}
/// <inheritdoc />
protected override void Down(MigrationBuilder migrationBuilder)
{
migrationBuilder.DropTable(
name: "CleanupExceptions");
migrationBuilder.AlterColumn<string>(
name: "State",
table: "Users",
type: "longtext",
nullable: false,
oldClrType: typeof(string),
oldType: "varchar(64)",
oldMaxLength: 64)
.Annotation("MySql:CharSet", "utf8mb4")
.OldAnnotation("MySql:CharSet", "utf8mb4");
migrationBuilder.AlterColumn<string>(
name: "LastName",
table: "Users",
type: "longtext",
nullable: false,
oldClrType: typeof(string),
oldType: "varchar(64)",
oldMaxLength: 64)
.Annotation("MySql:CharSet", "utf8mb4")
.OldAnnotation("MySql:CharSet", "utf8mb4");
migrationBuilder.AlterColumn<string>(
name: "FirstName",
table: "Users",
type: "longtext",
nullable: false,
oldClrType: typeof(string),
oldType: "varchar(64)",
oldMaxLength: 64)
.Annotation("MySql:CharSet", "utf8mb4")
.OldAnnotation("MySql:CharSet", "utf8mb4");
migrationBuilder.AlterColumn<string>(
name: "Country",
table: "Users",
type: "longtext",
nullable: false,
oldClrType: typeof(string),
oldType: "varchar(64)",
oldMaxLength: 64)
.Annotation("MySql:CharSet", "utf8mb4")
.OldAnnotation("MySql:CharSet", "utf8mb4");
migrationBuilder.AlterColumn<string>(
name: "City",
table: "Users",
type: "longtext",
nullable: false,
oldClrType: typeof(string),
oldType: "varchar(128)",
oldMaxLength: 128)
.Annotation("MySql:CharSet", "utf8mb4")
.OldAnnotation("MySql:CharSet", "utf8mb4");
migrationBuilder.AlterColumn<string>(
name: "Address",
table: "Users",
type: "longtext",
nullable: false,
oldClrType: typeof(string),
oldType: "varchar(128)",
oldMaxLength: 128)
.Annotation("MySql:CharSet", "utf8mb4")
.OldAnnotation("MySql:CharSet", "utf8mb4");
}
}
}

36
Moonlight/App/Database/Migrations/DataContextModelSnapshot.cs
View file

@ -707,18 +707,21 @@ namespace Moonlight.App.Database.Migrations
b.Property<string>("Address")
.IsRequired()
.HasColumnType("longtext");
.HasMaxLength(128)
.HasColumnType("varchar(128)");
b.Property<bool>("Admin")
.HasColumnType("tinyint(1)");
b.Property<string>("City")
.IsRequired()
.HasColumnType("longtext");
.HasMaxLength(128)
.HasColumnType("varchar(128)");
b.Property<string>("Country")
.IsRequired()
.HasColumnType("longtext");
.HasMaxLength(64)
.HasColumnType("varchar(64)");
b.Property<DateTime>("CreatedAt")
.HasColumnType("datetime(6)");
@ -732,11 +735,13 @@ namespace Moonlight.App.Database.Migrations
b.Property<string>("FirstName")
.IsRequired()
.HasColumnType("longtext");
.HasMaxLength(64)
.HasColumnType("varchar(64)");
b.Property<string>("LastName")
.IsRequired()
.HasColumnType("longtext");
.HasMaxLength(64)
.HasColumnType("varchar(64)");
b.Property<string>("Password")
.IsRequired()
@ -744,7 +749,8 @@ namespace Moonlight.App.Database.Migrations
b.Property<string>("State")
.IsRequired()
.HasColumnType("longtext");
.HasMaxLength(64)
.HasColumnType("varchar(64)");
b.Property<int>("Status")
.HasColumnType("int");
@ -821,6 +827,24 @@ namespace Moonlight.App.Database.Migrations
b.ToTable("Websites");
});
modelBuilder.Entity("Moonlight.App.Models.Misc.CleanupException", b =>
{
b.Property<int>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("int");
b.Property<string>("Note")
.IsRequired()
.HasColumnType("longtext");
b.Property<int>("ServerId")
.HasColumnType("int");
b.HasKey("Id");
b.ToTable("CleanupExceptions");
});
modelBuilder.Entity("Moonlight.App.Database.Entities.Database", b =>
{
b.HasOne("Moonlight.App.Database.Entities.AaPanel", "AaPanel")

8
Moonlight/App/Models/Misc/CleanupException.cs Normal file
View file

@ -0,0 +1,8 @@
namespace Moonlight.App.Models.Misc;
public class CleanupException
{
public int Id { get; set; }
public int ServerId { get; set; }
public string Note { get; set; }
}

44
Moonlight/App/Repositories/CleanupExceptionRepository.cs Normal file
View file

@ -0,0 +1,44 @@
using Microsoft.EntityFrameworkCore;
using Moonlight.App.Database;
using Moonlight.App.Models.Misc;
namespace Moonlight.App.Repositories;
public class CleanupExceptionRepository : IDisposable
{
private readonly DataContext DataContext;
public CleanupExceptionRepository(DataContext dataContext)
{
DataContext = dataContext;
}
public DbSet<CleanupException> Get()
{
return DataContext.CleanupExceptions;
}
public CleanupException Add(CleanupException cleanupException)
{
var x = DataContext.CleanupExceptions.Add(cleanupException).Entity;
DataContext.SaveChanges();
return x;
}
public void Update(CleanupException cleanupException)
{
DataContext.CleanupExceptions.Update(cleanupException);
DataContext.SaveChanges();
}
public void Delete(CleanupException cleanupException)
{
DataContext.CleanupExceptions.Remove(cleanupException);
DataContext.SaveChanges();
}
public void Dispose()
{
DataContext.Dispose();
}
}

370
Moonlight/App/Services/CleanupService.cs Normal file
View file

@ -0,0 +1,370 @@
using System.Diagnostics;
using Microsoft.EntityFrameworkCore;
using MineStatLib;
using Moonlight.App.Database.Entities;
using Moonlight.App.Models.Daemon.Resources;
using Moonlight.App.Models.Wings;
using Moonlight.App.Repositories;
using Moonlight.App.Repositories.Servers;
using Logging.Net;
using Newtonsoft.Json;
namespace Moonlight.App.Services;
public class CleanupService
{
public DateTime StartedAt { get; private set; }
public DateTime CompletedAt { get; private set; }
public int ServersCleaned { get; private set; }
public int CleanupsPerformed { get; private set; }
public int ServersRunning { get; private set; }
public bool IsRunning { get; private set; }
public bool Activated { get; set; }
public int PercentProgress { get; private set; } = 100;
public string Status { get; private set; } = "N/A";
private Task PerformTask;
private readonly ConfigService ConfigService;
private readonly IServiceScopeFactory ServiceScopeFactory;
private int RequiredCpu;
private long RequiredMemory;
private int WaitTime;
public EventHandler OnUpdated;
public CleanupService(ConfigService configService, IServiceScopeFactory serviceScopeFactory)
{
ServiceScopeFactory = serviceScopeFactory;
ConfigService = configService;
IConfiguration configuration = new ConfigurationBuilder()
.AddJsonFile("appsettings.json")
.Build();
var config = configuration.GetSection("Cleanup");
RequiredCpu = config.GetValue<int>("Cpu");
RequiredMemory = config.GetValue<long>("Memory");
WaitTime = config.GetValue<int>("Wait");
if (!ConfigService.DebugMode)
Task.Run(Start);
}
private void Start()
{
StartedAt = DateTime.Now;
CompletedAt = DateTime.Now;
IsRunning = false;
Activated = true;
DoWaiting();
}
private async void DoWaiting()
{
while (true)
{
if (Activated)
await Perform();
try
{
await Task.Delay((int)TimeSpan.FromMinutes(WaitTime).TotalMilliseconds);
}
catch (Exception)
{
}
}
}
public async Task TriggerPerform()
{
if (IsRunning)
return;
PerformTask = new Task(async () => await Perform());
PerformTask.Start();
}
private async Task Perform()
{
if (IsRunning)
return;
IsRunning = true;
StartedAt = DateTime.Now;
ServersRunning = 0;
OnUpdated?.Invoke(this, null);
using (var scope = ServiceScopeFactory.CreateScope())
{
// Setup time measure
var watch = new Stopwatch();
watch.Start();
// Get repos from dependency injection
var serverRepository = scope.ServiceProvider.GetRequiredService<ServerRepository>();
var nodeRepository = scope.ServiceProvider.GetRequiredService<NodeRepository>();
var cleanupExceptionRepository = scope.ServiceProvider.GetRequiredService<CleanupExceptionRepository>();
var nodeService = scope.ServiceProvider.GetRequiredService<NodeService>();
var imageRepo = scope.ServiceProvider.GetRequiredService<ImageRepository>();
var serverService = scope.ServiceProvider.GetRequiredService<ServerService>();
// Fetching data from mysql
var servers = serverRepository.Get()
.Include(x => x.Image)
.ToArray();
var nodes = nodeRepository.Get().ToArray();
var exceptions = cleanupExceptionRepository.Get().ToArray();
var images = imageRepo.Get().ToArray();
var nodeCount = nodes.Count();
// We use this counter for the foreach loops
int counter = 0;
PercentProgress = 0;
// Fetching data from nodes so we know what nodes to scan
var nodeContainers = new Dictionary<Node, ContainerStats.Container[]>();
Status = "Checking Nodes";
counter = 0;
PercentProgress = 0;
OnUpdated?.Invoke(this, null);
foreach (var node in nodes)
{
try
{
var cpu = await nodeService.GetCpuStats(node);
var freeMemory = await nodeService.GetMemoryStats(node);
if (cpu.Usage > RequiredCpu || freeMemory.Free < RequiredMemory)
{
var c = await nodeService.GetContainerStats(node);
var containers = c.Containers;
nodeContainers.Add(node, containers.ToArray());
}
}
catch (Exception e)
{
Logger.Error($"Error fetching cleanup data from node {node.Id}");
Logger.Error(e);
}
counter++;
CalculateAndUpdateProgress(counter, nodeCount);
OnUpdated?.Invoke(this, null);
}
// Searching for servers we can actually stop because they have the cleanup tag
// and determine which servers we have to check for an illegal mc server
var serversToCheck = new List<Server>();
var serversToCheckForMc = new List<Server>();
Status = "Checking found servers";
counter = 0;
PercentProgress = 0;
OnUpdated?.Invoke(this, null);
// Count every container for progress calculation
var allContainers = 0;
foreach (var array in nodeContainers)
{
allContainers += array.Value.Length;
}
foreach (var nodeContainer in nodeContainers)
{
try
{
foreach (var container in nodeContainer.Value)
{
var server = servers.First(x => x.Uuid.ToString() == container.Name);
var tagsJson = imageRepo
.Get()
.First(x => x.Id == server.Image.Id).TagsJson;
var tags = JsonConvert.DeserializeObject<string[]>(tagsJson) ?? Array.Empty<string>();
if (tags.FirstOrDefault(x => x == "cleanup") != null)
{
serversToCheck.Add(server);
}
if (tags.FirstOrDefault(x => x == "illegalmc") != null)
{
serversToCheckForMc.Add(server);
}
}
}
catch (Exception e)
{
Logger.Error($"Error processing cleanup data from node {nodeContainer.Key.Id}");
Logger.Error(e);
}
counter++;
CalculateAndUpdateProgress(counter, allContainers);
OnUpdated?.Invoke(this, null);
}
// Now we gonna scan every tagged server
Status = "Scanning servers";
counter = 0;
PercentProgress = 0;
OnUpdated?.Invoke(this, null);
foreach (var server in serversToCheck)
{
try
{
var serverData = serverRepository
.Get()
.Include(x => x.MainAllocation)
.Include(x => x.Node)
.Include(x => x.Variables)
.First(x => x.Id == server.Id);
var players = GetPlayers(serverData.Node, serverData.MainAllocation);
var stats = await serverService.GetDetails(server);
var exception = exceptions.FirstOrDefault(x => x.ServerId == server.Id) != null;
if (stats != null)
{
if (exception)
{
if (players == 0 && stats.Utilization.Uptime > TimeSpan.FromHours(6).TotalMilliseconds)
{
await serverService.SetPowerState(server, PowerSignal.Restart);
ServersCleaned++;
OnUpdated?.Invoke(this, null);
}
else
{
ServersRunning++;
OnUpdated?.Invoke(this, null);
}
}
else
{
if (players == 0 && stats.Utilization.Uptime > TimeSpan.FromMinutes(10).TotalMilliseconds)
{
var cleanupVar = serverData.Variables.FirstOrDefault(x => x.Key == "J2S");
if (cleanupVar == null)
{
await serverService.SetPowerState(server, PowerSignal.Stop);
ServersCleaned++;
OnUpdated?.Invoke(this, null);
}
else
{
if (cleanupVar.Value == "1")
{
await serverService.SetPowerState(server, PowerSignal.Restart);
ServersCleaned++;
OnUpdated?.Invoke(this, null);
}
else
{
await serverService.SetPowerState(server, PowerSignal.Stop);
ServersCleaned++;
OnUpdated?.Invoke(this, null);
}
}
}
else
{
ServersRunning++;
OnUpdated?.Invoke(this, null);
}
}
}
}
catch (Exception e)
{
Logger.Error($"Error scanning {server.Name}");
Logger.Error(e);
}
counter++;
CalculateAndUpdateProgress(counter, serversToCheck.Count);
OnUpdated?.Invoke(this, null);
}
// Finally we have to check all code container allocations
// for illegal hosted mc servers
Status = "Scanning code containers";
counter = 0;
PercentProgress = 0;
OnUpdated?.Invoke(this, null);
foreach (var server in serversToCheckForMc)
{
try
{
var serverData = serverRepository
.Get()
.Include(x => x.Allocations)
.Include(x => x.Node)
.First(x => x.Id == server.Id);
foreach (var allocation in serverData.Allocations)
{
if (GetPlayers(server.Node, allocation) != -1)
{
// TODO: Suspend server
Logger.Warn("Found CC running mc: https://moonlight.endelon-hosting.de/server/" +
server.Uuid + "/");
}
}
}
catch (Exception e)
{
Logger.Error($"Error scanning (cc) {server.Name}");
Logger.Error(e);
}
counter++;
CalculateAndUpdateProgress(counter, serversToCheckForMc.Count);
OnUpdated?.Invoke(this, null);
}
watch.Stop();
Status = $"Cleanup finifhed. Duration: {Math.Round(TimeSpan.FromMilliseconds(watch.ElapsedMilliseconds).TotalMinutes, 2)} Minuten";
PercentProgress = 100;
OnUpdated?.Invoke(this, null);
}
IsRunning = false;
CompletedAt = DateTime.Now;
CleanupsPerformed++;
OnUpdated?.Invoke(this, null);
}
private int GetPlayers(Node node, NodeAllocation allocation)
{
var ms = new MineStat(node.Fqdn, (ushort)allocation.Port);
if (ms.ServerUp)
{
return ms.CurrentPlayersInt;
}
else
{
return -1;
}
}
private void CalculateAndUpdateProgress(int now, int all)
{
PercentProgress = (int)Math.Round((now / (double)all) * 100);
}
}

5
Moonlight/Program.cs
View file

@ -97,6 +97,8 @@ namespace Moonlight
builder.Services.AddScoped<GoogleOAuth2Service>();
builder.Services.AddScoped<DiscordOAuth2Service>();
builder.Services.AddSingleton<CleanupService>();
// Loggers
builder.Services.AddScoped<SecurityLogService>();
builder.Services.AddScoped<AuditLogService>();
@ -153,6 +155,9 @@ namespace Moonlight
// Support service
var supportServerService = app.Services.GetRequiredService<SupportServerService>();
// cleanup service
_ = app.Services.GetRequiredService<CleanupService>();
// Discord bot service
//var discordBotService = app.Services.GetRequiredService<DiscordBotService>();