Moved the Agents into their own repo. Kept the agent.pl just for reference

This commit is contained in:
Frank Harris 2025-09-11 13:27:32 -04:00
parent 22381be29a
commit 8680a02b13
18132 changed files with 0 additions and 2569420 deletions

View file

@ -1,206 +0,0 @@
#!/usr/bin/env python3
# collector.py (place in gameserver root; run via cron)
# On each machine: pip install psutil mysql-connector-python
import os, socket, time, subprocess
from datetime import datetime, timezone
from pathlib import Path
import psutil
import mysql.connector
# ======== CONFIG (edit for YOUR PANEL DB) =========
DB_HOST = "127.0.0.1"
DB_USER = "panel_user" # your panel DB user
DB_PASS = "REPLACE_ME"
DB_NAME = "panel_database" # your panel DB name (not a new DB)
TABLE_PREFIX = "gsp_" # don't change unless you want a different prefix
# ================================================
BASE_DIR = Path(__file__).resolve().parent
DISK_PATH = str(BASE_DIR)
NET_IFACE = None
MACHINE_ID = os.environ.get("GS_MACHINE_ID") or socket.gethostname()
CPU_SAMPLE_DELAY = 0.5
def utc_now():
return datetime.now(timezone.utc).replace(tzinfo=None)
def get_default_iface():
if NET_IFACE:
return NET_IFACE
try:
with open("/proc/net/route") as f:
for line in f.readlines()[1:]:
parts = line.strip().split()
if len(parts) >= 11 and parts[1] == '00000000' and int(parts[3], 16) & 2:
return parts[0]
except Exception:
pass
stats = psutil.net_if_stats()
for name, st in stats.items():
if st.isup:
return name
return None
def get_folder_size_bytes(path: Path) -> int:
try:
res = subprocess.run(["du", "-sb", str(path)], capture_output=True, text=True, check=True)
return int(res.stdout.split()[0])
except Exception:
total = 0
for root, dirs, files in os.walk(path, followlinks=False):
for fn in files:
fp = os.path.join(root, fn)
try: total += os.path.getsize(fp)
except Exception: pass
return total
def connect_db():
return mysql.connector.connect(
host=DB_HOST, user=DB_USER, password=DB_PASS, database=DB_NAME, autocommit=True
)
def ensure_machine(db, hostname):
cur = db.cursor()
cur.execute(
f"INSERT IGNORE INTO {TABLE_PREFIX}machines (machine_id, hostname) VALUES (%s, %s)",
(MACHINE_ID, hostname),
)
cur.close()
def insert_machine_sample(db, ts, load1, load5, load15, cpu_pct, vm, swap, du, iface, rx, tx, speed):
cur = db.cursor()
cur.execute(f"""
INSERT INTO {TABLE_PREFIX}machine_samples
(machine_id, ts, load1, load5, load15, cpu_pct,
mem_used_bytes, mem_total_bytes, mem_used_pct,
swap_used_bytes, swap_total_bytes,
disk_path, disk_total_bytes, disk_used_bytes, disk_used_pct,
net_iface, rx_bytes, tx_bytes, iface_speed_mbps)
VALUES (%s,%s,%s,%s,%s,%s,
%s,%s,%s,
%s,%s,
%s,%s,%s,%s,
%s,%s,%s,%s)
""", (MACHINE_ID, ts, load1, load5, load15, round(cpu_pct,2),
vm.used, vm.total, round(vm.percent,2),
swap.used, swap.total,
DISK_PATH, du.total, du.used, round(du.percent,2),
iface, rx, tx, speed))
cur.close()
def insert_process_sample(db, ts, server_name, server_path, proc, cpu_pct, folder_size):
try:
mi = proc.memory_info(); mem_pct = proc.memory_percent()
except Exception:
mi = None; mem_pct = None
rss = mi.rss if mi else None; vms = mi.vms if mi else None
try:
io = proc.io_counters(); rd, wr = io.read_bytes, io.write_bytes
except Exception:
rd = wr = None
try:
fds = proc.num_fds() if hasattr(proc, "num_fds") else None
except Exception:
fds = None
ports = set()
try:
for c in proc.connections(kind="inet"):
try:
if c.status == psutil.CONN_LISTEN or c.type == psutil.SOCK_DGRAM:
if c.laddr and c.laddr.port: ports.add(str(c.laddr.port))
except Exception: pass
except Exception: pass
ports_str = ",".join(sorted(ports)) if ports else None
cmd_str = None
try:
cmd = proc.cmdline()
if cmd: cmd_str = " ".join(cmd)[:1024]
except Exception: pass
cur = db.cursor()
cur.execute(f"""
INSERT INTO {TABLE_PREFIX}process_samples
(machine_id, ts, server_name, server_path,
pid, proc_name, cmd, cpu_pct,
rss_bytes, vms_bytes, mem_pct,
io_read_bytes, io_write_bytes, open_fds,
listening_ports, folder_size_bytes)
VALUES (%s,%s,%s,%s,
%s,%s,%s,%s,
%s,%s,%s,
%s,%s,%s,
%s,%s)
""", (MACHINE_ID, ts, server_name, str(server_path),
proc.pid, proc.name()[:255] if proc.name() else None,
cmd_str, round(cpu_pct,2) if cpu_pct is not None else None,
rss, vms, round(mem_pct,2) if mem_pct is not None else None,
rd, wr, fds, ports_str, folder_size))
cur.close()
def main():
ts = utc_now()
hostname = socket.gethostname()
iface = get_default_iface()
ifaces_stats = psutil.net_if_stats()
iface_speed = ifaces_stats.get(iface).speed if iface and iface in ifaces_stats else None
net_counters = psutil.net_io_counters(pernic=True)
rx = tx = None
if iface and iface in net_counters:
rx = net_counters[iface].bytes_recv
tx = net_counters[iface].bytes_sent
try:
load1, load5, load15 = os.getloadavg()
except Exception:
load1=load5=load15=0.0
# discover servers
server_dirs = [p for p in BASE_DIR.iterdir() if p.is_dir() and not p.name.startswith('.')]
server_procs = {str(d): [] for d in server_dirs}
plist = []
for p in psutil.process_iter(attrs=["pid","name","cwd","exe","cmdline"]):
try:
_ = p.status()
p.cpu_percent(interval=None)
plist.append(p)
except Exception:
pass
for d in server_dirs:
dstr = str(d)
for p in plist:
try:
cwd = p.info.get("cwd") or ""
exe = p.info.get("exe") or ""
cmd = " ".join(p.info.get("cmdline") or [])
if cwd.startswith(dstr) or exe.startswith(dstr) or dstr in cmd:
server_procs[str(d)].append(p)
except Exception:
continue
time.sleep(CPU_SAMPLE_DELAY)
proc_cpu = {}
for p in plist:
try: proc_cpu[p.pid] = p.cpu_percent(interval=None)
except Exception: proc_cpu[p.pid] = None
vm = psutil.virtual_memory()
swap = psutil.swap_memory()
du = psutil.disk_usage(DISK_PATH)
cpu_pct = psutil.cpu_percent(interval=0.0)
db = connect_db()
ensure_machine(db, hostname)
insert_machine_sample(db, ts, load1, load5, load15, cpu_pct, vm, swap, du, iface, rx, tx, iface_speed)
for sdir, procs in server_procs.items():
server_name = Path(sdir).name
folder_size = get_folder_size_bytes(Path(sdir))
for p in procs:
insert_process_sample(db, ts, server_name, sdir, p, proc_cpu.get(p.pid), folder_size)
db.close()
if __name__ == "__main__":
main()

View file

@ -1,61 +0,0 @@
-- Run these while connected to your current PANEL database (no CREATE DATABASE here)
-- Machines catalog
CREATE TABLE IF NOT EXISTS gsp_machines (
id INT AUTO_INCREMENT PRIMARY KEY,
machine_id VARCHAR(64) NOT NULL,
hostname VARCHAR(255) NOT NULL,
ip VARCHAR(45) DEFAULT NULL,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
UNIQUE KEY uniq_machine (machine_id)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
-- Host-level samples
CREATE TABLE IF NOT EXISTS gsp_machine_samples (
id BIGINT AUTO_INCREMENT PRIMARY KEY,
machine_id VARCHAR(64) NOT NULL,
ts DATETIME NOT NULL,
load1 DECIMAL(6,2),
load5 DECIMAL(6,2),
load15 DECIMAL(6,2),
cpu_pct DECIMAL(6,2),
mem_used_bytes BIGINT,
mem_total_bytes BIGINT,
mem_used_pct DECIMAL(6,2),
swap_used_bytes BIGINT,
swap_total_bytes BIGINT,
disk_path VARCHAR(255),
disk_total_bytes BIGINT,
disk_used_bytes BIGINT,
disk_used_pct DECIMAL(6,2),
net_iface VARCHAR(64),
rx_bytes BIGINT,
tx_bytes BIGINT,
iface_speed_mbps INT NULL,
KEY idx_machine_ts (machine_id, ts),
KEY idx_ts (ts)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
-- Per-process/per-server samples
CREATE TABLE IF NOT EXISTS gsp_process_samples (
id BIGINT AUTO_INCREMENT PRIMARY KEY,
machine_id VARCHAR(64) NOT NULL,
ts DATETIME NOT NULL,
server_name VARCHAR(255) NOT NULL,
server_path VARCHAR(512) NOT NULL,
pid INT NOT NULL,
proc_name VARCHAR(255),
cmd TEXT,
cpu_pct DECIMAL(7,2),
rss_bytes BIGINT,
vms_bytes BIGINT,
mem_pct DECIMAL(6,2),
io_read_bytes BIGINT,
io_write_bytes BIGINT,
open_fds INT,
listening_ports VARCHAR(255),
folder_size_bytes BIGINT,
KEY idx_proc_server (machine_id, server_name, ts),
KEY idx_proc_pid (machine_id, pid, ts),
KEY idx_ts (ts)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;

View file

@ -1,275 +0,0 @@
<?php
// gsp_stats_dashboard.php
// Visual dashboard for GSP metrics (uses gsp_ tables in your PANEL DB).
/********** CONFIG (panel DB) **********/
$db = [
'host' => '127.0.0.1',
'user' => 'panel_user',
'pass' => 'REPLACE_ME',
'name' => 'panel_database'
];
$TABLE_PREFIX = 'gsp_';
$AUTO_REFRESH_SECONDS = 30; // set 0 to disable auto-refresh
/***************************************/
$mysqli = @new mysqli($db['host'], $db['user'], $db['pass'], $db['name']);
if ($mysqli->connect_errno) {
http_response_code(500);
echo "<pre>DB connect failed: " . htmlspecialchars($mysqli->connect_error) . "</pre>";
exit;
}
$mysqli->set_charset("utf8mb4");
function q($mysqli, $sql, $params=[]) {
$stmt = $mysqli->prepare($sql);
if(!$stmt){ throw new Exception($mysqli->error); }
if(!empty($params)) {
// infer types (all strings for simplicity)
$types = str_repeat('s', count($params));
$stmt->bind_param($types, ...$params);
}
$stmt->execute();
$res = $stmt->get_result();
$rows = $res ? $res->fetch_all(MYSQLI_ASSOC) : [];
$stmt->close();
return $rows;
}
function fmt_bytes($b) {
if ($b===null) return '—';
$u = ['B','KB','MB','GB','TB','PB']; $i=0;
while ($b>=1024 && $i<count($u)-1) { $b/=1024; $i++; }
return sprintf('%.1f %s', $b, $u[$i]);
}
function pct_class($p) {
if ($p===null) return 'bar';
if ($p>=80) return 'bar danger';
if ($p>=60) return 'bar warn';
return 'bar ok';
}
function pct($v) { return $v===null ? '—' : sprintf('%.1f%%', $v); }
function num0($v) { return $v===null ? '—' : number_format($v,0); }
$machine = isset($_GET['machine']) ? trim($_GET['machine']) : '';
$windows = ['1h'=>1, '24h'=>24, '7d'=>168];
?><!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8">
<title>GSP Stats Dashboard<?= $machine ? " ".htmlspecialchars($machine) : "" ?></title>
<?php if ($AUTO_REFRESH_SECONDS>0): ?>
<meta http-equiv="refresh" content="<?= (int)$AUTO_REFRESH_SECONDS ?>">
<?php endif; ?>
<meta name="viewport" content="width=device-width, initial-scale=1">
<style>
:root { --bg:#0b0f14; --panel:#121821; --muted:#9bb0c3; --text:#e6eef6; --ok:#1f9d55; --warn:#d4a017; --danger:#d9534f; --accent:#4ea1ff; }
body{background:var(--bg);color:var(--text);font:14px/1.45 system-ui,Segoe UI,Roboto,Helvetica,Arial,sans-serif;margin:0;padding:24px;}
a{color:var(--accent);text-decoration:none}
h1,h2,h3{margin:0 0 12px 0}
.wrap{max-width:1200px;margin:0 auto}
.topbar{display:flex;align-items:center;gap:12px;margin-bottom:16px}
.card{background:var(--panel);border-radius:14px;padding:16px;box-shadow:0 2px 12px rgba(0,0,0,.25)}
.grid{display:grid;grid-template-columns:repeat(12,1fr);gap:16px}
.col-3{grid-column:span 3} .col-4{grid-column:span 4} .col-6{grid-column:span 6} .col-12{grid-column:span 12}
.muted{color:var(--muted)}
.pill{display:inline-block;padding:2px 8px;border-radius:999px;background:#1a2330;color:#cfe2ff;border:1px solid #2a3a52}
.kpi{font-size:22px;font-weight:700}
.barwrap{background:#0e141d;border-radius:10px;height:12px;overflow:hidden}
.bar{height:100%;display:block;width:0%;transition:width .5s ease;background:var(--ok)}
.bar.warn{background:var(--warn)} .bar.danger{background:var(--danger)}
table{width:100%;border-collapse:separate;border-spacing:0 8px}
thead th{font-weight:600;color:#bcd; text-align:left; padding:8px}
tbody td{padding:8px;background:var(--panel)}
tbody tr{border-radius:10px}
tbody tr td:first-child{border-top-left-radius:10px;border-bottom-left-radius:10px}
tbody tr td:last-child{border-top-right-radius:10px;border-bottom-right-radius:10px}
.right{text-align:right}
.small{font-size:12px}
.row{display:flex;gap:10px;flex-wrap:wrap}
.btn{display:inline-block;padding:8px 10px;border:1px solid #2a3a52;border-radius:10px;background:#0e141d;color:#dfeaff}
.btn.active{background:#1a2330}
.split{display:flex;gap:16px;flex-wrap:wrap}
.split > div{flex:1 1 280px}
.subtle{opacity:.8}
.hdr{display:flex;justify-content:space-between;align-items:center;margin-bottom:10px}
.sep{height:1px;background:#223143;margin:16px 0}
.mono{font-family: ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, "Liberation Mono", "Courier New", monospace}
</style>
</head>
<body>
<div class="wrap">
<div class="topbar">
<h1>GSP Stats Dashboard</h1>
<span class="pill"><?= $AUTO_REFRESH_SECONDS>0 ? "auto-refresh {$AUTO_REFRESH_SECONDS}s" : "manual refresh" ?></span>
<a class="btn" href="?">All machines</a>
<?php if($machine): ?>
<a class="btn" href="?machine=<?= urlencode($machine) ?>">Refresh</a>
<a class="btn" target="_blank" href="stats_aggregate.php?machine=<?= urlencode($machine) ?>">View raw JSON</a>
<?php endif; ?>
</div>
<?php
try {
if (!$machine) {
// list machines with last-ts
$rows = q($mysqli, "SELECT m.machine_id, m.hostname,
(SELECT MAX(ts) FROM {$TABLE_PREFIX}machine_samples s WHERE s.machine_id=m.machine_id) AS last_ts
FROM {$TABLE_PREFIX}machines m ORDER BY m.created_at DESC");
echo '<div class="grid">';
foreach ($rows as $r) {
echo '<div class="col-4 card">';
echo '<div class="hdr"><h3 class="mono">'.htmlspecialchars($r['machine_id']).'</h3><span class="muted">'.htmlspecialchars($r['hostname']).'</span></div>';
echo '<div class="small muted">Last sample: '.htmlspecialchars($r['last_ts'] ?: '—').'</div>';
echo '<div style="margin-top:10px"><a class="btn" href="?machine='.urlencode($r['machine_id']).'">Open</a></div>';
echo '</div>';
}
echo '</div>';
echo '</div></body></html>'; exit;
}
// LAST sample
$last = q($mysqli, "SELECT * FROM {$TABLE_PREFIX}machine_samples WHERE machine_id=? ORDER BY ts DESC LIMIT 1", [$machine]);
$last = $last ? $last[0] : null;
// Windows
$agg = [];
foreach($windows as $label=>$hours){
$aggM = q($mysqli, "SELECT
COUNT(*) n,
AVG(cpu_pct) cpu_avg,
AVG(mem_used_pct) mem_avg,
AVG(disk_used_pct) disk_avg
FROM {$TABLE_PREFIX}machine_samples
WHERE machine_id=? AND ts >= (NOW() - INTERVAL {$hours} HOUR)", [$machine]);
$netRows = q($mysqli, "SELECT ts, rx_bytes, tx_bytes, iface_speed_mbps
FROM {$TABLE_PREFIX}machine_samples
WHERE machine_id=? AND ts >= (NOW() - INTERVAL {$hours} HOUR)
ORDER BY ts ASC", [$machine]);
$net = ['avg_rx_Bps'=>null,'avg_tx_Bps'=>null,'avg_total_Bps'=>null,'avg_util_pct'=>null];
if (count($netRows)>=2) {
$first = $netRows[0]; $lastn = $netRows[count($netRows)-1];
$secs = max(1, strtotime($lastn['ts']) - strtotime($first['ts']));
$rx_bps = ((int)$lastn['rx_bytes'] - (int)$first['rx_bytes']) / $secs;
$tx_bps = ((int)$lastn['tx_bytes'] - (int)$first['tx_bytes']) / $secs;
$speed_mbps = $lastn['iface_speed_mbps'] ? (int)$lastn['iface_speed_mbps'] : null;
$util_pct = null;
if ($speed_mbps && $speed_mbps>0) {
$capacity_Bps = ($speed_mbps * 1000000) / 8.0;
$util_pct = (($rx_bps + $tx_bps) / $capacity_Bps) * 100.0;
}
$net = ['avg_rx_Bps'=>$rx_bps,'avg_tx_Bps'=>$tx_bps,'avg_total_Bps'=>$rx_bps+$tx_bps,'avg_util_pct'=>$util_pct];
}
$aggS = q($mysqli, "SELECT server_name,
AVG(cpu_pct) cpu_avg,
AVG(mem_pct) mem_avg,
MAX(folder_size_bytes) folder_size_bytes
FROM {$TABLE_PREFIX}process_samples
WHERE machine_id=? AND ts >= (NOW() - INTERVAL {$hours} HOUR)
GROUP BY server_name
ORDER BY server_name ASC", [$machine]);
$agg[$label] = ['machine'=>$aggM[0], 'net'=>$net, 'servers'=>$aggS];
}
} catch (Throwable $e) {
echo '<div class="card">Error: '.htmlspecialchars($e->getMessage()).'</div></div></body></html>'; exit;
}
?>
<div class="grid">
<div class="col-12 card">
<div class="hdr">
<h2 class="mono"><?= htmlspecialchars($machine) ?></h2>
<div class="muted small">Last sample: <?= htmlspecialchars($last['ts'] ?? '—') ?> • IF: <?= htmlspecialchars($last['net_iface'] ?? '—') ?></div>
</div>
<div class="split">
<div>
<div class="muted small">CPU (last)</div>
<div class="kpi"><?= pct($last ? (float)$last['cpu_pct'] : null) ?></div>
<div class="barwrap"><span class="<?= pct_class($last ? (float)$last['cpu_pct'] : null) ?>" style="width:<?= $last? min(100,max(0,(float)$last['cpu_pct'])):0 ?>%"></span></div>
</div>
<div>
<div class="muted small">Memory used (last)</div>
<div class="kpi"><?= pct($last ? (float)$last['mem_used_pct'] : null) ?></div>
<div class="barwrap"><span class="<?= pct_class($last ? (float)$last['mem_used_pct'] : null) ?>" style="width:<?= $last? min(100,max(0,(float)$last['mem_used_pct'])):0 ?>%"></span></div>
</div>
<div>
<div class="muted small">Disk used (last)</div>
<div class="kpi"><?= pct($last ? (float)$last['disk_used_pct'] : null) ?></div>
<div class="barwrap"><span class="<?= pct_class($last ? (float)$last['disk_used_pct'] : null) ?>" style="width:<?= $last? min(100,max(0,(float)$last['disk_used_pct'])):0 ?>%"></span></div>
<div class="small subtle mono"><?= htmlspecialchars($last['disk_path'] ?? '') ?> • used <?= fmt_bytes($last['disk_used_bytes'] ?? null) ?> / <?= fmt_bytes($last['disk_total_bytes'] ?? null) ?></div>
</div>
<div>
<div class="muted small">Net avg util (1h)</div>
<?php $nu = $agg['1h']['net']['avg_util_pct']; ?>
<div class="kpi"><?= $nu===null?'—':sprintf('%.1f%%',$nu) ?></div>
<div class="barwrap"><span class="<?= pct_class($nu) ?>" style="width:<?= $nu!==null ? min(100,max(0,$nu)) : 0 ?>%"></span></div>
<div class="small subtle mono">rx <?= fmt_bytes($agg['1h']['net']['avg_rx_Bps']) ?>/s • tx <?= fmt_bytes($agg['1h']['net']['avg_tx_Bps']) ?>/s</div>
</div>
</div>
</div>
<?php foreach ($windows as $label=>$hours): $m=$agg[$label]['machine']; ?>
<div class="col-12 card">
<div class="hdr">
<h3>Window: <?= htmlspecialchars($label) ?></h3>
<div class="row">
<span class="small muted">AVG CPU</span>
<div class="barwrap" style="width:160px;"><span class="<?= pct_class($m['cpu_avg']) ?>" style="width:<?= $m['cpu_avg']!==null?min(100,max(0,$m['cpu_avg'])):0 ?>%"></span></div>
<span class="small"><?= pct($m['cpu_avg']) ?></span>
<span class="small muted" style="margin-left:14px;">AVG MEM</span>
<div class="barwrap" style="width:160px;"><span class="<?= pct_class($m['mem_avg']) ?>" style="width:<?= $m['mem_avg']!==null?min(100,max(0,$m['mem_avg'])):0 ?>%"></span></div>
<span class="small"><?= pct($m['mem_avg']) ?></span>
<span class="small muted" style="margin-left:14px;">AVG DISK</span>
<div class="barwrap" style="width:160px;"><span class="<?= pct_class($m['disk_avg']) ?>" style="width:<?= $m['disk_avg']!==null?min(100,max(0,$m['disk_avg'])):0 ?>%"></span></div>
<span class="small"><?= pct($m['disk_avg']) ?></span>
</div>
</div>
<div class="sep"></div>
<div class="tablewrap">
<table>
<thead>
<tr>
<th>Server</th>
<th class="right">CPU avg</th>
<th style="width:220px"></th>
<th class="right">Mem avg</th>
<th style="width:220px"></th>
<th class="right">Folder size</th>
</tr>
</thead>
<tbody>
<?php
$rows = $agg[$label]['servers'];
if (!$rows) {
echo '<tr><td colspan="6" class="small muted">No server samples in this window.</td></tr>';
} else {
foreach ($rows as $s) {
$cpu = $s['cpu_avg']!==null ? (float)$s['cpu_avg'] : null;
$mem = $s['mem_avg']!==null ? (float)$s['mem_avg'] : null;
echo '<tr>';
echo '<td class="mono">'.htmlspecialchars($s['server_name']).'</td>';
echo '<td class="right">'.pct($cpu).'</td>';
echo '<td><div class="barwrap"><span class="'.pct_class($cpu).'" style="width:'.($cpu!==null?min(100,max(0,$cpu)):0).'%"></span></div></td>';
echo '<td class="right">'.pct($mem).'</td>';
echo '<td><div class="barwrap"><span class="'.pct_class($mem).'" style="width:'.($mem!==null?min(100,max(0,$mem)):0).'%"></span></div></td>';
echo '<td class="right">'.fmt_bytes($s['folder_size_bytes']).'</td>';
echo '</tr>';
}
}
?>
</tbody>
</table>
</div>
</div>
<?php endforeach; ?>
</div>
</div>
</body>
</html>