GSP/Panel
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Fix Steam Workshop search scraping flow

Browse source 
Co-authored-by: iaretechnician <2749183+iaretechnician@users.noreply.github.com>
This commit is contained in:
copilot-swe-agent[bot] 2026-01-31 20:16:28 +00:00
parent 2a509fab03
commit 18b6bc1a14
8 changed files with 98 additions and 174 deletions
Download patch file Download diff file Expand all files Collapse all files

1
CHANGELOG.md
View file

@ -1,6 +1,7 @@
# Changelog
## 2026-01-31
- Rebuilt the Steam Workshop picker search to rely solely on Steam Community scraping (matching the working curl flow) and updated the request preview to show the actual Steam URL instead of a local panel endpoint.
- Added adapter AppID lookup for Workshop search so the picker can query Steam even when the server XML lacks a clear default installer entry.
- Switched Workshop picker results to checkbox selection, letting customers toggle multiple mods directly from the search list.
- Added a hardened Workshop scraping helper (the same HTML workflow we validated manually) and wired it into the Steam Workshop service as a fallback whenever the official API errors out or returns empty data.

3
modules/steam_workshop/controllers/SteamWorkshopController.php
View file

@ -169,6 +169,8 @@ class SteamWorkshopController
private function renderEdit(array $home, array $config, bool $isAdmin, bool $adapterLocked): void
{
$gameKey = (string)($home['game_key'] ?? '');
$appId = $gameKey !== '' ? $this->service->getSteamAppIdForGameKey($gameKey) : null;
$this->render('edit', [
'lang' => $this->lang,
'home' => $home,
@ -176,6 +178,7 @@ class SteamWorkshopController
'isAdmin' => $isAdmin,
'adapterOptions' => $this->service->getAdapterOptions(),
'adapterLocked' => $adapterLocked,
'appId' => $appId,
]);
}

2
modules/steam_workshop/lang/en_US.php
View file

@ -85,7 +85,7 @@ return [
'mod_picker_status_need_query' => 'Enter a Workshop ID or keyword before searching.',
'mod_picker_toggle_label' => 'Sync',
'mod_picker_request_label' => 'Submitting request',
'mod_picker_request_hint' => 'Exact URL preview. The input shows the text that will be submitted.',
'mod_picker_request_hint' => 'Exact Steam URL preview. The input shows the text that will be submitted.',
'mod_picker_request_input_label' => 'Workshop query preview',
'error_game_key_required' => 'Select a valid game key before editing the adapter.',
'error_adapter_delete_failed' => 'Adapter could not be deleted.',

224
modules/steam_workshop/lib/SteamWorkshopService.php
View file

@ -682,125 +682,84 @@ class SteamWorkshopService
}
if (ctype_digit($query)) {
$item = $this->fetchWorkshopItemById($query);
if ($item !== null) {
$payload['results'][] = $item;
$detail = $this->fetchWorkshopItemByScrape($query);
$payload['request'] = $detail['request'];
if ($detail['error'] !== null) {
$payload['error'] = $detail['error'];
return $payload;
}
if ($detail['item'] !== null) {
$payload['results'][] = $detail['item'];
$payload['pagination']['total'] = 1;
}
return $payload;
}
$postFields = [
'query_type' => 0,
'page' => $payload['pagination']['page'],
'numperpage' => $payload['pagination']['per_page'],
'appid' => $appId,
'search_text' => $query,
'return_details' => true,
'return_metadata' => false,
];
$response = $this->executeSteamApiRequest('https://api.steampowered.com/IPublishedFileService/QueryFiles/v1/', $postFields);
$requestContext = [
'backend' => 'api',
'url' => $response['url'],
'params' => $response['fields'],
'http_code' => $response['http_code'],
'transport_error' => $response['error'],
];
$requestContext['summary'] = $this->formatRequestSummary($requestContext);
$requestAttempts = [$requestContext];
$payload['request'] = $requestContext;
$apiFailed = false;
if ($response['error'] !== null || $response['http_code'] < 200 || $response['http_code'] >= 300) {
$apiFailed = true;
$reason = $response['error'] !== null ? $response['error'] : 'HTTP ' . $response['http_code'];
$this->logApiFailure(sprintf('Steam API search failed (app=%s query="%s" page=%d): %s', $appId, $query, $payload['pagination']['page'], $reason));
$payload['error'] = sprintf('Steam API request failed (%s). URL: %s Params: %s', $reason, $response['url'], http_build_query($response['fields'], '', '&'));
$scrapeResult = $this->scrapeWorkshopItems($appId, $query, $payload['pagination']['per_page'], $payload['pagination']['page']);
$payload['request'] = $scrapeResult['request'];
if (!empty($scrapeResult['attempts'])) {
$payload['request']['attempts'] = $scrapeResult['attempts'];
}
if ($scrapeResult['success']) {
$payload['results'] = $scrapeResult['results'];
$payload['pagination']['total'] = $scrapeResult['total'];
$payload['pagination']['has_more'] = $scrapeResult['has_more'];
} else {
$data = json_decode((string)$response['body'], true);
if (!is_array($data) || !isset($data['response'])) {
$apiFailed = true;
$this->logApiFailure(sprintf('Steam API search returned invalid payload (app=%s query="%s")', $appId, $query));
$payload['error'] = sprintf('Steam API returned invalid data. URL: %s Params: %s', $response['url'], http_build_query($response['fields'], '', '&'));
} else {
$details = $data['response']['publishedfiledetails'] ?? [];
$total = (int)($data['response']['total'] ?? count($details));
foreach ($details as $item) {
$id = isset($item['publishedfileid']) ? preg_replace('/[^0-9]/', '', (string)$item['publishedfileid']) : '';
if ($id === '') {
continue;
}
$title = isset($item['title']) ? trim((string)$item['title']) : '';
if ($title === '') {
$title = '@' . $id;
}
$payload['results'][] = [
'id' => $id,
'label' => $title,
'author' => isset($item['creator']) ? (string)$item['creator'] : '',
'preview_url' => isset($item['preview_url']) ? (string)$item['preview_url'] : '',
'time_updated' => isset($item['time_updated']) ? (int)$item['time_updated'] : null,
'subscriptions' => isset($item['subscriptions']) ? (int)$item['subscriptions'] : 0,
'source' => 'search',
];
}
$payload['pagination']['total'] = $total;
$payload['pagination']['has_more'] = ($payload['pagination']['page'] * $payload['pagination']['per_page']) < $total;
}
$payload['error'] = $scrapeResult['error'] ?? 'Steam Workshop scrape failed.';
$this->logApiFailure(sprintf('Steam Workshop scrape failed (app=%s query="%s" page=%d): %s', $appId, $query, $payload['pagination']['page'], $payload['error']));
}
if ($this->shouldAttemptScraper($query, $payload)) {
$scrapeResult = $this->scrapeWorkshopItems($appId, $query, $payload['pagination']['per_page'], $payload['pagination']['page']);
$scrapeContext = $scrapeResult['request'];
$attemptContexts = $scrapeResult['attempts'] ?? [$scrapeContext];
foreach ($attemptContexts as $attemptContext) {
$requestAttempts[] = $attemptContext;
}
if ($scrapeResult['success'] && !empty($scrapeResult['results'])) {
$payload['results'] = $scrapeResult['results'];
$payload['pagination']['total'] = $scrapeResult['total'];
$payload['pagination']['has_more'] = $scrapeResult['has_more'];
$payload['error'] = null;
$payload['request'] = $scrapeContext;
} elseif (!$scrapeResult['success']) {
$fallbackError = $scrapeResult['error'] ?? 'Steam Workshop scrape failed.';
if ($payload['error'] === null) {
$payload['error'] = $fallbackError;
} else {
$payload['error'] .= ' Scraper fallback failed: ' . $fallbackError;
}
}
}
$payload['request']['attempts'] = $requestAttempts;
return $payload;
}
private function shouldAttemptScraper(string $query, array $payload): bool
private function fetchWorkshopItemByScrape(string $id): array
{
if (ctype_digit($query)) {
return false;
}
if (!$this->hasAnyScraperTransport()) {
return false;
$id = preg_replace('/[^0-9]/', '', $id);
$request = [
'backend' => 'scraper_http',
'url' => 'https://steamcommunity.com/sharedfiles/filedetails/',
'params' => ['id' => $id],
'http_code' => null,
'transport_error' => null,
];
if ($id === '') {
$request['summary'] = $this->formatRequestSummary($request);
return ['item' => null, 'request' => $request, 'error' => 'Invalid Workshop ID.'];
}
$hasResults = !empty($payload['results']);
return $payload['error'] !== null || $hasResults === false;
}
$response = $this->httpGet($request['url'], $request['params'], $this->getScraperUserAgent());
$request['url'] = $response['url'] ?? $request['url'];
$request['http_code'] = $response['http_code'];
$request['transport_error'] = $response['error'];
$request['summary'] = $this->formatRequestSummary($request);
private function hasAnyScraperTransport(): bool
{
if ($this->isScraperAvailable()) {
return true;
if ($response['error'] !== null || $response['http_code'] < 200 || $response['http_code'] >= 300 || $response['body'] === null) {
$reason = $response['error'] !== null ? $response['error'] : 'HTTP ' . $response['http_code'];
return [
'item' => null,
'request' => $request,
'error' => 'Steam Community detail request failed: ' . $reason,
];
}
return function_exists('curl_init');
$title = $this->parseWorkshopTitle((string)$response['body']);
if ($title === '') {
$title = '@' . $id;
}
return [
'item' => [
'id' => $id,
'label' => $title,
'author' => '',
'preview_url' => '',
'enabled' => true,
'source' => 'search',
],
'request' => $request,
'error' => null,
];
}
private function scrapeWorkshopItems(string $appId, string $query, int $perPage, int $page): array
@ -1396,67 +1355,6 @@ class SteamWorkshopService
}
private function fetchWorkshopItemById(string $id): ?array
{
$response = $this->executeSteamApiRequest('https://api.steampowered.com/ISteamRemoteStorage/GetPublishedFileDetails/v1/', [
'itemcount' => 1,
'publishedfileids[0]' => $id,
]);
if ($response['error'] !== null || $response['http_code'] < 200 || $response['http_code'] >= 300) {
$reason = $response['error'] ?? ('HTTP ' . $response['http_code']);
$this->logApiFailure(sprintf('Steam API detail lookup failed (id=%s): %s', $id, $reason));
return null;
}
$data = json_decode((string)$response['body'], true);
$details = $data['response']['publishedfiledetails'][0] ?? null;
if (!is_array($details) || (int)($details['result'] ?? 0) !== 1) {
return null;
}
$title = trim((string)($details['title'] ?? ''));
if ($title === '') {
$title = '@' . $id;
}
return [
'id' => $id,
'label' => $title,
'author' => (string)($details['creator'] ?? ''),
'preview_url' => (string)($details['preview_url'] ?? ''),
'enabled' => true,
'source' => 'search',
];
}
private function executeSteamApiRequest(string $url, array $fields): array
{
if (!function_exists('curl_init')) {
return ['body' => null, 'http_code' => 0, 'error' => 'PHP cURL extension is required', 'url' => $url, 'fields' => $fields];
}
$ch = curl_init($url);
if ($ch === false) {
return ['body' => null, 'http_code' => 0, 'error' => 'Unable to initialize cURL', 'url' => $url, 'fields' => $fields];
}
curl_setopt_array($ch, [
CURLOPT_RETURNTRANSFER => true,
CURLOPT_POST => true,
CURLOPT_POSTFIELDS => http_build_query($fields, '', '&'),
CURLOPT_TIMEOUT => 15,
CURLOPT_USERAGENT => 'GSP-Workshop/1.0 (+https://github.com/GameServerPanel/GSP)',
CURLOPT_HTTPHEADER => ['Accept: application/json', 'Content-Type: application/x-www-form-urlencoded'],
]);
$body = curl_exec($ch);
$error = curl_errno($ch) ? curl_error($ch) : null;
$status = (int)curl_getinfo($ch, CURLINFO_HTTP_CODE);
curl_close($ch);
return ['body' => $error === null ? $body : null, 'http_code' => $status, 'error' => $error, 'url' => $url, 'fields' => $fields];
}
private function httpGet(string $url, array $params = [], ?string $userAgent = null): array
{
if (!function_exists('curl_init')) {

5
modules/steam_workshop/main.php
View file

@ -10,7 +10,10 @@ function exec_ogp_module(): void
{
global $db;
echo '<h2>' . get_lang('steam_workshop') . '</h2>';
$action = $_GET['action'] ?? '';
if ($action !== 'search') {
echo '<h2>' . get_lang('steam_workshop') . '</h2>';
}
$controller = new SteamWorkshopController($db);
$controller->handle();

31
modules/steam_workshop/steam_workshop.js
View file

@ -241,16 +241,31 @@
}
};
Picker.prototype.updateRequestPreview = function () {
if (this.requestInput && this.searchInput) {
this.requestInput.value = this.searchInput.value;
if (!this.searchInput) {
return;
}
if (this.requestSummary) {
var encoded = '';
if (this.searchInput && this.searchInput.value.trim() !== '') {
encoded = encodeURIComponent(this.searchInput.value.trim());
}
this.requestSummary.textContent = (this.requestSummaryBase || '') + encoded;
var term = this.searchInput.value.trim();
if (this.requestInput) {
this.requestInput.value = term;
}
if (!this.requestSummary) {
return;
}
var base = this.requestSummaryBase || '';
if (!base) {
this.requestSummary.textContent = '';
return;
}
if (!term) {
this.requestSummary.textContent = base;
return;
}
var isId = /^\d+$/.test(term);
if (isId) {
this.requestSummary.textContent = 'https://steamcommunity.com/sharedfiles/filedetails/?id=' + encodeURIComponent(term);
return;
}
this.requestSummary.textContent = base + encodeURIComponent(term);
};
Picker.prototype.isSelected = function (id) {
return this.state.selected.some(function (item) { return item.id === id; });

1
modules/steam_workshop/views/edit.php
View file

@ -4,6 +4,7 @@ declare(strict_types=1);
/** @var array $config */
/** @var array $lang */
/** @var array $adapterOptions */
/** @var string|null $appId */
$homeName = htmlspecialchars($home['home_name'] ?? ('#' . $home['home_id']));
$homeId = (int)$home['home_id'];
?>

5
modules/steam_workshop/views/partials/mod_picker.php
View file

@ -4,6 +4,7 @@ declare(strict_types=1);
/** @var array $config */
/** @var array $home */
/** @var int $homeId */
/** @var string|null $appId */
$homeId = (int)($home['home_id'] ?? 0);
$scriptPath = (string)($_SERVER['PHP_SELF'] ?? '/index.php');
if ($scriptPath === '') {
@ -13,6 +14,8 @@ if ($scriptPath[0] !== '/') {
$scriptPath = '/' . ltrim($scriptPath, '/');
}
$endpoint = sprintf('%s?m=steam_workshop&p=main&action=search&home_id=%d', $scriptPath, $homeId);
$steamBase = 'https://steamcommunity.com/workshop/browse/?appid=';
$steamAppId = isset($appId) && $appId !== '' ? $appId : '0';
$initialItems = [];
foreach ($config['workshop_items'] ?? [] as $item) {
if (!is_array($item)) {
@ -62,7 +65,7 @@ $langAttrs = [
<span class="sw-picker__request-label"><?php echo htmlspecialchars($lang['mod_picker_request_label'] ?? 'Submitting request'); ?></span>
<small class="sw-picker__request-hint"><?php echo htmlspecialchars($lang['mod_picker_request_hint'] ?? 'Exact URL preview. The field below mirrors your search text.'); ?></small>
<div class="sw-picker__request-line">
<?php $baseRequest = $endpoint . '&q='; ?>
<?php $baseRequest = $steamBase . $steamAppId . '&browsesort=textsearch&section=readytouseitems&searchtext='; ?>
<code class="sw-picker__request-summary js-sw-request-summary" data-base="<?php echo htmlspecialchars($baseRequest, ENT_QUOTES, 'UTF-8'); ?>"><?php echo htmlspecialchars($baseRequest, ENT_QUOTES, 'UTF-8'); ?></code>
<input type="text" class="sw-picker__request-input js-sw-request-input" value="" readonly aria-label="<?php echo htmlspecialchars($lang['mod_picker_request_input_label'] ?? 'Workshop search text preview'); ?>" />
</div>