Fix Steam Workshop search scraping flow
Co-authored-by: iaretechnician <2749183+iaretechnician@users.noreply.github.com>
This commit is contained in:
copilot-swe-agent[bot]
parent
2a509fab03
commit
18b6bc1a14
8 changed files with 98 additions and 174 deletions
|
|
@ -169,6 +169,8 @@ class SteamWorkshopController
|
|||
|
||||
private function renderEdit(array $home, array $config, bool $isAdmin, bool $adapterLocked): void
|
||||
{
|
||||
$gameKey = (string)($home['game_key'] ?? '');
|
||||
$appId = $gameKey !== '' ? $this->service->getSteamAppIdForGameKey($gameKey) : null;
|
||||
$this->render('edit', [
|
||||
'lang' => $this->lang,
|
||||
'home' => $home,
|
||||
|
|
@ -176,6 +178,7 @@ class SteamWorkshopController
|
|||
'isAdmin' => $isAdmin,
|
||||
'adapterOptions' => $this->service->getAdapterOptions(),
|
||||
'adapterLocked' => $adapterLocked,
|
||||
'appId' => $appId,
|
||||
]);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -85,7 +85,7 @@ return [
|
|||
'mod_picker_status_need_query' => 'Enter a Workshop ID or keyword before searching.',
|
||||
'mod_picker_toggle_label' => 'Sync',
|
||||
'mod_picker_request_label' => 'Submitting request',
|
||||
'mod_picker_request_hint' => 'Exact URL preview. The input shows the text that will be submitted.',
|
||||
'mod_picker_request_hint' => 'Exact Steam URL preview. The input shows the text that will be submitted.',
|
||||
'mod_picker_request_input_label' => 'Workshop query preview',
|
||||
'error_game_key_required' => 'Select a valid game key before editing the adapter.',
|
||||
'error_adapter_delete_failed' => 'Adapter could not be deleted.',
|
||||
|
|
|
|||
|
|
@ -682,125 +682,84 @@ class SteamWorkshopService
|
|||
}
|
||||
|
||||
if (ctype_digit($query)) {
|
||||
$item = $this->fetchWorkshopItemById($query);
|
||||
if ($item !== null) {
|
||||
$payload['results'][] = $item;
|
||||
$detail = $this->fetchWorkshopItemByScrape($query);
|
||||
$payload['request'] = $detail['request'];
|
||||
if ($detail['error'] !== null) {
|
||||
$payload['error'] = $detail['error'];
|
||||
return $payload;
|
||||
}
|
||||
if ($detail['item'] !== null) {
|
||||
$payload['results'][] = $detail['item'];
|
||||
$payload['pagination']['total'] = 1;
|
||||
}
|
||||
return $payload;
|
||||
}
|
||||
|
||||
$postFields = [
|
||||
'query_type' => 0,
|
||||
'page' => $payload['pagination']['page'],
|
||||
'numperpage' => $payload['pagination']['per_page'],
|
||||
'appid' => $appId,
|
||||
'search_text' => $query,
|
||||
'return_details' => true,
|
||||
'return_metadata' => false,
|
||||
];
|
||||
|
||||
$response = $this->executeSteamApiRequest('https://api.steampowered.com/IPublishedFileService/QueryFiles/v1/', $postFields);
|
||||
$requestContext = [
|
||||
'backend' => 'api',
|
||||
'url' => $response['url'],
|
||||
'params' => $response['fields'],
|
||||
'http_code' => $response['http_code'],
|
||||
'transport_error' => $response['error'],
|
||||
];
|
||||
$requestContext['summary'] = $this->formatRequestSummary($requestContext);
|
||||
$requestAttempts = [$requestContext];
|
||||
$payload['request'] = $requestContext;
|
||||
|
||||
$apiFailed = false;
|
||||
if ($response['error'] !== null || $response['http_code'] < 200 || $response['http_code'] >= 300) {
|
||||
$apiFailed = true;
|
||||
$reason = $response['error'] !== null ? $response['error'] : 'HTTP ' . $response['http_code'];
|
||||
$this->logApiFailure(sprintf('Steam API search failed (app=%s query="%s" page=%d): %s', $appId, $query, $payload['pagination']['page'], $reason));
|
||||
$payload['error'] = sprintf('Steam API request failed (%s). URL: %s Params: %s', $reason, $response['url'], http_build_query($response['fields'], '', '&'));
|
||||
$scrapeResult = $this->scrapeWorkshopItems($appId, $query, $payload['pagination']['per_page'], $payload['pagination']['page']);
|
||||
$payload['request'] = $scrapeResult['request'];
|
||||
if (!empty($scrapeResult['attempts'])) {
|
||||
$payload['request']['attempts'] = $scrapeResult['attempts'];
|
||||
}
|
||||
if ($scrapeResult['success']) {
|
||||
$payload['results'] = $scrapeResult['results'];
|
||||
$payload['pagination']['total'] = $scrapeResult['total'];
|
||||
$payload['pagination']['has_more'] = $scrapeResult['has_more'];
|
||||
} else {
|
||||
$data = json_decode((string)$response['body'], true);
|
||||
if (!is_array($data) || !isset($data['response'])) {
|
||||
$apiFailed = true;
|
||||
$this->logApiFailure(sprintf('Steam API search returned invalid payload (app=%s query="%s")', $appId, $query));
|
||||
$payload['error'] = sprintf('Steam API returned invalid data. URL: %s Params: %s', $response['url'], http_build_query($response['fields'], '', '&'));
|
||||
} else {
|
||||
$details = $data['response']['publishedfiledetails'] ?? [];
|
||||
$total = (int)($data['response']['total'] ?? count($details));
|
||||
|
||||
foreach ($details as $item) {
|
||||
$id = isset($item['publishedfileid']) ? preg_replace('/[^0-9]/', '', (string)$item['publishedfileid']) : '';
|
||||
if ($id === '') {
|
||||
continue;
|
||||
}
|
||||
$title = isset($item['title']) ? trim((string)$item['title']) : '';
|
||||
if ($title === '') {
|
||||
$title = '@' . $id;
|
||||
}
|
||||
$payload['results'][] = [
|
||||
'id' => $id,
|
||||
'label' => $title,
|
||||
'author' => isset($item['creator']) ? (string)$item['creator'] : '',
|
||||
'preview_url' => isset($item['preview_url']) ? (string)$item['preview_url'] : '',
|
||||
'time_updated' => isset($item['time_updated']) ? (int)$item['time_updated'] : null,
|
||||
'subscriptions' => isset($item['subscriptions']) ? (int)$item['subscriptions'] : 0,
|
||||
'source' => 'search',
|
||||
];
|
||||
}
|
||||
|
||||
$payload['pagination']['total'] = $total;
|
||||
$payload['pagination']['has_more'] = ($payload['pagination']['page'] * $payload['pagination']['per_page']) < $total;
|
||||
}
|
||||
$payload['error'] = $scrapeResult['error'] ?? 'Steam Workshop scrape failed.';
|
||||
$this->logApiFailure(sprintf('Steam Workshop scrape failed (app=%s query="%s" page=%d): %s', $appId, $query, $payload['pagination']['page'], $payload['error']));
|
||||
}
|
||||
|
||||
if ($this->shouldAttemptScraper($query, $payload)) {
|
||||
$scrapeResult = $this->scrapeWorkshopItems($appId, $query, $payload['pagination']['per_page'], $payload['pagination']['page']);
|
||||
$scrapeContext = $scrapeResult['request'];
|
||||
$attemptContexts = $scrapeResult['attempts'] ?? [$scrapeContext];
|
||||
foreach ($attemptContexts as $attemptContext) {
|
||||
$requestAttempts[] = $attemptContext;
|
||||
}
|
||||
if ($scrapeResult['success'] && !empty($scrapeResult['results'])) {
|
||||
$payload['results'] = $scrapeResult['results'];
|
||||
$payload['pagination']['total'] = $scrapeResult['total'];
|
||||
$payload['pagination']['has_more'] = $scrapeResult['has_more'];
|
||||
$payload['error'] = null;
|
||||
$payload['request'] = $scrapeContext;
|
||||
} elseif (!$scrapeResult['success']) {
|
||||
$fallbackError = $scrapeResult['error'] ?? 'Steam Workshop scrape failed.';
|
||||
if ($payload['error'] === null) {
|
||||
$payload['error'] = $fallbackError;
|
||||
} else {
|
||||
$payload['error'] .= ' Scraper fallback failed: ' . $fallbackError;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
$payload['request']['attempts'] = $requestAttempts;
|
||||
|
||||
return $payload;
|
||||
}
|
||||
|
||||
private function shouldAttemptScraper(string $query, array $payload): bool
|
||||
private function fetchWorkshopItemByScrape(string $id): array
|
||||
{
|
||||
if (ctype_digit($query)) {
|
||||
return false;
|
||||
}
|
||||
if (!$this->hasAnyScraperTransport()) {
|
||||
return false;
|
||||
$id = preg_replace('/[^0-9]/', '', $id);
|
||||
$request = [
|
||||
'backend' => 'scraper_http',
|
||||
'url' => 'https://steamcommunity.com/sharedfiles/filedetails/',
|
||||
'params' => ['id' => $id],
|
||||
'http_code' => null,
|
||||
'transport_error' => null,
|
||||
];
|
||||
|
||||
if ($id === '') {
|
||||
$request['summary'] = $this->formatRequestSummary($request);
|
||||
return ['item' => null, 'request' => $request, 'error' => 'Invalid Workshop ID.'];
|
||||
}
|
||||
|
||||
$hasResults = !empty($payload['results']);
|
||||
return $payload['error'] !== null || $hasResults === false;
|
||||
}
|
||||
$response = $this->httpGet($request['url'], $request['params'], $this->getScraperUserAgent());
|
||||
$request['url'] = $response['url'] ?? $request['url'];
|
||||
$request['http_code'] = $response['http_code'];
|
||||
$request['transport_error'] = $response['error'];
|
||||
$request['summary'] = $this->formatRequestSummary($request);
|
||||
|
||||
private function hasAnyScraperTransport(): bool
|
||||
{
|
||||
if ($this->isScraperAvailable()) {
|
||||
return true;
|
||||
if ($response['error'] !== null || $response['http_code'] < 200 || $response['http_code'] >= 300 || $response['body'] === null) {
|
||||
$reason = $response['error'] !== null ? $response['error'] : 'HTTP ' . $response['http_code'];
|
||||
return [
|
||||
'item' => null,
|
||||
'request' => $request,
|
||||
'error' => 'Steam Community detail request failed: ' . $reason,
|
||||
];
|
||||
}
|
||||
|
||||
return function_exists('curl_init');
|
||||
$title = $this->parseWorkshopTitle((string)$response['body']);
|
||||
if ($title === '') {
|
||||
$title = '@' . $id;
|
||||
}
|
||||
|
||||
return [
|
||||
'item' => [
|
||||
'id' => $id,
|
||||
'label' => $title,
|
||||
'author' => '',
|
||||
'preview_url' => '',
|
||||
'enabled' => true,
|
||||
'source' => 'search',
|
||||
],
|
||||
'request' => $request,
|
||||
'error' => null,
|
||||
];
|
||||
}
|
||||
|
||||
private function scrapeWorkshopItems(string $appId, string $query, int $perPage, int $page): array
|
||||
|
|
@ -1396,67 +1355,6 @@ class SteamWorkshopService
|
|||
}
|
||||
|
||||
|
||||
private function fetchWorkshopItemById(string $id): ?array
|
||||
{
|
||||
$response = $this->executeSteamApiRequest('https://api.steampowered.com/ISteamRemoteStorage/GetPublishedFileDetails/v1/', [
|
||||
'itemcount' => 1,
|
||||
'publishedfileids[0]' => $id,
|
||||
]);
|
||||
if ($response['error'] !== null || $response['http_code'] < 200 || $response['http_code'] >= 300) {
|
||||
$reason = $response['error'] ?? ('HTTP ' . $response['http_code']);
|
||||
$this->logApiFailure(sprintf('Steam API detail lookup failed (id=%s): %s', $id, $reason));
|
||||
return null;
|
||||
}
|
||||
|
||||
$data = json_decode((string)$response['body'], true);
|
||||
$details = $data['response']['publishedfiledetails'][0] ?? null;
|
||||
if (!is_array($details) || (int)($details['result'] ?? 0) !== 1) {
|
||||
return null;
|
||||
}
|
||||
|
||||
$title = trim((string)($details['title'] ?? ''));
|
||||
if ($title === '') {
|
||||
$title = '@' . $id;
|
||||
}
|
||||
|
||||
return [
|
||||
'id' => $id,
|
||||
'label' => $title,
|
||||
'author' => (string)($details['creator'] ?? ''),
|
||||
'preview_url' => (string)($details['preview_url'] ?? ''),
|
||||
'enabled' => true,
|
||||
'source' => 'search',
|
||||
];
|
||||
}
|
||||
|
||||
private function executeSteamApiRequest(string $url, array $fields): array
|
||||
{
|
||||
if (!function_exists('curl_init')) {
|
||||
return ['body' => null, 'http_code' => 0, 'error' => 'PHP cURL extension is required', 'url' => $url, 'fields' => $fields];
|
||||
}
|
||||
|
||||
$ch = curl_init($url);
|
||||
if ($ch === false) {
|
||||
return ['body' => null, 'http_code' => 0, 'error' => 'Unable to initialize cURL', 'url' => $url, 'fields' => $fields];
|
||||
}
|
||||
|
||||
curl_setopt_array($ch, [
|
||||
CURLOPT_RETURNTRANSFER => true,
|
||||
CURLOPT_POST => true,
|
||||
CURLOPT_POSTFIELDS => http_build_query($fields, '', '&'),
|
||||
CURLOPT_TIMEOUT => 15,
|
||||
CURLOPT_USERAGENT => 'GSP-Workshop/1.0 (+https://github.com/GameServerPanel/GSP)',
|
||||
CURLOPT_HTTPHEADER => ['Accept: application/json', 'Content-Type: application/x-www-form-urlencoded'],
|
||||
]);
|
||||
|
||||
$body = curl_exec($ch);
|
||||
$error = curl_errno($ch) ? curl_error($ch) : null;
|
||||
$status = (int)curl_getinfo($ch, CURLINFO_HTTP_CODE);
|
||||
curl_close($ch);
|
||||
|
||||
return ['body' => $error === null ? $body : null, 'http_code' => $status, 'error' => $error, 'url' => $url, 'fields' => $fields];
|
||||
}
|
||||
|
||||
private function httpGet(string $url, array $params = [], ?string $userAgent = null): array
|
||||
{
|
||||
if (!function_exists('curl_init')) {
|
||||
|
|
|
|||
|
|
@ -10,7 +10,10 @@ function exec_ogp_module(): void
|
|||
{
|
||||
global $db;
|
||||
|
||||
echo '<h2>' . get_lang('steam_workshop') . '</h2>';
|
||||
$action = $_GET['action'] ?? '';
|
||||
if ($action !== 'search') {
|
||||
echo '<h2>' . get_lang('steam_workshop') . '</h2>';
|
||||
}
|
||||
|
||||
$controller = new SteamWorkshopController($db);
|
||||
$controller->handle();
|
||||
|
|
|
|||
|
|
@ -241,16 +241,31 @@
|
|||
}
|
||||
};
|
||||
Picker.prototype.updateRequestPreview = function () {
|
||||
if (this.requestInput && this.searchInput) {
|
||||
this.requestInput.value = this.searchInput.value;
|
||||
if (!this.searchInput) {
|
||||
return;
|
||||
}
|
||||
if (this.requestSummary) {
|
||||
var encoded = '';
|
||||
if (this.searchInput && this.searchInput.value.trim() !== '') {
|
||||
encoded = encodeURIComponent(this.searchInput.value.trim());
|
||||
}
|
||||
this.requestSummary.textContent = (this.requestSummaryBase || '') + encoded;
|
||||
var term = this.searchInput.value.trim();
|
||||
if (this.requestInput) {
|
||||
this.requestInput.value = term;
|
||||
}
|
||||
if (!this.requestSummary) {
|
||||
return;
|
||||
}
|
||||
var base = this.requestSummaryBase || '';
|
||||
if (!base) {
|
||||
this.requestSummary.textContent = '';
|
||||
return;
|
||||
}
|
||||
if (!term) {
|
||||
this.requestSummary.textContent = base;
|
||||
return;
|
||||
}
|
||||
var isId = /^\d+$/.test(term);
|
||||
if (isId) {
|
||||
this.requestSummary.textContent = 'https://steamcommunity.com/sharedfiles/filedetails/?id=' + encodeURIComponent(term);
|
||||
return;
|
||||
}
|
||||
this.requestSummary.textContent = base + encodeURIComponent(term);
|
||||
};
|
||||
Picker.prototype.isSelected = function (id) {
|
||||
return this.state.selected.some(function (item) { return item.id === id; });
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ declare(strict_types=1);
|
|||
/** @var array $config */
|
||||
/** @var array $lang */
|
||||
/** @var array $adapterOptions */
|
||||
/** @var string|null $appId */
|
||||
$homeName = htmlspecialchars($home['home_name'] ?? ('#' . $home['home_id']));
|
||||
$homeId = (int)$home['home_id'];
|
||||
?>
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ declare(strict_types=1);
|
|||
/** @var array $config */
|
||||
/** @var array $home */
|
||||
/** @var int $homeId */
|
||||
/** @var string|null $appId */
|
||||
$homeId = (int)($home['home_id'] ?? 0);
|
||||
$scriptPath = (string)($_SERVER['PHP_SELF'] ?? '/index.php');
|
||||
if ($scriptPath === '') {
|
||||
|
|
@ -13,6 +14,8 @@ if ($scriptPath[0] !== '/') {
|
|||
$scriptPath = '/' . ltrim($scriptPath, '/');
|
||||
}
|
||||
$endpoint = sprintf('%s?m=steam_workshop&p=main&action=search&home_id=%d', $scriptPath, $homeId);
|
||||
$steamBase = 'https://steamcommunity.com/workshop/browse/?appid=';
|
||||
$steamAppId = isset($appId) && $appId !== '' ? $appId : '0';
|
||||
$initialItems = [];
|
||||
foreach ($config['workshop_items'] ?? [] as $item) {
|
||||
if (!is_array($item)) {
|
||||
|
|
@ -62,7 +65,7 @@ $langAttrs = [
|
|||
<span class="sw-picker__request-label"><?php echo htmlspecialchars($lang['mod_picker_request_label'] ?? 'Submitting request'); ?></span>
|
||||
<small class="sw-picker__request-hint"><?php echo htmlspecialchars($lang['mod_picker_request_hint'] ?? 'Exact URL preview. The field below mirrors your search text.'); ?></small>
|
||||
<div class="sw-picker__request-line">
|
||||
<?php $baseRequest = $endpoint . '&q='; ?>
|
||||
<?php $baseRequest = $steamBase . $steamAppId . '&browsesort=textsearch§ion=readytouseitems&searchtext='; ?>
|
||||
<code class="sw-picker__request-summary js-sw-request-summary" data-base="<?php echo htmlspecialchars($baseRequest, ENT_QUOTES, 'UTF-8'); ?>"><?php echo htmlspecialchars($baseRequest, ENT_QUOTES, 'UTF-8'); ?></code>
|
||||
<input type="text" class="sw-picker__request-input js-sw-request-input" value="" readonly aria-label="<?php echo htmlspecialchars($lang['mod_picker_request_input_label'] ?? 'Workshop search text preview'); ?>" />
|
||||
</div>
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue