Merge pull request #55 from GameServerPanel/copilot/create-steam-workshop-search

Switch Steam Workshop search to Steam Community scraping and update URL preview

CHANGELOG.md

@@ -1,6 +1,7 @@
 # Changelog
 
 ## 2026-01-31
+- Rebuilt the Steam Workshop picker search to rely solely on Steam Community scraping (matching the working curl flow) and updated the request preview to show the actual Steam URL instead of a local panel endpoint.
 - Added adapter AppID lookup for Workshop search so the picker can query Steam even when the server XML lacks a clear default installer entry.
 - Switched Workshop picker results to checkbox selection, letting customers toggle multiple mods directly from the search list.
 - Added a hardened Workshop scraping helper (the same HTML workflow we validated manually) and wired it into the Steam Workshop service as a fallback whenever the official API errors out or returns empty data.

modules/steam_workshop/controllers/SteamWorkshopController.php

@@ -169,6 +169,8 @@ class SteamWorkshopController
 
     private function renderEdit(array $home, array $config, bool $isAdmin, bool $adapterLocked): void
     {
+        $gameKey = (string)($home['game_key'] ?? '');
+        $appId = $gameKey !== '' ? $this->service->getSteamAppIdForGameKey($gameKey) : null;
         $this->render('edit', [
             'lang' => $this->lang,
             'home' => $home,
@@ -176,6 +178,7 @@ class SteamWorkshopController
             'isAdmin' => $isAdmin,
             'adapterOptions' => $this->service->getAdapterOptions(),
             'adapterLocked' => $adapterLocked,
+            'appId' => $appId,
         ]);
     }
 

modules/steam_workshop/lang/en_US.php

@@ -85,7 +85,7 @@ return [
     'mod_picker_status_need_query' => 'Enter a Workshop ID or keyword before searching.',
     'mod_picker_toggle_label' => 'Sync',
     'mod_picker_request_label' => 'Submitting request',
-    'mod_picker_request_hint' => 'Exact URL preview. The input shows the text that will be submitted.',
+    'mod_picker_request_hint' => 'Exact Steam URL preview. The input shows the text that will be submitted.',
     'mod_picker_request_input_label' => 'Workshop query preview',
     'error_game_key_required' => 'Select a valid game key before editing the adapter.',
     'error_adapter_delete_failed' => 'Adapter could not be deleted.',

modules/steam_workshop/lib/SteamWorkshopService.php

@@ -5,6 +5,7 @@ class SteamWorkshopService
 {
 	private const MIN_INTERVAL = 15;
 	private const MAX_INTERVAL = 360;
+	private const STEAM_WORKSHOP_DETAIL_URL = 'https://steamcommunity.com/sharedfiles/filedetails/';
 
 	private OGPDatabase $db;
 	private string $configDir;
@@ -682,125 +683,84 @@ class SteamWorkshopService
 		}
 
 		if (ctype_digit($query)) {
-			$item = $this->fetchWorkshopItemById($query);
+			$detail = $this->fetchWorkshopItemByScrape($query);
-			if ($item !== null) {
+			$payload['request'] = $detail['request'];
-				$payload['results'][] = $item;
+			if ($detail['error'] !== null) {
+				$payload['error'] = $detail['error'];
+				return $payload;
+			}
+			if ($detail['item'] !== null) {
+				$payload['results'][] = $detail['item'];
 				$payload['pagination']['total'] = 1;
 			}
 			return $payload;
 		}
 
-		$postFields = [
+		$scrapeResult = $this->scrapeWorkshopItems($appId, $query, $payload['pagination']['per_page'], $payload['pagination']['page']);
-			'query_type' => 0,
+		$payload['request'] = $scrapeResult['request'];
-			'page' => $payload['pagination']['page'],
+		if (!empty($scrapeResult['attempts'])) {
-			'numperpage' => $payload['pagination']['per_page'],
+			$payload['request']['attempts'] = $scrapeResult['attempts'];
-			'appid' => $appId,
+		}
-			'search_text' => $query,
+		if ($scrapeResult['success']) {
-			'return_details' => true,
+			$payload['results'] = $scrapeResult['results'];
-			'return_metadata' => false,
+			$payload['pagination']['total'] = $scrapeResult['total'];
-		];
+			$payload['pagination']['has_more'] = $scrapeResult['has_more'];
-
-		$response = $this->executeSteamApiRequest('https://api.steampowered.com/IPublishedFileService/QueryFiles/v1/', $postFields);
-		$requestContext = [
-			'backend' => 'api',
-			'url' => $response['url'],
-			'params' => $response['fields'],
-			'http_code' => $response['http_code'],
-			'transport_error' => $response['error'],
-		];
-		$requestContext['summary'] = $this->formatRequestSummary($requestContext);
-		$requestAttempts = [$requestContext];
-		$payload['request'] = $requestContext;
-
-		$apiFailed = false;
-		if ($response['error'] !== null || $response['http_code'] < 200 || $response['http_code'] >= 300) {
-			$apiFailed = true;
-			$reason = $response['error'] !== null ? $response['error'] : 'HTTP ' . $response['http_code'];
-			$this->logApiFailure(sprintf('Steam API search failed (app=%s query="%s" page=%d): %s', $appId, $query, $payload['pagination']['page'], $reason));
-			$payload['error'] = sprintf('Steam API request failed (%s). URL: %s Params: %s', $reason, $response['url'], http_build_query($response['fields'], '', '&'));
 		} else {
-			$data = json_decode((string)$response['body'], true);
+			$payload['error'] = $scrapeResult['error'] ?? 'Steam Workshop scrape failed.';
-			if (!is_array($data) || !isset($data['response'])) {
+			$this->logApiFailure(sprintf('Steam Workshop scrape failed (app=%s query="%s" page=%d): %s', $appId, $query, $payload['pagination']['page'], $payload['error']));
-				$apiFailed = true;
-				$this->logApiFailure(sprintf('Steam API search returned invalid payload (app=%s query="%s")', $appId, $query));
-				$payload['error'] = sprintf('Steam API returned invalid data. URL: %s Params: %s', $response['url'], http_build_query($response['fields'], '', '&'));
-			} else {
-				$details = $data['response']['publishedfiledetails'] ?? [];
-				$total = (int)($data['response']['total'] ?? count($details));
-
-				foreach ($details as $item) {
-					$id = isset($item['publishedfileid']) ? preg_replace('/[^0-9]/', '', (string)$item['publishedfileid']) : '';
-					if ($id === '') {
-						continue;
-					}
-					$title = isset($item['title']) ? trim((string)$item['title']) : '';
-					if ($title === '') {
-						$title = '@' . $id;
-					}
-					$payload['results'][] = [
-						'id' => $id,
-						'label' => $title,
-						'author' => isset($item['creator']) ? (string)$item['creator'] : '',
-						'preview_url' => isset($item['preview_url']) ? (string)$item['preview_url'] : '',
-						'time_updated' => isset($item['time_updated']) ? (int)$item['time_updated'] : null,
-						'subscriptions' => isset($item['subscriptions']) ? (int)$item['subscriptions'] : 0,
-						'source' => 'search',
-					];
-				}
-
-				$payload['pagination']['total'] = $total;
-				$payload['pagination']['has_more'] = ($payload['pagination']['page'] * $payload['pagination']['per_page']) < $total;
-			}
 		}
 
-		if ($this->shouldAttemptScraper($query, $payload)) {
-			$scrapeResult = $this->scrapeWorkshopItems($appId, $query, $payload['pagination']['per_page'], $payload['pagination']['page']);
-			$scrapeContext = $scrapeResult['request'];
-			$attemptContexts = $scrapeResult['attempts'] ?? [$scrapeContext];
-			foreach ($attemptContexts as $attemptContext) {
-				$requestAttempts[] = $attemptContext;
-			}
-			if ($scrapeResult['success'] && !empty($scrapeResult['results'])) {
-				$payload['results'] = $scrapeResult['results'];
-				$payload['pagination']['total'] = $scrapeResult['total'];
-				$payload['pagination']['has_more'] = $scrapeResult['has_more'];
-				$payload['error'] = null;
-				$payload['request'] = $scrapeContext;
-			} elseif (!$scrapeResult['success']) {
-				$fallbackError = $scrapeResult['error'] ?? 'Steam Workshop scrape failed.';
-				if ($payload['error'] === null) {
-					$payload['error'] = $fallbackError;
-				} else {
-					$payload['error'] .= ' Scraper fallback failed: ' . $fallbackError;
-				}
-			}
-		}
-
-		$payload['request']['attempts'] = $requestAttempts;
-
 		return $payload;
 	}
 
-	private function shouldAttemptScraper(string $query, array $payload): bool
+	private function fetchWorkshopItemByScrape(string $id): array
 	{
-		if (ctype_digit($query)) {
+		$sanitizedId = preg_replace('/[^0-9]/', '', $id);
-			return false;
+		$request = [
-		}
+			'backend' => 'scraper_http',
-		if (!$this->hasAnyScraperTransport()) {
+			'url' => self::STEAM_WORKSHOP_DETAIL_URL,
-			return false;
+			'params' => ['id' => $sanitizedId],
+			'http_code' => null,
+			'transport_error' => null,
+		];
+
+		if ($sanitizedId === '') {
+			$request['summary'] = $this->formatRequestSummary($request);
+			return ['item' => null, 'request' => $request, 'error' => 'Invalid Workshop ID.'];
 		}
 
-		$hasResults = !empty($payload['results']);
+		$response = $this->httpGet($request['url'], $request['params'], $this->getScraperUserAgent());
-		return $payload['error'] !== null || $hasResults === false;
+		$request['url'] = $response['url'] ?? $request['url'];
-	}
+		$request['http_code'] = $response['http_code'];
+		$request['transport_error'] = $response['error'];
+		$request['summary'] = $this->formatRequestSummary($request);
 
-	private function hasAnyScraperTransport(): bool
+		if ($response['error'] !== null || $response['http_code'] < 200 || $response['http_code'] >= 300 || $response['body'] === null) {
-	{
+			$reason = $response['error'] !== null ? $response['error'] : 'HTTP ' . $response['http_code'];
-		if ($this->isScraperAvailable()) {
+			return [
-			return true;
+				'item' => null,
+				'request' => $request,
+				'error' => 'Steam Community detail request failed: ' . $reason,
+			];
 		}
 
-		return function_exists('curl_init');
+		$title = $this->parseWorkshopTitle((string)$response['body']);
+		if ($title === '') {
+			$title = '@' . $sanitizedId;
+		}
+
+		return [
+			'item' => [
+				'id' => $sanitizedId,
+				'label' => $title,
+				'author' => '',
+				'preview_url' => '',
+				'enabled' => true,
+				'source' => 'search',
+			],
+			'request' => $request,
+			'error' => null,
+		];
 	}
 
 	private function scrapeWorkshopItems(string $appId, string $query, int $perPage, int $page): array
@@ -1001,7 +961,7 @@ class SteamWorkshopService
 
 		$results = [];
 		foreach ($sliceIds as $id) {
-			$detailResponse = $this->httpGet('https://steamcommunity.com/sharedfiles/filedetails/', ['id' => $id], $this->getScraperUserAgent());
+			$detailResponse = $this->httpGet(self::STEAM_WORKSHOP_DETAIL_URL, ['id' => $id], $this->getScraperUserAgent());
 			$title = '';
 			if ($detailResponse['error'] === null && $detailResponse['http_code'] >= 200 && $detailResponse['http_code'] < 300 && $detailResponse['body'] !== null) {
 				$title = $this->parseWorkshopTitle((string)$detailResponse['body']);
@@ -1396,67 +1356,6 @@ class SteamWorkshopService
 	}
 
 
-	private function fetchWorkshopItemById(string $id): ?array
-	{
-		$response = $this->executeSteamApiRequest('https://api.steampowered.com/ISteamRemoteStorage/GetPublishedFileDetails/v1/', [
-			'itemcount' => 1,
-			'publishedfileids[0]' => $id,
-		]);
-		if ($response['error'] !== null || $response['http_code'] < 200 || $response['http_code'] >= 300) {
-			$reason = $response['error'] ?? ('HTTP ' . $response['http_code']);
-			$this->logApiFailure(sprintf('Steam API detail lookup failed (id=%s): %s', $id, $reason));
-			return null;
-		}
-
-		$data = json_decode((string)$response['body'], true);
-		$details = $data['response']['publishedfiledetails'][0] ?? null;
-		if (!is_array($details) || (int)($details['result'] ?? 0) !== 1) {
-			return null;
-		}
-
-		$title = trim((string)($details['title'] ?? ''));
-		if ($title === '') {
-			$title = '@' . $id;
-		}
-
-		return [
-			'id' => $id,
-			'label' => $title,
-			'author' => (string)($details['creator'] ?? ''),
-			'preview_url' => (string)($details['preview_url'] ?? ''),
-			'enabled' => true,
-			'source' => 'search',
-		];
-	}
-
-	private function executeSteamApiRequest(string $url, array $fields): array
-	{
-		if (!function_exists('curl_init')) {
-			return ['body' => null, 'http_code' => 0, 'error' => 'PHP cURL extension is required', 'url' => $url, 'fields' => $fields];
-		}
-
-		$ch = curl_init($url);
-		if ($ch === false) {
-			return ['body' => null, 'http_code' => 0, 'error' => 'Unable to initialize cURL', 'url' => $url, 'fields' => $fields];
-		}
-
-		curl_setopt_array($ch, [
-			CURLOPT_RETURNTRANSFER => true,
-			CURLOPT_POST => true,
-			CURLOPT_POSTFIELDS => http_build_query($fields, '', '&'),
-			CURLOPT_TIMEOUT => 15,
-			CURLOPT_USERAGENT => 'GSP-Workshop/1.0 (+https://github.com/GameServerPanel/GSP)',
-			CURLOPT_HTTPHEADER => ['Accept: application/json', 'Content-Type: application/x-www-form-urlencoded'],
-		]);
-
-		$body = curl_exec($ch);
-		$error = curl_errno($ch) ? curl_error($ch) : null;
-		$status = (int)curl_getinfo($ch, CURLINFO_HTTP_CODE);
-		curl_close($ch);
-
-		return ['body' => $error === null ? $body : null, 'http_code' => $status, 'error' => $error, 'url' => $url, 'fields' => $fields];
-	}
-
 	private function httpGet(string $url, array $params = [], ?string $userAgent = null): array
 	{
 		if (!function_exists('curl_init')) {

modules/steam_workshop/main.php

@@ -10,7 +10,10 @@ function exec_ogp_module(): void
 {
     global $db;
 
-    echo '<h2>' . get_lang('steam_workshop') . '</h2>';
+    $action = $_GET['action'] ?? '';
+    if ($action !== 'search') {
+        echo '<h2>' . get_lang('steam_workshop') . '</h2>';
+    }
 
     $controller = new SteamWorkshopController($db);
     $controller->handle();

modules/steam_workshop/steam_workshop.js

@@ -5,6 +5,7 @@
         function Picker(root) {
             this.root = root;
             this.endpoint = root.getAttribute('data-endpoint') || '';
+            this.detailBase = root.getAttribute('data-detail-base') || 'https://steamcommunity.com/sharedfiles/filedetails/?id=';
             this.lang = {
                 add: root.getAttribute('data-lang-add') || 'Add',
                 remove: root.getAttribute('data-lang-remove') || 'Remove',
@@ -241,16 +242,32 @@
             }
         };
         Picker.prototype.updateRequestPreview = function () {
-            if (this.requestInput && this.searchInput) {
+            if (!this.searchInput) {
-                this.requestInput.value = this.searchInput.value;
+                return;
             }
-            if (this.requestSummary) {
+            var term = this.searchInput.value.trim();
-                var encoded = '';
+            if (this.requestInput) {
-                if (this.searchInput && this.searchInput.value.trim() !== '') {
+                this.requestInput.value = term;
-                    encoded = encodeURIComponent(this.searchInput.value.trim());
-                }
-                this.requestSummary.textContent = (this.requestSummaryBase || '') + encoded;
             }
+            if (!this.requestSummary) {
+                return;
+            }
+            var base = this.requestSummaryBase || '';
+            if (!base) {
+                this.requestSummary.textContent = '';
+                return;
+            }
+            if (!term) {
+                this.requestSummary.textContent = base;
+                return;
+            }
+            // Numeric-only terms are treated as Workshop item IDs and link to detail pages instead of search.
+            var isWorkshopId = /^\d+$/.test(term);
+            if (isWorkshopId) {
+                this.requestSummary.textContent = this.detailBase + encodeURIComponent(term);
+                return;
+            }
+            this.requestSummary.textContent = base + encodeURIComponent(term);
         };
         Picker.prototype.isSelected = function (id) {
             return this.state.selected.some(function (item) { return item.id === id; });

modules/steam_workshop/views/edit.php

@@ -4,6 +4,7 @@ declare(strict_types=1);
 /** @var array $config */
 /** @var array $lang */
 /** @var array $adapterOptions */
+/** @var string|null $appId */
 $homeName = htmlspecialchars($home['home_name'] ?? ('#' . $home['home_id']));
 $homeId = (int)$home['home_id'];
 ?>

modules/steam_workshop/views/partials/mod_picker.php

@@ -4,6 +4,7 @@ declare(strict_types=1);
 /** @var array $config */
 /** @var array $home */
 /** @var int $homeId */
+/** @var string|null $appId */
 $homeId = (int)($home['home_id'] ?? 0);
 $scriptPath = (string)($_SERVER['PHP_SELF'] ?? '/index.php');
 if ($scriptPath === '') {
@@ -13,6 +14,8 @@ if ($scriptPath[0] !== '/') {
     $scriptPath = '/' . ltrim($scriptPath, '/');
 }
 $endpoint = sprintf('%s?m=steam_workshop&p=main&action=search&home_id=%d', $scriptPath, $homeId);
+$steamBase = 'https://steamcommunity.com/workshop/browse/?appid=';
+$steamAppIdParam = $appId ?? '';
 $initialItems = [];
 foreach ($config['workshop_items'] ?? [] as $item) {
     if (!is_array($item)) {
@@ -43,7 +46,7 @@ $langAttrs = [
     'sync' => $lang['mod_picker_toggle_label'] ?? 'Sync',
 ];
 ?>
-<div class="sw-picker" id="<?php echo $pickerId; ?>" data-endpoint="<?php echo htmlspecialchars($endpoint, ENT_QUOTES, 'UTF-8'); ?>"
+    <div class="sw-picker" id="<?php echo $pickerId; ?>" data-endpoint="<?php echo htmlspecialchars($endpoint, ENT_QUOTES, 'UTF-8'); ?>" data-detail-base="https://steamcommunity.com/sharedfiles/filedetails/?id="
     <?php foreach ($langAttrs as $key => $value): ?>data-lang-<?php echo $key; ?>="<?php echo htmlspecialchars($value, ENT_QUOTES, 'UTF-8'); ?>" <?php endforeach; ?>>
     <div class="sw-picker__header">
         <h4><?php echo htmlspecialchars($lang['mod_picker_heading'] ?? 'Workshop library'); ?></h4>
@@ -62,7 +65,7 @@ $langAttrs = [
         <span class="sw-picker__request-label"><?php echo htmlspecialchars($lang['mod_picker_request_label'] ?? 'Submitting request'); ?></span>
         <small class="sw-picker__request-hint"><?php echo htmlspecialchars($lang['mod_picker_request_hint'] ?? 'Exact URL preview. The field below mirrors your search text.'); ?></small>
         <div class="sw-picker__request-line">
-            <?php $baseRequest = $endpoint . '&q='; ?>
+            <?php $baseRequest = $steamAppIdParam !== '' ? $steamBase . $steamAppIdParam . '&browsesort=textsearch&section=readytouseitems&searchtext=' : ''; ?>
             <code class="sw-picker__request-summary js-sw-request-summary" data-base="<?php echo htmlspecialchars($baseRequest, ENT_QUOTES, 'UTF-8'); ?>"><?php echo htmlspecialchars($baseRequest, ENT_QUOTES, 'UTF-8'); ?></code>
             <input type="text" class="sw-picker__request-input js-sw-request-input" value="" readonly aria-label="<?php echo htmlspecialchars($lang['mod_picker_request_input_label'] ?? 'Workshop search text preview'); ?>" />
         </div>