apiKey = $source['api_key']; $this->baseUrl = rtrim($source['api_url'], '/'); $this->httpClient = new Client([ 'timeout' => 60, // Stash can be slow 'headers' => [ 'User-Agent' => 'MediaCollector/1.0', 'Content-Type' => 'application/json', 'ApiKey' => $this->apiKey // Stash API key for authentication ], 'verify' => false // Disable SSL verification for problematic servers ]); $this->imageDownloader = new ImageDownloader('public/images', $this->apiKey); } protected function executeSync(string $syncType): void { if (empty($this->apiKey) || empty($this->baseUrl)) { throw new Exception('Stash API key and URL not configured'); } $this->logProgress('Starting Stash library sync...'); // Sync scenes (movies) $this->syncScenes(); // Sync movies (if Stash has movie support) $this->syncMovies(); $this->logProgress("Processed {$this->processedCount} Stash items"); } private function syncScenes(): void { try { $this->logProgress('Fetching Stash scenes...'); // First, get the total count to determine how many pages we need $totalCount = $this->getStashScenesCount(); if ($totalCount === 0) { $this->logProgress('No scenes found in Stash'); return; } $this->logProgress("Found {$totalCount} scenes in Stash"); // Use pagination to handle large libraries $perPage = 50; // Smaller batch size for reliability $totalPages = ceil($totalCount / $perPage); for ($page = 0; $page < $totalPages; $page++) { try { $offset = $page * $perPage; $scenes = $this->getStashScenes($offset, $perPage); if (empty($scenes)) { $this->logProgress("No scenes returned for page {$page}"); continue; } $this->logProgress("Processing page {$page} with " . count($scenes) . " scenes..."); foreach ($scenes as $sceneData) { try { $this->logProgress("Processing scene: {$sceneData['title']} (ID: {$sceneData['id']})"); $this->syncScene($sceneData); $this->processedCount++; // Update progress in real-time $this->updateSyncLog($this->currentSyncLogId, 'running', [ 'processed_items' => $this->processedCount, 'new_items' => $this->newCount, 'updated_items' => $this->updatedCount, 'message' => "Processed {$this->processedCount} of ~{$totalCount} scenes" ]); } catch (Exception $e) { $this->logProgress("Error processing scene {$sceneData['id']} ({$sceneData['title']}): " . $e->getMessage()); $this->processedCount++; // Still count as processed even if failed // Update progress even for failed items $this->updateSyncLog($this->currentSyncLogId, 'running', [ 'processed_items' => $this->processedCount, 'message' => "Error on scene {$sceneData['id']}: " . $e->getMessage() ]); } } } catch (Exception $e) { $this->logProgress("Error fetching page {$page}: " . $e->getMessage()); // Continue with next page even if this page fails $this->updateSyncLog($this->currentSyncLogId, 'running', [ 'message' => "Failed to fetch page {$page}, continuing with next page" ]); } // Add a small delay between pages to avoid overwhelming the server if ($page < $totalPages - 1) { sleep(1); } } $this->logProgress("Completed syncing Stash scenes"); } catch (Exception $e) { $this->logProgress('Error syncing scenes: ' . $e->getMessage()); throw $e; } } private function getStashScenesCount(): int { try { $query = ' query FindScenes($filter: FindFilterType) { findScenes(filter: $filter) { count } } '; $variables = [ 'filter' => [ 'per_page' => 1, 'page' => 1, 'sort' => 'created_at', 'direction' => 'DESC' ] ]; $response = $this->httpClient->post("{$this->baseUrl}/graphql", [ 'json' => [ 'query' => $query, 'variables' => $variables ], 'timeout' => 30 ]); $data = json_decode($response->getBody(), true); if (!isset($data['data']['findScenes']['count'])) { $this->logProgress('No count data in Stash response'); return 0; } return (int) $data['data']['findScenes']['count']; } catch (Exception $e) { $this->logProgress('Failed to get Stash scenes count: ' . $e->getMessage()); return 0; } } private function getStashScenes(int $offset = 0, int $limit = 50): array { try { $query = ' query FindScenes($filter: FindFilterType) { findScenes(filter: $filter) { scenes { id title details url date rating100 organized o_counter created_at updated_at paths { screenshot preview stream webp vtt sprite funscript caption } files { size duration video_codec audio_codec width height } performers { id name disambiguation url gender birthdate ethnicity country eye_color height_cm measurements fake_tits penis_length circumcised career_length tattoos piercings alias_list favorite ignore_auto_tag created_at updated_at details death_date hair_color weight image_path scene_count } } count } } '; $variables = [ 'filter' => [ 'per_page' => $limit, 'page' => $offset / $limit + 1, 'sort' => 'created_at', 'direction' => 'DESC' ] ]; $this->logProgress("Fetching Stash scenes: offset={$offset}, limit={$limit}"); $response = $this->httpClient->post("{$this->baseUrl}/graphql", [ 'json' => [ 'query' => $query, 'variables' => $variables ], 'timeout' => 60, // Increased timeout 'connect_timeout' => 30 ]); $data = json_decode($response->getBody(), true); if (!isset($data['data']['findScenes']['scenes'])) { $this->logProgress('No scenes data in Stash response'); return []; } $scenes = $data['data']['findScenes']['scenes']; $this->logProgress("Received " . count($scenes) . " scenes from Stash"); return $scenes; } catch (Exception $e) { $this->logProgress('Failed to fetch Stash scenes: ' . $e->getMessage()); $this->logProgress('Request details: ' . $e->getMessage()); throw new Exception('Failed to fetch Stash scenes: ' . $e->getMessage()); } } private function syncMovies(): void { try { $movies = $this->getStashMovies(); foreach ($movies as $movieData) { $this->syncMovie($movieData); $this->processedCount++; } } catch (Exception $e) { $this->logProgress('Error syncing movies: ' . $e->getMessage()); } } private function getStashMovies(): array { try { $query = ' query FindMovies($filter: FindFilterType) { findMovies(filter: $filter) { movies { id name aliases duration date rating100 director synopsis url created_at updated_at front_image_path back_image_path } count } } '; $variables = [ 'filter' => [ 'per_page' => 100, 'sort' => 'created_at', 'direction' => 'DESC' ] ]; $response = $this->httpClient->post("{$this->baseUrl}/graphql", [ 'json' => [ 'query' => $query, 'variables' => $variables ] ]); $data = json_decode($response->getBody(), true); if (!isset($data['data']['findMovies']['movies'])) { return []; // No movies found } return $data['data']['findMovies']['movies']; } catch (Exception $e) { // Return empty array if movies can't be fetched return []; } } private function syncScene(array $sceneData): void { $adultVideoModel = new AdultVideo($this->pdo); // Check if scene already exists by stash_id in metadata $stmt = $this->pdo->prepare(" SELECT id, metadata FROM adult_videos WHERE source_id = :source_id "); $stmt->execute(['source_id' => $this->source['id']]); $existingScenes = $stmt->fetchAll(\PDO::FETCH_ASSOC); $existingScene = null; foreach ($existingScenes as $scene) { $metadata = json_decode($scene['metadata'], true); if (isset($metadata['stash_id']) && $metadata['stash_id'] === $sceneData['id']) { $existingScene = $scene; break; } } // Download images locally $coverFilename = null; $screenshotFilename = null; // Extract image URLs from Stash API response $coverUrl = null; $screenshotUrl = null; // Stash provides paths.screenshot for screenshot if (!empty($sceneData['paths']['screenshot'])) { $screenshotPath = $sceneData['paths']['screenshot']; // Handle different path formats from Stash if (strpos($screenshotPath, 'http') === 0) { // Already a full URL $screenshotUrl = $screenshotPath; } elseif (strpos($screenshotPath, '/') === 0) { // Absolute path from Stash root $screenshotUrl = "{$this->baseUrl}" . $screenshotPath; } else { // Relative path - assume it's in a standard location $screenshotUrl = "{$this->baseUrl}/scene/" . $sceneData['id'] . "/" . $screenshotPath; } $this->logProgress("Screenshot URL: " . $screenshotUrl); } // For cover, we'll use the screenshot as cover if available if ($screenshotUrl) { $coverUrl = $screenshotUrl; } // Check if this is an existing scene and if images already exist $shouldDownloadImages = true; if ($existingScene) { $existingMetadata = json_decode($existingScene['metadata'], true); $hasExistingCover = !empty($existingMetadata['local_cover_path']); $hasExistingScreenshot = !empty($existingMetadata['local_screenshot_path']); if ($hasExistingCover && $hasExistingScreenshot) { $shouldDownloadImages = false; $this->logProgress("Scene {$sceneData['id']} already has images, skipping download"); } else { $this->logProgress("Scene {$sceneData['id']} missing images - downloading"); } } if ($shouldDownloadImages) { if (!empty($coverUrl)) { // Validate URL before attempting download if (filter_var($coverUrl, FILTER_VALIDATE_URL)) { try { $coverFilename = $this->imageDownloader->generateFilename($coverUrl, 'cover'); $localCoverPath = $this->imageDownloader->downloadImage($coverUrl, $coverFilename, 'adult_videos'); if ($localCoverPath) { $sceneData['local_cover_path'] = $this->imageDownloader->getPublicUrl($localCoverPath); $this->logProgress("Downloaded cover: " . $localCoverPath); } else { $this->logProgress("Failed to download cover from: " . $coverUrl); } } catch (Exception $e) { $this->logProgress("Exception downloading cover from {$coverUrl}: " . $e->getMessage()); } } else { $this->logProgress("Invalid cover URL: " . $coverUrl); } } if (!empty($screenshotUrl)) { // Validate URL before attempting download if (filter_var($screenshotUrl, FILTER_VALIDATE_URL)) { try { $screenshotFilename = $this->imageDownloader->generateFilename($screenshotUrl, 'screenshot'); $localScreenshotPath = $this->imageDownloader->downloadImage($screenshotUrl, $screenshotFilename, 'adult_videos'); if ($localScreenshotPath) { $sceneData['local_screenshot_path'] = $this->imageDownloader->getPublicUrl($localScreenshotPath); $this->logProgress("Downloaded screenshot: " . $localScreenshotPath); } else { $this->logProgress("Failed to download screenshot from: " . $screenshotUrl); } } catch (Exception $e) { $this->logProgress("Exception downloading screenshot from {$screenshotUrl}: " . $e->getMessage()); } } else { $this->logProgress("Invalid screenshot URL: " . $screenshotUrl); } } } else { // Use existing image paths $sceneData['local_cover_path'] = $existingMetadata['local_cover_path'] ?? null; $sceneData['local_screenshot_path'] = $existingMetadata['local_screenshot_path'] ?? null; } // Handle performers/actors $performers = $sceneData['performers'] ?? []; $actorNames = []; $performerImages = []; foreach ($performers as $performer) { $actorNames[] = $performer['name']; $performerImages[$performer['name']] = $performer['image_path'] ?? null; } $actors = $this->syncActors($actorNames, $performerImages); $sceneData = [ 'title' => $sceneData['title'] ?: 'Untitled Scene', 'overview' => $sceneData['details'] ?? null, 'release_date' => $sceneData['date'] ? date('Y-m-d', strtotime($sceneData['date'])) : null, 'runtime_minutes' => !empty($sceneData['files'][0]['duration']) ? round($sceneData['files'][0]['duration'] / 60) : null, 'rating' => $sceneData['rating100'] ? $sceneData['rating100'] / 100 : null, // Convert from 0-100 to 0-10 'source_id' => $this->source['id'], 'external_id' => $sceneData['id'], 'metadata' => json_encode([ 'stash_id' => $sceneData['id'], 'stash_url' => $sceneData['url'] ?? null, 'organized' => $sceneData['organized'] ?? false, 'o_counter' => $sceneData['o_counter'] ?? 0, 'performers' => $performers, 'actors' => $actors, 'file_info' => $sceneData['files'][0] ?? null, 'paths' => $sceneData['paths'] ?? null, 'cover_url' => $coverUrl, 'local_cover_path' => $sceneData['local_cover_path'] ?? null, 'screenshot_url' => $screenshotUrl, 'local_screenshot_path' => $sceneData['local_screenshot_path'] ?? null ]) ]; if ($existingScene) { // For existing scenes, check if we need to update images $existingMetadata = json_decode($existingScene['metadata'], true); // Only download images if they don't already exist locally if (empty($existingMetadata['local_cover_path']) && !empty($coverUrl)) { // Validate URL before attempting download if (filter_var($coverUrl, FILTER_VALIDATE_URL)) { try { $coverFilename = $this->imageDownloader->generateFilename($coverUrl, 'cover'); $localCoverPath = $this->imageDownloader->downloadImage($coverUrl, $coverFilename, 'adult_videos'); if ($localCoverPath) { $sceneData['local_cover_path'] = $this->imageDownloader->getPublicUrl($localCoverPath); $this->logProgress("Downloaded cover: " . $localCoverPath); } else { $this->logProgress("Failed to download cover from: " . $coverUrl); } } catch (Exception $e) { $this->logProgress("Exception downloading cover from {$coverUrl}: " . $e->getMessage()); } } else { $this->logProgress("Invalid cover URL: " . $coverUrl); } } else { // Keep existing local cover path $sceneData['local_cover_path'] = $existingMetadata['local_cover_path'] ?? null; if (!empty($sceneData['local_cover_path'])) { $this->logProgress("Using existing cover: " . $sceneData['local_cover_path']); } } if (empty($existingMetadata['local_screenshot_path']) && !empty($screenshotUrl)) { // Validate URL before attempting download if (filter_var($screenshotUrl, FILTER_VALIDATE_URL)) { try { $screenshotFilename = $this->imageDownloader->generateFilename($screenshotUrl, 'screenshot'); $localScreenshotPath = $this->imageDownloader->downloadImage($screenshotUrl, $screenshotFilename, 'adult_videos'); if ($localScreenshotPath) { $sceneData['local_screenshot_path'] = $this->imageDownloader->getPublicUrl($localScreenshotPath); $this->logProgress("Downloaded screenshot: " . $localScreenshotPath); } else { $this->logProgress("Failed to download screenshot from: " . $screenshotUrl); } } catch (Exception $e) { $this->logProgress("Exception downloading screenshot from {$screenshotUrl}: " . $e->getMessage()); } } else { $this->logProgress("Invalid screenshot URL: " . $screenshotUrl); } } else { // Keep existing local screenshot path $sceneData['local_screenshot_path'] = $existingMetadata['local_screenshot_path'] ?? null; if (!empty($sceneData['local_screenshot_path'])) { $this->logProgress("Using existing screenshot: " . $sceneData['local_screenshot_path']); } } $adultVideoModel->update($existingScene['id'], $sceneData); $adultVideoId = $existingScene['id']; $this->updatedCount++; // Create actor relationships for existing scene $this->createActorRelationships($adultVideoId, $actors); } else { $adultVideoModel->create($sceneData); $adultVideoId = $this->pdo->lastInsertId(); $this->newCount++; // Create actor relationships for new scene $this->createActorRelationships($adultVideoId, $actors); } } private function syncMovie(array $movieData): void { $adultVideoModel = new AdultVideo($this->pdo); // Check if movie already exists by stash_movie_id in metadata $stmt = $this->pdo->prepare(" SELECT id, metadata FROM adult_videos WHERE source_id = :source_id "); $stmt->execute(['source_id' => $this->source['id']]); $existingMovies = $stmt->fetchAll(\PDO::FETCH_ASSOC); $existingMovie = null; foreach ($existingMovies as $movie) { $metadata = json_decode($movie['metadata'], true); if (isset($metadata['stash_movie_id']) && $metadata['stash_movie_id'] === $movieData['id']) { $existingMovie = $movie; break; } } $movieData = [ 'title' => $movieData['name'] ?: 'Untitled Movie', 'overview' => $movieData['synopsis'] ?? null, 'director' => $movieData['director'] ?? null, 'release_date' => $movieData['date'] ? date('Y-m-d', strtotime($movieData['date'])) : null, 'runtime_minutes' => $movieData['duration'] ?? null, 'rating' => $movieData['rating100'] ? $movieData['rating100'] / 100 : null, 'source_id' => $this->source['id'], 'external_id' => $movieData['id'], 'metadata' => json_encode([ 'stash_movie_id' => $movieData['id'], 'aliases' => $movieData['aliases'] ?? null, 'url' => $movieData['url'] ?? null ]) ]; if ($existingMovie) { $adultVideoModel->update($existingMovie['id'], $movieData); $this->updatedCount++; } else { $adultVideoModel->create($movieData); $this->newCount++; } } private function syncActors(array $actorNames, array $performerImages = []): array { $actors = []; foreach ($actorNames as $actorName) { if (empty($actorName)) continue; $imagePath = $performerImages[$actorName] ?? null; $actor = $this->getOrCreateActor($actorName, $imagePath); if ($actor) { $actors[] = $actor; } } return $actors; } private function getOrCreateActor(string $name, ?string $imagePath = null): ?array { // Check if actor already exists $stmt = $this->pdo->prepare(" SELECT id, name, thumbnail_path FROM actors WHERE name = :name "); $stmt->execute(['name' => $name]); $existingActor = $stmt->fetch(PDO::FETCH_ASSOC); if ($existingActor) { return [ 'id' => $existingActor['id'], 'name' => $existingActor['name'], 'thumbnail_path' => $existingActor['thumbnail_path'] ]; } // Try to download performer image if available $thumbnailPath = null; if ($imagePath) { // Validate image path before constructing URL if (!empty(trim($imagePath))) { try { // Handle different image path formats from Stash if (strpos($imagePath, 'http') === 0) { // Already a full URL $imageUrl = $imagePath; } elseif (strpos($imagePath, '/') === 0) { // Absolute path from Stash root $imageUrl = "{$this->baseUrl}" . $imagePath; } else { // Relative path - assume it's in performer images directory $imageUrl = "{$this->baseUrl}/performer/" . $imagePath; } // Validate the constructed URL if (filter_var($imageUrl, FILTER_VALIDATE_URL)) { $this->logProgress("Performer image URL for {$name}: " . $imageUrl); $thumbnailFilename = $this->imageDownloader->generateFilename($imageUrl, 'actor'); $localImagePath = $this->imageDownloader->downloadImage($imageUrl, $thumbnailFilename, 'actors'); if ($localImagePath) { $thumbnailPath = $this->imageDownloader->getPublicUrl($localImagePath); $this->logProgress("Downloaded performer image: " . $localImagePath); } else { $this->logProgress("Failed to download performer image from: " . $imageUrl); } } else { $this->logProgress("Invalid performer image URL constructed: " . $imageUrl); } } catch (Exception $e) { $this->logProgress("Exception downloading performer image for {$name} from {$imagePath}: " . $e->getMessage()); } } } try { $stmt = $this->pdo->prepare(" INSERT INTO actors (name, thumbnail_path, created_at, updated_at) VALUES (:name, :thumbnail_path, NOW(), NOW()) "); $stmt->execute([ 'name' => $name, 'thumbnail_path' => $thumbnailPath ]); $actorId = $this->pdo->lastInsertId(); return [ 'id' => $actorId, 'name' => $name, 'thumbnail_path' => $thumbnailPath ]; } catch (Exception $e) { $this->logProgress("Failed to create actor {$name}: " . $e->getMessage()); return null; } } protected function getProcessedCount(): int { return $this->processedCount; } protected function getNewCount(): int { return $this->newCount; } protected function getUpdatedCount(): int { return $this->updatedCount; } private function createActorRelationships(int $adultVideoId, array $actors): void { foreach ($actors as $actor) { if (!isset($actor['id'])) continue; try { // Insert relationship into pivot table (ignore duplicates) $stmt = $this->pdo->prepare(" INSERT IGNORE INTO actor_adult_video (adult_video_id, actor_id, created_at, updated_at) VALUES (:adult_video_id, :actor_id, NOW(), NOW()) "); $stmt->execute([ 'adult_video_id' => $adultVideoId, 'actor_id' => $actor['id'] ]); $this->logProgress("Created relationship: Adult Video {$adultVideoId} -> Actor {$actor['name']} ({$actor['id']})"); } catch (Exception $e) { $this->logProgress("Failed to create relationship for Adult Video {$adultVideoId} and Actor {$actor['name']}: " . $e->getMessage()); } } } protected function executeCleanup(): void { $this->logProgress("Starting cleanup - detecting deleted media in Stash..."); // Clean up scenes $this->cleanupScenes(); // Clean up movies $this->cleanupMovies(); $this->logProgress("Cleanup completed. Deleted {$this->deletedCount} items."); } private function cleanupScenes(): void { $this->logProgress("Checking for deleted scenes..."); try { // Get all scenes from Stash $stashScenes = $this->getStashScenes(0, 1000); // Get up to 1000 scenes for cleanup $stashSceneIds = array_column($stashScenes, 'id'); $this->logProgress("Found " . count($stashSceneIds) . " scenes in Stash"); // Get all scenes from local database for this source $stmt = $this->pdo->prepare(" SELECT id, metadata FROM adult_videos WHERE source_id = :source_id "); $stmt->execute(['source_id' => $this->source['id']]); $localScenes = $stmt->fetchAll(\PDO::FETCH_ASSOC); $this->logProgress("Found " . count($localScenes) . " scenes in local database"); $deletedCount = 0; foreach ($localScenes as $localScene) { $metadata = json_decode($localScene['metadata'], true); $stashId = $metadata['stash_id'] ?? null; if ($stashId && !in_array($stashId, $stashSceneIds)) { // Scene exists in local DB but not in Stash - delete it $this->deleteAdultVideo($localScene['id']); $deletedCount++; $this->deletedCount++; } } $this->logProgress("Deleted {$deletedCount} scenes that no longer exist in Stash"); } catch (Exception $e) { $this->logProgress("Error during scene cleanup: " . $e->getMessage()); } } private function cleanupMovies(): void { $this->logProgress("Checking for deleted movies..."); try { // Get all movies from Stash $stashMovies = $this->getStashMovies(); $stashMovieIds = array_column($stashMovies, 'id'); $this->logProgress("Found " . count($stashMovieIds) . " movies in Stash"); // Get all movies from local database for this source $stmt = $this->pdo->prepare(" SELECT id, metadata FROM adult_videos WHERE source_id = :source_id "); $stmt->execute(['source_id' => $this->source['id']]); $localVideos = $stmt->fetchAll(\PDO::FETCH_ASSOC); $this->logProgress("Found " . count($localVideos) . " videos in local database"); $deletedCount = 0; foreach ($localVideos as $localVideo) { $metadata = json_decode($localVideo['metadata'], true); $stashMovieId = $metadata['stash_movie_id'] ?? null; if ($stashMovieId && !in_array($stashMovieId, $stashMovieIds)) { // Movie exists in local DB but not in Stash - delete it $this->deleteAdultVideo($localVideo['id']); $deletedCount++; $this->deletedCount++; } } $this->logProgress("Deleted {$deletedCount} movies that no longer exist in Stash"); } catch (Exception $e) { $this->logProgress("Error during movie cleanup: " . $e->getMessage()); } } private function deleteAdultVideo(int $videoId): void { try { // Delete actor relationships first $stmt = $this->pdo->prepare("DELETE FROM actor_adult_video WHERE adult_video_id = :adult_video_id"); $stmt->execute(['adult_video_id' => $videoId]); // Delete the video $stmt = $this->pdo->prepare("DELETE FROM adult_videos WHERE id = :id"); $stmt->execute(['id' => $videoId]); $this->logProgress("Deleted adult video with ID: {$videoId}"); } catch (Exception $e) { $this->logProgress("Error deleting adult video {$videoId}: " . $e->getMessage()); } } }