using System.Text.Json; using System.Threading.RateLimiting; using Serilog; namespace dbsync.Services; public class Downloader { private static readonly HttpClient _httpClient = new(); // 20 requests every 10 seconds (global limiter) private static readonly RateLimiter _rateLimiter = new FixedWindowRateLimiter(new FixedWindowRateLimiterOptions { PermitLimit = 20, Window = TimeSpan.FromSeconds(10), QueueProcessingOrder = QueueProcessingOrder.OldestFirst, QueueLimit = 2000 }); #region Public API public async Task> DownloadAsync( string url, string outputDir, string outputFile, ILogger log) { var episodeIds = new List(); try { Directory.CreateDirectory(outputDir); var jsonString = await GetStringWithRateLimitAsync(url); if (url.Contains("episodes", StringComparison.OrdinalIgnoreCase)) { episodeIds = ExtractEpisodeIds(jsonString); log.Information("Extracted {Count} episode IDs.", episodeIds.Count); } await SaveFormattedJsonAsync(jsonString, outputDir, outputFile); log.Information("Downloaded and formatted {OutputFile} successfully.", outputFile); } catch (Exception ex) { log.Error(ex, "Failed to download {OutputFile} from {Url}", outputFile, url); } return episodeIds; } public async Task DownloadEpisodeExtrasAsync( List episodeIds, Dictionary directories, ILogger log) { var guestCastDir = directories["GUESTCAST"]; var guestCrewDir = directories["GUESTCREW"]; Directory.CreateDirectory(guestCastDir); Directory.CreateDirectory(guestCrewDir); var tasks = episodeIds.Select(async episodeId => { try { var guestCastUrl = $"https://api.tvmaze.com/episodes/{episodeId}/guestcast"; var guestCrewUrl = $"https://api.tvmaze.com/episodes/{episodeId}/guestcrew"; await DownloadAndSaveAsync( guestCastUrl, guestCastDir, $"{episodeId}_guestcast.json", log); await DownloadAndSaveAsync( guestCrewUrl, guestCrewDir, $"{episodeId}_guestcrew.json", log); } catch (Exception ex) { log.Error(ex, "Failed downloading guest data for episode {EpisodeId}", episodeId); } }); await Task.WhenAll(tasks); log.Information("Completed guestcast and guestcrew downloads for {Count} episodes.", episodeIds.Count); } #endregion #region Core HTTP Logic (Rate Limited) private async Task GetStringWithRateLimitAsync(string url) { using var lease = await _rateLimiter.AcquireAsync(); if (!lease.IsAcquired) throw new Exception("Rate limit exceeded."); var response = await _httpClient.GetAsync(url); response.EnsureSuccessStatusCode(); return await response.Content.ReadAsStringAsync(); } private async Task DownloadAndSaveAsync( string url, string outputDir, string outputFile, ILogger log) { try { var jsonString = await GetStringWithRateLimitAsync(url); await SaveFormattedJsonAsync(jsonString, outputDir, outputFile); log.Information("Saved {File}", outputFile); } catch (Exception ex) { log.Warning(ex, "Request failed for {Url}", url); } } #endregion #region JSON Helpers private static List ExtractEpisodeIds(string jsonString) { var episodeIds = new List(); using var jsonDoc = JsonDocument.Parse(jsonString); if (jsonDoc.RootElement.ValueKind == JsonValueKind.Array) foreach (var element in jsonDoc.RootElement.EnumerateArray()) if (element.TryGetProperty("id", out var idProperty) && idProperty.TryGetInt32(out var episodeId)) episodeIds.Add(episodeId.ToString()); return episodeIds; } private static async Task SaveFormattedJsonAsync( string jsonString, string outputDir, string outputFile) { using var jsonDoc = JsonDocument.Parse(jsonString); var formattedJson = JsonSerializer.Serialize( jsonDoc.RootElement, new JsonSerializerOptions { WriteIndented = true }); var filePath = Path.Combine(outputDir, outputFile); await File.WriteAllTextAsync(filePath, formattedJson); } #endregion }