dotnet_dbsync/Services/Downloader.cs

using System.Text.Json;
using System.Threading.RateLimiting;
using Serilog;

namespace dbsync.Services;

public class Downloader
{
    private static readonly HttpClient _httpClient = new();

    // 20 requests every 10 seconds (global limiter)
    private static readonly RateLimiter _rateLimiter =
        new FixedWindowRateLimiter(new FixedWindowRateLimiterOptions
        {
            PermitLimit = 20,
            Window = TimeSpan.FromSeconds(10),
            QueueProcessingOrder = QueueProcessingOrder.OldestFirst,
            QueueLimit = 2000
        });

    #region Public API

    public async Task<List<string>> DownloadAsync(
        string url,
        string outputDir,
        string outputFile,
        ILogger log)
    {
        var episodeIds = new List<string>();

        try
        {
            Directory.CreateDirectory(outputDir);

            var jsonString = await GetStringWithRateLimitAsync(url);

            if (url.Contains("episodes", StringComparison.OrdinalIgnoreCase))
            {
                episodeIds = ExtractEpisodeIds(jsonString);
                log.Information("Extracted {Count} episode IDs.", episodeIds.Count);
            }

            await SaveFormattedJsonAsync(jsonString, outputDir, outputFile);

            log.Information("Downloaded and formatted {OutputFile} successfully.", outputFile);
        }
        catch (Exception ex)
        {
            log.Error(ex, "Failed to download {OutputFile} from {Url}", outputFile, url);
        }

        return episodeIds;
    }

    public async Task DownloadEpisodeExtrasAsync(
        List<string> episodeIds,
        Dictionary<string, string> directories,
        ILogger log)
    {
        var guestCastDir = directories["GUESTCAST"];
        var guestCrewDir = directories["GUESTCREW"];

        Directory.CreateDirectory(guestCastDir);
        Directory.CreateDirectory(guestCrewDir);

        var tasks = episodeIds.Select(async episodeId =>
        {
            try
            {
                var guestCastUrl = $"https://api.tvmaze.com/episodes/{episodeId}/guestcast";
                var guestCrewUrl = $"https://api.tvmaze.com/episodes/{episodeId}/guestcrew";

                await DownloadAndSaveAsync(
                    guestCastUrl,
                    guestCastDir,
                    $"{episodeId}_guestcast.json",
                    log);

                await DownloadAndSaveAsync(
                    guestCrewUrl,
                    guestCrewDir,
                    $"{episodeId}_guestcrew.json",
                    log);
            }
            catch (Exception ex)
            {
                log.Error(ex, "Failed downloading guest data for episode {EpisodeId}", episodeId);
            }
        });

        await Task.WhenAll(tasks);

        log.Information("Completed guestcast and guestcrew downloads for {Count} episodes.", episodeIds.Count);
    }

    #endregion

    #region Core HTTP Logic (Rate Limited)

    private async Task<string> GetStringWithRateLimitAsync(string url)
    {
        using var lease = await _rateLimiter.AcquireAsync();

        if (!lease.IsAcquired)
            throw new Exception("Rate limit exceeded.");

        var response = await _httpClient.GetAsync(url);
        response.EnsureSuccessStatusCode();

        return await response.Content.ReadAsStringAsync();
    }

    private async Task DownloadAndSaveAsync(
        string url,
        string outputDir,
        string outputFile,
        ILogger log)
    {
        try
        {
            var jsonString = await GetStringWithRateLimitAsync(url);
            await SaveFormattedJsonAsync(jsonString, outputDir, outputFile);

            log.Information("Saved {File}", outputFile);
        }
        catch (Exception ex)
        {
            log.Warning(ex, "Request failed for {Url}", url);
        }
    }

    #endregion

    #region JSON Helpers

    private static List<string> ExtractEpisodeIds(string jsonString)
    {
        var episodeIds = new List<string>();

        using var jsonDoc = JsonDocument.Parse(jsonString);

        if (jsonDoc.RootElement.ValueKind == JsonValueKind.Array)
            foreach (var element in jsonDoc.RootElement.EnumerateArray())
                if (element.TryGetProperty("id", out var idProperty) &&
                    idProperty.TryGetInt32(out var episodeId))
                    episodeIds.Add(episodeId.ToString());

        return episodeIds;
    }

    private static async Task SaveFormattedJsonAsync(
        string jsonString,
        string outputDir,
        string outputFile)
    {
        using var jsonDoc = JsonDocument.Parse(jsonString);

        var formattedJson = JsonSerializer.Serialize(
            jsonDoc.RootElement,
            new JsonSerializerOptions { WriteIndented = true });

        var filePath = Path.Combine(outputDir, outputFile);

        await File.WriteAllTextAsync(filePath, formattedJson);
    }

    #endregion
}