added rate limiter on the downloader and updated comparison to only look for files from updated or missing series ids.
This commit is contained in:
+86
-23
@@ -1,35 +1,43 @@
|
||||
using Npgsql;
|
||||
using Serilog;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Text.Json;
|
||||
using System.Threading.Tasks;
|
||||
using dbsync.Config;
|
||||
using dbsync.Services;
|
||||
using TvSyncApp.Services;
|
||||
using System.Collections.Concurrent;
|
||||
|
||||
namespace TvSyncApp
|
||||
{
|
||||
public class Program
|
||||
|
||||
{
|
||||
|
||||
public static Dictionary<string, string> directories { get; private set; } = new();
|
||||
|
||||
public static async Task<int> Main(string[] args)
|
||||
{
|
||||
string settingsPath = Path.Combine(Directory.GetCurrentDirectory(), "appsettings.json");
|
||||
var settings = TvSyncApp.Settings.Load(settingsPath)
|
||||
var settings = Settings.Load(settingsPath)
|
||||
?? throw new Exception("Failed to load settings from appsettings.json");
|
||||
|
||||
Console.WriteLine($"Log level: {settings.Global.LogLevel}");
|
||||
Console.WriteLine($"Database type: {settings.Database.DbType}");
|
||||
Console.WriteLine($"Log level: {settings.Global!.LogLevel}");
|
||||
Console.WriteLine($"Database type: {settings.Database!.DbType}");
|
||||
// Console.WriteLine($"TVMaze series API: {settings.Json.SeriesApi}");
|
||||
|
||||
string rootDir = Directory.GetCurrentDirectory();
|
||||
|
||||
var directories = new Dictionary<string, string>
|
||||
directories = new Dictionary<string, string>
|
||||
{
|
||||
["SERIESDIR"] = Path.Combine(rootDir, "cache", "series"),
|
||||
["EPISODEDIR"] = Path.Combine(rootDir, "cache", "episodes"),
|
||||
["CASTDIR"] = Path.Combine(rootDir, "cache", "cast"),
|
||||
// ["GUESTCASTDIR"] = Path.Combine(rootDir, "cache", "guestcast"),
|
||||
["CREWDIR"] = Path.Combine(rootDir, "cache", "crew"),
|
||||
["UNKNOWN"] = Path.Combine(rootDir, "cache", "unknown")
|
||||
["ALIASES"] = Path.Combine(rootDir, "cache", "aliases"),
|
||||
["CREDITS"] = Path.Combine(rootDir, "cache", "credits"),
|
||||
["GUESTCAST"] = Path.Combine(rootDir, "cache", "guestcast"),
|
||||
["GUESTCREW"] = Path.Combine(rootDir, "cache", "guestcrew"),
|
||||
["CONFIG"] = Path.Combine(rootDir, "config"),
|
||||
["TEMP"] = Path.Combine(rootDir, "temp")
|
||||
};
|
||||
|
||||
CreateDirectories(rootDir);
|
||||
@@ -43,19 +51,46 @@ namespace TvSyncApp
|
||||
Log.Information("Starting sync process...");
|
||||
|
||||
// --- Fetch series IDs from PostgreSQL ---
|
||||
string connectionString = $"Host={settings.DbSettings.Hostname};Port={settings.DbSettings.PgsqlPort};Username={settings.DbSettings.PgsqlUsername};Password={settings.DbSettings.PgsqlPassword};Database={settings.DbSettings.DbName}";
|
||||
string connectionString = $"Host={settings.DbSettings!.Hostname};Port={settings.DbSettings.PgsqlPort};Username={settings.DbSettings.PgsqlUsername};Password={settings.DbSettings.PgsqlPassword};Database={settings.DbSettings.DbName}";
|
||||
var service = new TvSyncService(connectionString)!;
|
||||
List<SeriesUpdate> seriesUpdates = GetSeriesIdsFromDb(connectionString);
|
||||
|
||||
List<string> seriesIds = seriesUpdates
|
||||
.Select(x => x.SeriesId.ToString())
|
||||
.ToList();
|
||||
|
||||
var downloadedUpdates = await service.DownloadTvMazeUpdatesAsync(
|
||||
settings.Json!.UpdatesApi!,
|
||||
directories["CONFIG"]);
|
||||
service.UpdateTvUpdatesTable(downloadedUpdates);
|
||||
|
||||
var previousDict = seriesUpdates
|
||||
.ToDictionary(x => x.SeriesId, x => x.Timestamp);
|
||||
|
||||
Log.Information("Fetched {Count} series IDs from tvupdates table.", seriesIds.Count);
|
||||
var updatedSeries = GetUpdatedSeriesOptimized(previousDict, downloadedUpdates);
|
||||
|
||||
|
||||
if (updatedSeries.Count == 0)
|
||||
{
|
||||
Log.Information("No new or updated series found. Exiting.");
|
||||
Log.CloseAndFlush();
|
||||
return 0; // success exit code
|
||||
}
|
||||
|
||||
var filesNeeded = service.DetermineFilesNeeded(
|
||||
downloadedUpdates,
|
||||
previousDict,
|
||||
directories);
|
||||
|
||||
var skipIds = new Dictionary<string, string>(); // load skip file here
|
||||
|
||||
List<int> seriesIdsNeeded = updatedSeries.Keys.ToList();
|
||||
|
||||
|
||||
// Convert to string list if your Loader expects strings
|
||||
List<string> updateList = seriesIds.ConvertAll(id => id.ToString());
|
||||
|
||||
List<string> updateList = seriesIdsNeeded.ConvertAll(id => id.ToString());
|
||||
|
||||
// Optionally, you can also fetch timestamps if needed
|
||||
Dictionary<string, long> updateDict = new Dictionary<string, long>();
|
||||
|
||||
@@ -77,10 +112,7 @@ namespace TvSyncApp
|
||||
using var conn = new NpgsqlConnection(connectionString);
|
||||
conn.Open();
|
||||
|
||||
const string query = @"
|
||||
SELECT seriesid, timestamp
|
||||
FROM updates.tvupdates
|
||||
ORDER BY seriesid";
|
||||
const string query = @"SELECT seriesid, timestamp FROM updates.tvupdates ORDER BY seriesid";
|
||||
|
||||
using var cmd = new NpgsqlCommand(query, conn);
|
||||
using var reader = cmd.ExecuteReader();
|
||||
@@ -109,17 +141,14 @@ namespace TvSyncApp
|
||||
|
||||
private static void SaveSeriesUpdatesToFile(List<SeriesUpdate> data)
|
||||
{
|
||||
string configDir = Path.Combine(AppContext.BaseDirectory, "config");
|
||||
Directory.CreateDirectory(configDir);
|
||||
|
||||
string fileName = "previous_tvmaze_updates.json";
|
||||
string filePath = Path.Combine(configDir, fileName);
|
||||
string filePath = Path.Combine(directories["CONFIG"], fileName);
|
||||
|
||||
if (File.Exists(filePath))
|
||||
{
|
||||
string timestamp = DateTime.UtcNow.ToString("yyyyMMddHHmmss");
|
||||
string archiveName = $"previous_tvmaze_updates_{timestamp}.json";
|
||||
string archivePath = Path.Combine(configDir, archiveName);
|
||||
string archivePath = Path.Combine(directories["CONFIG"], archiveName);
|
||||
|
||||
File.Move(filePath, archivePath);
|
||||
}
|
||||
@@ -132,7 +161,7 @@ namespace TvSyncApp
|
||||
string json = JsonSerializer.Serialize(data, options);
|
||||
File.WriteAllText(filePath, json);
|
||||
|
||||
var archives = Directory.GetFiles(configDir, "previous_tvmaze_updates_*.json")
|
||||
var archives = Directory.GetFiles(directories["CONFIG"], "previous_tvmaze_updates_*.json")
|
||||
.Select(f => new FileInfo(f))
|
||||
.OrderByDescending(f => f.CreationTimeUtc)
|
||||
.ToList();
|
||||
@@ -145,6 +174,40 @@ namespace TvSyncApp
|
||||
Log.Information("Saved {Count} series updates to {FilePath}", data.Count, filePath);
|
||||
}
|
||||
|
||||
public static Dictionary<int, long> GetUpdatedSeriesOptimized(
|
||||
IReadOnlyDictionary<int, long> previousDict,
|
||||
IReadOnlyDictionary<int, long> downloadedUpdates)
|
||||
{
|
||||
var result = new Dictionary<int, long>(downloadedUpdates.Count);
|
||||
|
||||
int updatedCount = 0;
|
||||
int newCount = 0;
|
||||
|
||||
foreach (var (seriesId, newTimestamp) in downloadedUpdates)
|
||||
{
|
||||
if (previousDict.TryGetValue(seriesId, out var oldTimestamp))
|
||||
{
|
||||
if (newTimestamp > oldTimestamp)
|
||||
{
|
||||
result[seriesId] = newTimestamp;
|
||||
updatedCount++;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
result[seriesId] = newTimestamp;
|
||||
newCount++;
|
||||
}
|
||||
}
|
||||
|
||||
Log.Information(
|
||||
"Series comparison complete. Updated: {Updated}, New: {New}, Total Returned: {Total}",
|
||||
updatedCount,
|
||||
newCount,
|
||||
result.Count);
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
private static void CreateDirectories(string rootDir)
|
||||
{
|
||||
|
||||
Reference in New Issue
Block a user