From bdcb596e39933847b0fd37af5295db72459415f6 Mon Sep 17 00:00:00 2001 From: Wendell Jones Date: Mon, 16 Feb 2026 17:12:34 -0500 Subject: [PATCH] Initial commit --- Downloader.cs | 47 ++++++++++++ Loader.cs | 108 +++++++++++++++++++++++++++ MongoLoader.cs | 187 +++++++++++++++++++++++++++++++++++++++++++++++ Settings.cs | 102 ++++++++++++++++++++++++++ TvSyncApp.cs | 105 ++++++++++++++++++++++++++ appsettings.json | 53 ++++++++++++++ dbsync.csproj | 19 +++++ dbsync.sln | 25 +++++++ 8 files changed, 646 insertions(+) create mode 100644 Downloader.cs create mode 100644 Loader.cs create mode 100644 MongoLoader.cs create mode 100644 Settings.cs create mode 100644 TvSyncApp.cs create mode 100644 appsettings.json create mode 100644 dbsync.csproj create mode 100644 dbsync.sln diff --git a/Downloader.cs b/Downloader.cs new file mode 100644 index 0000000..3b697d4 --- /dev/null +++ b/Downloader.cs @@ -0,0 +1,47 @@ +using System; +using System.IO; +using System.Net.Http; +using System.Text.Json; +using Serilog; + +namespace TvSyncApp +{ + public class Downloader + { + private static readonly HttpClient _httpClient = new HttpClient(); + + public void Download(string url, string outputDir, string outputFile, ILogger log) + { + try + { + // log.Information("Downloading {Url} to {OutputFile}", url, outputFile); + + // Ensure the output directory exists + Directory.CreateDirectory(outputDir); + + // Get JSON content as string + var response = _httpClient.GetAsync(url).Result; + response.EnsureSuccessStatusCode(); + string jsonString = response.Content.ReadAsStringAsync().Result; + + // Parse and pretty-print JSON + var jsonDoc = JsonDocument.Parse(jsonString); + var formattedJson = JsonSerializer.Serialize(jsonDoc.RootElement, new JsonSerializerOptions + { + WriteIndented = true + }); + + // Write to file + string filePath = Path.Combine(outputDir, outputFile); + File.WriteAllText(filePath, formattedJson); + + //log.Information("Downloaded and formatted {OutputFile} successfully.", outputFile); + } + catch (Exception ex) + { + log.Error(ex, "Failed to download {OutputFile} from {Url}", outputFile, url); + } + } + } +} + diff --git a/Loader.cs b/Loader.cs new file mode 100644 index 0000000..5ef4095 --- /dev/null +++ b/Loader.cs @@ -0,0 +1,108 @@ +using Serilog; + +namespace TvSyncApp +{ + public class Loader + { + private readonly Downloader _downloader; + + public Loader() + { + _downloader = new Downloader(); + } + + public async Task<(int Success, int Failed)> LoadJsonAsync( + Dictionary directories, + List updateList, + Dictionary updateDict, + Settings settings, + int maxParallelDownloads = 5) + { + int success = 0; + int failed = 0; + int completed = 0; + + + var semaphore = new System.Threading.SemaphoreSlim(maxParallelDownloads); + + var filesNeeded = new List<(string Directory, string FileName)>(); + + foreach (var seriesId in updateList) + { + foreach (var dir in directories.Values) + { + string filePath = Path.Combine(dir, $"{seriesId}.json"); + if (!File.Exists(filePath)) + filesNeeded.Add((dir, $"{seriesId}.json")); + } + } + + Log.Information("Downloading {Count} files...", filesNeeded.Count); + var totalFiles = filesNeeded.Count; + var progress = new Progress(value => + { + completed += value; + Console.Write($"\rDownloading files: {completed}/{totalFiles}"); + }); + + var tasks = new List(); + foreach (var file in filesNeeded) + { + tasks.Add(DownloadWithThrottleAsync(file.Directory, file.FileName, settings, semaphore, progress)); + } + + await Task.WhenAll(tasks); + Console.WriteLine("\nDownload complete."); + + foreach (var file in filesNeeded) + { + string path = Path.Combine(file.Directory, file.FileName); + if (File.Exists(path)) success++; + else failed++; + } + + return (success, failed); + } + + private async Task DownloadWithThrottleAsync( + string directory, + string fileName, + Settings settings, + System.Threading.SemaphoreSlim semaphore, + IProgress progress) + { + await semaphore.WaitAsync(); + try + { + string seriesId = Path.GetFileNameWithoutExtension(fileName); + string url; + + if (directory.Contains("series", StringComparison.OrdinalIgnoreCase)) + url = settings.Json.ShowApi.Replace("", seriesId); + else if (directory.Contains("episodes", StringComparison.OrdinalIgnoreCase)) + url = settings.Json.EpisodeApi.Replace("", seriesId); + else if (directory.Contains("cast", StringComparison.OrdinalIgnoreCase)) + url = settings.Json.CastApi.Replace("", seriesId); + else if (directory.Contains("crew", StringComparison.OrdinalIgnoreCase)) + url = settings.Json.CrewApi.Replace("", seriesId); + else + url = $"https://api.tvmaze.com/shows/{seriesId}"; + + // Download the file + await Task.Run(() => _downloader.Download(url, directory, fileName, Log.Logger)); + + // Report progress + progress?.Report(1); + } + catch (Exception ex) + { + Log.Error(ex, "Failed to download {FileName}", fileName); + progress?.Report(1); // still report so progress moves forward + } + finally + { + semaphore.Release(); + } + } + } +} diff --git a/MongoLoader.cs b/MongoLoader.cs new file mode 100644 index 0000000..8c37d15 --- /dev/null +++ b/MongoLoader.cs @@ -0,0 +1,187 @@ +using MongoDB.Bson; +using MongoDB.Bson.Serialization; +using MongoDB.Driver; +using Serilog; +using System.Text.Json; + +public class MongoLoader +{ + private readonly IMongoCollection _series; + private readonly IMongoCollection _episodes; + private readonly IMongoCollection _actors; + private readonly IMongoCollection _characters; + private readonly IMongoCollection _crew; + + public MongoLoader(IMongoDatabase db) + { + _series = db.GetCollection("series"); + _episodes = db.GetCollection("episodes"); + _actors = db.GetCollection("actors"); + _characters = db.GetCollection("characters"); + _crew = db.GetCollection("crew"); + } + + public async Task<(int Success, int Failed)> LoadJsonToMongoAsync( + Dictionary directories, + List updateList, + Dictionary updateDict) + { + int success = 0; + int failed = 0; + + var options = new ParallelOptions + { + MaxDegreeOfParallelism = Environment.ProcessorCount // or set manually (e.g. 4) + }; + + await Parallel.ForEachAsync(updateList, options, async (seriesId, cancellationToken) => + { + try + { + var seriesFile = Path.Combine(directories["SERIESDIR"], $"{seriesId}.json"); + if (!File.Exists(seriesFile)) + return; + + var json = await File.ReadAllTextAsync(seriesFile, cancellationToken); + var doc = BsonDocument.Parse(json); + + await UpsertWithTimestamp(_series, doc, updateDict, seriesId); + + await ProcessEpisodes(seriesId, directories, updateDict); + await ProcessCast(seriesId, directories, updateDict); + await ProcessCrew(seriesId, directories, updateDict); + + Interlocked.Increment(ref success); + + Log.Information("Processed series {SeriesId}", seriesId); + } + catch (Exception ex) + { + Interlocked.Increment(ref failed); + Log.Error(ex, "Failed processing series {SeriesId}", seriesId); + } + }); + + return (success, failed); + } + + + private async Task UpsertWithTimestamp( + IMongoCollection collection, + BsonDocument doc, + Dictionary updateDict, + string seriesId) + { + var id = doc.GetValue("id", null); + if (id == null) return; + + var filter = Builders.Filter.Eq("id", id); + var existing = await collection.Find(filter).FirstOrDefaultAsync(); + + if (updateDict.TryGetValue(seriesId, out var newTimestamp)) + { + if (existing == null || + !existing.Contains("updated") || + existing["updated"].AsInt64 < newTimestamp) + { + await collection.ReplaceOneAsync(filter, doc, + new ReplaceOptions { IsUpsert = true }); + } + } + } + + private async Task ProcessEpisodes( + string seriesId, + Dictionary directories, + Dictionary updateDict) + { + var file = Path.Combine(directories["EPISODEDIR"], $"{seriesId}.json"); + if (!File.Exists(file)) return; + + var json = await File.ReadAllTextAsync(file); + var array = JsonSerializer.Deserialize>>(json); + + if (array == null) return; + + foreach (var episode in array) + { + episode["seriesid"] = int.Parse(seriesId); + var doc = BsonDocument.Parse(JsonSerializer.Serialize(episode)); + await UpsertWithTimestamp(_episodes, doc, updateDict, seriesId); + } + } + + private async Task ProcessCast( + string seriesId, + Dictionary directories, + Dictionary updateDict) + { + var file = Path.Combine(directories["CASTDIR"], $"{seriesId}.json"); + if (!File.Exists(file)) return; + + var json = await File.ReadAllTextAsync(file); + + // ✅ Correct way to parse root JSON array + var array = BsonSerializer.Deserialize(json); + + foreach (var item in array) + { + if (!item.IsBsonDocument) + continue; + + var doc = item.AsBsonDocument; + + if (doc.Contains("person") && doc["person"].IsBsonDocument) + { + var person = doc["person"].AsBsonDocument; + person["seriesid"] = int.Parse(seriesId); + + await UpsertWithTimestamp(_actors, person, updateDict, seriesId); + } + + if (doc.Contains("character") && doc["character"].IsBsonDocument) + { + var character = doc["character"].AsBsonDocument; + character["seriesid"] = int.Parse(seriesId); + + await UpsertWithTimestamp(_characters, character, updateDict, seriesId); + } + } + } + + private async Task ProcessCrew( + string seriesId, + Dictionary directories, + Dictionary updateDict) + { + var file = Path.Combine(directories["CREWDIR"], $"{seriesId}.json"); + if (!File.Exists(file)) return; + + var json = await File.ReadAllTextAsync(file); + + // ✅ Correct way to parse root JSON array + var array = BsonSerializer.Deserialize(json); + + foreach (var item in array) + { + if (!item.IsBsonDocument) + continue; + + var doc = item.AsBsonDocument; + + if (doc.Contains("person") && doc["person"].IsBsonDocument) + { + var person = doc["person"].AsBsonDocument; + + person["seriesid"] = int.Parse(seriesId); + + // Safely get crew type + person["crew_type"] = doc.Contains("type") + ? doc["type"] + : "unknown"; + + await UpsertWithTimestamp(_crew, person, updateDict, seriesId); + } + } + } +} diff --git a/Settings.cs b/Settings.cs new file mode 100644 index 0000000..9728f8c --- /dev/null +++ b/Settings.cs @@ -0,0 +1,102 @@ +using System; +using System.IO; +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace TvSyncApp +{ + public class Settings + { + public GlobalSettings? Global { get; set; } + public DbSettings? DbSettings { get; set; } + public ApiSettings? Api { get; set; } + public JsonSettings? Json { get; set; } + public DatabaseSettings? Database { get; set; } + public UpdateTypeSettings? UpdateType { get; set; } + + public static Settings Load(string filePath) + { + if (!File.Exists(filePath)) + throw new FileNotFoundException($"Settings file not found: {filePath}"); + + var json = File.ReadAllText(filePath); + + var options = new JsonSerializerOptions + { + PropertyNameCaseInsensitive = true, + ReadCommentHandling = JsonCommentHandling.Skip, + AllowTrailingCommas = true + }; + + return JsonSerializer.Deserialize(json, options); + } + } + + public class GlobalSettings + { + public string? LogLevel { get; set; } + } + + public class DbSettings + { + public string? SqlServerDriver { get; set; } + public string? Hostname { get; set; } + public string? MgHostname { get; set; } + public string? DbName { get; set; } + public string? MgDbName { get; set; } + public string? MysqlUsername { get; set; } + public string? PgsqlUsername { get; set; } + public string? MssqlUsername { get; set; } + public string? MysqlPassword { get; set; } + public string? PgsqlPassword { get; set; } + public string? MssqlPassword { get; set; } + public int PgsqlPort { get; set; } + public int MysqlPort { get; set; } + public int MssqlPort { get; set; } + public int MgPort { get; set; } + } + + public class ApiSettings + { + public string? Type { get; set; } + } + + public class JsonSettings + { + public string? SeriesApi { get; set; } + public string? ShowApi { get; set; } + public string? EpisodeApi { get; set; } + public string? AliasApi { get; set; } + public string? CastApi { get; set; } + public string? CrewApi { get; set; } + public string? CreditsApi { get; set; } + public string? UpdatesApi { get; set; } + } + + public class DatabaseSettings + { + public string? DbType { get; set; } + public string? UpdateSchema { get; set; } + public int Initialize { get; set; } + public int InitLoad { get; set; } + } + + public class UpdateTypeSettings + { + public string? Type { get; set; } + public string? Full { get; set; } + public long Monthly { get; set; } + public long Weekly { get; set; } + public long Daily { get; set; } + + [JsonPropertyName("12h")] + public long H12 { get; set; } + + [JsonPropertyName("6h")] + public long H6 { get; set; } + + public long Custom { get; set; } + public int CacheSkip { get; set; } + public int RefreshCache { get; set; } + } +} diff --git a/TvSyncApp.cs b/TvSyncApp.cs new file mode 100644 index 0000000..63e37b9 --- /dev/null +++ b/TvSyncApp.cs @@ -0,0 +1,105 @@ +using System; +using System.IO; +using System.Collections.Generic; +using System.Threading.Tasks; +using Npgsql; +using Serilog; + +namespace TvSyncApp +{ + public class Program + { + public static async Task Main(string[] args) + { + string settingsPath = Path.Combine(Directory.GetCurrentDirectory(), "appsettings.json"); + var settings = TvSyncApp.Settings.Load(settingsPath) + ?? throw new Exception("Failed to load settings from appsettings.json"); + + Console.WriteLine($"Log level: {settings.Global.LogLevel}"); + Console.WriteLine($"Database type: {settings.Database.DbType}"); + // Console.WriteLine($"TVMaze series API: {settings.Json.SeriesApi}"); + + string rootDir = Directory.GetCurrentDirectory(); + + var directories = new Dictionary + { + ["SERIESDIR"] = Path.Combine(rootDir, "cache", "series"), + ["EPISODEDIR"] = Path.Combine(rootDir, "cache", "episodes"), + ["CASTDIR"] = Path.Combine(rootDir, "cache", "cast"), + // ["GUESTCASTDIR"] = Path.Combine(rootDir, "cache", "guestcast"), + ["CREWDIR"] = Path.Combine(rootDir, "cache", "crew"), + ["UNKNOWN"] = Path.Combine(rootDir, "cache", "unknown") + }; + + CreateDirectories(rootDir); + + Log.Logger = new LoggerConfiguration() + .MinimumLevel.Debug() + .WriteTo.Console() + .WriteTo.File(Path.Combine(rootDir, "log", "sync.log")) + .CreateLogger(); + + Log.Information("Starting sync process..."); + + // --- Fetch series IDs from PostgreSQL --- + string connectionString = $"Host={settings.DbSettings.Hostname};Port={settings.DbSettings.PgsqlPort};Username={settings.DbSettings.PgsqlUsername};Password={settings.DbSettings.PgsqlPassword};Database={settings.DbSettings.DbName}"; + List seriesIds = GetSeriesIdsFromDb(connectionString); + + Log.Information("Fetched {Count} series IDs from tvupdates table.", seriesIds.Count); + + // Convert to string list if your Loader expects strings + List updateList = seriesIds.ConvertAll(id => id.ToString()); + + // Optionally, you can also fetch timestamps if needed + Dictionary updateDict = new Dictionary(); + + var loader = new Loader(); + + var result = await loader.LoadJsonAsync(directories, updateList, updateDict, settings); + + Log.Information("Sync complete. Success: {Success}, Failed: {Failed}", result.Success, result.Failed); + Log.CloseAndFlush(); + + return 0; + } + + private static List GetSeriesIdsFromDb(string connectionString) + { + var seriesIds = new List(); + + try + { + using var conn = new NpgsqlConnection(connectionString); + conn.Open(); + + string query = "SELECT seriesid FROM updates.tvupdates ORDER BY seriesid"; + + using var cmd = new NpgsqlCommand(query, conn); + using var reader = cmd.ExecuteReader(); + + while (reader.Read()) + { + if (!reader.IsDBNull(0)) + seriesIds.Add(reader.GetInt32(0)); + } + } + catch (Exception ex) + { + Log.Error(ex, "Failed to fetch series IDs from PostgreSQL."); + } + + return seriesIds; + } + + private static void CreateDirectories(string rootDir) + { + Directory.CreateDirectory(Path.Combine(rootDir, "log")); + Directory.CreateDirectory(Path.Combine(rootDir, "temp")); + Directory.CreateDirectory(Path.Combine(rootDir, "cache", "series")); + Directory.CreateDirectory(Path.Combine(rootDir, "cache", "episodes")); + Directory.CreateDirectory(Path.Combine(rootDir, "cache", "cast")); + Directory.CreateDirectory(Path.Combine(rootDir, "cache", "guestcast")); + Directory.CreateDirectory(Path.Combine(rootDir, "cache", "crew")); + } + } +} diff --git a/appsettings.json b/appsettings.json new file mode 100644 index 0000000..5991af1 --- /dev/null +++ b/appsettings.json @@ -0,0 +1,53 @@ +{ + "global": { + "loglevel": "info" + }, + "dbsettings": { + "sqlserver_driver": "ODBC Driver 13 for SQL Server", + "hostname": "192.168.129.248", + "mghostname": "192.168.128.24", + "dbname": "media_dbsync", + "mgdbname": "tvdata", + "mysqlusername": "root", + "pgsqlusername": "postgres", + "mssqlusername": "sa", + "mysqlpassword": "Optimus0329", + "pgsqlpassword": "Optimus0329", + "mssqlpassword": "Optimus0329", + "pgsqlport": 5432, + "mysqlport": 3306, + "mssqlport": 1433, + "mgport": 27017 + }, + "api": { + "type": "tvmaze" + }, + "json": { + // "seriesapi": "http://api.tvmaze.com/shows?page=", + "showapi": "http://api.tvmaze.com/shows/", + "episodeapi": "http://api.tvmaze.com/shows//episodes", + "aliasapi": "http://api.tvmaze.com/shows//akas", + "castapi": "http://api.tvmaze.com/shows//cast", + "crewapi": "http://api.tvmaze.com/shows//crew", + "creditsapi": "http://api.tvmaze.com/people//castcredits", + "updatesapi": "http://api.tvmaze.com/updates/shows" + }, + "database": { + "dbtype": "pgsql", + "updateschema": "updates", + "initialize": 0, + "initload": 0 + }, + "updatetype": { + "type": "full", + "full": "full", + "monthly": 1219200, + "weekly": 604800, + "daily": 86400, + "12h": 23200, + "6h": 16600, + "custom": 4150, + "cacheskip": 1, + "refreshcache": 0 + } +} \ No newline at end of file diff --git a/dbsync.csproj b/dbsync.csproj new file mode 100644 index 0000000..90eb1e6 --- /dev/null +++ b/dbsync.csproj @@ -0,0 +1,19 @@ + + + + Exe + net9.0 + enable + enable + + + + + + + + + + + + diff --git a/dbsync.sln b/dbsync.sln new file mode 100644 index 0000000..8b0943b --- /dev/null +++ b/dbsync.sln @@ -0,0 +1,25 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Version 17 +VisualStudioVersion = 17.14.36930.0 d17.14 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "dbsync", "dbsync.csproj", "{06905DC5-E7BA-4CD0-9F8F-FD0E60C9DB71}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {06905DC5-E7BA-4CD0-9F8F-FD0E60C9DB71}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {06905DC5-E7BA-4CD0-9F8F-FD0E60C9DB71}.Debug|Any CPU.Build.0 = Debug|Any CPU + {06905DC5-E7BA-4CD0-9F8F-FD0E60C9DB71}.Release|Any CPU.ActiveCfg = Release|Any CPU + {06905DC5-E7BA-4CD0-9F8F-FD0E60C9DB71}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(ExtensibilityGlobals) = postSolution + SolutionGuid = {5F508F17-C255-49C1-B01D-13B2B739B1FF} + EndGlobalSection +EndGlobal