diff --git a/Downloader.cs b/Downloader.cs new file mode 100644 index 0000000..3b697d4 --- /dev/null +++ b/Downloader.cs @@ -0,0 +1,47 @@ +using System; +using System.IO; +using System.Net.Http; +using System.Text.Json; +using Serilog; + +namespace TvSyncApp +{ + public class Downloader + { + private static readonly HttpClient _httpClient = new HttpClient(); + + public void Download(string url, string outputDir, string outputFile, ILogger log) + { + try + { + // log.Information("Downloading {Url} to {OutputFile}", url, outputFile); + + // Ensure the output directory exists + Directory.CreateDirectory(outputDir); + + // Get JSON content as string + var response = _httpClient.GetAsync(url).Result; + response.EnsureSuccessStatusCode(); + string jsonString = response.Content.ReadAsStringAsync().Result; + + // Parse and pretty-print JSON + var jsonDoc = JsonDocument.Parse(jsonString); + var formattedJson = JsonSerializer.Serialize(jsonDoc.RootElement, new JsonSerializerOptions + { + WriteIndented = true + }); + + // Write to file + string filePath = Path.Combine(outputDir, outputFile); + File.WriteAllText(filePath, formattedJson); + + //log.Information("Downloaded and formatted {OutputFile} successfully.", outputFile); + } + catch (Exception ex) + { + log.Error(ex, "Failed to download {OutputFile} from {Url}", outputFile, url); + } + } + } +} + diff --git a/Loader.cs b/Loader.cs new file mode 100644 index 0000000..5ef4095 --- /dev/null +++ b/Loader.cs @@ -0,0 +1,108 @@ +using Serilog; + +namespace TvSyncApp +{ + public class Loader + { + private readonly Downloader _downloader; + + public Loader() + { + _downloader = new Downloader(); + } + + public async Task<(int Success, int Failed)> LoadJsonAsync( + Dictionary directories, + List updateList, + Dictionary updateDict, + Settings settings, + int maxParallelDownloads = 5) + { + int success = 0; + int failed = 0; + int completed = 0; + + + var semaphore = new System.Threading.SemaphoreSlim(maxParallelDownloads); + + var filesNeeded = new List<(string Directory, string FileName)>(); + + foreach (var seriesId in updateList) + { + foreach (var dir in directories.Values) + { + string filePath = Path.Combine(dir, $"{seriesId}.json"); + if (!File.Exists(filePath)) + filesNeeded.Add((dir, $"{seriesId}.json")); + } + } + + Log.Information("Downloading {Count} files...", filesNeeded.Count); + var totalFiles = filesNeeded.Count; + var progress = new Progress(value => + { + completed += value; + Console.Write($"\rDownloading files: {completed}/{totalFiles}"); + }); + + var tasks = new List(); + foreach (var file in filesNeeded) + { + tasks.Add(DownloadWithThrottleAsync(file.Directory, file.FileName, settings, semaphore, progress)); + } + + await Task.WhenAll(tasks); + Console.WriteLine("\nDownload complete."); + + foreach (var file in filesNeeded) + { + string path = Path.Combine(file.Directory, file.FileName); + if (File.Exists(path)) success++; + else failed++; + } + + return (success, failed); + } + + private async Task DownloadWithThrottleAsync( + string directory, + string fileName, + Settings settings, + System.Threading.SemaphoreSlim semaphore, + IProgress progress) + { + await semaphore.WaitAsync(); + try + { + string seriesId = Path.GetFileNameWithoutExtension(fileName); + string url; + + if (directory.Contains("series", StringComparison.OrdinalIgnoreCase)) + url = settings.Json.ShowApi.Replace("", seriesId); + else if (directory.Contains("episodes", StringComparison.OrdinalIgnoreCase)) + url = settings.Json.EpisodeApi.Replace("", seriesId); + else if (directory.Contains("cast", StringComparison.OrdinalIgnoreCase)) + url = settings.Json.CastApi.Replace("", seriesId); + else if (directory.Contains("crew", StringComparison.OrdinalIgnoreCase)) + url = settings.Json.CrewApi.Replace("", seriesId); + else + url = $"https://api.tvmaze.com/shows/{seriesId}"; + + // Download the file + await Task.Run(() => _downloader.Download(url, directory, fileName, Log.Logger)); + + // Report progress + progress?.Report(1); + } + catch (Exception ex) + { + Log.Error(ex, "Failed to download {FileName}", fileName); + progress?.Report(1); // still report so progress moves forward + } + finally + { + semaphore.Release(); + } + } + } +} diff --git a/MongoLoader.cs b/MongoLoader.cs new file mode 100644 index 0000000..8c37d15 --- /dev/null +++ b/MongoLoader.cs @@ -0,0 +1,187 @@ +using MongoDB.Bson; +using MongoDB.Bson.Serialization; +using MongoDB.Driver; +using Serilog; +using System.Text.Json; + +public class MongoLoader +{ + private readonly IMongoCollection _series; + private readonly IMongoCollection _episodes; + private readonly IMongoCollection _actors; + private readonly IMongoCollection _characters; + private readonly IMongoCollection _crew; + + public MongoLoader(IMongoDatabase db) + { + _series = db.GetCollection("series"); + _episodes = db.GetCollection("episodes"); + _actors = db.GetCollection("actors"); + _characters = db.GetCollection("characters"); + _crew = db.GetCollection("crew"); + } + + public async Task<(int Success, int Failed)> LoadJsonToMongoAsync( + Dictionary directories, + List updateList, + Dictionary updateDict) + { + int success = 0; + int failed = 0; + + var options = new ParallelOptions + { + MaxDegreeOfParallelism = Environment.ProcessorCount // or set manually (e.g. 4) + }; + + await Parallel.ForEachAsync(updateList, options, async (seriesId, cancellationToken) => + { + try + { + var seriesFile = Path.Combine(directories["SERIESDIR"], $"{seriesId}.json"); + if (!File.Exists(seriesFile)) + return; + + var json = await File.ReadAllTextAsync(seriesFile, cancellationToken); + var doc = BsonDocument.Parse(json); + + await UpsertWithTimestamp(_series, doc, updateDict, seriesId); + + await ProcessEpisodes(seriesId, directories, updateDict); + await ProcessCast(seriesId, directories, updateDict); + await ProcessCrew(seriesId, directories, updateDict); + + Interlocked.Increment(ref success); + + Log.Information("Processed series {SeriesId}", seriesId); + } + catch (Exception ex) + { + Interlocked.Increment(ref failed); + Log.Error(ex, "Failed processing series {SeriesId}", seriesId); + } + }); + + return (success, failed); + } + + + private async Task UpsertWithTimestamp( + IMongoCollection collection, + BsonDocument doc, + Dictionary updateDict, + string seriesId) + { + var id = doc.GetValue("id", null); + if (id == null) return; + + var filter = Builders.Filter.Eq("id", id); + var existing = await collection.Find(filter).FirstOrDefaultAsync(); + + if (updateDict.TryGetValue(seriesId, out var newTimestamp)) + { + if (existing == null || + !existing.Contains("updated") || + existing["updated"].AsInt64 < newTimestamp) + { + await collection.ReplaceOneAsync(filter, doc, + new ReplaceOptions { IsUpsert = true }); + } + } + } + + private async Task ProcessEpisodes( + string seriesId, + Dictionary directories, + Dictionary updateDict) + { + var file = Path.Combine(directories["EPISODEDIR"], $"{seriesId}.json"); + if (!File.Exists(file)) return; + + var json = await File.ReadAllTextAsync(file); + var array = JsonSerializer.Deserialize>>(json); + + if (array == null) return; + + foreach (var episode in array) + { + episode["seriesid"] = int.Parse(seriesId); + var doc = BsonDocument.Parse(JsonSerializer.Serialize(episode)); + await UpsertWithTimestamp(_episodes, doc, updateDict, seriesId); + } + } + + private async Task ProcessCast( + string seriesId, + Dictionary directories, + Dictionary updateDict) + { + var file = Path.Combine(directories["CASTDIR"], $"{seriesId}.json"); + if (!File.Exists(file)) return; + + var json = await File.ReadAllTextAsync(file); + + // ✅ Correct way to parse root JSON array + var array = BsonSerializer.Deserialize(json); + + foreach (var item in array) + { + if (!item.IsBsonDocument) + continue; + + var doc = item.AsBsonDocument; + + if (doc.Contains("person") && doc["person"].IsBsonDocument) + { + var person = doc["person"].AsBsonDocument; + person["seriesid"] = int.Parse(seriesId); + + await UpsertWithTimestamp(_actors, person, updateDict, seriesId); + } + + if (doc.Contains("character") && doc["character"].IsBsonDocument) + { + var character = doc["character"].AsBsonDocument; + character["seriesid"] = int.Parse(seriesId); + + await UpsertWithTimestamp(_characters, character, updateDict, seriesId); + } + } + } + + private async Task ProcessCrew( + string seriesId, + Dictionary directories, + Dictionary updateDict) + { + var file = Path.Combine(directories["CREWDIR"], $"{seriesId}.json"); + if (!File.Exists(file)) return; + + var json = await File.ReadAllTextAsync(file); + + // ✅ Correct way to parse root JSON array + var array = BsonSerializer.Deserialize(json); + + foreach (var item in array) + { + if (!item.IsBsonDocument) + continue; + + var doc = item.AsBsonDocument; + + if (doc.Contains("person") && doc["person"].IsBsonDocument) + { + var person = doc["person"].AsBsonDocument; + + person["seriesid"] = int.Parse(seriesId); + + // Safely get crew type + person["crew_type"] = doc.Contains("type") + ? doc["type"] + : "unknown"; + + await UpsertWithTimestamp(_crew, person, updateDict, seriesId); + } + } + } +} diff --git a/Settings.cs b/Settings.cs new file mode 100644 index 0000000..9728f8c --- /dev/null +++ b/Settings.cs @@ -0,0 +1,102 @@ +using System; +using System.IO; +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace TvSyncApp +{ + public class Settings + { + public GlobalSettings? Global { get; set; } + public DbSettings? DbSettings { get; set; } + public ApiSettings? Api { get; set; } + public JsonSettings? Json { get; set; } + public DatabaseSettings? Database { get; set; } + public UpdateTypeSettings? UpdateType { get; set; } + + public static Settings Load(string filePath) + { + if (!File.Exists(filePath)) + throw new FileNotFoundException($"Settings file not found: {filePath}"); + + var json = File.ReadAllText(filePath); + + var options = new JsonSerializerOptions + { + PropertyNameCaseInsensitive = true, + ReadCommentHandling = JsonCommentHandling.Skip, + AllowTrailingCommas = true + }; + + return JsonSerializer.Deserialize(json, options); + } + } + + public class GlobalSettings + { + public string? LogLevel { get; set; } + } + + public class DbSettings + { + public string? SqlServerDriver { get; set; } + public string? Hostname { get; set; } + public string? MgHostname { get; set; } + public string? DbName { get; set; } + public string? MgDbName { get; set; } + public string? MysqlUsername { get; set; } + public string? PgsqlUsername { get; set; } + public string? MssqlUsername { get; set; } + public string? MysqlPassword { get; set; } + public string? PgsqlPassword { get; set; } + public string? MssqlPassword { get; set; } + public int PgsqlPort { get; set; } + public int MysqlPort { get; set; } + public int MssqlPort { get; set; } + public int MgPort { get; set; } + } + + public class ApiSettings + { + public string? Type { get; set; } + } + + public class JsonSettings + { + public string? SeriesApi { get; set; } + public string? ShowApi { get; set; } + public string? EpisodeApi { get; set; } + public string? AliasApi { get; set; } + public string? CastApi { get; set; } + public string? CrewApi { get; set; } + public string? CreditsApi { get; set; } + public string? UpdatesApi { get; set; } + } + + public class DatabaseSettings + { + public string? DbType { get; set; } + public string? UpdateSchema { get; set; } + public int Initialize { get; set; } + public int InitLoad { get; set; } + } + + public class UpdateTypeSettings + { + public string? Type { get; set; } + public string? Full { get; set; } + public long Monthly { get; set; } + public long Weekly { get; set; } + public long Daily { get; set; } + + [JsonPropertyName("12h")] + public long H12 { get; set; } + + [JsonPropertyName("6h")] + public long H6 { get; set; } + + public long Custom { get; set; } + public int CacheSkip { get; set; } + public int RefreshCache { get; set; } + } +} diff --git a/TvSyncApp.cs b/TvSyncApp.cs new file mode 100644 index 0000000..63e37b9 --- /dev/null +++ b/TvSyncApp.cs @@ -0,0 +1,105 @@ +using System; +using System.IO; +using System.Collections.Generic; +using System.Threading.Tasks; +using Npgsql; +using Serilog; + +namespace TvSyncApp +{ + public class Program + { + public static async Task Main(string[] args) + { + string settingsPath = Path.Combine(Directory.GetCurrentDirectory(), "appsettings.json"); + var settings = TvSyncApp.Settings.Load(settingsPath) + ?? throw new Exception("Failed to load settings from appsettings.json"); + + Console.WriteLine($"Log level: {settings.Global.LogLevel}"); + Console.WriteLine($"Database type: {settings.Database.DbType}"); + // Console.WriteLine($"TVMaze series API: {settings.Json.SeriesApi}"); + + string rootDir = Directory.GetCurrentDirectory(); + + var directories = new Dictionary + { + ["SERIESDIR"] = Path.Combine(rootDir, "cache", "series"), + ["EPISODEDIR"] = Path.Combine(rootDir, "cache", "episodes"), + ["CASTDIR"] = Path.Combine(rootDir, "cache", "cast"), + // ["GUESTCASTDIR"] = Path.Combine(rootDir, "cache", "guestcast"), + ["CREWDIR"] = Path.Combine(rootDir, "cache", "crew"), + ["UNKNOWN"] = Path.Combine(rootDir, "cache", "unknown") + }; + + CreateDirectories(rootDir); + + Log.Logger = new LoggerConfiguration() + .MinimumLevel.Debug() + .WriteTo.Console() + .WriteTo.File(Path.Combine(rootDir, "log", "sync.log")) + .CreateLogger(); + + Log.Information("Starting sync process..."); + + // --- Fetch series IDs from PostgreSQL --- + string connectionString = $"Host={settings.DbSettings.Hostname};Port={settings.DbSettings.PgsqlPort};Username={settings.DbSettings.PgsqlUsername};Password={settings.DbSettings.PgsqlPassword};Database={settings.DbSettings.DbName}"; + List seriesIds = GetSeriesIdsFromDb(connectionString); + + Log.Information("Fetched {Count} series IDs from tvupdates table.", seriesIds.Count); + + // Convert to string list if your Loader expects strings + List updateList = seriesIds.ConvertAll(id => id.ToString()); + + // Optionally, you can also fetch timestamps if needed + Dictionary updateDict = new Dictionary(); + + var loader = new Loader(); + + var result = await loader.LoadJsonAsync(directories, updateList, updateDict, settings); + + Log.Information("Sync complete. Success: {Success}, Failed: {Failed}", result.Success, result.Failed); + Log.CloseAndFlush(); + + return 0; + } + + private static List GetSeriesIdsFromDb(string connectionString) + { + var seriesIds = new List(); + + try + { + using var conn = new NpgsqlConnection(connectionString); + conn.Open(); + + string query = "SELECT seriesid FROM updates.tvupdates ORDER BY seriesid"; + + using var cmd = new NpgsqlCommand(query, conn); + using var reader = cmd.ExecuteReader(); + + while (reader.Read()) + { + if (!reader.IsDBNull(0)) + seriesIds.Add(reader.GetInt32(0)); + } + } + catch (Exception ex) + { + Log.Error(ex, "Failed to fetch series IDs from PostgreSQL."); + } + + return seriesIds; + } + + private static void CreateDirectories(string rootDir) + { + Directory.CreateDirectory(Path.Combine(rootDir, "log")); + Directory.CreateDirectory(Path.Combine(rootDir, "temp")); + Directory.CreateDirectory(Path.Combine(rootDir, "cache", "series")); + Directory.CreateDirectory(Path.Combine(rootDir, "cache", "episodes")); + Directory.CreateDirectory(Path.Combine(rootDir, "cache", "cast")); + Directory.CreateDirectory(Path.Combine(rootDir, "cache", "guestcast")); + Directory.CreateDirectory(Path.Combine(rootDir, "cache", "crew")); + } + } +} diff --git a/appsettings.json b/appsettings.json new file mode 100644 index 0000000..5991af1 --- /dev/null +++ b/appsettings.json @@ -0,0 +1,53 @@ +{ + "global": { + "loglevel": "info" + }, + "dbsettings": { + "sqlserver_driver": "ODBC Driver 13 for SQL Server", + "hostname": "192.168.129.248", + "mghostname": "192.168.128.24", + "dbname": "media_dbsync", + "mgdbname": "tvdata", + "mysqlusername": "root", + "pgsqlusername": "postgres", + "mssqlusername": "sa", + "mysqlpassword": "Optimus0329", + "pgsqlpassword": "Optimus0329", + "mssqlpassword": "Optimus0329", + "pgsqlport": 5432, + "mysqlport": 3306, + "mssqlport": 1433, + "mgport": 27017 + }, + "api": { + "type": "tvmaze" + }, + "json": { + // "seriesapi": "http://api.tvmaze.com/shows?page=", + "showapi": "http://api.tvmaze.com/shows/", + "episodeapi": "http://api.tvmaze.com/shows//episodes", + "aliasapi": "http://api.tvmaze.com/shows//akas", + "castapi": "http://api.tvmaze.com/shows//cast", + "crewapi": "http://api.tvmaze.com/shows//crew", + "creditsapi": "http://api.tvmaze.com/people//castcredits", + "updatesapi": "http://api.tvmaze.com/updates/shows" + }, + "database": { + "dbtype": "pgsql", + "updateschema": "updates", + "initialize": 0, + "initload": 0 + }, + "updatetype": { + "type": "full", + "full": "full", + "monthly": 1219200, + "weekly": 604800, + "daily": 86400, + "12h": 23200, + "6h": 16600, + "custom": 4150, + "cacheskip": 1, + "refreshcache": 0 + } +} \ No newline at end of file diff --git a/dbsync.csproj b/dbsync.csproj new file mode 100644 index 0000000..90eb1e6 --- /dev/null +++ b/dbsync.csproj @@ -0,0 +1,19 @@ + + + + Exe + net9.0 + enable + enable + + + + + + + + + + + + diff --git a/dbsync.sln b/dbsync.sln new file mode 100644 index 0000000..8b0943b --- /dev/null +++ b/dbsync.sln @@ -0,0 +1,25 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Version 17 +VisualStudioVersion = 17.14.36930.0 d17.14 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "dbsync", "dbsync.csproj", "{06905DC5-E7BA-4CD0-9F8F-FD0E60C9DB71}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {06905DC5-E7BA-4CD0-9F8F-FD0E60C9DB71}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {06905DC5-E7BA-4CD0-9F8F-FD0E60C9DB71}.Debug|Any CPU.Build.0 = Debug|Any CPU + {06905DC5-E7BA-4CD0-9F8F-FD0E60C9DB71}.Release|Any CPU.ActiveCfg = Release|Any CPU + {06905DC5-E7BA-4CD0-9F8F-FD0E60C9DB71}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(ExtensibilityGlobals) = postSolution + SolutionGuid = {5F508F17-C255-49C1-B01D-13B2B739B1FF} + EndGlobalSection +EndGlobal