Initial commit

2026-02-16 17:12:34 -05:00
parent 77eb3db44f
commit bdcb596e39
8 changed files with 646 additions and 0 deletions
@@ -0,0 +1,47 @@
+using System;
+using System.IO;
+using System.Net.Http;
+using System.Text.Json;
+using Serilog;
+
+namespace TvSyncApp
+{
+    public class Downloader
+    {
+        private static readonly HttpClient _httpClient = new HttpClient();
+
+        public void Download(string url, string outputDir, string outputFile, ILogger log)
+        {
+            try
+            {
+                // log.Information("Downloading {Url} to {OutputFile}", url, outputFile);
+
+                // Ensure the output directory exists
+                Directory.CreateDirectory(outputDir);
+
+                // Get JSON content as string
+                var response = _httpClient.GetAsync(url).Result;
+                response.EnsureSuccessStatusCode();
+                string jsonString = response.Content.ReadAsStringAsync().Result;
+
+                // Parse and pretty-print JSON
+                var jsonDoc = JsonDocument.Parse(jsonString);
+                var formattedJson = JsonSerializer.Serialize(jsonDoc.RootElement, new JsonSerializerOptions
+                {
+                    WriteIndented = true
+                });
+
+                // Write to file
+                string filePath = Path.Combine(outputDir, outputFile);
+                File.WriteAllText(filePath, formattedJson);
+
+                //log.Information("Downloaded and formatted {OutputFile} successfully.", outputFile);
+            }
+            catch (Exception ex)
+            {
+                log.Error(ex, "Failed to download {OutputFile} from {Url}", outputFile, url);
+            }
+        }
+    }
+}
+
@@ -0,0 +1,108 @@
+using Serilog;
+
+namespace TvSyncApp
+{
+    public class Loader
+    {
+        private readonly Downloader _downloader;
+
+        public Loader()
+        {
+            _downloader = new Downloader();
+        }
+
+        public async Task<(int Success, int Failed)> LoadJsonAsync(
+            Dictionary<string, string> directories,
+            List<string> updateList,
+            Dictionary<string, long> updateDict,
+            Settings settings,
+            int maxParallelDownloads = 5)
+        {
+            int success = 0;
+            int failed = 0;
+            int completed = 0;
+
+
+            var semaphore = new System.Threading.SemaphoreSlim(maxParallelDownloads);
+
+            var filesNeeded = new List<(string Directory, string FileName)>();
+            
+            foreach (var seriesId in updateList)
+            {
+                foreach (var dir in directories.Values)
+                {
+                    string filePath = Path.Combine(dir, $"{seriesId}.json");
+                    if (!File.Exists(filePath))
+                        filesNeeded.Add((dir, $"{seriesId}.json"));
+                }
+            }
+
+            Log.Information("Downloading {Count} files...", filesNeeded.Count);
+            var totalFiles = filesNeeded.Count;
+            var progress = new Progress<int>(value =>
+            {
+                completed += value;
+                Console.Write($"\rDownloading files: {completed}/{totalFiles}");
+            });
+
+            var tasks = new List<Task>();
+            foreach (var file in filesNeeded)
+            {
+                tasks.Add(DownloadWithThrottleAsync(file.Directory, file.FileName, settings, semaphore, progress));
+            }
+
+            await Task.WhenAll(tasks);
+            Console.WriteLine("\nDownload complete.");
+
+            foreach (var file in filesNeeded)
+            {
+                string path = Path.Combine(file.Directory, file.FileName);
+                if (File.Exists(path)) success++;
+                else failed++;
+            }
+
+            return (success, failed);
+        }
+
+        private async Task DownloadWithThrottleAsync(
+            string directory,
+            string fileName,
+            Settings settings,
+            System.Threading.SemaphoreSlim semaphore,
+            IProgress<int> progress)
+        {
+            await semaphore.WaitAsync();
+            try
+            {
+                string seriesId = Path.GetFileNameWithoutExtension(fileName);
+                string url;
+
+                if (directory.Contains("series", StringComparison.OrdinalIgnoreCase))
+                    url = settings.Json.ShowApi.Replace("<tvmazeid>", seriesId);
+                else if (directory.Contains("episodes", StringComparison.OrdinalIgnoreCase))
+                    url = settings.Json.EpisodeApi.Replace("<tvmazeid>", seriesId);
+                else if (directory.Contains("cast", StringComparison.OrdinalIgnoreCase))
+                    url = settings.Json.CastApi.Replace("<tvmazeid>", seriesId);
+                else if (directory.Contains("crew", StringComparison.OrdinalIgnoreCase))
+                    url = settings.Json.CrewApi.Replace("<tvmazeid>", seriesId);
+                else
+                    url = $"https://api.tvmaze.com/shows/{seriesId}";
+
+                // Download the file
+                await Task.Run(() => _downloader.Download(url, directory, fileName, Log.Logger));
+
+                // Report progress
+                progress?.Report(1);
+            }
+            catch (Exception ex)
+            {
+                Log.Error(ex, "Failed to download {FileName}", fileName);
+                progress?.Report(1); // still report so progress moves forward
+            }
+            finally
+            {
+                semaphore.Release();
+            }
+        }
+    }
+}
@@ -0,0 +1,187 @@
+using MongoDB.Bson;
+using MongoDB.Bson.Serialization;
+using MongoDB.Driver;
+using Serilog;
+using System.Text.Json;
+
+public class MongoLoader
+{
+    private readonly IMongoCollection<BsonDocument> _series;
+    private readonly IMongoCollection<BsonDocument> _episodes;
+    private readonly IMongoCollection<BsonDocument> _actors;
+    private readonly IMongoCollection<BsonDocument> _characters;
+    private readonly IMongoCollection<BsonDocument> _crew;
+
+    public MongoLoader(IMongoDatabase db)
+    {
+        _series = db.GetCollection<BsonDocument>("series");
+        _episodes = db.GetCollection<BsonDocument>("episodes");
+        _actors = db.GetCollection<BsonDocument>("actors");
+        _characters = db.GetCollection<BsonDocument>("characters");
+        _crew = db.GetCollection<BsonDocument>("crew");
+    }
+
+    public async Task<(int Success, int Failed)> LoadJsonToMongoAsync(
+        Dictionary<string, string> directories,
+        List<string> updateList,
+        Dictionary<string, long> updateDict)
+    {
+        int success = 0;
+        int failed = 0;
+
+        var options = new ParallelOptions
+        {
+            MaxDegreeOfParallelism = Environment.ProcessorCount // or set manually (e.g. 4)
+        };
+
+        await Parallel.ForEachAsync(updateList, options, async (seriesId, cancellationToken) =>
+        {
+            try
+            {
+                var seriesFile = Path.Combine(directories["SERIESDIR"], $"{seriesId}.json");
+                if (!File.Exists(seriesFile))
+                    return;
+
+                var json = await File.ReadAllTextAsync(seriesFile, cancellationToken);
+                var doc = BsonDocument.Parse(json);
+
+                await UpsertWithTimestamp(_series, doc, updateDict, seriesId);
+
+                await ProcessEpisodes(seriesId, directories, updateDict);
+                await ProcessCast(seriesId, directories, updateDict);
+                await ProcessCrew(seriesId, directories, updateDict);
+
+                Interlocked.Increment(ref success);
+
+                Log.Information("Processed series {SeriesId}", seriesId);
+            }
+            catch (Exception ex)
+            {
+                Interlocked.Increment(ref failed);
+                Log.Error(ex, "Failed processing series {SeriesId}", seriesId);
+            }
+        });
+
+        return (success, failed);
+    }
+
+
+    private async Task UpsertWithTimestamp(
+        IMongoCollection<BsonDocument> collection,
+        BsonDocument doc,
+        Dictionary<string, long> updateDict,
+        string seriesId)
+    {
+        var id = doc.GetValue("id", null);
+        if (id == null) return;
+
+        var filter = Builders<BsonDocument>.Filter.Eq("id", id);
+        var existing = await collection.Find(filter).FirstOrDefaultAsync();
+
+        if (updateDict.TryGetValue(seriesId, out var newTimestamp))
+        {
+            if (existing == null ||
+                !existing.Contains("updated") ||
+                existing["updated"].AsInt64 < newTimestamp)
+            {
+                await collection.ReplaceOneAsync(filter, doc,
+                    new ReplaceOptions { IsUpsert = true });
+            }
+        }
+    }
+
+    private async Task ProcessEpisodes(
+        string seriesId,
+        Dictionary<string, string> directories,
+        Dictionary<string, long> updateDict)
+    {
+        var file = Path.Combine(directories["EPISODEDIR"], $"{seriesId}.json");
+        if (!File.Exists(file)) return;
+
+        var json = await File.ReadAllTextAsync(file);
+        var array = JsonSerializer.Deserialize<List<Dictionary<string, object>>>(json);
+
+        if (array == null) return;
+
+        foreach (var episode in array)
+        {
+            episode["seriesid"] = int.Parse(seriesId);
+            var doc = BsonDocument.Parse(JsonSerializer.Serialize(episode));
+            await UpsertWithTimestamp(_episodes, doc, updateDict, seriesId);
+        }
+    }
+
+    private async Task ProcessCast(
+        string seriesId,
+        Dictionary<string, string> directories,
+        Dictionary<string, long> updateDict)
+    {
+        var file = Path.Combine(directories["CASTDIR"], $"{seriesId}.json");
+        if (!File.Exists(file)) return;
+
+        var json = await File.ReadAllTextAsync(file);
+
+        // ✅ Correct way to parse root JSON array
+        var array = BsonSerializer.Deserialize<BsonArray>(json);
+
+        foreach (var item in array)
+        {
+            if (!item.IsBsonDocument)
+                continue;
+
+            var doc = item.AsBsonDocument;
+
+            if (doc.Contains("person") && doc["person"].IsBsonDocument)
+            {
+                var person = doc["person"].AsBsonDocument;
+                person["seriesid"] = int.Parse(seriesId);
+
+                await UpsertWithTimestamp(_actors, person, updateDict, seriesId);
+            }
+
+            if (doc.Contains("character") && doc["character"].IsBsonDocument)
+            {
+                var character = doc["character"].AsBsonDocument;
+                character["seriesid"] = int.Parse(seriesId);
+
+                await UpsertWithTimestamp(_characters, character, updateDict, seriesId);
+            }
+        }
+    }
+
+    private async Task ProcessCrew(
+        string seriesId,
+        Dictionary<string, string> directories,
+        Dictionary<string, long> updateDict)
+    {
+        var file = Path.Combine(directories["CREWDIR"], $"{seriesId}.json");
+        if (!File.Exists(file)) return;
+
+        var json = await File.ReadAllTextAsync(file);
+
+        // ✅ Correct way to parse root JSON array
+        var array = BsonSerializer.Deserialize<BsonArray>(json);
+
+        foreach (var item in array)
+        {
+            if (!item.IsBsonDocument)
+                continue;
+
+            var doc = item.AsBsonDocument;
+
+            if (doc.Contains("person") && doc["person"].IsBsonDocument)
+            {
+                var person = doc["person"].AsBsonDocument;
+
+                person["seriesid"] = int.Parse(seriesId);
+
+                // Safely get crew type
+                person["crew_type"] = doc.Contains("type")
+                    ? doc["type"]
+                    : "unknown";
+
+                await UpsertWithTimestamp(_crew, person, updateDict, seriesId);
+            }
+        }
+    }
+}
@@ -0,0 +1,102 @@
+using System;
+using System.IO;
+using System.Text.Json;
+using System.Text.Json.Serialization;
+
+namespace TvSyncApp
+{
+    public class Settings
+    {
+        public GlobalSettings? Global { get; set; }
+        public DbSettings? DbSettings { get; set; }
+        public ApiSettings? Api { get; set; }
+        public JsonSettings? Json { get; set; }
+        public DatabaseSettings? Database { get; set; }
+        public UpdateTypeSettings? UpdateType { get; set; }
+
+        public static Settings Load(string filePath)
+        {
+            if (!File.Exists(filePath))
+                throw new FileNotFoundException($"Settings file not found: {filePath}");
+
+            var json = File.ReadAllText(filePath);
+
+            var options = new JsonSerializerOptions
+            {
+                PropertyNameCaseInsensitive = true,
+                ReadCommentHandling = JsonCommentHandling.Skip,
+                AllowTrailingCommas = true
+            };
+
+            return JsonSerializer.Deserialize<Settings>(json, options);
+        }
+    }
+
+    public class GlobalSettings
+    {
+        public string? LogLevel { get; set; }
+    }
+
+    public class DbSettings
+    {
+        public string? SqlServerDriver { get; set; }
+        public string? Hostname { get; set; }
+        public string? MgHostname { get; set; }
+        public string? DbName { get; set; }
+        public string? MgDbName { get; set; }
+        public string? MysqlUsername { get; set; }
+        public string? PgsqlUsername { get; set; }
+        public string? MssqlUsername { get; set; }
+        public string? MysqlPassword { get; set; }
+        public string? PgsqlPassword { get; set; }
+        public string? MssqlPassword { get; set; }
+        public int PgsqlPort { get; set; }
+        public int MysqlPort { get; set; }
+        public int MssqlPort { get; set; }
+        public int MgPort { get; set; }
+    }
+
+    public class ApiSettings
+    {
+        public string? Type { get; set; }
+    }
+
+    public class JsonSettings
+    {
+        public string? SeriesApi { get; set; }
+        public string? ShowApi { get; set; }
+        public string? EpisodeApi { get; set; }
+        public string? AliasApi { get; set; }
+        public string? CastApi { get; set; }
+        public string? CrewApi { get; set; }
+        public string? CreditsApi { get; set; }
+        public string? UpdatesApi { get; set; }
+    }
+
+    public class DatabaseSettings
+    {
+        public string? DbType { get; set; }
+        public string? UpdateSchema { get; set; }
+        public int Initialize { get; set; }
+        public int InitLoad { get; set; }
+    }
+
+    public class UpdateTypeSettings
+    {
+        public string? Type { get; set; }
+        public string? Full { get; set; }
+        public long Monthly { get; set; }
+        public long Weekly { get; set; }
+        public long Daily { get; set; }
+
+        [JsonPropertyName("12h")]
+        public long H12 { get; set; }
+
+        [JsonPropertyName("6h")]
+        public long H6 { get; set; }
+
+        public long Custom { get; set; }
+        public int CacheSkip { get; set; }
+        public int RefreshCache { get; set; }
+    }
+}
@@ -0,0 +1,105 @@
+using System;
+using System.IO;
+using System.Collections.Generic;
+using System.Threading.Tasks;
+using Npgsql;
+using Serilog;
+
+namespace TvSyncApp
+{
+    public class Program
+    {
+        public static async Task<int> Main(string[] args)
+        {
+            string settingsPath = Path.Combine(Directory.GetCurrentDirectory(), "appsettings.json");
+            var settings = TvSyncApp.Settings.Load(settingsPath)
+               ?? throw new Exception("Failed to load settings from appsettings.json");
+
+            Console.WriteLine($"Log level: {settings.Global.LogLevel}");
+            Console.WriteLine($"Database type: {settings.Database.DbType}");
+            // Console.WriteLine($"TVMaze series API: {settings.Json.SeriesApi}");
+            
+            string rootDir = Directory.GetCurrentDirectory();
+
+            var directories = new Dictionary<string, string>
+            {
+                ["SERIESDIR"] = Path.Combine(rootDir, "cache", "series"),
+                ["EPISODEDIR"] = Path.Combine(rootDir, "cache", "episodes"),
+                ["CASTDIR"] = Path.Combine(rootDir, "cache", "cast"),
+                // ["GUESTCASTDIR"] = Path.Combine(rootDir, "cache", "guestcast"),
+                ["CREWDIR"] = Path.Combine(rootDir, "cache", "crew"),
+                ["UNKNOWN"] = Path.Combine(rootDir, "cache", "unknown")
+            };
+
+            CreateDirectories(rootDir);
+
+            Log.Logger = new LoggerConfiguration()
+                .MinimumLevel.Debug()
+                .WriteTo.Console()
+                .WriteTo.File(Path.Combine(rootDir, "log", "sync.log"))
+                .CreateLogger();
+
+            Log.Information("Starting sync process...");
+
+            // --- Fetch series IDs from PostgreSQL ---
+            string connectionString = $"Host={settings.DbSettings.Hostname};Port={settings.DbSettings.PgsqlPort};Username={settings.DbSettings.PgsqlUsername};Password={settings.DbSettings.PgsqlPassword};Database={settings.DbSettings.DbName}";
+            List<int> seriesIds = GetSeriesIdsFromDb(connectionString);
+
+            Log.Information("Fetched {Count} series IDs from tvupdates table.", seriesIds.Count);
+
+            // Convert to string list if your Loader expects strings
+            List<string> updateList = seriesIds.ConvertAll(id => id.ToString());
+
+            // Optionally, you can also fetch timestamps if needed
+            Dictionary<string, long> updateDict = new Dictionary<string, long>();
+
+            var loader = new Loader();
+
+            var result = await loader.LoadJsonAsync(directories, updateList, updateDict, settings);
+
+            Log.Information("Sync complete. Success: {Success}, Failed: {Failed}", result.Success, result.Failed);
+            Log.CloseAndFlush();
+
+            return 0;
+        }
+
+        private static List<int> GetSeriesIdsFromDb(string connectionString)
+        {
+            var seriesIds = new List<int>();
+
+            try
+            {
+                using var conn = new NpgsqlConnection(connectionString);
+                conn.Open();
+
+                string query = "SELECT seriesid FROM updates.tvupdates ORDER BY seriesid";
+
+                using var cmd = new NpgsqlCommand(query, conn);
+                using var reader = cmd.ExecuteReader();
+
+                while (reader.Read())
+                {
+                    if (!reader.IsDBNull(0))
+                        seriesIds.Add(reader.GetInt32(0));
+                }
+            }
+            catch (Exception ex)
+            {
+                Log.Error(ex, "Failed to fetch series IDs from PostgreSQL.");
+            }
+
+            return seriesIds;
+        }
+
+        private static void CreateDirectories(string rootDir)
+        {
+            Directory.CreateDirectory(Path.Combine(rootDir, "log"));
+            Directory.CreateDirectory(Path.Combine(rootDir, "temp"));
+            Directory.CreateDirectory(Path.Combine(rootDir, "cache", "series"));
+            Directory.CreateDirectory(Path.Combine(rootDir, "cache", "episodes"));
+            Directory.CreateDirectory(Path.Combine(rootDir, "cache", "cast"));
+            Directory.CreateDirectory(Path.Combine(rootDir, "cache", "guestcast"));
+            Directory.CreateDirectory(Path.Combine(rootDir, "cache", "crew"));
+        }
+    }
+}
@@ -0,0 +1,53 @@
+{
+  "global": {
+    "loglevel": "info"
+  },
+  "dbsettings": {
+    "sqlserver_driver": "ODBC Driver 13 for SQL Server",
+    "hostname": "192.168.129.248",
+    "mghostname": "192.168.128.24",
+    "dbname": "media_dbsync",
+    "mgdbname": "tvdata",
+    "mysqlusername": "root",
+    "pgsqlusername": "postgres",
+    "mssqlusername": "sa",
+    "mysqlpassword": "Optimus0329",
+    "pgsqlpassword": "Optimus0329",
+    "mssqlpassword": "Optimus0329",
+    "pgsqlport": 5432,
+    "mysqlport": 3306,
+    "mssqlport": 1433,
+    "mgport": 27017
+  },
+  "api": {
+    "type": "tvmaze"
+  },
+  "json": {
+    // "seriesapi": "http://api.tvmaze.com/shows?page=<pagenum>",
+    "showapi": "http://api.tvmaze.com/shows/<tvmazeid>",
+    "episodeapi": "http://api.tvmaze.com/shows/<tvmazeid>/episodes",
+    "aliasapi": "http://api.tvmaze.com/shows/<tvmazeid>/akas",
+    "castapi": "http://api.tvmaze.com/shows/<tvmazeid>/cast",
+    "crewapi": "http://api.tvmaze.com/shows/<tvmazeid>/crew",
+    "creditsapi": "http://api.tvmaze.com/people/<actorid>/castcredits",
+    "updatesapi": "http://api.tvmaze.com/updates/shows"
+  },
+  "database": {
+    "dbtype": "pgsql",
+    "updateschema": "updates",
+    "initialize": 0,
+    "initload": 0
+  },
+  "updatetype": {
+    "type": "full",
+    "full": "full",
+    "monthly": 1219200,
+    "weekly": 604800,
+    "daily": 86400,
+    "12h": 23200,
+    "6h": 16600,
+    "custom": 4150,
+    "cacheskip": 1,
+    "refreshcache": 0
+  }
+}
@@ -0,0 +1,19 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net9.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="MongoDB.driver" Version="3.6.0" />
+    <PackageReference Include="Npgsql" Version="10.0.1" />
+    <PackageReference Include="Serilog" Version="4.3.1" />
+    <PackageReference Include="Serilog.Sinks.Console" Version="6.1.1" />
+    <PackageReference Include="Serilog.Sinks.File" Version="7.0.0" />
+    <PackageReference Include="System.Text.Json" Version="10.0.3" />
+  </ItemGroup>
+
+</Project>
@@ -0,0 +1,25 @@
+
+Microsoft Visual Studio Solution File, Format Version 12.00
+# Visual Studio Version 17
+VisualStudioVersion = 17.14.36930.0 d17.14
+MinimumVisualStudioVersion = 10.0.40219.1
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "dbsync", "dbsync.csproj", "{06905DC5-E7BA-4CD0-9F8F-FD0E60C9DB71}"
+EndProject
+Global
+	GlobalSection(SolutionConfigurationPlatforms) = preSolution
+		Debug|Any CPU = Debug|Any CPU
+		Release|Any CPU = Release|Any CPU
+	EndGlobalSection
+	GlobalSection(ProjectConfigurationPlatforms) = postSolution
+		{06905DC5-E7BA-4CD0-9F8F-FD0E60C9DB71}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{06905DC5-E7BA-4CD0-9F8F-FD0E60C9DB71}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{06905DC5-E7BA-4CD0-9F8F-FD0E60C9DB71}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{06905DC5-E7BA-4CD0-9F8F-FD0E60C9DB71}.Release|Any CPU.Build.0 = Release|Any CPU
+	EndGlobalSection
+	GlobalSection(SolutionProperties) = preSolution
+		HideSolutionNode = FALSE
+	EndGlobalSection
+	GlobalSection(ExtensibilityGlobals) = postSolution
+		SolutionGuid = {5F508F17-C255-49C1-B01D-13B2B739B1FF}
+	EndGlobalSection
+EndGlobal