wjones/dotnet_dbsync
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Initial commit

Browse Source
This commit is contained in:
Wendell Jones
2026-02-16 17:12:34 -05:00
parent 77eb3db44f
commit bdcb596e39
8 changed files with 646 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Downloader.cs
+47
View File
@@ -0,0 +1,47 @@
using System;
using System.IO;
using System.Net.Http;
using System.Text.Json;
using Serilog;
namespace TvSyncApp
{
public class Downloader
{
private static readonly HttpClient _httpClient = new HttpClient();
public void Download(string url, string outputDir, string outputFile, ILogger log)
{
try
{
// log.Information("Downloading {Url} to {OutputFile}", url, outputFile);
// Ensure the output directory exists
Directory.CreateDirectory(outputDir);
// Get JSON content as string
var response = _httpClient.GetAsync(url).Result;
response.EnsureSuccessStatusCode();
string jsonString = response.Content.ReadAsStringAsync().Result;
// Parse and pretty-print JSON
var jsonDoc = JsonDocument.Parse(jsonString);
var formattedJson = JsonSerializer.Serialize(jsonDoc.RootElement, new JsonSerializerOptions
{
WriteIndented = true
});
// Write to file
string filePath = Path.Combine(outputDir, outputFile);
File.WriteAllText(filePath, formattedJson);
//log.Information("Downloaded and formatted {OutputFile} successfully.", outputFile);
}
catch (Exception ex)
{
log.Error(ex, "Failed to download {OutputFile} from {Url}", outputFile, url);
}
}
}
}
Loader.cs
+108
View File
@@ -0,0 +1,108 @@
using Serilog;
namespace TvSyncApp
{
public class Loader
{
private readonly Downloader _downloader;
public Loader()
{
_downloader = new Downloader();
}
public async Task<(int Success, int Failed)> LoadJsonAsync(
Dictionary<string, string> directories,
List<string> updateList,
Dictionary<string, long> updateDict,
Settings settings,
int maxParallelDownloads = 5)
{
int success = 0;
int failed = 0;
int completed = 0;
var semaphore = new System.Threading.SemaphoreSlim(maxParallelDownloads);
var filesNeeded = new List<(string Directory, string FileName)>();
foreach (var seriesId in updateList)
{
foreach (var dir in directories.Values)
{
string filePath = Path.Combine(dir, $"{seriesId}.json");
if (!File.Exists(filePath))
filesNeeded.Add((dir, $"{seriesId}.json"));
}
}
Log.Information("Downloading {Count} files...", filesNeeded.Count);
var totalFiles = filesNeeded.Count;
var progress = new Progress<int>(value =>
{
completed += value;
Console.Write($"\rDownloading files: {completed}/{totalFiles}");
});
var tasks = new List<Task>();
foreach (var file in filesNeeded)
{
tasks.Add(DownloadWithThrottleAsync(file.Directory, file.FileName, settings, semaphore, progress));
}
await Task.WhenAll(tasks);
Console.WriteLine("\nDownload complete.");
foreach (var file in filesNeeded)
{
string path = Path.Combine(file.Directory, file.FileName);
if (File.Exists(path)) success++;
else failed++;
}
return (success, failed);
}
private async Task DownloadWithThrottleAsync(
string directory,
string fileName,
Settings settings,
System.Threading.SemaphoreSlim semaphore,
IProgress<int> progress)
{
await semaphore.WaitAsync();
try
{
string seriesId = Path.GetFileNameWithoutExtension(fileName);
string url;
if (directory.Contains("series", StringComparison.OrdinalIgnoreCase))
url = settings.Json.ShowApi.Replace("<tvmazeid>", seriesId);
else if (directory.Contains("episodes", StringComparison.OrdinalIgnoreCase))
url = settings.Json.EpisodeApi.Replace("<tvmazeid>", seriesId);
else if (directory.Contains("cast", StringComparison.OrdinalIgnoreCase))
url = settings.Json.CastApi.Replace("<tvmazeid>", seriesId);
else if (directory.Contains("crew", StringComparison.OrdinalIgnoreCase))
url = settings.Json.CrewApi.Replace("<tvmazeid>", seriesId);
else
url = $"https://api.tvmaze.com/shows/{seriesId}";
// Download the file
await Task.Run(() => _downloader.Download(url, directory, fileName, Log.Logger));
// Report progress
progress?.Report(1);
}
catch (Exception ex)
{
Log.Error(ex, "Failed to download {FileName}", fileName);
progress?.Report(1); // still report so progress moves forward
}
finally
{
semaphore.Release();
}
}
}
}
MongoLoader.cs
+187
View File
@@ -0,0 +1,187 @@
using MongoDB.Bson;
using MongoDB.Bson.Serialization;
using MongoDB.Driver;
using Serilog;
using System.Text.Json;
public class MongoLoader
{
private readonly IMongoCollection<BsonDocument> _series;
private readonly IMongoCollection<BsonDocument> _episodes;
private readonly IMongoCollection<BsonDocument> _actors;
private readonly IMongoCollection<BsonDocument> _characters;
private readonly IMongoCollection<BsonDocument> _crew;
public MongoLoader(IMongoDatabase db)
{
_series = db.GetCollection<BsonDocument>("series");
_episodes = db.GetCollection<BsonDocument>("episodes");
_actors = db.GetCollection<BsonDocument>("actors");
_characters = db.GetCollection<BsonDocument>("characters");
_crew = db.GetCollection<BsonDocument>("crew");
}
public async Task<(int Success, int Failed)> LoadJsonToMongoAsync(
Dictionary<string, string> directories,
List<string> updateList,
Dictionary<string, long> updateDict)
{
int success = 0;
int failed = 0;
var options = new ParallelOptions
{
MaxDegreeOfParallelism = Environment.ProcessorCount // or set manually (e.g. 4)
};
await Parallel.ForEachAsync(updateList, options, async (seriesId, cancellationToken) =>
{
try
{
var seriesFile = Path.Combine(directories["SERIESDIR"], $"{seriesId}.json");
if (!File.Exists(seriesFile))
return;
var json = await File.ReadAllTextAsync(seriesFile, cancellationToken);
var doc = BsonDocument.Parse(json);
await UpsertWithTimestamp(_series, doc, updateDict, seriesId);
await ProcessEpisodes(seriesId, directories, updateDict);
await ProcessCast(seriesId, directories, updateDict);
await ProcessCrew(seriesId, directories, updateDict);
Interlocked.Increment(ref success);
Log.Information("Processed series {SeriesId}", seriesId);
}
catch (Exception ex)
{
Interlocked.Increment(ref failed);
Log.Error(ex, "Failed processing series {SeriesId}", seriesId);
}
});
return (success, failed);
}
private async Task UpsertWithTimestamp(
IMongoCollection<BsonDocument> collection,
BsonDocument doc,
Dictionary<string, long> updateDict,
string seriesId)
{
var id = doc.GetValue("id", null);
if (id == null) return;
var filter = Builders<BsonDocument>.Filter.Eq("id", id);
var existing = await collection.Find(filter).FirstOrDefaultAsync();
if (updateDict.TryGetValue(seriesId, out var newTimestamp))
{
if (existing == null ||
!existing.Contains("updated") ||
existing["updated"].AsInt64 < newTimestamp)
{
await collection.ReplaceOneAsync(filter, doc,
new ReplaceOptions { IsUpsert = true });
}
}
}
private async Task ProcessEpisodes(
string seriesId,
Dictionary<string, string> directories,
Dictionary<string, long> updateDict)
{
var file = Path.Combine(directories["EPISODEDIR"], $"{seriesId}.json");
if (!File.Exists(file)) return;
var json = await File.ReadAllTextAsync(file);
var array = JsonSerializer.Deserialize<List<Dictionary<string, object>>>(json);
if (array == null) return;
foreach (var episode in array)
{
episode["seriesid"] = int.Parse(seriesId);
var doc = BsonDocument.Parse(JsonSerializer.Serialize(episode));
await UpsertWithTimestamp(_episodes, doc, updateDict, seriesId);
}
}
private async Task ProcessCast(
string seriesId,
Dictionary<string, string> directories,
Dictionary<string, long> updateDict)
{
var file = Path.Combine(directories["CASTDIR"], $"{seriesId}.json");
if (!File.Exists(file)) return;
var json = await File.ReadAllTextAsync(file);
// ✅ Correct way to parse root JSON array
var array = BsonSerializer.Deserialize<BsonArray>(json);
foreach (var item in array)
{
if (!item.IsBsonDocument)
continue;
var doc = item.AsBsonDocument;
if (doc.Contains("person") && doc["person"].IsBsonDocument)
{
var person = doc["person"].AsBsonDocument;
person["seriesid"] = int.Parse(seriesId);
await UpsertWithTimestamp(_actors, person, updateDict, seriesId);
}
if (doc.Contains("character") && doc["character"].IsBsonDocument)
{
var character = doc["character"].AsBsonDocument;
character["seriesid"] = int.Parse(seriesId);
await UpsertWithTimestamp(_characters, character, updateDict, seriesId);
}
}
}
private async Task ProcessCrew(
string seriesId,
Dictionary<string, string> directories,
Dictionary<string, long> updateDict)
{
var file = Path.Combine(directories["CREWDIR"], $"{seriesId}.json");
if (!File.Exists(file)) return;
var json = await File.ReadAllTextAsync(file);
// ✅ Correct way to parse root JSON array
var array = BsonSerializer.Deserialize<BsonArray>(json);
foreach (var item in array)
{
if (!item.IsBsonDocument)
continue;
var doc = item.AsBsonDocument;
if (doc.Contains("person") && doc["person"].IsBsonDocument)
{
var person = doc["person"].AsBsonDocument;
person["seriesid"] = int.Parse(seriesId);
// Safely get crew type
person["crew_type"] = doc.Contains("type")
? doc["type"]
: "unknown";
await UpsertWithTimestamp(_crew, person, updateDict, seriesId);
}
}
}
}
Settings.cs
+102
View File
@@ -0,0 +1,102 @@
using System;
using System.IO;
using System.Text.Json;
using System.Text.Json.Serialization;
namespace TvSyncApp
{
public class Settings
{
public GlobalSettings? Global { get; set; }
public DbSettings? DbSettings { get; set; }
public ApiSettings? Api { get; set; }
public JsonSettings? Json { get; set; }
public DatabaseSettings? Database { get; set; }
public UpdateTypeSettings? UpdateType { get; set; }
public static Settings Load(string filePath)
{
if (!File.Exists(filePath))
throw new FileNotFoundException($"Settings file not found: {filePath}");
var json = File.ReadAllText(filePath);
var options = new JsonSerializerOptions
{
PropertyNameCaseInsensitive = true,
ReadCommentHandling = JsonCommentHandling.Skip,
AllowTrailingCommas = true
};
return JsonSerializer.Deserialize<Settings>(json, options);
}
}
public class GlobalSettings
{
public string? LogLevel { get; set; }
}
public class DbSettings
{
public string? SqlServerDriver { get; set; }
public string? Hostname { get; set; }
public string? MgHostname { get; set; }
public string? DbName { get; set; }
public string? MgDbName { get; set; }
public string? MysqlUsername { get; set; }
public string? PgsqlUsername { get; set; }
public string? MssqlUsername { get; set; }
public string? MysqlPassword { get; set; }
public string? PgsqlPassword { get; set; }
public string? MssqlPassword { get; set; }
public int PgsqlPort { get; set; }
public int MysqlPort { get; set; }
public int MssqlPort { get; set; }
public int MgPort { get; set; }
}
public class ApiSettings
{
public string? Type { get; set; }
}
public class JsonSettings
{
public string? SeriesApi { get; set; }
public string? ShowApi { get; set; }
public string? EpisodeApi { get; set; }
public string? AliasApi { get; set; }
public string? CastApi { get; set; }
public string? CrewApi { get; set; }
public string? CreditsApi { get; set; }
public string? UpdatesApi { get; set; }
}
public class DatabaseSettings
{
public string? DbType { get; set; }
public string? UpdateSchema { get; set; }
public int Initialize { get; set; }
public int InitLoad { get; set; }
}
public class UpdateTypeSettings
{
public string? Type { get; set; }
public string? Full { get; set; }
public long Monthly { get; set; }
public long Weekly { get; set; }
public long Daily { get; set; }
[JsonPropertyName("12h")]
public long H12 { get; set; }
[JsonPropertyName("6h")]
public long H6 { get; set; }
public long Custom { get; set; }
public int CacheSkip { get; set; }
public int RefreshCache { get; set; }
}
}
TvSyncApp.cs
+105
View File
@@ -0,0 +1,105 @@
using System;
using System.IO;
using System.Collections.Generic;
using System.Threading.Tasks;
using Npgsql;
using Serilog;
namespace TvSyncApp
{
public class Program
{
public static async Task<int> Main(string[] args)
{
string settingsPath = Path.Combine(Directory.GetCurrentDirectory(), "appsettings.json");
var settings = TvSyncApp.Settings.Load(settingsPath)
?? throw new Exception("Failed to load settings from appsettings.json");
Console.WriteLine($"Log level: {settings.Global.LogLevel}");
Console.WriteLine($"Database type: {settings.Database.DbType}");
// Console.WriteLine($"TVMaze series API: {settings.Json.SeriesApi}");
string rootDir = Directory.GetCurrentDirectory();
var directories = new Dictionary<string, string>
{
["SERIESDIR"] = Path.Combine(rootDir, "cache", "series"),
["EPISODEDIR"] = Path.Combine(rootDir, "cache", "episodes"),
["CASTDIR"] = Path.Combine(rootDir, "cache", "cast"),
// ["GUESTCASTDIR"] = Path.Combine(rootDir, "cache", "guestcast"),
["CREWDIR"] = Path.Combine(rootDir, "cache", "crew"),
["UNKNOWN"] = Path.Combine(rootDir, "cache", "unknown")
};
CreateDirectories(rootDir);
Log.Logger = new LoggerConfiguration()
.MinimumLevel.Debug()
.WriteTo.Console()
.WriteTo.File(Path.Combine(rootDir, "log", "sync.log"))
.CreateLogger();
Log.Information("Starting sync process...");
// --- Fetch series IDs from PostgreSQL ---
string connectionString = $"Host={settings.DbSettings.Hostname};Port={settings.DbSettings.PgsqlPort};Username={settings.DbSettings.PgsqlUsername};Password={settings.DbSettings.PgsqlPassword};Database={settings.DbSettings.DbName}";
List<int> seriesIds = GetSeriesIdsFromDb(connectionString);
Log.Information("Fetched {Count} series IDs from tvupdates table.", seriesIds.Count);
// Convert to string list if your Loader expects strings
List<string> updateList = seriesIds.ConvertAll(id => id.ToString());
// Optionally, you can also fetch timestamps if needed
Dictionary<string, long> updateDict = new Dictionary<string, long>();
var loader = new Loader();
var result = await loader.LoadJsonAsync(directories, updateList, updateDict, settings);
Log.Information("Sync complete. Success: {Success}, Failed: {Failed}", result.Success, result.Failed);
Log.CloseAndFlush();
return 0;
}
private static List<int> GetSeriesIdsFromDb(string connectionString)
{
var seriesIds = new List<int>();
try
{
using var conn = new NpgsqlConnection(connectionString);
conn.Open();
string query = "SELECT seriesid FROM updates.tvupdates ORDER BY seriesid";
using var cmd = new NpgsqlCommand(query, conn);
using var reader = cmd.ExecuteReader();
while (reader.Read())
{
if (!reader.IsDBNull(0))
seriesIds.Add(reader.GetInt32(0));
}
}
catch (Exception ex)
{
Log.Error(ex, "Failed to fetch series IDs from PostgreSQL.");
}
return seriesIds;
}
private static void CreateDirectories(string rootDir)
{
Directory.CreateDirectory(Path.Combine(rootDir, "log"));
Directory.CreateDirectory(Path.Combine(rootDir, "temp"));
Directory.CreateDirectory(Path.Combine(rootDir, "cache", "series"));
Directory.CreateDirectory(Path.Combine(rootDir, "cache", "episodes"));
Directory.CreateDirectory(Path.Combine(rootDir, "cache", "cast"));
Directory.CreateDirectory(Path.Combine(rootDir, "cache", "guestcast"));
Directory.CreateDirectory(Path.Combine(rootDir, "cache", "crew"));
}
}
}
appsettings.json
+53
View File
@@ -0,0 +1,53 @@
{
"global": {
"loglevel": "info"
},
"dbsettings": {
"sqlserver_driver": "ODBC Driver 13 for SQL Server",
"hostname": "192.168.129.248",
"mghostname": "192.168.128.24",
"dbname": "media_dbsync",
"mgdbname": "tvdata",
"mysqlusername": "root",
"pgsqlusername": "postgres",
"mssqlusername": "sa",
"mysqlpassword": "Optimus0329",
"pgsqlpassword": "Optimus0329",
"mssqlpassword": "Optimus0329",
"pgsqlport": 5432,
"mysqlport": 3306,
"mssqlport": 1433,
"mgport": 27017
},
"api": {
"type": "tvmaze"
},
"json": {
// "seriesapi": "http://api.tvmaze.com/shows?page=<pagenum>",
"showapi": "http://api.tvmaze.com/shows/<tvmazeid>",
"episodeapi": "http://api.tvmaze.com/shows/<tvmazeid>/episodes",
"aliasapi": "http://api.tvmaze.com/shows/<tvmazeid>/akas",
"castapi": "http://api.tvmaze.com/shows/<tvmazeid>/cast",
"crewapi": "http://api.tvmaze.com/shows/<tvmazeid>/crew",
"creditsapi": "http://api.tvmaze.com/people/<actorid>/castcredits",
"updatesapi": "http://api.tvmaze.com/updates/shows"
},
"database": {
"dbtype": "pgsql",
"updateschema": "updates",
"initialize": 0,
"initload": 0
},
"updatetype": {
"type": "full",
"full": "full",
"monthly": 1219200,
"weekly": 604800,
"daily": 86400,
"12h": 23200,
"6h": 16600,
"custom": 4150,
"cacheskip": 1,
"refreshcache": 0
}
}
dbsync.csproj
+19
View File
@@ -0,0 +1,19 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFramework>net9.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="MongoDB.driver" Version="3.6.0" />
<PackageReference Include="Npgsql" Version="10.0.1" />
<PackageReference Include="Serilog" Version="4.3.1" />
<PackageReference Include="Serilog.Sinks.Console" Version="6.1.1" />
<PackageReference Include="Serilog.Sinks.File" Version="7.0.0" />
<PackageReference Include="System.Text.Json" Version="10.0.3" />
</ItemGroup>
</Project>
dbsync.sln
+25
View File
@@ -0,0 +1,25 @@
Microsoft Visual Studio Solution File, Format Version 12.00
# Visual Studio Version 17
VisualStudioVersion = 17.14.36930.0 d17.14
MinimumVisualStudioVersion = 10.0.40219.1
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "dbsync", "dbsync.csproj", "{06905DC5-E7BA-4CD0-9F8F-FD0E60C9DB71}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
Release|Any CPU = Release|Any CPU
EndGlobalSection
GlobalSection(ProjectConfigurationPlatforms) = postSolution
{06905DC5-E7BA-4CD0-9F8F-FD0E60C9DB71}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{06905DC5-E7BA-4CD0-9F8F-FD0E60C9DB71}.Debug|Any CPU.Build.0 = Debug|Any CPU
{06905DC5-E7BA-4CD0-9F8F-FD0E60C9DB71}.Release|Any CPU.ActiveCfg = Release|Any CPU
{06905DC5-E7BA-4CD0-9F8F-FD0E60C9DB71}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
EndGlobalSection
GlobalSection(ExtensibilityGlobals) = postSolution
SolutionGuid = {5F508F17-C255-49C1-B01D-13B2B739B1FF}
EndGlobalSection
EndGlobal