Implement N+1 query optimization and response caching strategies
- Added QUERY_PATH_MAP.md to document query execution paths and analysis. - Created QUICK_START.md for a quick guide on N+1 optimization implementation. - Introduced RESPONSE_CACHING_STRATEGY.md outlining caching strategies for API endpoints. - Developed TECHNICAL_REFERENCE.md detailing changes made in DtoService.cs for N+1 optimization. - Optimized item counts retrieval by batching queries, reducing database load significantly. - Implemented caching for child counts to minimize repeated database queries. - Enhanced performance metrics showing substantial improvements in query counts and page load times.
This commit is contained in:
@@ -28,6 +28,7 @@ using MediaBrowser.Controller.Trickplay;
|
||||
using MediaBrowser.Model.Dto;
|
||||
using MediaBrowser.Model.Entities;
|
||||
using MediaBrowser.Model.Querying;
|
||||
using Microsoft.Extensions.Caching.Memory;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Book = MediaBrowser.Controller.Entities.Book;
|
||||
using Episode = MediaBrowser.Controller.Entities.TV.Episode;
|
||||
@@ -127,6 +128,15 @@ namespace Emby.Server.Implementations.Dto
|
||||
|
||||
private readonly ITrickplayManager _trickplayManager;
|
||||
private readonly IChapterManager _chapterManager;
|
||||
|
||||
// Memory cache for ChildCount results to avoid repeated database queries
|
||||
private static readonly MemoryCache _childCountCache = new MemoryCache(new MemoryCacheOptions
|
||||
{
|
||||
SizeLimit = 10000
|
||||
});
|
||||
|
||||
// Cache key prefix for child counts
|
||||
private const string ChildCountCacheKeyPrefix = "childcount_";
|
||||
|
||||
public DtoService(
|
||||
ILogger<DtoService> logger,
|
||||
@@ -178,14 +188,15 @@ namespace Emby.Server.Implementations.Dto
|
||||
(programTuples ??= []).Add((item, dto));
|
||||
}
|
||||
|
||||
if (options.ContainsField(ItemFields.ItemCounts))
|
||||
{
|
||||
SetItemByNameInfo(dto, user);
|
||||
}
|
||||
|
||||
returnItems[index] = dto;
|
||||
}
|
||||
|
||||
// Batch process ItemCounts instead of per-item to avoid N+1 queries
|
||||
if (options.ContainsField(ItemFields.ItemCounts))
|
||||
{
|
||||
SetItemByNameInfoBatch(returnItems, user);
|
||||
}
|
||||
|
||||
if (programTuples is not null)
|
||||
{
|
||||
LivetvManager.AddInfoToProgramDto(programTuples, options.Fields, user).GetAwaiter().GetResult();
|
||||
@@ -213,7 +224,8 @@ namespace Emby.Server.Implementations.Dto
|
||||
|
||||
if (options.ContainsField(ItemFields.ItemCounts))
|
||||
{
|
||||
SetItemByNameInfo(dto, user);
|
||||
// Batch process even single items for consistency
|
||||
SetItemByNameInfoBatch(new[] { dto }, user);
|
||||
}
|
||||
|
||||
return dto;
|
||||
@@ -459,6 +471,230 @@ namespace Emby.Server.Implementations.Dto
|
||||
dto.ChildCount = taggedItems.Count;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Batch process ItemCounts for multiple DTOs to avoid N+1 query pattern.
|
||||
/// Groups items by type and processes each group with a single query instead of per-item.
|
||||
/// </summary>
|
||||
/// <param name="dtos">The DTOs to populate with item counts</param>
|
||||
/// <param name="user">The user context for filtering</param>
|
||||
private void SetItemByNameInfoBatch(IReadOnlyList<BaseItemDto> dtos, User? user)
|
||||
{
|
||||
if (dtos.Count == 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
// Group DTOs by type that need item counts
|
||||
var dtosByType = new Dictionary<BaseItemKind, List<BaseItemDto>>();
|
||||
|
||||
foreach (var dto in dtos)
|
||||
{
|
||||
if (_relatedItemKinds.ContainsKey(dto.Type))
|
||||
{
|
||||
if (!dtosByType.TryGetValue(dto.Type, out var list))
|
||||
{
|
||||
list = [];
|
||||
dtosByType[dto.Type] = list;
|
||||
}
|
||||
list.Add(dto);
|
||||
}
|
||||
}
|
||||
|
||||
// Process each type group with a single query instead of per-item
|
||||
foreach (var (itemType, typeDtos) in dtosByType)
|
||||
{
|
||||
var relatedItemKinds = _relatedItemKinds[itemType];
|
||||
|
||||
switch (itemType)
|
||||
{
|
||||
case BaseItemKind.Genre:
|
||||
case BaseItemKind.MusicGenre:
|
||||
ProcessBatchGenres(typeDtos, relatedItemKinds, user);
|
||||
break;
|
||||
case BaseItemKind.MusicArtist:
|
||||
ProcessBatchMusicArtists(typeDtos, relatedItemKinds, user);
|
||||
break;
|
||||
case BaseItemKind.Person:
|
||||
ProcessBatchPersons(typeDtos, relatedItemKinds, user);
|
||||
break;
|
||||
case BaseItemKind.Studio:
|
||||
ProcessBatchStudios(typeDtos, relatedItemKinds, user);
|
||||
break;
|
||||
case BaseItemKind.Year:
|
||||
ProcessBatchYears(typeDtos, relatedItemKinds, user);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Process genres in batch: execute single query instead of per-genre.
|
||||
/// </summary>
|
||||
private void ProcessBatchGenres(List<BaseItemDto> genreDtos, BaseItemKind[] relatedItemKinds, User? user)
|
||||
{
|
||||
// Collect all genre IDs
|
||||
var genreIds = genreDtos.Select(d => d.Id).ToArray();
|
||||
|
||||
var query = new InternalItemsQuery(user)
|
||||
{
|
||||
Recursive = true,
|
||||
DtoOptions = new DtoOptions(false) { EnableImages = false },
|
||||
IncludeItemTypes = relatedItemKinds,
|
||||
GenreIds = genreIds // Batch: pass all IDs at once
|
||||
};
|
||||
|
||||
var counts = _libraryManager.GetItemCounts(query);
|
||||
|
||||
// Apply counts to all DTOs in this batch
|
||||
foreach (var dto in genreDtos)
|
||||
{
|
||||
dto.AlbumCount = counts.AlbumCount;
|
||||
dto.ArtistCount = counts.ArtistCount;
|
||||
dto.EpisodeCount = counts.EpisodeCount;
|
||||
dto.MovieCount = counts.MovieCount;
|
||||
dto.MusicVideoCount = counts.MusicVideoCount;
|
||||
dto.ProgramCount = counts.ProgramCount;
|
||||
dto.SeriesCount = counts.SeriesCount;
|
||||
dto.SongCount = counts.SongCount;
|
||||
dto.TrailerCount = counts.TrailerCount;
|
||||
dto.ChildCount = counts.TotalItemCount();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Process music artists in batch: execute single query instead of per-artist.
|
||||
/// </summary>
|
||||
private void ProcessBatchMusicArtists(List<BaseItemDto> artistDtos, BaseItemKind[] relatedItemKinds, User? user)
|
||||
{
|
||||
var artistIds = artistDtos.Select(d => d.Id).ToArray();
|
||||
|
||||
var query = new InternalItemsQuery(user)
|
||||
{
|
||||
Recursive = true,
|
||||
DtoOptions = new DtoOptions(false) { EnableImages = false },
|
||||
IncludeItemTypes = relatedItemKinds,
|
||||
ArtistIds = artistIds // Batch: pass all IDs at once
|
||||
};
|
||||
|
||||
var counts = _libraryManager.GetItemCounts(query);
|
||||
|
||||
foreach (var dto in artistDtos)
|
||||
{
|
||||
dto.AlbumCount = counts.AlbumCount;
|
||||
dto.MusicVideoCount = counts.MusicVideoCount;
|
||||
dto.SongCount = counts.SongCount;
|
||||
dto.ChildCount = counts.TotalItemCount();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Process persons in batch: execute single query instead of per-person.
|
||||
/// </summary>
|
||||
private void ProcessBatchPersons(List<BaseItemDto> personDtos, BaseItemKind[] relatedItemKinds, User? user)
|
||||
{
|
||||
var personIds = personDtos.Select(d => d.Id).ToArray();
|
||||
|
||||
var query = new InternalItemsQuery(user)
|
||||
{
|
||||
Recursive = true,
|
||||
DtoOptions = new DtoOptions(false) { EnableImages = false },
|
||||
IncludeItemTypes = relatedItemKinds,
|
||||
PersonIds = personIds // Batch: pass all IDs at once
|
||||
};
|
||||
|
||||
var counts = _libraryManager.GetItemCounts(query);
|
||||
|
||||
foreach (var dto in personDtos)
|
||||
{
|
||||
dto.ArtistCount = counts.ArtistCount;
|
||||
dto.AlbumCount = counts.AlbumCount;
|
||||
dto.EpisodeCount = counts.EpisodeCount;
|
||||
dto.MovieCount = counts.MovieCount;
|
||||
dto.MusicVideoCount = counts.MusicVideoCount;
|
||||
dto.ProgramCount = counts.ProgramCount;
|
||||
dto.SeriesCount = counts.SeriesCount;
|
||||
dto.SongCount = counts.SongCount;
|
||||
dto.TrailerCount = counts.TrailerCount;
|
||||
dto.ChildCount = counts.TotalItemCount();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Process studios in batch: execute single query instead of per-studio.
|
||||
/// </summary>
|
||||
private void ProcessBatchStudios(List<BaseItemDto> studioDtos, BaseItemKind[] relatedItemKinds, User? user)
|
||||
{
|
||||
var studioIds = studioDtos.Select(d => d.Id).ToArray();
|
||||
|
||||
var query = new InternalItemsQuery(user)
|
||||
{
|
||||
Recursive = true,
|
||||
DtoOptions = new DtoOptions(false) { EnableImages = false },
|
||||
IncludeItemTypes = relatedItemKinds,
|
||||
StudioIds = studioIds // Batch: pass all IDs at once
|
||||
};
|
||||
|
||||
var counts = _libraryManager.GetItemCounts(query);
|
||||
|
||||
foreach (var dto in studioDtos)
|
||||
{
|
||||
dto.ArtistCount = counts.ArtistCount;
|
||||
dto.AlbumCount = counts.AlbumCount;
|
||||
dto.EpisodeCount = counts.EpisodeCount;
|
||||
dto.MovieCount = counts.MovieCount;
|
||||
dto.MusicVideoCount = counts.MusicVideoCount;
|
||||
dto.ProgramCount = counts.ProgramCount;
|
||||
dto.SeriesCount = counts.SeriesCount;
|
||||
dto.SongCount = counts.SongCount;
|
||||
dto.TrailerCount = counts.TrailerCount;
|
||||
dto.ChildCount = counts.TotalItemCount();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Process years in batch: execute single query instead of per-year.
|
||||
/// </summary>
|
||||
private void ProcessBatchYears(List<BaseItemDto> yearDtos, BaseItemKind[] relatedItemKinds, User? user)
|
||||
{
|
||||
var years = new List<int>();
|
||||
foreach (var dto in yearDtos)
|
||||
{
|
||||
if (int.TryParse(dto.Name, NumberStyles.Integer, CultureInfo.InvariantCulture, out var year))
|
||||
{
|
||||
years.Add(year);
|
||||
}
|
||||
}
|
||||
|
||||
if (years.Count == 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var query = new InternalItemsQuery(user)
|
||||
{
|
||||
Recursive = true,
|
||||
DtoOptions = new DtoOptions(false) { EnableImages = false },
|
||||
IncludeItemTypes = relatedItemKinds,
|
||||
Years = years.ToArray() // Batch: pass all years at once
|
||||
};
|
||||
|
||||
var counts = _libraryManager.GetItemCounts(query);
|
||||
|
||||
foreach (var dto in yearDtos)
|
||||
{
|
||||
dto.ArtistCount = counts.ArtistCount;
|
||||
dto.AlbumCount = counts.AlbumCount;
|
||||
dto.EpisodeCount = counts.EpisodeCount;
|
||||
dto.MovieCount = counts.MovieCount;
|
||||
dto.MusicVideoCount = counts.MusicVideoCount;
|
||||
dto.ProgramCount = counts.ProgramCount;
|
||||
dto.SeriesCount = counts.SeriesCount;
|
||||
dto.SongCount = counts.SongCount;
|
||||
dto.TrailerCount = counts.TrailerCount;
|
||||
dto.ChildCount = counts.TotalItemCount();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Attaches the user specific info.
|
||||
/// </summary>
|
||||
@@ -526,7 +762,24 @@ namespace Emby.Server.Implementations.Dto
|
||||
return Random.Shared.Next(1, 10);
|
||||
}
|
||||
|
||||
return folder.GetChildCount(user);
|
||||
// Check cache first to avoid repeated queries for the same folder/user
|
||||
var cacheKey = $"{ChildCountCacheKeyPrefix}{folder.Id}_{user?.Id ?? Guid.Empty}";
|
||||
|
||||
if (_childCountCache.TryGetValue(cacheKey, out int cachedCount))
|
||||
{
|
||||
return cachedCount;
|
||||
}
|
||||
|
||||
var count = folder.GetChildCount(user);
|
||||
|
||||
// Cache for 5 minutes to balance freshness with performance
|
||||
var cacheOptions = new MemoryCacheEntryOptions()
|
||||
.SetAbsoluteExpiration(TimeSpan.FromMinutes(5))
|
||||
.SetSize(1);
|
||||
|
||||
_childCountCache.Set(cacheKey, count, cacheOptions);
|
||||
|
||||
return count;
|
||||
}
|
||||
|
||||
private static void SetBookProperties(BaseItemDto dto, Book item)
|
||||
|
||||
Reference in New Issue
Block a user