Skip to content

Commit

Permalink
feat: schedule anime updates for files with missing data during import
Browse files Browse the repository at this point in the history
  • Loading branch information
@revam
revam committed Feb 5, 2025
1 parent de84656 commit 977424d
Show file tree
Hide file tree
Showing 3 changed files with 116 additions and 95 deletions.
96 changes: 4 additions & 92 deletions Shoko.Server/API/v3/Controllers/ActionController.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -345,99 +345,11 @@ public async Task<ActionResult> AVDumpMismatchedFiles()
/// <returns></returns>
[Authorize("admin")]
[HttpGet("DownloadMissingAniDBAnimeData")]
public async Task<ActionResult> UpdateMissingAnidbXml()
{
// Check existing anime.
var index = 0;
var queuedAnimeSet = new HashSet<int>();
var localAnimeSet = RepoFactory.AniDB_Anime.GetAll()
.Select(a => a.AnimeID)
.OrderBy(a => a)
.ToHashSet();
_logger.LogInformation("Checking {AllAnimeCount} anime for missing XML files…", localAnimeSet.Count);
foreach (var animeID in localAnimeSet)
{
if (++index % 10 == 1)
_logger.LogInformation("Checking {AllAnimeCount} anime for missing XML files — {CurrentCount}/{AllAnimeCount}", localAnimeSet.Count, index + 1, localAnimeSet.Count);

var xmlUtils = HttpContext.RequestServices.GetRequiredService<HttpXmlUtils>();
var rawXml = await xmlUtils.LoadAnimeHTTPFromFile(animeID);

if (rawXml != null)
continue;

await _seriesService.QueueAniDBRefresh(animeID, true, false, false, skipTmdbUpdate: true);
queuedAnimeSet.Add(animeID);
}

// Attempt to fix cross-references with incomplete data.
index = 0;
var videos = RepoFactory.VideoLocal.GetVideosWithMissingCrossReferenceData();
var unknownEpisodeDict = videos
.SelectMany(file => file.EpisodeCrossReferences)
.Where(xref => xref.AnimeID is 0)
.GroupBy(xref => xref.EpisodeID)
.ToDictionary(groupBy => groupBy.Key, groupBy => groupBy.ToList());
_logger.LogInformation("Attempting to fix {MissingAnimeCount} cross-references with unknown anime…", unknownEpisodeDict.Count);
foreach (var (episodeId, xrefs) in unknownEpisodeDict)
{
if (++index % 10 == 1)
_logger.LogInformation("Attempting to fix {MissingAnimeCount} cross-references with unknown anime — {CurrentCount}/{MissingAnimeCount}", unknownEpisodeDict.Count, index + 1, unknownEpisodeDict.Count);

var episode = RepoFactory.AniDB_Episode.GetByEpisodeID(episodeId);
if (episode is not null)
{
foreach (var xref in xrefs)
{
xref.AnimeID = episode.AnimeID;
}
RepoFactory.CrossRef_File_Episode.Save(xrefs);
continue;
}
int? epAnimeID = null;
var epRequest = _requestFactory.Create<RequestGetEpisode>(r => r.EpisodeID = episodeId);
try
{
var epResponse = epRequest.Send();
epAnimeID = epResponse.Response?.AnimeID;
}
catch (Exception e)
{
_logger.LogError(e, "Could not get Episode Info for {EpisodeID}", episode.EpisodeID);
}

if (epAnimeID is not null)
{
foreach (var xref in xrefs)
{
xref.AnimeID = epAnimeID.Value;
}
RepoFactory.CrossRef_File_Episode.Save(xrefs);
}
}

// Queue missing anime needed by existing files.
index = 0;
var localEpisodeSet = RepoFactory.AniDB_Episode.GetAll()
.Select(episode => episode.EpisodeID)
.ToHashSet();
var missingAnimeSet = videos
.SelectMany(file => file.EpisodeCrossReferences)
.Where(xref => xref.AnimeID > 0 && !queuedAnimeSet.Contains(xref.AnimeID) && (!localAnimeSet.Contains(xref.AnimeID) || !localEpisodeSet.Contains(xref.EpisodeID)))
.Select(xref => xref.AnimeID)
.ToHashSet();
var settings = SettingsProvider.GetSettings();
_logger.LogInformation("Queueing {MissingAnimeCount} anime that needs an update…", missingAnimeSet.Count);
foreach (var animeID in missingAnimeSet)
{
if (++index % 10 == 1)
_logger.LogInformation("Queueing {MissingAnimeCount} anime that needs an update — {CurrentCount}/{MissingAnimeCount}", missingAnimeSet.Count, index + 1, missingAnimeSet.Count);

await _seriesService.QueueAniDBRefresh(animeID, false, settings.AniDb.DownloadRelatedAnime, true);
queuedAnimeSet.Add(animeID);
}
public ActionResult UpdateMissingAnidbXml()
{
Task.Run(_actionService.DownloadMissingAnidbAnimeXmls);
Task.Run(_actionService.ScheduleMissingAnidbAnimeForFiles);

_logger.LogInformation("Queued {QueuedAnimeCount} anime for an online refresh", queuedAnimeSet.Count);
return Ok();
}

Expand Down
2 changes: 2 additions & 0 deletions Shoko.Server/Scheduling/Jobs/Actions/ImportJob.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,8 @@ public override async Task Process()

// Check for previously ignored files
_service.CheckForPreviouslyIgnored();

await _service.ScheduleMissingAnidbAnimeForFiles();
}

public ImportJob(ActionService service)
Expand Down
113 changes: 110 additions & 3 deletions Shoko.Server/Services/ActionService.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,9 @@
using Shoko.Server.Databases;
using Shoko.Server.Extensions;
using Shoko.Server.Models;
using Shoko.Server.Providers.AniDB;
using Shoko.Server.Providers.AniDB.Interfaces;
using Shoko.Server.Providers.AniDB.UDP.Info;
using Shoko.Server.Providers.TMDB;
using Shoko.Server.Providers.TraktTV;
using Shoko.Server.Repositories;
Expand All @@ -35,32 +38,41 @@ public class ActionService
{
private readonly ILogger<ActionService> _logger;
private readonly ISchedulerFactory _schedulerFactory;
private readonly IRequestFactory _requestFactory;
private readonly ISettingsProvider _settingsProvider;
private readonly VideoLocal_PlaceService _placeService;
private readonly TmdbMetadataService _tmdbService;
private readonly AnimeSeriesService _seriesService;
private readonly TraktTVHelper _traktHelper;
private readonly ImportFolderRepository _importFolders;
private readonly DatabaseFactory _databaseFactory;
private readonly HttpXmlUtils _xmlUtils;

public ActionService(
ILogger<ActionService> logger,
ISchedulerFactory schedulerFactory,
IRequestFactory requestFactory,
ISettingsProvider settingsProvider,
VideoLocal_PlaceService placeService,
TraktTVHelper traktHelper,
TmdbMetadataService tmdbService,
AnimeSeriesService seriesService,
TraktTVHelper traktHelper,
ImportFolderRepository importFolders,
DatabaseFactory databaseFactory
DatabaseFactory databaseFactory,
HttpXmlUtils xmlUtils
)
{
_logger = logger;
_schedulerFactory = schedulerFactory;
_requestFactory = requestFactory;
_settingsProvider = settingsProvider;
_placeService = placeService;
_traktHelper = traktHelper;
_tmdbService = tmdbService;
_seriesService = seriesService;
_traktHelper = traktHelper;
_importFolders = importFolders;
_databaseFactory = databaseFactory;
_xmlUtils = xmlUtils;
}

public async Task RunImport_IntegrityCheck()
Expand Down Expand Up @@ -1087,6 +1099,101 @@ public void CheckForPreviouslyIgnored()
}
}

public async Task DownloadMissingAnidbAnimeXmls()
{
// Check existing anime.
var index = 0;
var queuedAnimeSet = new HashSet<int>();
var localAnimeSet = RepoFactory.AniDB_Anime.GetAll()
.Select(a => a.AnimeID)
.OrderBy(a => a)
.ToHashSet();
_logger.LogInformation("Checking {AllAnimeCount} anime for missing XML files…", localAnimeSet.Count);
foreach (var animeID in localAnimeSet)
{
if (++index % 10 == 1 || index == localAnimeSet.Count)
_logger.LogInformation("Checking {AllAnimeCount} anime for missing XML files — {CurrentCount}/{AllAnimeCount}", localAnimeSet.Count, index + 1, localAnimeSet.Count);

var rawXml = await _xmlUtils.LoadAnimeHTTPFromFile(animeID);
if (rawXml != null)
continue;

_logger.LogDebug("Found anime {AnimeID} with missing XML", animeID);
await _seriesService.QueueAniDBRefresh(animeID, true, false, false, skipTmdbUpdate: true);
queuedAnimeSet.Add(animeID);
}
}

public async Task ScheduleMissingAnidbAnimeForFiles()
{
// Attempt to fix cross-references with incomplete data.
var index = 0;
var videos = RepoFactory.VideoLocal.GetVideosWithMissingCrossReferenceData();
var unknownEpisodeDict = videos
.SelectMany(file => file.EpisodeCrossReferences)
.Where(xref => xref.AnimeID is 0)
.GroupBy(xref => xref.EpisodeID)
.ToDictionary(groupBy => groupBy.Key, groupBy => groupBy.ToList());
_logger.LogInformation("Attempting to fix {MissingAnimeCount} cross-references with unknown anime…", unknownEpisodeDict.Count);
foreach (var (episodeId, xrefs) in unknownEpisodeDict)
{
if (++index % 10 == 1)
_logger.LogInformation("Attempting to fix {MissingAnimeCount} cross-references with unknown anime — {CurrentCount}/{MissingAnimeCount}", unknownEpisodeDict.Count, index + 1, unknownEpisodeDict.Count);

var episode = RepoFactory.AniDB_Episode.GetByEpisodeID(episodeId);
if (episode is not null)
{
foreach (var xref in xrefs)
xref.AnimeID = episode.AnimeID;
RepoFactory.CrossRef_File_Episode.Save(xrefs);
continue;
}

int? epAnimeID = null;
var epRequest = _requestFactory.Create<RequestGetEpisode>(r => r.EpisodeID = episodeId);
try
{
var epResponse = epRequest.Send();
epAnimeID = epResponse.Response?.AnimeID;
}
catch (Exception e)
{
_logger.LogError(e, "Could not get Episode Info for {EpisodeID}", episode.EpisodeID);
}

if (epAnimeID is not null)
{
foreach (var xref in xrefs)
xref.AnimeID = epAnimeID.Value;
RepoFactory.CrossRef_File_Episode.Save(xrefs);
}
}

// Queue missing anime needed by existing files.
index = 0;
var localAnimeSet = RepoFactory.AniDB_Anime.GetAll()
.Select(a => a.AnimeID)
.OrderBy(a => a)
.ToHashSet();
var localEpisodeSet = RepoFactory.AniDB_Episode.GetAll()
.Select(episode => episode.EpisodeID)
.ToHashSet();
var missingAnimeSet = videos
.SelectMany(file => file.EpisodeCrossReferences)
.Where(xref => xref.AnimeID > 0 && (!localAnimeSet.Contains(xref.AnimeID) || !localEpisodeSet.Contains(xref.EpisodeID)))
.Select(xref => xref.AnimeID)
.ToHashSet();
var settings = _settingsProvider.GetSettings();
_logger.LogInformation("Queueing {MissingAnimeCount} anime that needs an update…", missingAnimeSet.Count);
foreach (var animeID in missingAnimeSet)
{
if (++index % 10 == 1 || index == missingAnimeSet.Count)
_logger.LogInformation("Queueing {MissingAnimeCount} anime that needs an update — {CurrentCount}/{MissingAnimeCount}", missingAnimeSet.Count, index + 1, missingAnimeSet.Count);

await _seriesService.QueueAniDBRefresh(animeID, false, settings.AniDb.DownloadRelatedAnime, true);
}
}

public async Task ScheduleMissingAnidbCreators()
{
if (!_settingsProvider.GetSettings().AniDb.DownloadCreators) return;
Expand Down

0 comments on commit 977424d

Please sign in to comment.