Skip to content

Commit

Permalink
Some commit with not compiled code.
Browse files Browse the repository at this point in the history
  • Loading branch information
MonkAlex committed Dec 14, 2018
1 parent b16d6a4 commit 843ba7e
Show file tree
Hide file tree
Showing 15 changed files with 167 additions and 60 deletions.
3 changes: 2 additions & 1 deletion MangaReader.Core/Account/Login.cs
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,8 @@ protected internal CookieClient GetClient()
public virtual async Task<bool> Logout()
{
IsLogined = false;
await Page.GetPageAsync(LogoutUri, GetClient());
#warning FAIL
// await Page.GetPageAsync(LogoutUri, GetClient());
return true;
}

Expand Down
5 changes: 3 additions & 2 deletions MangaReader.Core/Manga/Chapter.cs
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ public override string Folder
/// Скачать главу.
/// </summary>
/// <param name="downloadFolder">Папка для файлов.</param>
public override async Task Download(string downloadFolder = null)
public override async Task Download(string downloadFolder, Throttler throttler)
{
await DownloadManager.CheckPause();
var chapterFolder = Path.Combine(downloadFolder, this.Folder);
Expand All @@ -62,7 +62,8 @@ public override async Task Download(string downloadFolder = null)
if (!Directory.Exists(chapterFolder))
Directory.CreateDirectory(chapterFolder);

var pTasks = this.InDownloading.Select(page => page.Download(chapterFolder).LogException(string.Empty, $"Не удалось скачать изображение {page.ImageLink} со страницы {page.Uri}"));
var pTasks = this.InDownloading.Select(page => page.Download(chapterFolder, throttler)
.LogException(string.Empty, $"Не удалось скачать изображение {page.ImageLink} со страницы {page.Uri}"));
await Task.WhenAll(pTasks.ToArray());
}
catch (AggregateException ae)
Expand Down
2 changes: 1 addition & 1 deletion MangaReader.Core/Manga/DownloadableContainerImpl.cs
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ protected virtual string GetNormalizedName(string newName)

public DateTime? DownloadedAt { get; set; }

public abstract Task Download(string folder = null);
public abstract Task Download(string folder, Throttler throttler);

public virtual void ClearHistory()
{
Expand Down
3 changes: 2 additions & 1 deletion MangaReader.Core/Manga/IDownloadable.cs
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
using System;
using System.Collections.Generic;
using System.Threading.Tasks;
using MangaReader.Core.Services;

namespace MangaReader.Core.Manga
{
Expand Down Expand Up @@ -54,7 +55,7 @@ public interface IDownloadable
/// <summary>
/// Скачать все главы.
/// </summary>
Task Download(string folder = null);
Task Download(string folder, Throttler throttler);

/// <summary>
/// Clear history.
Expand Down
2 changes: 2 additions & 0 deletions MangaReader.Core/Manga/IManga.cs
Original file line number Diff line number Diff line change
Expand Up @@ -54,5 +54,7 @@ public interface IManga : IDownloadable, ICompressible, IEntity, INotifyProperty
void UpdateContent();

void RefreshFolder();

System.Threading.Tasks.Task Download();
}
}
7 changes: 4 additions & 3 deletions MangaReader.Core/Manga/MangaPage.cs
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,8 @@ public double Downloaded
/// Скачать страницу.
/// </summary>
/// <param name="chapterFolder">Папка для файлов.</param>
public async Task Download(string chapterFolder)
/// <param name="throttler">Limiter for parallel load.</param>
public async Task Download(string chapterFolder, Throttler throttler)
{
this.IsDownloaded = false;

Expand All @@ -83,7 +84,7 @@ public async Task Download(string chapterFolder)
try
{
await DownloadManager.CheckPause();
using (await ThrottleService.WaitAsync())
using (await throttler.WaitAsync())
{
await DownloadManager.CheckPause();
chapterFolder = DirectoryHelpers.MakeValidPath(chapterFolder);
Expand All @@ -102,7 +103,7 @@ public async Task Download(string chapterFolder)
{
Log.Exception(ex, this.Uri.OriginalString);
++restartCounter;
await Download(chapterFolder);
await Download(chapterFolder, throttler);
}
}

Expand Down
14 changes: 10 additions & 4 deletions MangaReader.Core/Manga/Mangas.cs
Original file line number Diff line number Diff line change
Expand Up @@ -305,7 +305,13 @@ protected void AddToHistory(params IDownloadable[] downloadables)
}
}

public virtual async Task Download(string mangaFolder = null)
public virtual async Task Download()
{
#warning тут какой то троттлер от настроек
await Download(null, null);
}

public virtual async Task Download(string mangaFolder, Throttler throttler)
{
if (!this.NeedUpdate)
return;
Expand Down Expand Up @@ -357,7 +363,7 @@ public virtual async Task Download(string mangaFolder = null)
v =>
{
v.OnlyUpdate = this.Setting.OnlyUpdate;
return v.Download(mangaFolder).ContinueWith(t =>
return v.Download(mangaFolder, throttler).ContinueWith(t =>
{
if (t.Exception != null)
Log.Exception(t.Exception, v.Uri?.ToString());
Expand All @@ -373,7 +379,7 @@ public virtual async Task Download(string mangaFolder = null)
ch =>
{
ch.OnlyUpdate = this.Setting.OnlyUpdate;
return ch.Download(mangaFolder).ContinueWith(t =>
return ch.Download(mangaFolder, throttler).ContinueWith(t =>
{
if (t.Exception != null)
Log.Exception(t.Exception, ch.Uri?.ToString());
Expand All @@ -388,7 +394,7 @@ public virtual async Task Download(string mangaFolder = null)
var pTasks = this.ActivePages.Select(
p =>
{
return p.Download(mangaFolder).ContinueWith(t =>
return p.Download(mangaFolder, throttler).ContinueWith(t =>
{
if (t.Exception != null)
Log.Exception(t.Exception, $"Не удалось скачать изображение {p.ImageLink} со страницы {p.Uri}");
Expand Down
4 changes: 2 additions & 2 deletions MangaReader.Core/Manga/Volume.cs
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ public override string Folder

public virtual bool OnlyUpdate { get; set; }

public override Task Download(string mangaFolder = null)
public override Task Download(string mangaFolder, Throttler throttler)
{
var volumeFolder = Path.Combine(mangaFolder, this.Folder);

Expand All @@ -31,7 +31,7 @@ public override Task Download(string mangaFolder = null)
var tasks = this.InDownloading.Select(c =>
{
c.OnlyUpdate = this.OnlyUpdate;
return c.Download(volumeFolder);
return c.Download(volumeFolder, throttler);
});
return Task.WhenAll(tasks);
}
Expand Down
30 changes: 29 additions & 1 deletion MangaReader.Core/Services/DownloadManager.cs
Original file line number Diff line number Diff line change
@@ -1,10 +1,13 @@
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Net;
using System.Reflection;
using System.Text;
using System.Threading.Tasks;
using MangaReader.Core.Manga;

namespace MangaReader.Core.Services
{
Expand All @@ -20,7 +23,32 @@ public static async Task CheckPause()
while (IsPaused)
{
await Task.Delay(1000);
}
}
}

public static async Task TestDownloadSpeed(IManga manga)
{
manga.UpdateContent();
var pages = manga.Pages.ToList();
var chapters = manga.Chapters.ToList();
chapters.AddRange(manga.Volumes.SelectMany(c => c.Container));
MethodInfo methodInfo = null;
if (chapters.Any())
foreach (var chapter in chapters.Where(c => c.Container == null || !c.Container.Any()))
{
methodInfo = methodInfo ?? chapter.GetType().GetMethod("UpdatePages", BindingFlags.NonPublic | BindingFlags.Instance);
methodInfo.Invoke(chapter, null);
}
pages.AddRange(chapters.SelectMany(c => c.Container));

var sw = new Stopwatch();
sw.Start();
var tasks = pages.Select(c => DownloadImage(c.ImageLink));
await Task.WhenAll(tasks);
sw.Stop();
var size = tasks.Sum(t => t.Result.Body.LongLength);
var log = $"Загружено {pages.Count} картинок, размером {size.HumanizeByteSize()} за {sw.Elapsed.TotalSeconds} секунд, итого {(size / sw.Elapsed.TotalSeconds).HumanizeByteSize()} в секунду";
Log.Info(log);
}

/// <summary>
Expand Down
11 changes: 11 additions & 0 deletions MangaReader.Core/Services/IThrottler.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
using System;
using System.Threading.Tasks;

namespace MangaReader.Core.Services
{
public interface IThrottler : IDisposable
{
IDisposable Wait();
Task<IDisposable> WaitAsync();
}
}
7 changes: 7 additions & 0 deletions MangaReader.Core/Services/MangaSetting.cs
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,13 @@ public class MangaSetting : Entity.Entity
/// </summary>
public Guid FolderNamingStrategy { get; set; }

public ushort TrottleLimit { get; set; }

public virtual IThrottler Throttler
{
get { return Services.Throttler.GetThrottler(this.GetType()); }
}

public override void BeforeSave(ChangeTrackerArgs args)
{
var folderIndex = args.PropertyNames.ToList().IndexOf(nameof(Folder));
Expand Down
14 changes: 8 additions & 6 deletions MangaReader.Core/Services/Page.cs
Original file line number Diff line number Diff line change
Expand Up @@ -19,17 +19,18 @@ public class Page
/// Получить текст страницы.
/// </summary>
/// <param name="url">Ссылка на страницу.</param>
/// <param name="throttler">Limiter.</param>
/// <param name="client">Клиент, если нужен специфичный.</param>
/// <param name="restartCounter">Попыток скачивания.</param>
/// <returns>Исходный код страницы.</returns>
public static Page GetPage(Uri url, CookieClient client = null, int restartCounter = 0)
public static Page GetPage(Uri url, Throttler throttler, CookieClient client = null, int restartCounter = 0)
{
try
{
if (restartCounter > 3)
throw new DownloadAttemptFailed(restartCounter, url);

using (ThrottleService.Wait())
using (throttler.Wait())
{
var webClient = client ?? new CookieClient();
var content = webClient.DownloadString(url);
Expand All @@ -49,7 +50,7 @@ public static Page GetPage(Uri url, CookieClient client = null, int restartCount
if (ex.Status != WebExceptionStatus.Timeout && !DelayOnExpectationFailed(ex) && ex.HResult != -2146893023 && ex.HResult != 2147012721)
return new Page(url);
++restartCounter;
return GetPage(url, client, restartCounter);
return GetPage(url, throttler, client, restartCounter);
}
catch (System.Exception ex)
{
Expand All @@ -75,17 +76,18 @@ public static bool DelayOnExpectationFailed(WebException ex)
/// Получить текст страницы.
/// </summary>
/// <param name="url">Ссылка на страницу.</param>
/// <param name="throttler">Limiter.</param>
/// <param name="client">Клиент, если нужен специфичный.</param>
/// <param name="restartCounter">Попыток скачивания.</param>
/// <returns>Исходный код страницы.</returns>
public static async Task<Page> GetPageAsync(Uri url, CookieClient client = null, int restartCounter = 0)
public static async Task<Page> GetPageAsync(Uri url, Throttler throttler, CookieClient client = null, int restartCounter = 0)
{
try
{
if (restartCounter > 3)
throw new DownloadAttemptFailed(restartCounter, url);

using (await ThrottleService.WaitAsync())
using (await throttler.WaitAsync())
{
var webClient = client ?? new CookieClient();
var task = webClient.DownloadStringTaskAsync(url).ConfigureAwait(false);
Expand All @@ -103,7 +105,7 @@ public static async Task<Page> GetPageAsync(Uri url, CookieClient client = null,
if (ex.Status != WebExceptionStatus.Timeout)
return new Page(url);
++restartCounter;
return await GetPageAsync(url, client, restartCounter).ConfigureAwait(false);
return await GetPageAsync(url, throttler, client, restartCounter).ConfigureAwait(false);
}
catch (System.Exception ex)
{
Expand Down
39 changes: 0 additions & 39 deletions MangaReader.Core/Services/ThrottleService.cs

This file was deleted.

Loading

0 comments on commit 843ba7e

Please sign in to comment.