Skip to content

Commit

Permalink
Fix #395 : SftpClient Enumerates Rather Than Accumulates Directory Items
Browse files Browse the repository at this point in the history
  • Loading branch information
@znamenap
znamenap committed Sep 20, 2023
1 parent cefdc20 commit c1b6b9c
Show file tree
Hide file tree
Showing 4 changed files with 347 additions and 18 deletions.
209 changes: 209 additions & 0 deletions src/Renci.SshNet.Tests/Classes/SftpClientTest.EnumerateDirectory.cs
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,209 @@
using Microsoft.VisualStudio.TestTools.UnitTesting;
using Renci.SshNet.Common;
using Renci.SshNet.Tests.Common;
using Renci.SshNet.Tests.Properties;
using System;
using System.Diagnostics;
using System.Linq;

namespace Renci.SshNet.Tests.Classes
{
/// <summary>
/// Implementation of the SSH File Transfer Protocol (SFTP) over SSH.
/// </summary>
public partial class SftpClientTest : TestBase
{
[TestMethod]
[TestCategory("Sftp")]
[ExpectedException(typeof(SshConnectionException))]
public void Test_Sftp_EnumerateDirectory_Without_Connecting()
{
using (var sftp = new SftpClient(Resources.HOST, Resources.USERNAME, Resources.PASSWORD))
{
var files = sftp.EnumerateDirectory(".");
foreach (var file in files)
{
Debug.WriteLine(file.FullName);
}
}
}

[TestMethod]
[TestCategory("Sftp")]
[TestCategory("integration")]
[ExpectedException(typeof(SftpPermissionDeniedException))]
public void Test_Sftp_EnumerateDirectory_Permission_Denied()
{
using (var sftp = new SftpClient(Resources.HOST, Resources.USERNAME, Resources.PASSWORD))
{
sftp.Connect();

var files = sftp.EnumerateDirectory("/root");
foreach (var file in files)
{
Debug.WriteLine(file.FullName);
}

sftp.Disconnect();
}
}

[TestMethod]
[TestCategory("Sftp")]
[TestCategory("integration")]
[ExpectedException(typeof(SftpPathNotFoundException))]
public void Test_Sftp_EnumerateDirectory_Not_Exists()
{
using (var sftp = new SftpClient(Resources.HOST, Resources.USERNAME, Resources.PASSWORD))
{
sftp.Connect();

var files = sftp.EnumerateDirectory("/asdfgh");
foreach (var file in files)
{
Debug.WriteLine(file.FullName);
}

sftp.Disconnect();
}
}

[TestMethod]
[TestCategory("Sftp")]
[TestCategory("integration")]
public void Test_Sftp_EnumerateDirectory_Current()
{
using (var sftp = new SftpClient(Resources.HOST, Resources.USERNAME, Resources.PASSWORD))
{
sftp.Connect();

var files = sftp.EnumerateDirectory(".");

Assert.IsTrue(files.Count() > 0);

foreach (var file in files)
{
Debug.WriteLine(file.FullName);
}

sftp.Disconnect();
}
}

[TestMethod]
[TestCategory("Sftp")]
[TestCategory("integration")]
public void Test_Sftp_EnumerateDirectory_Empty()
{
using (var sftp = new SftpClient(Resources.HOST, Resources.USERNAME, Resources.PASSWORD))
{
sftp.Connect();

var files = sftp.EnumerateDirectory(string.Empty);

Assert.IsTrue(files.Count() > 0);

foreach (var file in files)
{
Debug.WriteLine(file.FullName);
}

sftp.Disconnect();
}
}

[TestMethod]
[TestCategory("Sftp")]
[TestCategory("integration")]
[Description("Test passing null to EnumerateDirectory.")]
[ExpectedException(typeof(ArgumentNullException))]
public void Test_Sftp_EnumerateDirectory_Null()
{
using (var sftp = new SftpClient(Resources.HOST, Resources.USERNAME, Resources.PASSWORD))
{
sftp.Connect();

var files = sftp.EnumerateDirectory(null);

Assert.IsTrue(files.Count() > 0);

foreach (var file in files)
{
Debug.WriteLine(file.FullName);
}

sftp.Disconnect();
}
}

[TestMethod]
[TestCategory("Sftp")]
[TestCategory("integration")]
public void Test_Sftp_EnumerateDirectory_HugeDirectory()
{
var stopwatch = Stopwatch.StartNew();
try
{
using (var sftp = new SftpClient(Resources.HOST, Resources.USERNAME, Resources.PASSWORD))
{
sftp.Connect();
sftp.ChangeDirectory("/home/" + Resources.USERNAME);

var count = 10000;
// Create 10000 directory items
for (int i = 0; i < count; i++)
{
sftp.CreateDirectory(string.Format("test_{0}", i));
}
Debug.WriteLine(string.Format("Created {0} directories within {1} seconds", count, stopwatch.Elapsed.TotalSeconds));

stopwatch.Reset(); stopwatch.Start();
var files = sftp.EnumerateDirectory(".");
Debug.WriteLine(string.Format("Listed {0} directories within {1} seconds", count, stopwatch.Elapsed.TotalSeconds));

// Ensure that directory has at least 10000 items
stopwatch.Reset(); stopwatch.Start();
var actualCount = files.Count();
Assert.IsTrue(actualCount >= count);
Debug.WriteLine(string.Format("Used {0} items within {1} seconds", actualCount, stopwatch.Elapsed.TotalSeconds));

sftp.Disconnect();
}
}
finally
{
stopwatch.Reset(); stopwatch.Start();
RemoveAllFiles();
stopwatch.Stop();
Debug.WriteLine(string.Format("Removed all files within {0} seconds", stopwatch.Elapsed.TotalSeconds));
}
}

[TestMethod]
[TestCategory("Sftp")]
[TestCategory("integration")]
[ExpectedException(typeof(SshConnectionException))]
public void Test_Sftp_EnumerateDirectory_After_Disconnected()
{
try {
using (var sftp = new SftpClient(Resources.HOST, Resources.USERNAME, Resources.PASSWORD))
{
sftp.Connect();

sftp.CreateDirectory("test_at_dsiposed");

var files = sftp.EnumerateDirectory(".").Take(1);

sftp.Disconnect();

// Must fail on disconnected session.
var count = files.Count();
}
}
finally
{
RemoveAllFiles();
}
}
}
}
48 changes: 32 additions & 16 deletions src/Renci.SshNet.Tests/Classes/SftpClientTest.ListDirectory.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -140,26 +140,42 @@ public void Test_Sftp_ListDirectory_Null()
[TestCategory("integration")]
public void Test_Sftp_ListDirectory_HugeDirectory()
{
using (var sftp = new SftpClient(Resources.HOST, Resources.USERNAME, Resources.PASSWORD))
var stopwatch = Stopwatch.StartNew();
try
{
sftp.Connect();

// Create 10000 directory items
for (int i = 0; i < 10000; i++)
using (var sftp = new SftpClient(Resources.HOST, Resources.USERNAME, Resources.PASSWORD))
{
sftp.CreateDirectory(string.Format("test_{0}", i));
Debug.WriteLine("Created " + i);
sftp.Connect();
sftp.ChangeDirectory("/home/" + Resources.USERNAME);

var count = 10000;
// Create 10000 directory items
for (int i = 0; i < count; i++)
{
sftp.CreateDirectory(string.Format("test_{0}", i));
}
Debug.WriteLine(string.Format("Created {0} directories within {1} seconds", count, stopwatch.Elapsed.TotalSeconds));

stopwatch.Reset(); stopwatch.Start();
var files = sftp.ListDirectory(".");
Debug.WriteLine(string.Format("Listed {0} directories within {1} seconds", count, stopwatch.Elapsed.TotalSeconds));

// Ensure that directory has at least 10000 items
stopwatch.Reset(); stopwatch.Start();
var actualCount = files.Count();
Assert.IsTrue(actualCount >= count);
Debug.WriteLine(string.Format("Used {0} items within {1} seconds", actualCount, stopwatch.Elapsed.TotalSeconds));

sftp.Disconnect();
}

var files = sftp.ListDirectory(".");

// Ensure that directory has at least 10000 items
Assert.IsTrue(files.Count() > 10000);

sftp.Disconnect();
}

RemoveAllFiles();
finally
{
stopwatch.Reset(); stopwatch.Start();
RemoveAllFiles();
stopwatch.Stop();
Debug.WriteLine(string.Format("Removed all files within {0} seconds", stopwatch.Elapsed.TotalSeconds));
}
}

[TestMethod]
Expand Down
22 changes: 22 additions & 0 deletions src/Renci.SshNet/ISftpClient.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -668,6 +668,28 @@ public interface ISftpClient
/// <exception cref="ObjectDisposedException">The method was called after the client was disposed.</exception>
IEnumerable<SftpFile> ListDirectory(string path, Action<int> listCallback = null);

/// <summary>
/// Enumerates files and directories in remote directory.
/// </summary>
/// <remarks>
/// This method differs to <see cref="ListDirectory(string, Action{int})"/> in the way how the items are returned.
/// It yields the items to the last moment for the enumerator to decide if it needs to continue or stop enumerating the items.
/// It is handy in case of really huge directory contents at remote server - meaning really huge 65 thousand files and more.
/// It also decrease the memory footprint and avoids LOH allocation as happen per call to <see cref="ListDirectory(string, Action{int})"/> method.
/// There aren't asynchronous counterpart methods to this because enumerating should happen in your specific asynchronous block.
/// </remarks>
/// <param name="path">The path.</param>
/// <param name="listCallback">The list callback.</param>
/// <returns>
/// An <see cref="System.Collections.Generic.IEnumerable{SftpFile}"/> of files and directories ready to be enumerated.
/// </returns>
/// <exception cref="ArgumentNullException"><paramref name="path" /> is <b>null</b>.</exception>
/// <exception cref="SshConnectionException">Client is not connected.</exception>
/// <exception cref="SftpPermissionDeniedException">Permission to list the contents of the directory was denied by the remote host. <para>-or-</para> A SSH command was denied by the server.</exception>
/// <exception cref="SshException">A SSH error where <see cref="Exception.Message" /> is the message from the remote host.</exception>
/// <exception cref="ObjectDisposedException">The method was called after the client was disposed.</exception>
IEnumerable<SftpFile> EnumerateDirectory(string path, Action<int> listCallback = null);

/// <summary>
/// Opens a <see cref="SftpFileStream"/> on the specified path with read/write access.
/// </summary>
Expand Down
Loading

0 comments on commit c1b6b9c

Please sign in to comment.