SftpClient Enumerates Rather Than Accumulates Directory Items (#395)

src/Renci.SshNet.IntegrationTests/OldIntegrationTests/SftpClientTest.EnumerateDirectory.cs

@@ -0,0 +1,209 @@
+using System.Diagnostics;
+
+using Renci.SshNet.Common;
+
+namespace Renci.SshNet.IntegrationTests.OldIntegrationTests
+{
+    /// <summary>
+    /// Implementation of the SSH File Transfer Protocol (SFTP) over SSH.
+    /// </summary>
+    public partial class SftpClientTest : IntegrationTestBase
+    {
+        [TestMethod]
+        [TestCategory("Sftp")]
+        [ExpectedException(typeof(SshConnectionException))]
+        public void Test_Sftp_EnumerateDirectory_Without_Connecting()
+        {
+            using (var sftp = new SftpClient(SshServerHostName, SshServerPort, User.UserName, User.Password))
+            {
+                var files = sftp.EnumerateDirectory(".");
+                foreach (var file in files)
+                {
+                    Debug.WriteLine(file.FullName);
+                }
+            }
+        }
+
+        [TestMethod]
+        [TestCategory("Sftp")]
+        [TestCategory("integration")]
+        [ExpectedException(typeof(SftpPermissionDeniedException))]
+        public void Test_Sftp_EnumerateDirectory_Permission_Denied()
+        {
+            using (var sftp = new SftpClient(SshServerHostName, SshServerPort, User.UserName, User.Password))
+            {
+                sftp.Connect();
+
+                var files = sftp.EnumerateDirectory("/root");
+                foreach (var file in files)
+                {
+                    Debug.WriteLine(file.FullName);
+                }
+
+                sftp.Disconnect();
+            }
+        }
+
+        [TestMethod]
+        [TestCategory("Sftp")]
+        [TestCategory("integration")]
+        [ExpectedException(typeof(SftpPathNotFoundException))]
+        public void Test_Sftp_EnumerateDirectory_Not_Exists()
+        {
+            using (var sftp = new SftpClient(SshServerHostName, SshServerPort, User.UserName, User.Password))
+            {
+                sftp.Connect();
+
+                var files = sftp.EnumerateDirectory("/asdfgh");
+                foreach (var file in files)
+                {
+                    Debug.WriteLine(file.FullName);
+                }
+
+                sftp.Disconnect();
+            }
+        }
+
+        [TestMethod]
+        [TestCategory("Sftp")]
+        [TestCategory("integration")]
+        public void Test_Sftp_EnumerateDirectory_Current()
+        {
+            using (var sftp = new SftpClient(SshServerHostName, SshServerPort, User.UserName, User.Password))
+            {
+                sftp.Connect();
+
+                var files = sftp.EnumerateDirectory(".");
+
+                Assert.IsTrue(files.Count() > 0);
+
+                foreach (var file in files)
+                {
+                    Debug.WriteLine(file.FullName);
+                }
+
+                sftp.Disconnect();
+            }
+        }
+
+        [TestMethod]
+        [TestCategory("Sftp")]
+        [TestCategory("integration")]
+        public void Test_Sftp_EnumerateDirectory_Empty()
+        {
+            using (var sftp = new SftpClient(SshServerHostName, SshServerPort, User.UserName, User.Password))
+            {
+                sftp.Connect();
+
+                var files = sftp.EnumerateDirectory(string.Empty);
+
+                Assert.IsTrue(files.Count() > 0);
+
+                foreach (var file in files)
+                {
+                    Debug.WriteLine(file.FullName);
+                }
+
+                sftp.Disconnect();
+            }
+        }
+
+        [TestMethod]
+        [TestCategory("Sftp")]
+        [TestCategory("integration")]
+        [Description("Test passing null to EnumerateDirectory.")]
+        [ExpectedException(typeof(ArgumentNullException))]
+        public void Test_Sftp_EnumerateDirectory_Null()
+        {
+            using (var sftp = new SftpClient(SshServerHostName, SshServerPort, User.UserName, User.Password))
+            {
+                sftp.Connect();
+
+                var files = sftp.EnumerateDirectory(null);
+
+                Assert.IsTrue(files.Count() > 0);
+
+                foreach (var file in files)
+                {
+                    Debug.WriteLine(file.FullName);
+                }
+
+                sftp.Disconnect();
+            }
+        }
+
+        [TestMethod]
+        [TestCategory("Sftp")]
+        [TestCategory("integration")]
+        public void Test_Sftp_EnumerateDirectory_HugeDirectory()
+        {
+            var stopwatch = Stopwatch.StartNew();
+            try
+            {
+                using (var sftp = new SftpClient(SshServerHostName, SshServerPort, User.UserName, User.Password))
+                {
+                    sftp.Connect();
+                    sftp.ChangeDirectory("/home/" + User.UserName);
+
+                    var count = 10000;
+                    //  Create 10000 directory items
+                    for (int i = 0; i < count; i++)
+                    {
+                        sftp.CreateDirectory(string.Format("test_{0}", i));
+                    }
+                    Debug.WriteLine(string.Format("Created {0} directories within {1} seconds", count, stopwatch.Elapsed.TotalSeconds));
+
+                    stopwatch.Reset();
+                    stopwatch.Start();
+                    var files = sftp.EnumerateDirectory(".");
+                    Debug.WriteLine(string.Format("Listed {0} directories within {1} seconds", count, stopwatch.Elapsed.TotalSeconds));
+
+                    //  Ensure that directory has at least 10000 items
+                    stopwatch.Reset();
+                    stopwatch.Start();
+                    var actualCount = files.Count();
+                    Assert.IsTrue(actualCount >= count);
+                    Debug.WriteLine(string.Format("Used {0} items within {1} seconds", actualCount, stopwatch.Elapsed.TotalSeconds));
+
+                    sftp.Disconnect();
+                }
+            }
+            finally
+            {
+                stopwatch.Reset();
+                stopwatch.Start();
+                RemoveAllFiles();
+                stopwatch.Stop();
+                Debug.WriteLine(string.Format("Removed all files within {0} seconds", stopwatch.Elapsed.TotalSeconds));
+            }
+        }
+
+        [TestMethod]
+        [TestCategory("Sftp")]
+        [TestCategory("integration")]
+        [ExpectedException(typeof(SshConnectionException))]
+        public void Test_Sftp_EnumerateDirectory_After_Disconnected()
+        {
+            try
+            {
+                using (var sftp = new SftpClient(SshServerHostName, SshServerPort, User.UserName, User.Password))
+                {
+                    sftp.Connect();
+
+                    sftp.CreateDirectory("test_at_dsiposed");
+
+                    var files = sftp.EnumerateDirectory(".").Take(1);
+
+                    sftp.Disconnect();
+
+                    // Must fail on disconnected session.
+                    var count = files.Count();
+                }
+            }
+            finally
+            {
+                RemoveAllFiles();
+            }
+        }
+    }
+}

src/Renci.SshNet/ISftpClient.cs

@@ -717,6 +717,28 @@ public interface ISftpClient : IBaseClient, IDisposable
         IAsyncEnumerable<ISftpFile> ListDirectoryAsync(string path, CancellationToken cancellationToken);
 #endif //FEATURE_ASYNC_ENUMERABLE
 
+        /// <summary>
+        /// Enumerates files and directories in remote directory.
+        /// </summary>
+        /// <remarks>
+        /// This method differs to <see cref="ListDirectory(string, Action{int})"/> in the way how the items are returned. 
+        /// It yields the items to the last moment for the enumerator to decide if it needs to continue or stop enumerating the items. 
+        /// It is handy in case of really huge directory contents at remote server - meaning really huge 65 thousand files and more.
+        /// It also decrease the memory footprint and avoids LOH allocation as happen per call to <see cref="ListDirectory(string, Action{int})"/> method.
+        /// There aren't asynchronous counterpart methods to this because enumerating should happen in your specific asynchronous block.
+        /// </remarks>
+        /// <param name="path">The path.</param>
+        /// <param name="listCallback">The list callback.</param>
+        /// <returns>
+        /// An <see cref="System.Collections.Generic.IEnumerable{SftpFile}"/> of files and directories ready to be enumerated.
+        /// </returns>
+        /// <exception cref="ArgumentNullException"><paramref name="path" /> is <b>null</b>.</exception>
+        /// <exception cref="SshConnectionException">Client is not connected.</exception>
+        /// <exception cref="SftpPermissionDeniedException">Permission to list the contents of the directory was denied by the remote host. <para>-or-</para> A SSH command was denied by the server.</exception>
+        /// <exception cref="SshException">A SSH error where <see cref="Exception.Message" /> is the message from the remote host.</exception>
+        /// <exception cref="ObjectDisposedException">The method was called after the client was disposed.</exception>
+        IEnumerable<SftpFile> EnumerateDirectory(string path, Action<int> listCallback = null);
+
         /// <summary>
         /// Opens a <see cref="SftpFileStream"/> on the specified path with read/write access.
         /// </summary>

src/Renci.SshNet/SftpClient.cs

@@ -1,15 +1,16 @@
 using System;
 using System.Collections.Generic;
 using System.Diagnostics.CodeAnalysis;
-using System.IO;
 using System.Globalization;
+using System.IO;
 using System.Net;
 using System.Text;
 using System.Threading;
+using System.Threading.Tasks;
+
 using Renci.SshNet.Abstractions;
 using Renci.SshNet.Common;
 using Renci.SshNet.Sftp;
-using System.Threading.Tasks;
 #if FEATURE_ASYNC_ENUMERABLE
 using System.Runtime.CompilerServices;
 #endif
@@ -706,6 +707,33 @@ public IEnumerable<ISftpFile> EndListDirectory(IAsyncResult asyncResult)
             return ar.EndInvoke();
         }
 
+        /// <summary>
+        /// Enumerates files and directories in remote directory.
+        /// </summary>
+        /// <remarks>
+        /// This method differs to <see cref="ListDirectory(string, Action{int})"/> in the way how the items are returned. 
+        /// It yields the items to the last moment for the enumerator to decide if it needs to continue or stop enumerating the items. 
+        /// It is handy in case of really huge directory contents at remote server - meaning really huge 65 thousand files and more.
+        /// It also decrease the memory footprint and avoids LOH allocation as happen per call to <see cref="ListDirectory(string, Action{int})"/> method.
+        /// There aren't asynchronous counterpart methods to this because enumerating should happen in your specific asynchronous block.
+        /// </remarks>
+        /// <param name="path">The path.</param>
+        /// <param name="listCallback">The list callback.</param>
+        /// <returns>
+        /// An <see cref="System.Collections.Generic.IEnumerable{SftpFile}"/> of files and directories ready to be enumerated.
+        /// </returns>
+        /// <exception cref="ArgumentNullException"><paramref name="path" /> is <b>null</b>.</exception>
+        /// <exception cref="SshConnectionException">Client is not connected.</exception>
+        /// <exception cref="SftpPermissionDeniedException">Permission to list the contents of the directory was denied by the remote host. <para>-or-</para> A SSH command was denied by the server.</exception>
+        /// <exception cref="SshException">A SSH error where <see cref="Exception.Message" /> is the message from the remote host.</exception>
+        /// <exception cref="ObjectDisposedException">The method was called after the client was disposed.</exception>
+        public IEnumerable<SftpFile> EnumerateDirectory(string path, Action<int> listCallback = null)
+        {
+            CheckDisposed();
+
+            return InternalEnumerateDirectory(path, listCallback);
+        }
+
         /// <summary>
         /// Gets reference to remote file or directory.
         /// </summary>
@@ -1613,7 +1641,7 @@ public Task<SftpFileStream> OpenAsync(string path, FileMode mode, FileAccess acc
 
             cancellationToken.ThrowIfCancellationRequested();
 
-            return SftpFileStream.OpenAsync(_sftpSession, path, mode, access, (int)_bufferSize, cancellationToken);
+            return SftpFileStream.OpenAsync(_sftpSession, path, mode, access, (int) _bufferSize, cancellationToken);
         }
 
         /// <summary>
@@ -2269,7 +2297,7 @@ private IEnumerable<FileInfo> InternalSynchronizeDirectories(string sourcePath,
         /// <param name="path">The path.</param>
         /// <param name="listCallback">The list callback.</param>
         /// <returns>
-        /// A list of files in the specfied directory.
+        /// A list of files in the specified directory.
         /// </returns>
         /// <exception cref="ArgumentNullException"><paramref name="path" /> is <b>null</b>.</exception>
         /// <exception cref="SshConnectionException">Client not connected.</exception>
@@ -2324,6 +2352,67 @@ private IEnumerable<ISftpFile> InternalListDirectory(string path, Action<int> li
             return result;
         }
 
+        /// <summary>
+        /// Internals the list directory.
+        /// </summary>
+        /// <param name="path">The path.</param>
+        /// <param name="listCallback">The list callback.</param>
+        /// <returns>
+        /// A list of files in the specified directory.
+        /// </returns>
+        /// <exception cref="ArgumentNullException"><paramref name="path" /> is <b>null</b>.</exception>
+        /// <exception cref="SshConnectionException">Client not connected.</exception>
+        private IEnumerable<SftpFile> InternalEnumerateDirectory(string path, Action<int> listCallback)
+        {
+            if (path == null)
+            {
+                throw new ArgumentNullException("path");
+            }
+
+            if (_sftpSession == null)
+            {
+                throw new SshConnectionException("Client not connected.");
+            }
+
+            var fullPath = _sftpSession.GetCanonicalPath(path);
+
+            var handle = _sftpSession.RequestOpenDir(fullPath);
+
+            var basePath = fullPath;
+
+            if (!basePath.EndsWith("/"))
+            {
+                basePath = string.Format("{0}/", fullPath);
+            }
+
+            try
+            {
+                var count = 0;
+                var files = _sftpSession.RequestReadDir(handle);
+
+                while (files != null)
+                {
+                    count += files.Length;
+                    //  Call callback to report number of files read
+                    if (listCallback != null)
+                    {
+                        //  Execute callback on different thread
+                        ThreadAbstraction.ExecuteThread(() => listCallback(count));
+                    }
+                    foreach (var file in files)
+                    {
+                        var fullName = string.Format(CultureInfo.InvariantCulture, "{0}{1}", basePath, file.Key);
+                        yield return new SftpFile(_sftpSession, fullName, file.Value);
+                    }
+                    files = _sftpSession.RequestReadDir(handle);
+                }
+            }
+            finally
+            {
+                _sftpSession.RequestClose(handle);
+            }
+        }
+
         /// <summary>
         /// Internals the download file.
         /// </summary>