Skip to content

Commit

Permalink
Functional test that verifies the order of packages in the feed (#3398)
Browse files Browse the repository at this point in the history 
* PackagesAppearInFeedInOrderTest

* remove isListed check from CheckIfPackageVersionExistsInSource

* remove FindPackagesById url comment

* fix to small code error

* pr fixes

* increase number of tries to give test more leeway

* improve runtime of test by bypassing search service

* remove unused references

* fix name of PackagesInOrderRefreshTimeMs

* removed unnecessary checking, tidied the code
  • Loading branch information
Scott Bommarito authored Dec 8, 2016
1 parent 190aab5 commit b31a5dd
Show file tree
Hide file tree
Showing 4 changed files with 226 additions and 28 deletions.
65 changes: 56 additions & 9 deletions tests/NuGetGallery.FunctionalTests.Core/Helpers/ClientSdkHelper.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,7 @@ public bool CheckIfPackageVersionExistsInSource(string packageId, string version

WriteLine("[verification attempt {0}]: Checking if package {1} with version {2} exists in source {3}... ", i, packageId, version, sourceUrl);
IPackage package = repo.FindPackage(packageId, semVersion);
found = (package != null);
found = package != null;
if (found)
{
WriteLine("Found!");
Expand All @@ -110,10 +110,19 @@ public bool CheckIfPackageVersionExistsInSource(string packageId, string version
}

/// <summary>
/// Creates a package with the specified Id and Version and uploads it and checks if the upload has suceeded.
/// This will be used by test classes which tests scenarios on top of upload.
/// Creates a package with the specified Id and Version and uploads it and checks if the upload has succeeded.
/// Throws if the upload fails or cannot be verified in the source.
/// </summary>
public async Task UploadNewPackageAndVerify(string packageId, string version = "1.0.0", string minClientVersion = null, string title = null, string tags = null, string description = null, string licenseUrl = null, string dependencies = null)
{
await UploadNewPackage(packageId, version, minClientVersion, title, tags, description, licenseUrl, dependencies);

VerifyPackageExistsInSource(packageId, version);
}

public async Task UploadNewPackage(string packageId, string version = "1.0.0", string minClientVersion = null,
string title = null, string tags = null, string description = null, string licenseUrl = null,
string dependencies = null)
{
if (string.IsNullOrEmpty(packageId))
{
Expand All @@ -128,20 +137,58 @@ public async Task UploadNewPackageAndVerify(string packageId, string version = "
var commandlineHelper = new CommandlineHelper(TestOutputHelper);
var processResult = await commandlineHelper.UploadPackageAsync(packageFullPath, UrlHelper.V2FeedPushSourceUrl);

Assert.True(processResult.ExitCode == 0, "The package upload via Nuget.exe did not succeed properly. Check the logs to see the process error and output stream. Exit Code: " + processResult.ExitCode + ". Error message: \"" + processResult.StandardError + "\"");
Assert.True(processResult.ExitCode == 0,
"The package upload via Nuget.exe did not succeed properly. Check the logs to see the process error and output stream. Exit Code: " +
processResult.ExitCode + ". Error message: \"" + processResult.StandardError + "\"");

var packageExistsInSource = CheckIfPackageVersionExistsInSource(packageId, version, UrlHelper.V2FeedRootUrl);
var userMessage = string.Format("Package {0} with version {1} is not found in the site {2} after uploading.", packageId, version, UrlHelper.V2FeedRootUrl);
Assert.True(packageExistsInSource, userMessage);

// Delete package from local disk so once it gets uploaded
// Delete package from local disk once it gets uploaded
if (File.Exists(packageFullPath))
{
File.Delete(packageFullPath);
Directory.Delete(Path.GetFullPath(Path.GetDirectoryName(packageFullPath)), true);
}
}

/// <summary>
/// Unlists a package with the specified Id and Version and checks if the unlist has succeeded.
/// Throws if the unlist fails or cannot be verified in the source.
/// </summary>
public async Task UnlistPackageAndVerify(string packageId, string version = "1.0.0")
{
await UnlistPackage(packageId, version);

VerifyPackageExistsInSource(packageId, version);
}

public async Task UnlistPackage(string packageId, string version = "1.0.0")
{
if (string.IsNullOrEmpty(packageId))
{
throw new ArgumentException($"{nameof(packageId)} cannot be null or empty!");
}

WriteLine("Unlisting package '{0}', version '{1}'", packageId, version);

var commandlineHelper = new CommandlineHelper(TestOutputHelper);
var processResult = await commandlineHelper.DeletePackageAsync(packageId, version, UrlHelper.V2FeedPushSourceUrl);

Assert.True(processResult.ExitCode == 0,
"The package unlist via Nuget.exe did not succeed properly. Check the logs to see the process error and output stream. Exit Code: " +
processResult.ExitCode + ". Error message: \"" + processResult.StandardError + "\"");
}

/// <summary>
/// Throws if the specified package cannot be found in the source.
/// </summary>
/// <param name="packageId">Id of the package.</param>
/// <param name="version">Version of the package.</param>
public void VerifyPackageExistsInSource(string packageId, string version = "1.0.0")
{
var packageExistsInSource = CheckIfPackageVersionExistsInSource(packageId, version, UrlHelper.V2FeedRootUrl);
Assert.True(packageExistsInSource,
$"Package {packageId} with version {version} is not found on the site {UrlHelper.V2FeedRootUrl}.");
}

/// <summary>
/// Returns the latest stable version string for the given package.
/// </summary>
Expand Down
14 changes: 14 additions & 0 deletions tests/NuGetGallery.FunctionalTests.Core/Helpers/CommandlineHelper.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ public class CommandlineHelper
internal static string PackCommandString = " pack ";
internal static string UpdateCommandString = " update ";
internal static string InstallCommandString = " install ";
internal static string DeleteCommandString = " delete ";
internal static string PushCommandString = " push ";
internal static string OutputDirectorySwitchString = " -OutputDirectory ";
internal static string PreReleaseSwitchString = " -Prerelease ";
Expand Down Expand Up @@ -51,6 +52,19 @@ public async Task<ProcessResult> UploadPackageAsync(string packageFullPath, stri
return await InvokeNugetProcess(arguments);
}

/// <summary>
/// Delete the specified package using Nuget.exe
/// </summary>
/// <param name="packageId">package to be deleted</param>
/// <param name="version">version of package to be deleted</param>
/// <param name="sourceName">source url</param>
/// <returns></returns>
public async Task<ProcessResult> DeletePackageAsync(string packageId, string version, string sourceName)
{
var arguments = string.Join(string.Empty, DeleteCommandString, packageId, " ", version, SourceSwitchString, sourceName, ApiKeySwitchString, EnvironmentSettings.TestAccountApiKey);
return await InvokeNugetProcess(arguments);
}

/// <summary>
/// Install the specified package using Nuget.exe
/// </summary>
Expand Down
83 changes: 67 additions & 16 deletions tests/NuGetGallery.FunctionalTests.Core/Helpers/ODataHelper.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,17 +59,61 @@ public async Task<string> TryDownloadPackageFromFeed(string packageId, string ve
}
}

public async Task<bool> ContainsResponseText(string url, params string[] expectedTexts)
public async Task<DateTime?> GetTimestampOfPackageFromResponse(string url, string propertyName, string packageId, string version = "1.0.0")
{
var request = WebRequest.Create(url);
var response = await request.GetResponseAsync().ConfigureAwait(false);
WriteLine($"Getting '{propertyName}' timestamp of package '{packageId}' with version '{version}'.");

string responseText;
using (var sr = new StreamReader(response.GetResponseStream()))
var packageResponse = await GetPackageDataInResponse(url, packageId, version);
if (string.IsNullOrEmpty(packageResponse))
{
return null;
}

var timestampStartTag = "<d:" + propertyName + " m:type=\"Edm.DateTime\">";
var timestampEndTag = "</d:" + propertyName + ">";

var timestampTagIndex = packageResponse.IndexOf(timestampStartTag);
if (timestampTagIndex < 0)
{
responseText = await sr.ReadToEndAsync().ConfigureAwait(false);
WriteLine($"Package data does not contain '{propertyName}' timestamp!");
return null;
}

var timestampStartIndex = timestampTagIndex + timestampStartTag.Length;
var timestampLength = packageResponse.Substring(timestampStartIndex).IndexOf(timestampEndTag);

var timestamp =
DateTime.Parse(packageResponse.Substring(timestampStartIndex, timestampLength));
WriteLine($"'{propertyName}' timestamp of package '{packageId}' with version '{version}' is '{timestamp}'");
return timestamp;
}

public async Task<string> GetPackageDataInResponse(string url, string packageId, string version = "1.0.0")
{
WriteLine($"Getting data for package '{packageId}' with version '{version}'.");

var responseText = await GetResponseText(url);

var packageString = @"<id>" + UrlHelper.V2FeedRootUrl + @"Packages(Id='" + packageId + @"',Version='" + (string.IsNullOrEmpty(version) ? "" : version + "')</id>");
var endEntryTag = "</entry>";

var startingIndex = responseText.IndexOf(packageString);

if (startingIndex < 0)
{
WriteLine("Package not found in response text!");
return null;
}

var endingIndex = responseText.IndexOf(endEntryTag, startingIndex);

return responseText.Substring(startingIndex, endingIndex - startingIndex);
}

public async Task<bool> ContainsResponseText(string url, params string[] expectedTexts)
{
var responseText = await GetResponseText(url);

foreach (string s in expectedTexts)
{
if (!responseText.Contains(s))
Expand All @@ -83,14 +127,7 @@ public async Task<bool> ContainsResponseText(string url, params string[] expecte

public async Task<bool> ContainsResponseTextIgnoreCase(string url, params string[] expectedTexts)
{
var request = WebRequest.Create(url);
var response = await request.GetResponseAsync();

string responseText;
using (var sr = new StreamReader(response.GetResponseStream()))
{
responseText = (await sr.ReadToEndAsync()).ToLowerInvariant();
}
var responseText = (await GetResponseText(url)).ToLowerInvariant();

foreach (string s in expectedTexts)
{
Expand All @@ -103,15 +140,29 @@ public async Task<bool> ContainsResponseTextIgnoreCase(string url, params string
return true;
}

private async Task<string> GetResponseText(string url)
{
var request = WebRequest.Create(url);
var response = await request.GetResponseAsync();

string responseText;
using (var sr = new StreamReader(response.GetResponseStream()))
{
responseText = await sr.ReadToEndAsync();
}

return responseText;
}

public async Task DownloadPackageFromV2FeedWithOperation(string packageId, string version, string operation)
{
string filename = await DownloadPackageFromFeed(packageId, version, operation);

//check if the file exists.
// Check if the file exists.
Assert.True(File.Exists(filename), Constants.PackageDownloadFailureMessage);
var clientSdkHelper = new ClientSdkHelper(TestOutputHelper);
string downloadedPackageId = clientSdkHelper.GetPackageIdFromNupkgFile(filename);
//Check that the downloaded Nupkg file is not corrupt and it indeed corresponds to the package which we were trying to download.
// Check that the downloaded Nupkg file is not corrupt and it indeed corresponds to the package which we were trying to download.
Assert.True(downloadedPackageId.Equals(packageId), Constants.UnableToZipError);
}

Expand Down
92 changes: 89 additions & 3 deletions tests/NuGetGallery.FunctionalTests/ODataFeeds/V2FeedExtendedTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,11 @@
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.

using System;
using System.Collections.Generic;
using System.ComponentModel;
using System.IO;
using System.Net;
using System.Threading;
using System.Threading.Tasks;
using Xunit;
using Xunit.Abstractions;
Expand All @@ -28,21 +30,27 @@ public V2FeedExtendedTests(ITestOutputHelper testOutputHelper)
_packageCreationHelper = new PackageCreationHelper(TestOutputHelper);
}

private bool CanUploadToSite()
{
// Temporary workaround for the SSL issue, which keeps the upload test from working with cloudapp.net sites
return UrlHelper.BaseUrl.Contains("nugettest.org") || UrlHelper.BaseUrl.Contains("nuget.org") ||
UrlHelper.BaseUrl.Contains("nuget.localtest.me");
}

[Fact]
[Description("Upload two packages and then issue the FindPackagesById request, expect to return both versions")]
[Priority(1)]
[Category("P0Tests")]
public async Task FindPackagesByIdTest()
{
// Temporary workaround for the SSL issue, which keeps the upload test from working with cloudapp.net sites
if (UrlHelper.BaseUrl.Contains("nugettest.org") || UrlHelper.BaseUrl.Contains("nuget.org"))
if (CanUploadToSite())
{
string packageId = string.Format("TestV2FeedFindPackagesById.{0}", DateTime.UtcNow.Ticks);

TestOutputHelper.WriteLine("Uploading package '{0}'", packageId);
await _clientSdkHelper.UploadNewPackageAndVerify(packageId);
TestOutputHelper.WriteLine("Uploaded package '{0}'", packageId);

TestOutputHelper.WriteLine("Uploaded package '{0}'", packageId);
await _clientSdkHelper.UploadNewPackageAndVerify(packageId, "2.0.0");

string url = UrlHelper.V2FeedRootUrl + @"/FindPackagesById()?id='" + packageId + "'";
Expand All @@ -56,6 +64,84 @@ public async Task FindPackagesByIdTest()
}
}

private const int PackagesInOrderNumPackages = 10;

[Fact]
[Description("Upload multiple packages and then unlist them and verify that they appear in the feed in the correct order")]
[Priority(1)]
[Category("P0Tests")]
public async Task PackagesAppearInFeedInOrderTest()
{
// This test uploads/unlists packages in a particular order to test the timestamps of the packages in the feed.
// Because it waits for previous requests to finish before starting new ones, it will only catch ordering issues if these issues are greater than a second or two.
// This is consistent with the time frame in which we've seen these issues in the past, but if new issues arise that are on a smaller scale, this test will not catch it!

if (CanUploadToSite())
{
var packageIds = new List<string>(PackagesInOrderNumPackages);
var startingTime = DateTime.UtcNow;

// Upload the packages in order.
var uploadStartTimestamp = DateTime.UtcNow.AddMinutes(-1);
for (var i = 0; i < PackagesInOrderNumPackages; i++)
{
var packageId = GetPackagesAppearInFeedInOrderPackageId(startingTime, i);
await _clientSdkHelper.UploadNewPackage(packageId);
packageIds.Add(packageId);
}

await CheckPackageTimestampsInOrder(packageIds, "Created", uploadStartTimestamp);

// Unlist the packages in order.
var unlistStartTimestamp = DateTime.UtcNow.AddMinutes(-1);
for (var i = 0; i < PackagesInOrderNumPackages; i++)
{
await _clientSdkHelper.UnlistPackage(packageIds[i]);
}

await CheckPackageTimestampsInOrder(packageIds, "LastEdited", unlistStartTimestamp);
}
}

private static string GetPackagesAppearInFeedInOrderPackageId(DateTime startingTime, int i)
{
return $"TestV2FeedPackagesAppearInFeedInOrderTest.{startingTime.Ticks}.{i}";
}

private static string GetPackagesAppearInFeedInOrderUrl(DateTime time, string timestamp)
{
return $"{UrlHelper.V2FeedRootUrl}/Packages?$filter={timestamp} gt DateTime'{time:o}'&$orderby={timestamp} desc&$select={timestamp}";
}

/// <summary>
/// Verifies if a set of packages in the feed have timestamps in a particular order.
/// </summary>
/// <param name="packageIds">An ordered list of package ids. Each package id in the list must have a timestamp in the feed earlier than all package ids after it.</param>
/// <param name="timestampPropertyName">The timestamp property to test the ordering of. For example, "Created" or "LastEdited".</param>
/// <param name="operationStartTimestamp">A timestamp that is before all of the timestamps expected to be found in the feed. This is used in a request to the feed.</param>
private async Task CheckPackageTimestampsInOrder(List<string> packageIds, string timestampPropertyName,
DateTime operationStartTimestamp)
{
var lastTimestamp = DateTime.MinValue;
for (var i = 0; i < PackagesInOrderNumPackages; i++)
{
var packageId = packageIds[i];
TestOutputHelper.WriteLine($"Attempting to check order of package #{i} {timestampPropertyName} timestamp in feed.");

var newTimestamp =
await
_odataHelper.GetTimestampOfPackageFromResponse(
GetPackagesAppearInFeedInOrderUrl(operationStartTimestamp, timestampPropertyName),
timestampPropertyName,
packageId);

Assert.True(newTimestamp.HasValue);
Assert.True(newTimestamp.Value > lastTimestamp,
$"Package #{i} was last modified after package #{i - 1} but has an earlier {timestampPropertyName} timestamp ({newTimestamp} should be greater than {lastTimestamp}).");
lastTimestamp = newTimestamp.Value;
}
}

/// <summary>
/// Regression test for #1199, also covers #1052
/// </summary>
Expand Down

0 comments on commit b31a5dd

Please sign in to comment.