CI/CD Adding automatic pipeline tests

pipeline/azure-pipelines-emulator.yml

@@ -0,0 +1,45 @@
+trigger: none
+pr:
+  branches:
+    include:
+    - master
+    - releases/*
+
+pool:
+  vmImage: 'windows-latest'
+
+steps:
+- pwsh:  |
+        Write-Host "Downloading Cosmos Emulator - $env:EMULATORMSIURL" -ForegroundColor green 
+        Invoke-WebRequest "$env:EMULATORMSIURL" -OutFile "$env:temp\azure-cosmosdb-emulator.msi"
+        Write-Host "Finished Downloading Cosmos Emulator - $env:temp\azure-cosmosdb-emulator.msi" -ForegroundColor green 
+        dir "$env:temp" 
+        choco install lessmsi
+        choco upgrade lessmsi
+        mkdir "$env:temp\Azure Cosmos DB Emulator"
+        lessmsi x "$env:temp\azure-cosmosdb-emulator.msi" "$env:temp\Azure Cosmos DB Emulator\"
+        Write-Host "Starting Cosmos DB Emulator" -ForegroundColor green 
+        Start-Process "$env:temp\Azure Cosmos DB Emulator\SourceDir\Azure Cosmos DB Emulator\CosmosDB.Emulator.exe" "/NoExplorer /NoUI /DisableRateLimiting /PartitionCount=100 /Consistency=Strong /enableRio /overrides=sqlAllowGroupByClause:true" -Verb RunAs
+        Import-Module "$env:temp\Azure Cosmos DB Emulator\SourceDir\Azure Cosmos DB Emulator\PSModules\Microsoft.Azure.CosmosDB.Emulator"
+        Get-Item env:* | Sort-Object -Property Name
+        for ($i=0; $i -lt 10; $i++) {
+          $status=Get-CosmosDbEmulatorStatus 
+          if ($status -ne "Running") {
+            sleep 30;
+            Write-Host "Cosmos DB Emulator Status: $status" -ForegroundColor yellow
+          } else {
+            break;
+          }
+        }
+- task: DotNetCoreCLI@2
+  displayName: Build
+  inputs:
+    command: build
+    projects: '**/*.csproj'
+- script: dotnet test ./src/DocumentDB.ChangeFeedProcessor.IntegrationTests/DocumentDB.ChangeFeedProcessor.IntegrationTests.csproj --logger trx
+  displayName: 'Running tests'
+- task: PublishTestResults@2
+  condition: succeededOrFailed()
+  inputs:
+    testRunner: VSTest
+    testResultsFiles: '**/*.trx'

pipeline/azure-pipelines.yml

@@ -0,0 +1,29 @@
+trigger: none
+pr:
+  branches:
+    include:
+    - master
+    - releases/*
+
+strategy:
+  matrix:
+    windows:
+      imageName: 'windows-latest'
+
+pool:
+  vmImage: $(imageName)
+
+steps:
+- task: DotNetCoreCLI@2
+  displayName: Build
+  inputs:
+    command: build
+    projects: '**/*.csproj'
+    arguments: '--configuration Release'
+- script: dotnet test ./src/DocumentDB.ChangeFeedProcessor.UnitTests/DocumentDB.ChangeFeedProcessor.UnitTests.csproj --logger trx
+  displayName: 'Running tests'
+- task: PublishTestResults@2
+  condition: succeededOrFailed()
+  inputs:
+    testRunner: VSTest
+    testResultsFiles: '**/*.trx'

src/DocumentDB.ChangeFeedProcessor.IntegrationTests/EstimatorTests.cs

@@ -25,8 +25,6 @@ public EstimatorTests(IntegrationTestFixture fixture) : base(fixture, typeof(Est
         [Fact]
         public async Task CountPendingDocuments()
         {
-            // Cleanup the test collection to avoid other tests' documents causing issues with StartFromBeginning
-            await this.ResetTestCollection();
             int documentCount = 1;
             int partitionCount = await IntegrationTestsHelper.GetPartitionCount(this.ClassData.monitoredCollectionInfo);
             int openedCount = 0, processedCount = 0;
@@ -132,8 +130,6 @@ await IntegrationTestsHelper.CreateDocumentsAsync(
         [Fact]
         public async Task WhenNoLeasesExistReturn1()
         {
-            // Cleanup the test collection to avoid other tests' documents causing issues with StartFromBeginning
-            await this.ResetTestCollection();
             var hostName = Guid.NewGuid().ToString();
 
             var host = new ChangeFeedEventHost(
@@ -155,8 +151,6 @@ public async Task WhenNoLeasesExistReturn1()
         [Fact]
         public async Task WhenLeasesHaveContinuationTokenNullReturn0()
         {
-            // Cleanup the test collection to avoid other tests' documents causing issues with StartFromBeginning
-            await this.ResetTestCollection();
             int documentCount = 1;
             int partitionCount = await IntegrationTestsHelper.GetPartitionCount(this.ClassData.monitoredCollectionInfo);
             int openedCount = 0, processedCount = 0;
@@ -205,8 +199,6 @@ public async Task WhenLeasesHaveContinuationTokenNullReturn0()
         [Fact]
         public async Task WhenLeasesHaveContinuationTokenNullStartFromBeginning()
         {
-            // Cleanup the test collection to avoid other tests' documents causing issues with StartFromBeginning
-            await this.ResetTestCollection();
             int documentCount = 1;
             int partitionCount = await IntegrationTestsHelper.GetPartitionCount(this.ClassData.monitoredCollectionInfo);
             int openedCount = 0, processedCount = 0;

src/DocumentDB.ChangeFeedProcessor.IntegrationTests/IntegrationTest.cs

@@ -5,7 +5,6 @@
 using System;
 using System.Collections.Concurrent;
 using System.Collections.Generic;
-using System.Collections.ObjectModel;
 using System.Diagnostics;
 using System.Reflection;
 using System.Threading;
@@ -24,7 +23,6 @@ public class TestClassData
         internal readonly int testCount;
         internal readonly bool isPartitionedMonitoredCollection;
         internal readonly bool isPartitionedLeaseCollection;
-        internal volatile int executedTestCount;
         internal DocumentCollectionInfo monitoredCollectionInfo;
         internal DocumentCollectionInfo leaseCollectionInfoTemplate;
 
@@ -93,18 +91,13 @@ public class IntegrationTestCollection : ICollectionFixture<IntegrationTestFixtu
     /// </remarks>
     [Trait("Category", "Integration")]
     [Collection("Integration tests")]
-    public class IntegrationTest: IDisposable
+    public class IntegrationTest: IAsyncLifetime
     {
         private const string leaseCollectionInfoPropertyName = "leaseCollectionInfo";
         protected static int monitoredOfferThroughput;
         protected static int leaseOfferThroughput;
         protected static readonly TimeSpan changeWaitTimeout = TimeSpan.FromSeconds(30);
 
-        /// <summary>
-        /// This dictionary has one entry per derived class.
-        /// </summary>
-        private static object testClassesSyncRoot = new object();
-
         IntegrationTestFixture fixture;
 
         protected DocumentCollectionInfo LeaseCollectionInfo
@@ -138,28 +131,23 @@ public IntegrationTest(
                     isPartitionedCollection,
                     isPartitionedLeaseCollection);
             }
-
-            TestInitializeAsync().Wait();
         }
 
-        public async Task TestInitializeAsync()
+        public async Task InitializeAsync()
         {
-            if (this.ClassData.monitoredCollectionInfo == null)
+            try
             {
-                try
-                {
-                    if (this.ClassData.monitoredCollectionInfo == null)
-                    {
-                        this.ClassData.leaseCollectionInfoTemplate = await TestClassInitializeAsync(this, $"data_{this.GetType().Name}");
-                    }
-                }
-                catch(Exception ex)
-                {
-                    Debug.Write(ex);
-                    throw;
-                }
+                await IntegrationTest.CreateMonitoredCollection(this, $"data_{this.GetType().Name}");
+            }
+            catch(Exception ex)
+            {
+                Debug.WriteLine(ex);
+                throw;
             }
 
+            Debug.Assert(this.ClassData.monitoredCollectionInfo != null, "Monitored collection information missing.");
+            Debug.Assert(this.ClassData.leaseCollectionInfoTemplate != null, "Lease collection information missing.");
+
             this.LeaseCollectionInfo = new DocumentCollectionInfo(this.ClassData.leaseCollectionInfoTemplate);
             this.LeaseCollectionInfo.CollectionName = $"leases_{this.GetType().Name}_{Guid.NewGuid().ToString()}";
 
@@ -179,50 +167,24 @@ public async Task TestInitializeAsync()
             }
         }
 
-        public void Dispose()
-        {
-            TestCleanupAsync().Wait();
-        }
-
-        public async Task TestCleanupAsync()
+        public async Task DisposeAsync()
         {
             Debug.Assert(this.LeaseCollectionInfo != null);
             using (var client = new DocumentClient(this.LeaseCollectionInfo.Uri, this.LeaseCollectionInfo.MasterKey, this.LeaseCollectionInfo.ConnectionPolicy))
             {
                 await client.DeleteDocumentCollectionAsync(UriFactory.CreateDocumentCollectionUri(this.LeaseCollectionInfo.DatabaseName, this.LeaseCollectionInfo.CollectionName));
-            }
-
-            var executedTestCount = Interlocked.Increment(ref this.ClassData.executedTestCount);
-            if (this.ClassData.executedTestCount == this.ClassData.testCount)
-            {
-                await TestClassCleanupAsync(this);
+                await client.DeleteDocumentCollectionAsync(UriFactory.CreateDocumentCollectionUri(this.ClassData.monitoredCollectionInfo.DatabaseName, this.ClassData.monitoredCollectionInfo.CollectionName));
             }
         }
 
-        /// <summary>
-        /// Recreates the test collection
-        /// </summary>
-        /// <returns></returns>
-        public async Task ResetTestCollection()
-        {
-            await IntegrationTest.TestClassCleanupAsync(this);
-            await IntegrationTest.TestClassInitializeAsync(this, $"data_{this.GetType().Name}");
-        }
-
-        protected virtual Task FinishTestClassInitializeAsync()
-        {
-            return Task.CompletedTask;
-        }
-
-        private static async Task<DocumentCollectionInfo> TestClassInitializeAsync(IntegrationTest test, string monitoredCollectionName)
+        private static async Task CreateMonitoredCollection(IntegrationTest test, string monitoredCollectionName)
         {
             Debug.Assert(test != null);
             Debug.Assert(monitoredCollectionName != null);
 
-            DocumentCollectionInfo leaseCollectionInfo;
             IntegrationTestsHelper.GetConfigurationSettings(
                 out test.ClassData.monitoredCollectionInfo,
-                out leaseCollectionInfo,
+                out test.ClassData.leaseCollectionInfoTemplate,
                 out monitoredOfferThroughput,
                 out leaseOfferThroughput);
 
@@ -249,21 +211,6 @@ private static async Task<DocumentCollectionInfo> TestClassInitializeAsync(Integ
             {
                 await IntegrationTestsHelper.CreateDocumentCollectionAsync(client, test.ClassData.monitoredCollectionInfo.DatabaseName, monitoredCollection, monitoredOfferThroughput);
             }
-
-            test.FinishTestClassInitializeAsync().Wait();
-
-            return leaseCollectionInfo;
-        }
-
-        private static async Task TestClassCleanupAsync(IntegrationTest test)
-        {
-            Debug.Assert(test != null);
-
-            using (var client = new DocumentClient(test.ClassData.monitoredCollectionInfo.Uri, test.ClassData.monitoredCollectionInfo.MasterKey, test.ClassData.monitoredCollectionInfo.ConnectionPolicy))
-            {
-                await client.DeleteDocumentCollectionAsync(UriFactory.CreateDocumentCollectionUri(
-                    test.ClassData.monitoredCollectionInfo.DatabaseName, test.ClassData.monitoredCollectionInfo.CollectionName));
-            }
         }
 
         private static int GetTestCount(Type testType)

src/DocumentDB.ChangeFeedProcessor.IntegrationTests/StaticCollectionTests.cs

@@ -8,6 +8,7 @@
 using System.Threading.Tasks;
 using Microsoft.Azure.Documents.ChangeFeedProcessor.IntegrationTests.Utils;
 using Microsoft.Azure.Documents.Client;
+using Microsoft.Azure.Documents.Linq;
 using Xunit;
 
 namespace Microsoft.Azure.Documents.ChangeFeedProcessor.IntegrationTests
@@ -37,6 +38,7 @@ public StaticCollectionTests(
         [Fact]
         public async Task CountDocumentsInCollection_NormalCase()
         {
+            await this.InitializeDocumentsAsync();
             int partitionKeyRangeCount = await IntegrationTestsHelper.GetPartitionCount(this.ClassData.monitoredCollectionInfo);
             int openedCount = 0, closedCount = 0, processedCount = 0;
             var allDocsProcessed = new ManualResetEvent(false);
@@ -67,7 +69,7 @@ public async Task CountDocumentsInCollection_NormalCase()
             try
             {
                 Assert.True(partitionKeyRangeCount == openedCount, "Wrong openedCount");
-                Assert.True(documentCount == processedCount, "Wrong processedCount");
+                Assert.True(documentCount == processedCount, $"Wrong processedCount {documentCount} {processedCount}");
             }
             finally
             {
@@ -80,6 +82,7 @@ public async Task CountDocumentsInCollection_NormalCase()
         [Fact]
         public async Task CountDocumentsInCollection_ProcessChangesThrows()
         {
+            await this.InitializeDocumentsAsync();
             int processedCount = 0;
             var allDocsProcessed = new ManualResetEvent(false);
             bool isFirstChangeNotification = false; // Make sure there was at least one throw.
@@ -127,6 +130,7 @@ public async Task CountDocumentsInCollection_ProcessChangesThrows()
         [Fact]
         public async Task CountDocumentsInCollection_TwoHosts()
         {
+            await this.InitializeDocumentsAsync();
             int partitionKeyRangeCount = await IntegrationTestsHelper.GetPartitionCount(this.ClassData.monitoredCollectionInfo);
             Assert.True(partitionKeyRangeCount > 1, "Prerequisite failed: expected monitored collection with at least 2 partitions.");
 
@@ -164,7 +168,7 @@ public async Task CountDocumentsInCollection_TwoHosts()
 
             try
             {
-                Assert.True(documentCount == processedCount, "Wrong processedCount");
+                Assert.True(documentCount == processedCount, $"Wrong processedCount {documentCount} {processedCount}");
             }
             finally
             {
@@ -209,11 +213,12 @@ public async Task StopAtFullSpeed()
             Assert.True(partitionKeyRangeCount == closedCount, "Wrong closedCount");
         }
 
-        protected override async Task FinishTestClassInitializeAsync()
+        private async Task InitializeDocumentsAsync()
         {
             using (var client = new DocumentClient(this.ClassData.monitoredCollectionInfo.Uri, this.ClassData.monitoredCollectionInfo.MasterKey, this.ClassData.monitoredCollectionInfo.ConnectionPolicy))
             {
                 var collectionUri = UriFactory.CreateDocumentCollectionUri(this.ClassData.monitoredCollectionInfo.DatabaseName, this.ClassData.monitoredCollectionInfo.CollectionName);
+
                 await IntegrationTestsHelper.CreateDocumentsAsync(client, collectionUri, documentCount);
             }
         }

src/DocumentDB.ChangeFeedProcessor.IntegrationTests/Utils/IntegrationTestsHelper.cs

@@ -14,6 +14,9 @@ namespace Microsoft.Azure.Documents.ChangeFeedProcessor.IntegrationTests.Utils
 {
     internal class IntegrationTestsHelper
     {
+        // Used when tests are running in CI/CD pipeline
+        private const string GatewayEndpointEnvironmentName = "COSMOSDBEMULATOR_ENDPOINT";
+
         static readonly string Endpoint;
         static readonly string MasterKey;
         static readonly string DatabaseId;
@@ -26,7 +29,7 @@ static IntegrationTestsHelper()
                     .AddJsonFile("appsettings.json")
                     .Build();
 
-            Endpoint = config["IntegrationTests:endpoint"];
+            Endpoint = Environment.GetEnvironmentVariable(GatewayEndpointEnvironmentName) ?? config["IntegrationTests:endpoint"];
             MasterKey = config["IntegrationTests:masterKey"];
             DatabaseId = config["IntegrationTests:databaseId"];
             MonitoredOfferThroughput = config["IntegrationTests:monitoredOfferThroughput"];
@@ -77,7 +80,19 @@ internal static async Task CreateDocumentCollectionAsync(DocumentClient client,
             var database = new Database { Id = databaseId };
             database = await client.CreateDatabaseIfNotExistsAsync(database);
 
-            await client.CreateDocumentCollectionAsync(database.SelfLink, collection, new RequestOptions { OfferThroughput = offerThroughput });
+            int retryCount = 3;
+            while (retryCount-- > 0)
+            {
+                try
+                {
+                    await client.CreateDocumentCollectionAsync(database.SelfLink, collection, new RequestOptions { OfferThroughput = offerThroughput });
+                    break;
+                }
+                catch (DocumentClientException)
+                {
+                    // Public emulator might have transient
+                }
+            }
         }
 
         internal static async Task CreateDocumentsAsync(DocumentClient client, Uri collectionUri, int count)