Adding integration test runs for PRs (#2619)

* Inital integration test commit

* Changing password format

* Upodating global JSON location

* trying this

* removing working directory

* building code

* modern cake run

* updated cake with built-in Xliff transformer

* Slash direction changes

* with SRGen

* fixing cake tool install paths

* No-oping .NET Framework version of XUnit if no tests target Framework

* Adding log upload on failure

* trying some path normalization

* running precreation script

* Adding missing project

* handling release in addition to debug

* normalizing lang variants

* Not crashing when a late notification arrives to a completed queue.

* Fixing some slash directions

* adding logging

* timeout on test step

* Skipping problem test

* commenting out NonTsql tests

* debugging

* Tracking disposal

* lint error

* Disposals

* Skipping LangServiceTests for now

* working on PR pipeline now

* Platform independent nuget cli call

* Comment for nuget CLI

* Improving check

* removing experimental flag

* rename

* Copying changes to ADO pipeline

* default linux image

* Slimming build step

* Converting to pwsh

* Removing mono from bootstrapper

* Adding back RESX header comment and schema

* Reverting the translated tags

* Skipping broken tests

* fixing slash direction

* Some integration test fixes

* fixing pipeline names

* Removing dead xunit check

* reverting changes from debugging lang service

* PR feedback cleanup

* more PR cleanup

* moving xliff helpers to separate file

* consolidating test name sanitization

* Cleaning up changes

* Updating comment

* Cleaning up outdated comment

* Adding back graceful handling of thread queue exit

Author: Benjin

.github/workflows/integration-test.yml

@@ -0,0 +1,175 @@
+name: Integration Tests
+
+on:
+  schedule:
+    - cron: '0 0 * * *'
+  push:
+    branches:
+      - main
+      - release/*
+  pull_request:
+    branches:
+      - main
+      - release/*
+  workflow_dispatch:
+
+permissions:
+  contents: read
+  actions: read
+  checks: write
+
+jobs:
+  integration-tests:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Setup .NET SDK
+        uses: actions/setup-dotnet@v4
+        with:
+          global-json-file: global.json
+
+      - name: Generate SQL authentication password
+        shell: pwsh
+        run: |
+          $chars = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789'.ToCharArray()
+          $passwordBody = -join (1..24 | ForEach-Object { $chars[(Get-Random -Maximum $chars.Length)] })
+          $sqlPassword = "A1!b$passwordBody" # 'A1!' prefix is to meet password complexity requirements for SQL
+          Write-Output "::add-mask::$sqlPassword"
+          Add-Content -Path $env:GITHUB_ENV -Value "SQL_PASSWORD=$sqlPassword"
+          Add-Content -Path $env:GITHUB_ENV -Value "sqlOnPrem_password=$sqlPassword"
+          Add-Content -Path $env:GITHUB_ENV -Value "sqlAzure_password=$sqlPassword"
+
+      - name: Install Cake tools
+        shell: pwsh
+        run: |
+          dotnet tool install --tool-path ./.tools Cake.Tool
+          dotnet tool install --tool-path ./.tools dotnet-t4
+
+      - name: Build repository
+        shell: pwsh
+        run: |
+          ./.tools/dotnet-cake build.cake --target=all
+
+      - name: Upload build logs
+        if: failure()
+        uses: actions/upload-artifact@v4
+        with:
+          name: cake-build-logs
+          path: artifacts/logs
+          if-no-files-found: ignore
+
+      - name: Start SQL Server container
+        shell: pwsh
+        run: |
+          docker pull mcr.microsoft.com/mssql/server:2025-latest
+          docker rm -f sqltoolsservice-integration-tests *> $null
+          docker run `
+            --detach `
+            --name sqltoolsservice-integration-tests `
+            --hostname sqltoolsservice-integration-tests `
+            --publish 1433:1433 `
+            --env ACCEPT_EULA=Y `
+            --env MSSQL_PID=Developer `
+            --env MSSQL_AGENT_ENABLED=true `
+            --env MSSQL_SA_PASSWORD="$env:SQL_PASSWORD" `
+            mcr.microsoft.com/mssql/server:2025-latest
+
+      - name: Create SQL login for integration tests
+        shell: pwsh
+        run: |
+          $ready = $false
+          for ($attempt = 1; $attempt -le 30; $attempt++) {
+            docker exec sqltoolsservice-integration-tests /opt/mssql-tools18/bin/sqlcmd -S localhost -U sa -P $env:SQL_PASSWORD -C -Q "SELECT 1" *> $null
+            if ($LASTEXITCODE -eq 0) {
+              $ready = $true
+              break
+            }
+
+            Start-Sleep -Seconds 5
+          }
+
+          if (-not $ready) {
+            throw "SQL Server container did not become ready in time."
+          }
+
+          $sql = "IF NOT EXISTS (SELECT 1 FROM sys.sql_logins WHERE name = 'testAccount') BEGIN CREATE LOGIN [testAccount] WITH PASSWORD = '$($env:SQL_PASSWORD)', CHECK_POLICY = OFF; END; ALTER SERVER ROLE [sysadmin] ADD MEMBER [testAccount];"
+          docker exec sqltoolsservice-integration-tests /opt/mssql-tools18/bin/sqlcmd `
+            -S localhost `
+            -U sa `
+            -P $env:SQL_PASSWORD `
+            -C `
+            -Q $sql
+
+      - name: Create integration test connection instances file
+        shell: pwsh
+        run: |
+          @'
+          <?xml version="1.0" encoding="utf-8"?>
+          <Instances>
+            <Instance VersionKey="sqlOnPrem">
+              <DataSource>localhost</DataSource>
+              <UserId>testAccount</UserId>
+              <Password></Password>
+            </Instance>
+            <Instance VersionKey="sqlAzure">
+              <DataSource>localhost</DataSource>
+              <UserId>testAccount</UserId>
+              <Password></Password>
+            </Instance>
+          </Instances>
+          '@ | Set-Content -Path test/Microsoft.SqlTools.ServiceLayer.TestEnvConfig/SQLConnectionInstances.CI.xml
+
+      - name: Generate integration test settings
+        shell: pwsh
+        run: |
+          dotnet run `
+            --project test/Microsoft.SqlTools.ServiceLayer.TestEnvConfig/Microsoft.SqlTools.ServiceLayer.TestEnvConfig.csproj `
+            -- `
+            test/Microsoft.SqlTools.ServiceLayer.TestEnvConfig/SQLConnectionInstances.CI.xml
+
+      - name: Build integration tests
+        shell: pwsh
+        run: |
+          dotnet build test/Microsoft.SqlTools.ServiceLayer.IntegrationTests/Microsoft.SqlTools.ServiceLayer.IntegrationTests.csproj --configuration Release
+
+      - name: Run integration tests
+        id: integration_tests
+        continue-on-error: true
+        timeout-minutes: 60
+        shell: pwsh
+        run: |
+          dotnet test test/Microsoft.SqlTools.ServiceLayer.IntegrationTests/Microsoft.SqlTools.ServiceLayer.IntegrationTests.csproj `
+            --configuration Release `
+            --no-build `
+            --logger "trx;LogFileName=IntegrationTests.trx" `
+            --results-directory "$PWD/TestResults"
+
+      - name: Publish integration test report
+        if: always()
+        uses: dorny/test-reporter@v1
+        with:
+          name: Integration Tests
+          path: TestResults/*.trx
+          reporter: dotnet-trx
+          fail-on-error: false
+
+      - name: Upload integration test results
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: integration-test-results
+          path: TestResults/*.trx
+          if-no-files-found: ignore
+
+      - name: Stop SQL container
+        if: always()
+        shell: pwsh
+        run: docker rm -f sqltoolsservice-integration-tests *> $null
+
+      - name: Fail workflow when integration tests fail
+        if: steps.integration_tests.outcome == 'failure'
+        shell: pwsh
+        run: exit 1

.github/workflows/integration-tests.yml

@@ -4,4 +4,4 @@
 		<DebugType>full</DebugType>
 		<DebugSymbols>true</DebugSymbols>
 	</PropertyGroup>
-</Project>
+</Project>

Directory.Build.targets

@@ -69,28 +69,31 @@ steps:
   inputs:
     useGlobalJson: true
 
-- task: PowerShell@2
+- task: NuGetToolInstaller@1
+  displayName: 'Install NuGet'
+
+- pwsh: |
+    dotnet tool install --tool-path ./.tools Cake.Tool
+    dotnet tool install --tool-path ./.tools dotnet-t4
+  displayName: 'Install Cake tools'
+
+- pwsh: |
+    $projectFile = '$(Build.SourcesDirectory)/src/Microsoft.SqlTools.SqlCore/Microsoft.SqlTools.SqlCore.csproj'
+    $props = New-Object XML
+    $props.Load($projectFile)
+    $version = ($(Major)).ToString() + '.' + ($(Minor)).ToString() + '.' + $(Patch) + $(Build.BuildNumber);
+    $props.Project.PropertyGroup.Version = $version;
+    $props.Save($projectFile)
   displayName: 'Add version number to SqlCore project for packaging'
-  inputs:
-    targetType: 'inline'
-    script: |
-      $projectFile = '$(Build.SourcesDirectory)\src\Microsoft.SqlTools.SqlCore\Microsoft.SqlTools.SqlCore.csproj'
-      $props = New-Object XML
-      $props.Load($projectFile)
-      $version = ($(Major)).ToString() + '.' + ($(Minor)).ToString() + '.' + $(Patch) + $(Build.BuildNumber);
-      $props.Project.PropertyGroup.Version = $version;
-      $props.Save($projectFile)
 
 - task: CodeQL3000Init@0
   displayName: 'CodeQL Initialize'
   condition: eq(variables['Codeql.enabled'], 'True')
 
-- task: BatchScript@1
-  displayName: 'Run script build.cmd'
-  inputs:
-    filename: build.cmd
-    arguments: '-target=all -mono'
-    
+- pwsh: |
+    ./.tools/dotnet-cake build.cake --target=all
+  displayName: 'Run Cake build'
+
 - task: DotNetCoreCLI@2
   displayName: 'dotnet restore test/Microsoft.SqlTools.ServiceLayer.UnitTests'
   inputs:
@@ -125,7 +128,7 @@ steps:
   displayName: Create Code coverage report
 
 
-- powershell: |
+- pwsh: |
     $ProgressPreference = 'SilentlyContinue'
     Invoke-WebRequest -Uri https://keybase.io/codecovsecurity/pgp_keys.asc -OutFile codecov.asc
     gpg.exe --import codecov.asc
@@ -151,26 +154,23 @@ steps:
     codeCoverageTool: 'Cobertura'
     summaryFileLocation: '$(Agent.TempDirectory)/coverlet/reports/Cobertura.xml'
 
-- task: PowerShell@2
+- pwsh: |
+    $projectFile = '$(Build.SourcesDirectory)\src\Microsoft.SqlTools.ManagedBatchParser\Microsoft.SqlTools.ManagedBatchParser.csproj'
+    $props = New-Object XML
+    $props.Load($projectFile)
+    $propGroup = $props.Project.PropertyGroup;
+    $versionPre = $env:ManagedBatchParserMajor + '.' + $env:ManagedBatchParserMinor + '.'
+    if($env:StableRelease.Equals('true')) {
+        $version = $versionPre + '0';
+    } else {
+        $version = $versionPre + $(Build.BuildNumber) + '-' + $propGroup.VersionSuffix.'#text'
+    }
+    $nuspecFile = '$(Build.SourcesDirectory)\packages\Microsoft.SqlTools.ManagedBatchParser\Microsoft.SqlTools.ManagedBatchParser.nuspec'
+    $nuspec = New-Object XML
+    $nuspec.Load($nuspecFile)
+    $nuspec.package.metadata.version = $version;
+    $nuspec.Save($nuspecFile)
   displayName: 'Generate ManagedBatchParser Nuspec Version'
-  inputs:
-    targetType: 'inline'
-    script: |
-      $projectFile = '$(Build.SourcesDirectory)\src\Microsoft.SqlTools.ManagedBatchParser\Microsoft.SqlTools.ManagedBatchParser.csproj'
-      $props = New-Object XML
-      $props.Load($projectFile)
-      $propGroup = $props.Project.PropertyGroup;
-      $versionPre = $env:ManagedBatchParserMajor + '.' + $env:ManagedBatchParserMinor + '.'
-      if($env:StableRelease.Equals('true')) {
-          $version = $versionPre + '0';
-      } else {
-          $version = $versionPre + $(Build.BuildNumber) + '-' + $propGroup.VersionSuffix.'#text'
-      }
-      $nuspecFile = '$(Build.SourcesDirectory)\packages\Microsoft.SqlTools.ManagedBatchParser\Microsoft.SqlTools.ManagedBatchParser.nuspec'
-      $nuspec = New-Object XML
-      $nuspec.Load($nuspecFile)
-      $nuspec.package.metadata.version = $version;
-      $nuspec.Save($nuspecFile)
 
 - task: Npm@1
   displayName: 'npm install -g gulp-cli'
@@ -186,19 +186,15 @@ steps:
     custom: pack
     arguments: '--output $(Build.SourcesDirectory)\artifacts\nugetPackages --no-build --configuration $(buildConfiguration) -v n src\Microsoft.SqlTools.SqlCore\Microsoft.SqlTools.SqlCore.csproj'
 
-- task: BatchScript@1
+- pwsh: |
+    ./.tools/dotnet-cake build.cake --target=NugetPackNuspec
   displayName: 'Package nuspec projects'
-  inputs:
-    filename: build.cmd
-    arguments: '-target=NugetPackNuspec -mono'
-    
-- task: BatchScript@1
-  displayName: "Build and Package service tool projects"
+
+- pwsh: |
+    ./.tools/dotnet-cake build.cake --target=dotnetpackservicetools
+  displayName: 'Build and Package service tool projects'
   env:
     BUILD_DOTNET_TOOL: "true"
-  inputs:
-    filename: build.cmd
-    arguments: "-target=dotnetpackservicetools -mono"
 
 - ${{ each project in parameters.projects }}:
   - ${{ each platform in parameters.platforms }}:
@@ -253,4 +249,4 @@ steps:
 - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0
   displayName: 'Component Detection'
   inputs:
-    failOnAlert: true
+    failOnAlert: true

azure-pipelines/build.yml

@@ -17,7 +17,6 @@
 # The script need to run from the repo root
 
 dotnetProjectArray=(
-    "./src/Microsoft.Kusto.ServiceLayer"
     "./src/Microsoft.SqlTools.Credentials"
     "./src/Microsoft.SqlTools.Hosting"
     "./src/Microsoft.SqlTools.ResourceProvider"
@@ -26,7 +25,7 @@ dotnetProjectArray=(
     "./src/Microsoft.SqlTools.ServiceLayer"
     "./src/Microsoft.SqlTools.Migration"
     "./src/Microsoft.SqlTools.SqlCore"
-    "./test/Microsoft.Kusto.ServiceLayer.UnitTests"
+    "./test/Microsoft.SqlTools.Authentication.UnitTests"
     "./test/Microsoft.SqlTools.ManagedBatchParser.IntegrationTests"
     "./test/Microsoft.SqlTools.ServiceLayer.IntegrationTests"
     "./test/Microsoft.SqlTools.ServiceLayer.PerfTests"
@@ -38,8 +37,10 @@ dotnetProjectArray=(
     "./test/Microsoft.SqlTools.Test.CompletionExtension"
 )
 
-# Please update the framework vars when updating target framework for the projects
-framework8="/bin/Debug/net8.0/"
+configurations=(
+    "Debug"
+    "Release"
+)
 
 requiredLocDirectories=(
     "pt-br"
@@ -53,10 +54,13 @@ requiredLocDirectories=(
 for i in "${dotnetProjectArray[@]}"
 do
    : 
-   for k in "${requiredLocDirectories[@]}"
-    do
-        : 
-        output=`mkdir -v -p $i$framework8$k`
-        echo $output
+   for configuration in "${configurations[@]}"
+   do
+      for k in "${requiredLocDirectories[@]}"
+      do
+          :
+          output=`mkdir -v -p "$i/bin/$configuration/net8.0/$k"`
+          echo $output
+      done
     done
 done