diff --git a/Actions/.Modules/ReadSettings.psm1 b/Actions/.Modules/ReadSettings.psm1 index 3e483e4294..f0f3ed166d 100644 --- a/Actions/.Modules/ReadSettings.psm1 +++ b/Actions/.Modules/ReadSettings.psm1 @@ -203,7 +203,7 @@ function GetDefaultSettings "retentionDays" = 30 "mode" = "modifiedApps" # modifiedProjects, modifiedApps } - "microsoftTelemetryConnectionString" = "InstrumentationKey=cd2cc63e-0f37-4968-b99a-532411a314b8;IngestionEndpoint=https://northeurope-2.in.applicationinsights.azure.com/" + "microsoftTelemetryConnectionString" = "InstrumentationKey=cd2cc63e-0f37-4968-b99a-532411a314b8;IngestionEndpoint=https://northeurope-2.in.applicationinsights.azure.com/" #gitleaks:allow "partnerTelemetryConnectionString" = "" "sendExtendedTelemetryToMicrosoft" = $false "environments" = @() diff --git a/Actions/AL-Go-Helper.ps1 b/Actions/AL-Go-Helper.ps1 index 82829f4875..60d3e2a493 100644 --- a/Actions/AL-Go-Helper.ps1 +++ b/Actions/AL-Go-Helper.ps1 @@ -2197,10 +2197,55 @@ function ConnectAz { <# .SYNOPSIS - Output a message and an array of strings in a formatted way. + Checks whether nuget.org is added as a nuget source. +#> +function AssertNugetSourceIsAdded() { + $nugetSource = "https://api.nuget.org/v3/index.json" + $nugetSourceExists = dotnet nuget list source | Select-String -Pattern $nugetSource + if (-not $nugetSourceExists) { + throw "Nuget source $nugetSource is not added. Please add the source using 'dotnet nuget add source $nugetSource' or add another source with nuget.org as an upstream source." + } +} - Deprecated: Use OutputArray function from DebugLogHelper module. +<# + .SYNOPSIS + Installs a .NET tool in a temporary folder and returns the path to the folder + .DESCRIPTION + This function installs a .NET tool using 'dotnet tool install' in a temporary folder and returns the path to the folder where the tool is installed. + .PARAMETER PackageName + The name of the package to install + .RETURNS + The path to the folder where the tool is installed. #> +function Install-DotNetTool { + param( + [Parameter(Mandatory = $true)] + [string] $PackageName + ) + AssertNugetSourceIsAdded + $ToolPath = GetTemporaryPath + + $installationFolder = Join-Path -Path $ToolPath $PackageName + if (Test-Path -Path $installationFolder) { + # Tool is already installed + Write-Host "$PackageName is already installed in $installationFolder" + return $installationFolder + } + + # Get version of the package + $version = GetPackageVersion -PackageName $PackageName + + # Install the tool in the temp folder + Write-Host "Installing $PackageName ($version) in $installationFolder" + dotnet tool install $PackageName --version $version --tool-path $installationFolder | Out-Host + + if (-not (Test-Path -Path $installationFolder)) { + throw "Failed to install $PackageName. If you are using a self-hosted runner please make sure you've followed all the steps described in https://aka.ms/algosettings#runs-on." + } + + return $installationFolder +} + function OutputMessageAndArray { Param( [string] $message, diff --git a/Actions/Environment.Packages.proj b/Actions/Environment.Packages.proj index 74157f858d..72c6767c65 100644 --- a/Actions/Environment.Packages.proj +++ b/Actions/Environment.Packages.proj @@ -12,6 +12,7 @@ + diff --git a/Actions/RunPipeline/RunPipeline.ps1 b/Actions/RunPipeline/RunPipeline.ps1 index b039bdfebd..f72dfdd4aa 100644 --- a/Actions/RunPipeline/RunPipeline.ps1 +++ b/Actions/RunPipeline/RunPipeline.ps1 @@ -210,32 +210,42 @@ try { } # Replace secret names in install.apps and install.testApps + $tempDependenciesLocation = NewTemporaryFolder foreach($list in @('Apps','TestApps')) { $install."$list" = @($install."$list" | ForEach-Object { - $pattern = '.*(\$\{\{\s*([^}]+?)\s*\}\}).*' - $url = $_ - if ($url -match $pattern) { - $finalUrl = $url.Replace($matches[1],[System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($secrets."$($matches[2])"))) + $appFile = $_ + + # If the app file is not a URL, return it as is + if ($appFile -notlike 'http*://*') { + return $appFile } - else { - $finalUrl = $url + + # Else, check for secrets in the URL and replace them + $appFileUrl = $appFile + $pattern = '.*(\$\{\{\s*([^}]+?)\s*\}\}).*' + if ($appFile -match $pattern) { + $appFileFinalUrl = $appFileUrl.Replace($matches[1],[System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($secrets."$($matches[2])"))) } - # Check validity of URL - if ($finalUrl -like 'http*://*') { - try { - Invoke-WebRequest -Method Head -UseBasicParsing -Uri $finalUrl | Out-Null - } - catch { - throw "Setting: install$($list) contains an inaccessible URL: $($url). Error was: $($_.Exception.Message)" - } + + # Download the app file to a temporary location + try { + $appFile = Get-AppFileFromUrl -Url $appFileFinalUrl -DownloadPath $tempDependenciesLocation + } catch { + OutputError -message "Failed to download app from URL: $($appFileUrl). Please check that the URL is valid. Error was: $($_.Exception.Message)" } - return $finalUrl + + return $appFile }) } # Analyze app.json version dependencies before launching pipeline # Analyze InstallApps and InstallTestApps before launching pipeline + if ((-not $settings.doNotPublishApps)) { + # Test that InstallApps are not symbols packages + Import-Module (Join-Path -Path $PSScriptRoot -ChildPath ".\RunPipeline.psm1" -Resolve) + Test-InstallApps -AllInstallApps ($install.Apps + $install.TestApps) -ProjectPath $projectPath + } # Check if codeSignCertificateUrl+Password is used (and defined) if (!$settings.doNotSignApps -and $codeSignCertificateUrl -and $codeSignCertificatePassword -and !$settings.keyVaultCodesignCertificateName) { diff --git a/Actions/RunPipeline/RunPipeline.psm1 b/Actions/RunPipeline/RunPipeline.psm1 new file mode 100644 index 0000000000..4ed5fe9e07 --- /dev/null +++ b/Actions/RunPipeline/RunPipeline.psm1 @@ -0,0 +1,124 @@ +Import-Module (Join-Path $PSScriptRoot '..\TelemetryHelper.psm1' -Resolve) +. (Join-Path -Path $PSScriptRoot -ChildPath "..\AL-Go-Helper.ps1" -Resolve) + +<# + .SYNOPSIS + Installs the AL tool. + .DESCRIPTION + Installs the AL tool from the Microsoft.Dynamics.BusinessCentral.Development.Tools package. + .RETURNS + The path to the al.exe tool. +#> +function Install-ALTool { + $alToolFolder = Install-DotNetTool -PackageName "Microsoft.Dynamics.BusinessCentral.Development.Tools" + # Load the AL tool from the downloaded package + $alExe = Get-ChildItem -Path $alToolFolder -Filter "al*" | Where-Object { $_.Name -eq "al" -or $_.Name -eq "al.exe" } | Select-Object -First 1 -ExpandProperty FullName + if (-not $alExe) { + throw "Could not find al.exe in the development tools package." + } + return $alExe +} + +<# + .SYNOPSIS + Analyzes the install apps to check if they are symbols packages. + .DESCRIPTION + Analyzes the install apps to check if they are symbols packages. + If an app is a symbols package, it outputs a warning message. + .PARAMETER AllInstallApps + The list of all install apps to analyze. + .PARAMETER ProjectPath + The path to the project where the apps are located. +#> +function Test-InstallApps() { + Param( + [string[]] $AllInstallApps, + [string] $ProjectPath + ) + + if ($AllInstallApps.Count -eq 0) { + Write-Host "No install apps to analyze." + return + } + + try { + # Install the AL tool and get the path to al.exe + $alExe = Install-ALTool + + $symbolsOnlyCount = 0 + foreach ($app in $AllInstallApps) { + if (Test-Path -Path $app) { + $appFilePath = (Get-Item -Path $app).FullName + } else { + $appFilePath = Join-Path $ProjectPath $app -Resolve -ErrorAction SilentlyContinue + } + + if ($appFilePath) { + $appFile = Get-Item -Path $appFilePath + $appFileName = $appFile.Name + Write-Host "Analyzing app file $appFileName" + if (IsSymbolsOnlyPackage -AppFilePath $appFile -AlExePath $alExe) { + # If package is not a runtime package and has no source code files, it is a symbols package + # Symbols packages are not meant to be published to a BC Environment + $symbolsOnlyCount++ + OutputWarning -Message "App $appFileName is a symbols package and should not be published. The workflow may fail if you try to publish it." + } + } else { + Write-Host "App file path for $app could not be resolved. Skipping symbols check." + } + } + + if ($symbolsOnlyCount -gt 0) { + Trace-Warning -Message "$symbolsOnlyCount symbols-only package(s) detected in install apps. These packages should not be published." + } + } + catch { + Trace-Warning -Message "Something went wrong while analyzing install apps." + OutputDebug -message "Error: $_" + } +} + +function IsSymbolsOnlyPackage { + param( + [string] $AppFilePath, + [string] $AlExePath + ) + . $AlExePath IsSymbolOnly $AppFilePath | Out-Null + return $LASTEXITCODE -eq 0 +} + +<# + .SYNOPSIS + Downloads an app file from a URL to a specified download path. + .DESCRIPTION + Downloads an app file from a URL to a specified download path. + It handles URL decoding and sanitizes the file name. + .PARAMETER Url + The URL of the app file to download. + .PARAMETER DownloadPath + The path where the app file should be downloaded. + .OUTPUTS + The path to the downloaded app file. +#> +function Get-AppFileFromUrl { + Param( + [string] $Url, + [string] $DownloadPath + ) + # Get the file name from the URL + $urlWithoutQuery = $Url.Split('?')[0].TrimEnd('/') + $rawFileName = [System.IO.Path]::GetFileName($urlWithoutQuery) + $decodedFileName = [Uri]::UnescapeDataString($rawFileName) + $decodedFileName = [System.IO.Path]::GetFileName($decodedFileName) + + # Sanitize file name by removing invalid characters + $sanitizedFileName = $decodedFileName.Split([System.IO.Path]::getInvalidFileNameChars()) -join "" + $sanitizedFileName = $sanitizedFileName.Trim() + + # Get the final app file path + $appFile = Join-Path $DownloadPath $sanitizedFileName + Invoke-WebRequest -Method GET -UseBasicParsing -Uri $Url -OutFile $appFile -MaximumRetryCount 3 -RetryIntervalSec 5 | Out-Null + return $appFile +} + +Export-ModuleMember -Function Test-InstallApps, Get-AppFileFromUrl diff --git a/Actions/Sign/Sign.psm1 b/Actions/Sign/Sign.psm1 index 6ec3890be1..30a3bde5a3 100644 --- a/Actions/Sign/Sign.psm1 +++ b/Actions/Sign/Sign.psm1 @@ -1,15 +1,3 @@ -<# - .SYNOPSIS - Checks whether nuget.org is added as a nuget source. -#> -function AssertNugetSourceIsAdded() { - $nugetSource = "https://api.nuget.org/v3/index.json" - $nugetSourceExists = dotnet nuget list source | Select-String -Pattern $nugetSource - if (-not $nugetSourceExists) { - throw "Nuget source $nugetSource is not added. Please add the source using 'dotnet nuget add source $nugetSource' or add another source with nuget.org as an upstream source." - } -} - <# .SYNOPSIS Installs the dotnet signing tool. @@ -19,26 +7,10 @@ function AssertNugetSourceIsAdded() { function Install-SigningTool() { . (Join-Path -Path $PSScriptRoot -ChildPath "..\AL-Go-Helper.ps1" -Resolve) - # Create folder in temp directory with a unique name - $tempFolder = Join-Path -Path (GetTemporaryPath) "SigningTool-$(Get-Random)" - - # Get version of the signing tool - $version = GetPackageVersion -PackageName "sign" - - # Install the signing tool in the temp folder - Write-Host "Installing signing tool version $version in $tempFolder" - New-Item -ItemType Directory -Path $tempFolder | Out-Null - dotnet tool install sign --version $version --tool-path $tempFolder | Out-Host + $signToolFolder = Install-DotNetTool -PackageName sign # Return the path to the signing tool - $signingTool = Join-Path -Path $tempFolder "sign.exe" - if (-not (Test-Path -Path $signingTool)) { - # Check if nuget.org is added as a nuget source - AssertNugetSourceIsAdded - - # If the tool is not found, throw an error - throw "Failed to install signing tool. If you are using a self-hosted runner please make sure you've followed all the steps described in https://aka.ms/algosettings#runs-on." - } + $signingTool = Join-Path -Path $signToolFolder "sign.exe" return $signingTool } diff --git a/RELEASENOTES.md b/RELEASENOTES.md index 3c7cd00000..cad4b56dd5 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -1,5 +1,6 @@ ### Issues +- Issue 1512: Throw a warning before trying to publish symbols packages - Issue 2055 When using versioningStrategy 3+16, you get an error when building - AL-Go repositories with large amounts of projects may run into issues with too large environment variables diff --git a/Scenarios/DeliveryTargets.md b/Scenarios/DeliveryTargets.md index 87fe57bbf0..b53460f4f8 100644 --- a/Scenarios/DeliveryTargets.md +++ b/Scenarios/DeliveryTargets.md @@ -137,7 +137,7 @@ Unlike GitHub Packages, NuGet feeds configured with `NuGetContext` are not autom **Organizational Secret**: `GitHubPackagesContext` ```json -{"token":"ghp_1234567890abcdef","serverUrl":"https://nuget.pkg.github.com/contoso/index.json"} +{"token":"ghp_","serverUrl":"https://nuget.pkg.github.com/contoso/index.json"} ``` **AL-Go-Settings.json** (optional): diff --git a/Scenarios/settings.md b/Scenarios/settings.md index c650107348..4952bccb1d 100644 --- a/Scenarios/settings.md +++ b/Scenarios/settings.md @@ -101,7 +101,7 @@ The repository settings are only read from the repository settings file (.github | licenseFileUrlSecretName | Specify the name (**NOT the secret**) of the LicenseFileUrl secret. Default is LicenseFileUrl. AL-Go for GitHub will look for a secret with this name in GitHub Secrets or Azure KeyVault to use as LicenseFileUrl. A LicenseFileUrl is required when building AppSource apps for Business Central prior to version 22. Read [this](SetupCiCdForExistingAppSourceApp.md) for more information. | LicenseFileUrl | | ghTokenWorkflowSecretName | Specifies the name (**NOT the secret**) of the GhTokenWorkflow secret. Default is GhTokenWorkflow. AL-Go for GitHub will look for a secret with this name in GitHub Secrets or Azure KeyVault to use as Personal Access Token with permission to modify workflows when running the Update AL-Go System Files workflow. Read [this](UpdateAlGoSystemFiles.md) for more information. | GhTokenWorkflow | | adminCenterApiCredentialsSecretName | Specifies the name (**NOT the secret**) of the adminCenterApiCredentials secret. Default is adminCenterApiCredentials. AL-Go for GitHub will look for a secret with this name in GitHub Secrets or Azure KeyVault to use when connecting to the Admin Center API when creating Online Development Environments. Read [this](CreateOnlineDevEnv2.md) for more information. | AdminCenterApiCredentials | -| installApps | An array of 3rd party dependency apps, which you do not have access to through the appDependencyProbingPaths. The setting should be an array of either secure URLs or paths to folders or files relative to the project, where the CI/CD workflow can find and download the apps. The apps in installApps are downloaded and installed before compiling and installing the apps.
**Note:** If you specify ${{SECRETNAME}} as part of a URL, it will be replaced by the value of the secret SECRETNAME. | [ ] | +| installApps | An array of 3rd party dependency apps, which you do not have access to through the appDependencyProbingPaths. The setting should be an array of either secure URLs or paths to folders or files relative to the project, where the build workflow can find and download the apps. The apps in installApps are downloaded and installed before compiling and installing the apps in the current project. Before adding an app to the installApps property, please ensure it is not a symbols-only package. Symbols-only packages are not intended for publishing and installing and using them via installApps might lead to compilation or runtime errors.
**Note:** If you specify ${{SECRETNAME}} as part of a URL, it will be replaced by the value of the secret SECRETNAME. | [ ] | | installTestApps | An array of 3rd party dependency apps, which you do not have access to through the appDependencyProbingPaths. The setting should be an array of either secure URLs or paths to folders or files relative to the project, where the CI/CD workflow can find and download the apps. The apps in installTestApps are downloaded and installed before compiling and installing the test apps. Adding a parentheses around the setting indicates that the test in this app will NOT be run, only installed.
**Note:** If you specify ${{SECRETNAME}} as part of a URL, it will be replaced by the value of the secret SECRETNAME. | [ ] | | configPackages | An array of configuration packages to be applied to the build container before running tests. Configuration packages can be the relative path within the project or it can be STANDARD, EXTENDED or EVALUATION for the rapidstart packages, which comes with Business Central. | [ ] | | configPackages.country | An array of configuration packages to be applied to the build container for country **country** before running tests. Configuration packages can be the relative path within the project or it can be STANDARD, EXTENDED or EVALUATION for the rapidstart packages, which comes with Business Central. | [ ] | diff --git a/Tests/RunPipeline.Action.Test.ps1 b/Tests/RunPipeline.Action.Test.ps1 index d0c063c179..2a83e7b2c5 100644 --- a/Tests/RunPipeline.Action.Test.ps1 +++ b/Tests/RunPipeline.Action.Test.ps1 @@ -23,6 +23,29 @@ Describe "RunPipeline Action Tests" { YamlTest -scriptRoot $scriptRoot -actionName $actionName -actionScript $actionScript -outputs $outputs } + It 'Test warning for symbols packages' { + Import-Module (Join-Path $scriptRoot '.\RunPipeline.psm1' -Resolve) -Force + . (Join-Path $PSScriptRoot '../Actions/AL-Go-Helper.ps1') + Import-Module (Join-Path $PSScriptRoot '../Actions/TelemetryHelper.psm1') + + # Mock the OutputWarning and Trace-Warning functions + Mock -CommandName OutputWarning -MockWith { param($Message) Write-Host "OutputWarning: $Message" } -ModuleName RunPipeline + Mock -CommandName Trace-Warning -MockWith { param($Message) Write-Host "Trace-Information: $Message" } -ModuleName RunPipeline + + # Invoke the function with TestApp1 (a symbols package) and TestApp2 (a full app package) + $tempFolder = [System.IO.Path]::GetTempPath() + Test-InstallApps -AllInstallApps @(".\TestApps\EssentialBusinessHeadlinesFull.app", ".\TestApps\EssentialBusinessHeadlinesSymbols.app") -ProjectPath $PSScriptRoot -RunnerTempFolder $tempFolder + + # Assert that the warning was output + Should -Invoke -CommandName 'OutputWarning' -Times 1 -ModuleName RunPipeline + Should -Invoke -CommandName 'OutputWarning' -Times 1 -ModuleName RunPipeline -ParameterFilter { $Message -like "*App EssentialBusinessHeadlinesSymbols.app is a symbols package and should not be published. The workflow may fail if you try to publish it." } + + # Assert that Trace-Warning was called once with the count + Should -Invoke -CommandName 'Trace-Warning' -Times 1 -ModuleName RunPipeline -ParameterFilter { $Message -like "*1 symbols-only package(s) detected in install apps*" } + # Assert that Trace-Warning was not called + Should -Invoke -CommandName 'Trace-Warning' -Times 0 -ModuleName RunPipeline -ParameterFilter { $Message -like "App file path for * could not be resolved." } + } + # Call action } diff --git a/Tests/TestApps/EssentialBusinessHeadlinesFull.app b/Tests/TestApps/EssentialBusinessHeadlinesFull.app new file mode 100644 index 0000000000..73418c3519 Binary files /dev/null and b/Tests/TestApps/EssentialBusinessHeadlinesFull.app differ diff --git a/Tests/TestApps/EssentialBusinessHeadlinesSymbols.app b/Tests/TestApps/EssentialBusinessHeadlinesSymbols.app new file mode 100644 index 0000000000..d395933a91 Binary files /dev/null and b/Tests/TestApps/EssentialBusinessHeadlinesSymbols.app differ