diff --git a/Actions/.Modules/ReadSettings.psm1 b/Actions/.Modules/ReadSettings.psm1 index ab242c3adf..6945acacc2 100644 --- a/Actions/.Modules/ReadSettings.psm1 +++ b/Actions/.Modules/ReadSettings.psm1 @@ -251,9 +251,10 @@ function GetDefaultSettings "reportSuppressedDiagnostics" = $false "workflowDefaultInputs" = @() "customALGoFiles" = [ordered]@{ - "filesToInclude" = @() + "filesToInclude" = @() "filesToExclude" = @() } + "postponeProjectInBuildOrder" = $false } } diff --git a/Actions/.Modules/settings.schema.json b/Actions/.Modules/settings.schema.json index eafcb7bb64..af08720d34 100644 --- a/Actions/.Modules/settings.schema.json +++ b/Actions/.Modules/settings.schema.json @@ -744,6 +744,10 @@ } } }, + "postponeProjectInBuildOrder": { + "type": "boolean", + "description": "Indicates whether the project can be postponed in the build order to optimize build times. See https://aka.ms/ALGoSettings#postponeProjectInBuildOrder" + }, "workflowDefaultInputs": { "type": "array", "items": { diff --git a/Actions/AL-Go-Helper.ps1 b/Actions/AL-Go-Helper.ps1 index 82829f4875..5c97759506 100644 --- a/Actions/AL-Go-Helper.ps1 +++ b/Actions/AL-Go-Helper.ps1 @@ -1737,10 +1737,14 @@ Function AnalyzeProjectDependencies { # Loop through all projects # Get all apps in the project # Get all dependencies for the apps + $projectsThatCanBePostponed = @() foreach($project in $projects) { - Write-Host "- Analyzing project: $project" + Write-Host -NoNewline "Analyzing project: $project, " $projectSettings = ReadSettings -project $project -baseFolder $baseFolder + if ($projectSettings.postponeProjectInBuildOrder) { + $projectsThatCanBePostponed += $project + } ResolveProjectFolders -baseFolder $baseFolder -project $project -projectSettings ([ref] $projectSettings) # App folders are relative to the AL-Go project folder. Convert them to relative to the base folder @@ -1754,7 +1758,7 @@ Function AnalyzeProjectDependencies { Pop-Location } - OutputMessageAndArray -Message "Folders containing apps" -arrayOfStrings $folders + OutputMessageAndArray -Message "folders containing apps" -arrayOfStrings $folders $unknownDependencies = @() $apps = @() @@ -1785,6 +1789,11 @@ Function AnalyzeProjectDependencies { # } $no = 1 $projectsOrder = @() + # Collect projects without dependants, which can be build later + # This is done to avoid building projects at an earlier stage than needed and increase the time until next job subsequently + # For every time we have determined a set of projects that can be build in parallel, we check whether any of these projects has no dependants + # If so, we remove these projects from the build order and add them at the end of the build order (by adding them to projectsWithoutDependants) + $projectsWithoutDependents = @() Write-Host "Analyzing dependencies" while ($projects.Count -gt 0) { $thisJob = @() @@ -1799,6 +1808,11 @@ Function AnalyzeProjectDependencies { # Loop through all dependencies and locate the projects, containing the apps for which the current project has a dependency $foundDependencies = @() foreach($dependency in $dependencies) { + # Check whether dependency is already resolved by a previous build project + $depProject = @($projectsOrder | ForEach-Object { $_.Projects | Where-Object { $_ -ne $project -and $appDependencies."$_".apps -contains $dependency } }) + if ($depProject.Count -gt 0) { + continue + } # Find the project that contains the app for which the current project has a dependency $depProjects = @($projects | Where-Object { $_ -ne $project -and $appDependencies."$_".apps -contains $dependency }) # Add this project and all projects on which that project has a dependency to the list of dependencies for the current project @@ -1851,11 +1865,49 @@ Function AnalyzeProjectDependencies { if ($thisJob.Count -eq 0) { throw "Circular project reference encountered, cannot determine build order" } + + # Check whether any of the projects in $thisJob can be built later (has postponeProjectInBuildOrder set to true and no remaining dependents) + $projectsWithoutDependents += @($thisJob | Where-Object { $projectsThatCanBePostponed -contains $_ } | Where-Object { + $hasRemainingDependents = $false + foreach($otherProject in $projects) { + if ($otherProject -ne $_) { + # Grab dependencies from other project, which haven't been build yet + $otherDependencies = $appDependencies."$otherProject".dependencies | Where-Object { + $dependency = $_ + $alreadyBuilt = ($projectsOrder | ForEach-Object { $_.Projects | Where-Object { $appDependencies."$_".apps -contains $dependency } }) + return -not $alreadyBuilt + } + Write-Host "Other project $otherProject has unbuilt dependencies: $($otherDependencies -join ", ")" + foreach($dependency in $otherDependencies) { + if ($appDependencies."$_".apps -contains $dependency) { + Write-Host "Project $_ is still a dependency for project $otherProject" + $hasRemainingDependents = $true + } + } + } + } + if (!$hasRemainingDependents) { + Write-Host "Project $_ has no remaining dependents, can be built later" + } + return -not $hasRemainingDependents + }) + + # Remove projects in this job from the list of projects to be built (including the projects without dependents) + $projects = @($projects | Where-Object { $thisJob -notcontains $_ }) + + # Do not build jobs without dependents until the last job, remove them from this job + $thisJob = @($thisJob | Where-Object { $projectsWithoutDependents -notcontains $_ }) + + if ($projects.Count -eq 0) { + # Last job, add jobs without dependents + Write-Host "Adding jobs without dependents to last build job" + $thisJob += $projectsWithoutDependents + } + Write-Host "#$no - build projects: $($thisJob -join ", ")" $projectsOrder += @{'projects' = $thisJob; 'projectsCount' = $thisJob.Count } - $projects = @($projects | Where-Object { $thisJob -notcontains $_ }) $no++ } diff --git a/Actions/DownloadProjectDependencies/DownloadProjectDependencies.Action.ps1 b/Actions/DownloadProjectDependencies/DownloadProjectDependencies.Action.ps1 index be9f84fb75..6e0f8706e2 100644 --- a/Actions/DownloadProjectDependencies/DownloadProjectDependencies.Action.ps1 +++ b/Actions/DownloadProjectDependencies/DownloadProjectDependencies.Action.ps1 @@ -46,7 +46,7 @@ function DownloadDependenciesFromCurrentBuild { Write-Host "Dependency projects: $($dependencyProjects -join ', ')" # For each dependency project, calculate the corresponding probing path - $dependeciesProbingPaths = @() + $dependenciesProbingPaths = @() foreach($dependencyProject in $dependencyProjects) { Write-Host "Reading settings for project '$dependencyProject'" $dependencyProjectSettings = ReadSettings -baseFolder $baseFolder -project $dependencyProject @@ -70,7 +70,7 @@ function DownloadDependenciesFromCurrentBuild { $baseBranch = $ENV:GITHUB_REF_NAME } - $dependeciesProbingPaths += @(@{ + $dependenciesProbingPaths += @(@{ "release_status" = "thisBuild" "version" = "latest" "buildMode" = $dependencyBuildMode @@ -85,7 +85,7 @@ function DownloadDependenciesFromCurrentBuild { # For each probing path, download the dependencies $downloadedDependencies = @() - foreach($probingPath in $dependeciesProbingPaths) { + foreach($probingPath in $dependenciesProbingPaths) { $buildMode = $probingPath.buildMode $project = $probingPath.projects $branch = $probingPath.branch diff --git a/Actions/Github-Helper.psm1 b/Actions/Github-Helper.psm1 index a6f08091b6..4b46d44e37 100644 --- a/Actions/Github-Helper.psm1 +++ b/Actions/Github-Helper.psm1 @@ -141,9 +141,16 @@ function GetDependencies { $projects = $dependency.projects $buildMode = $dependency.buildMode + if ($mask -eq 'TestApps') { + $altMask = 'Apps' + } + else { + $altMask = 'TestApps' + } # change the mask to include the build mode if($buildMode -ne "Default") { $mask = "$buildMode$mask" + $altMask = "$buildMode$altMask" } Write-Host "Locating $mask artifacts for projects: $projects" @@ -169,8 +176,15 @@ function GetDependencies { } } elseif ($mask -like '*Apps') { - Write-Host "$project not built, downloading from artifacts" - $missingProjects += @($project) + # Check whether Apps/TestApps exists before determining that project isn't built + $altDownloadName = Join-Path $saveToPath "$project-$branchName-$altMask-*" + if (!(Test-Path $altDownloadName -PathType Container)) { + Write-Host "$project not built, downloading from artifacts" + $missingProjects += @($project) + } + else { + Write-Host "$project built, but $mask not found" + } } } if ($missingProjects -and $dependency.baselineWorkflowID) { @@ -1015,6 +1029,11 @@ function GetArtifactsFromWorkflowRun { # Get sanitized project names (the way they appear in the artifact names) $projectArr = @(@($projects.Split(',')) | ForEach-Object { $_.Replace('\','_').Replace('/','_') }) + # Get branch used in workflowRun + $workflowRunInfo = (InvokeWebRequest -Headers $headers -Uri "$api_url/repos/$repository/actions/runs/$workflowRun").Content | ConvertFrom-Json + $branch = $workflowRunInfo.head_branch.Replace('\', '_').Replace('/', '_') + Write-Host "Branch for workflow run $workflowRun is $branch" + # Get the artifacts from the the workflow run while($true) { $artifactsURI = "$api_url/repos/$repository/actions/runs/$workflowRun/artifacts?per_page=$per_page&page=$page" @@ -1026,14 +1045,40 @@ function GetArtifactsFromWorkflowRun { } foreach($project in $projectArr) { - $artifactPattern = "$project-*-$mask-*" # e.g. "MyProject-*-Apps-*", format is: "project-branch-mask-version" + # e.g. "MyProject-main-Apps-*", format is: "project-branch-mask-version" + # Mask might include buildMode like TranslatedTestApps + $artifactPattern = "$project-$branch-$mask-*" $matchingArtifacts = @($artifacts.artifacts | Where-Object { $_.name -like $artifactPattern }) if ($matchingArtifacts.Count -eq 0) { continue } - $matchingArtifacts = @($matchingArtifacts) #enforce array + # If there are reruns of the build we found, we might see artifacts like: + # Test DBC-BHG-SAF-T-main-TestApps-1.0.48.0 + # Test DBC-BHG-SAF-T-main-TestApps-1.0.48.1 + # We want to keep only the latest version of each artifact (based on the last segment of the version) + $matchingArtifacts = @($matchingArtifacts | ForEach-Object { + # Sort on version number object + if ($_.name -match '^(.*)-(\d+\.\d+\.\d+\.\d+)$') { + [PSCustomObject]@{ + Name = $Matches[1] + Version = [version]$Matches[2] + Obj = $_ + } + } + else { + # artifacts from PR builds doesn't match the versioning pattern but are sortable + [PSCustomObject]@{ + Name = $_.name + Version = $_.name + Obj = $_ + } + } + } | Group-Object Name | ForEach-Object { + $_.Group | Sort-Object Version -Descending | Select-Object -First 1 + } | + Select-Object -ExpandProperty Obj) foreach($artifact in $matchingArtifacts) { Write-Host "Found artifact $($artifact.name) (ID: $($artifact.id)) for mask $mask and project $project" diff --git a/Actions/PipelineCleanup/PipelineCleanup.ps1 b/Actions/PipelineCleanup/PipelineCleanup.ps1 index 81209b175f..339cb8e767 100644 --- a/Actions/PipelineCleanup/PipelineCleanup.ps1 +++ b/Actions/PipelineCleanup/PipelineCleanup.ps1 @@ -3,10 +3,15 @@ [string] $project = "." ) -. (Join-Path -Path $PSScriptRoot -ChildPath "..\AL-Go-Helper.ps1" -Resolve) -DownloadAndImportBcContainerHelper +try { + . (Join-Path -Path $PSScriptRoot -ChildPath "..\AL-Go-Helper.ps1" -Resolve) + DownloadAndImportBcContainerHelper -if ($project -eq ".") { $project = "" } + if ($project -eq ".") { $project = "" } -$containerName = GetContainerName($project) -Remove-Bccontainer $containerName + $containerName = GetContainerName($project) + Remove-Bccontainer $containerName +} +catch { + Write-Host "Pipeline Cleanup failed: $($_.Exception.Message)" +} diff --git a/RELEASENOTES.md b/RELEASENOTES.md index 4b9595bae1..814e307b29 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -1,5 +1,8 @@ ### Issues +- Issue 2084 Multiple artifacts failure if you re-run failed jobs after flaky tests +- Issue 2085 Projects that doesn't contain both Apps and TestApps are wrongly seen as not built. +- Issue 2086 Postpone jobs, which doesn't have any dependents to the end of the build order. - Issue 2082 Sign action no longer fails when repository is empty or no artifacts are generated - Issue 2078 Workflows run since January 14th '26 have space before CI/CD removed - Issue 2070 Support public GitHub Packages feeds without requiring a Personal Access Token (PAT) @@ -9,6 +12,10 @@ - AL-Go repositories with large amounts of projects may run into issues with too large environment variables - Discussion 1855 Add trigger 'workflow_call' to workflow 'Update AL-Go System Files' for reusability +### New Settings + +- `postponeProjectInBuildOrder` is a new project setting, which will (if set to true) cause the project to be postponed until the last build job when possible. If set on test projects, then all tests can be deferred until all builds have succeeded. + ### Set default values for workflow inputs The `workflowDefaultInputs` setting now also applies to `workflow_call` inputs when an input with the same name exists for `workflow_dispatch`. diff --git a/Scenarios/settings.md b/Scenarios/settings.md index 6f28ee5b0a..f8b2a2d586 100644 --- a/Scenarios/settings.md +++ b/Scenarios/settings.md @@ -48,6 +48,7 @@ When running a workflow or a local script, the settings are applied by reading s | appDependencyProbingPaths | Array of dependency specifications, from which apps will be downloaded when the CI/CD workflow is starting. Every dependency specification consists of the following properties:
**repo** = repository
**version** = version (default latest)
**release_status** = latestBuild/release/prerelease/draft (default release)
**projects** = projects (default * = all)
**branch** = branch (default main)
**AuthTokenSecret** = Name of secret containing auth token (default none)
| [ ] | | preprocessorSymbols | List of preprocessor symbols to use when building the apps. This setting can be specified in [workflow specific settings files](https://aka.ms/algosettings#where-are-the-settings-located) or in [conditional settings](https://aka.ms/algosettings#conditional-settings). | [ ] | | bcptThresholds | Structure with properties for the thresholds when running performance tests using the Business Central Performance Toolkit.
**DurationWarning** = a warning is shown if the duration of a bcpt test degrades more than this percentage (default 10)
**DurationError** - an error is shown if the duration of a bcpt test degrades more than this percentage (default 25)
**NumberOfSqlStmtsWarning** - a warning is shown if the number of SQL statements from a bcpt test increases more than this percentage (default 5)
**NumberOfSqlStmtsError** - an error is shown if the number of SQL statements from a bcpt test increases more than this percentage (default 10)
*Note that errors and warnings on the build in GitHub are only issued when a threshold is exceeded on the codeunit level, when an individual operation threshold is exceeded, it is only shown in the test results viewer.* | +| postponeProjectInBuildOrder | Setting this project setting to true will cause the project to be postponed until the last build job when possible (i.e. if no other projects depends on it). If set on test projects, then all tests can be deferred until all builds have succeeded. | false | ## AppSource specific basic project settings diff --git a/Tests/DetermineProjectsToBuild.Test.ps1 b/Tests/DetermineProjectsToBuild.Test.ps1 index 8445567cc8..575962910d 100644 --- a/Tests/DetermineProjectsToBuild.Test.ps1 +++ b/Tests/DetermineProjectsToBuild.Test.ps1 @@ -747,6 +747,131 @@ Describe "Get-ProjectsToBuild" { { Get-ProjectsToBuild -baseFolder $baseFolder -maxBuildDepth 1 } | Should -Throw "The build depth is too deep, the maximum build depth is 1. You need to run 'Update AL-Go System Files' to update the workflows" } + It 'postpones projects if postponeProjectInBuildOrder is set to true' { + # Add three dependent projects + # Project 1 + # Project 2 depends on Project 1, has postponeProjectInBuildOrder set to true + # Project 3 depends on Project 1, has postponeProjectInBuildOrder set to true + # Project 4 depends on Project 2 + $dependecyAppFile = @{ id = '83fb8305-4079-415d-a25d-8132f0436fd1'; name = 'First App'; publisher = 'Contoso'; version = '1.0.0.0'; dependencies = @() } + New-Item -Path "$baseFolder/Project1/.AL-Go/settings.json" -type File -Force + New-Item -Path "$baseFolder/Project1/app/app.json" -Value (ConvertTo-Json $dependecyAppFile -Depth 10) -type File -Force + + $dependantAppFile = @{ id = '83fb8305-4079-415d-a25d-8132f0436fd2'; name = 'Second App'; publisher = 'Contoso'; version = '1.0.0.0'; dependencies = @(@{id = '83fb8305-4079-415d-a25d-8132f0436fd1'; name = 'First App'; publisher = 'Contoso'; version = '1.0.0.0'} ) } + New-Item -Path "$baseFolder/Project2/.AL-Go/settings.json" -type File -Force + @{ postponeProjectInBuildOrder = $true } | ConvertTo-Json -Depth 99 -Compress | Out-File (Join-Path $baseFolder "Project2/.AL-Go/settings.json") -Encoding UTF8 + New-Item -Path "$baseFolder/Project2/app/app.json" -Value (ConvertTo-Json $dependantAppFile -Depth 10) -type File -Force + + $dependantAppFile3 = @{ id = '83fb8305-4079-415d-a25d-8132f0436fd3'; name = 'Third App'; publisher = 'Contoso'; version = '1.0.0.0'; dependencies = @(@{id = '83fb8305-4079-415d-a25d-8132f0436fd1'; name = 'First App'; publisher = 'Contoso'; version = '1.0.0.0'} ) } + New-Item -Path "$baseFolder/Project3/.AL-Go/settings.json" -type File -Force + @{ postponeProjectInBuildOrder = $true } | ConvertTo-Json -Depth 99 -Compress | Out-File (Join-Path $baseFolder "Project3/.AL-Go/settings.json") -Encoding UTF8 + New-Item -Path "$baseFolder/Project3/app/app.json" -Value (ConvertTo-Json $dependantAppFile3 -Depth 10) -type File -Force + + $depedantAppFile4 = @{ id = '83fb8305-4079-415d-a25d-8132f0436fd4'; name = 'Fourth App'; publisher = 'Contoso'; version = '1.0.0.0'; dependencies = @(@{id = '83fb8305-4079-415d-a25d-8132f0436fd2'; name = 'Second App'; publisher = 'Contoso'; version = '1.0.0.0'} ) } + New-Item -Path "$baseFolder/Project4/.AL-Go/settings.json" -type File -Force + New-Item -Path "$baseFolder/Project4/app/app.json" -Value (ConvertTo-Json $depedantAppFile4 -Depth 10) -type File -Force + + #Add settings file + $alGoSettings = @{ fullBuildPatterns = @(); projects = @(); powerPlatformSolutionFolder = ''; useProjectDependencies = $true } + New-Item -Path "$baseFolder/.github" -type Directory -Force + $alGoSettings | ConvertTo-Json -Depth 99 -Compress | Out-File (Join-Path $baseFolder ".github/AL-Go-Settings.json") -Encoding UTF8 + + # Add settings as environment variable to simulate we've run ReadSettings + $env:Settings = ConvertTo-Json $alGoSettings -Depth 99 -Compress + + $allProjects, $modifiedProjects, $projectsToBuild, $projectDependencies, $buildOrder = Get-ProjectsToBuild -baseFolder $baseFolder + + $allProjects | Should -BeExactly @("Project1", "Project2", "Project3", "Project4") + $modifiedProjects | Should -BeExactly @() + $projectsToBuild | Should -BeExactly @('Project1', 'Project2', 'Project3', 'Project4') + + $projectDependencies | Should -BeOfType System.Collections.Hashtable + $projectDependencies['Project1'] | Should -BeExactly @() + $projectDependencies['Project2'] | Should -BeExactly @("Project1") + $projectDependencies['Project3'] | Should -BeExactly @("Project1") + $projectDependencies['Project4'] | Should -BeExactly @("Project2", "Project1") + + # Build order should have the following structure: + #[ + #{ + # "buildDimensions": [ + # { + # "projectName": "Project1", + # "buildMode": "Default", + # "project": "Project1", + # "githubRunnerShell": "powershell", + # "gitHubRunner": "\"windows-latest\"" + # }, + # ], + # "projectsCount": 1, + # "projects": [ + # "Project1" + # ] + #}, + #{ + # "buildDimensions": [ + # { + # "projectName": "Project2", + # "buildMode": "Default", + # "project": "Project2", + # "githubRunnerShell": "powershell", + # "gitHubRunner": "\"windows-latest\"" + # } + # ], + # "projectsCount": 1, + # "projects": [ + # "Project2" + # ] + #} + #{ + # "buildDimensions": [ + # { + # "projectName": "Project3", + # "buildMode": "Default", + # "project": "Project3", + # "githubRunnerShell": "powershell", + # "gitHubRunner": "\"windows-latest\"" + # }, + # { + # "projectName": "Project4", + # "buildMode": "Default", + # "project": "Project4", + # "githubRunnerShell": "powershell", + # "gitHubRunner": "\"windows-latest\"" + # } + # ], + # "projectsCount": 2, + # "projects": [ + # "Project3", + # "Project4" + # ] + #} + #] + $buildOrder.Count | Should -BeExactly 3 + $buildOrder[0] | Should -BeOfType System.Collections.Hashtable + $buildOrder[0].projects | Should -BeExactly @("Project1") + $buildOrder[0].projectsCount | Should -BeExactly 1 + $buildOrder[0].buildDimensions.Count | Should -BeExactly 1 + $buildOrder[0].buildDimensions[0].buildMode | Should -BeExactly "Default" + $buildOrder[0].buildDimensions[0].project | Should -BeExactly "Project1" + + $buildOrder[1] | Should -BeOfType System.Collections.Hashtable + $buildOrder[1].projects | Should -BeExactly @("Project2") + $buildOrder[1].projectsCount | Should -BeExactly 1 + $buildOrder[1].buildDimensions.Count | Should -BeExactly 1 + $buildOrder[1].buildDimensions[0].buildMode | Should -BeExactly "Default" + $buildOrder[1].buildDimensions[0].project | Should -BeExactly "Project2" + + $buildOrder[2] | Should -BeOfType System.Collections.Hashtable + $buildOrder[2].projects | Should -BeExactly @("Project4", "Project3") + $buildOrder[2].projectsCount | Should -BeExactly 2 + $buildOrder[2].buildDimensions.Count | Should -BeExactly 2 + $buildOrder[2].buildDimensions[0].buildMode | Should -BeExactly "Default" + $buildOrder[2].buildDimensions[0].project | Should -BeExactly "Project4" + $buildOrder[2].buildDimensions[1].buildMode | Should -BeExactly "Default" + $buildOrder[2].buildDimensions[1].project | Should -BeExactly "Project3" + } + AfterEach { Remove-Item $baseFolder -Force -Recurse }