[release/9.0] Update dependencies from dotnet/arcade

eng/Version.Details.xml

@@ -13,18 +13,18 @@
       <Sha>02fe27cd6a9b001c8feb7938e6ef4b3799745759</Sha>
       <SourceBuild RepoName="command-line-api" ManagedOnly="true" />
     </Dependency>
-    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="9.0.0-beta.24368.9">
+    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="9.0.0-beta.24572.2">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>b4e499d1f6e6b3a981feabfed290d99261762382</Sha>
+      <Sha>b41381d5cd633471265e9cd72e933a7048e03062</Sha>
     </Dependency>
-    <Dependency Name="Microsoft.DotNet.XliffTasks" Version="9.0.0-beta.24368.9">
+    <Dependency Name="Microsoft.DotNet.XliffTasks" Version="9.0.0-beta.24572.2">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>b4e499d1f6e6b3a981feabfed290d99261762382</Sha>
+      <Sha>b41381d5cd633471265e9cd72e933a7048e03062</Sha>
     </Dependency>
     <!-- Intermediate is necessary for source build. -->
-    <Dependency Name="Microsoft.SourceBuild.Intermediate.arcade" Version="9.0.0-beta.24368.9">
+    <Dependency Name="Microsoft.SourceBuild.Intermediate.arcade" Version="9.0.0-beta.24572.2">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>b4e499d1f6e6b3a981feabfed290d99261762382</Sha>
+      <Sha>b41381d5cd633471265e9cd72e933a7048e03062</Sha>
       <SourceBuild RepoName="arcade" ManagedOnly="true" />
     </Dependency>
     <!-- Intermediate is necessary for source build. -->

eng/common/SetupNugetSources.ps1

@@ -157,7 +157,7 @@ if ($dotnet31Source -ne $null) {
     AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-transport/nuget/v2" -Creds $creds -Username $userName -pwd $Password
 }
 
-$dotnetVersions = @('5','6','7','8')
+$dotnetVersions = @('5','6','7','8','9')
 
 foreach ($dotnetVersion in $dotnetVersions) {
     $feedPrefix = "dotnet" + $dotnetVersion;

eng/common/SetupNugetSources.sh

@@ -99,7 +99,7 @@ if [ "$?" == "0" ]; then
     PackageSources+=('dotnet3.1-internal-transport')
 fi
 
-DotNetVersions=('5' '6' '7' '8')
+DotNetVersions=('5' '6' '7' '8' '9')
 
 for DotNetVersion in ${DotNetVersions[@]} ; do
     FeedPrefix="dotnet${DotNetVersion}";

eng/common/core-templates/job/job.yml

@@ -33,11 +33,6 @@ parameters:
   artifactPublishSteps: []
   runAsPublic: false
 
-# Sbom related params
-  enableSbom: true
-  PackageVersion: 9.0.0
-  BuildDropPath: '$(Build.SourcesDirectory)/artifacts'
-
 # 1es specific parameters
   is1ESPipeline: ''
 

eng/common/core-templates/job/source-index-stage1.yml

@@ -34,10 +34,12 @@ jobs:
     pool:
       ${{ if eq(variables['System.TeamProject'], 'public') }}:
         name: $(DncEngPublicBuildPool)
-        image: windows.vs2022.amd64.open
+        image: 1es-windows-2022-open
+        os: windows
       ${{ if eq(variables['System.TeamProject'], 'internal') }}:
         name: $(DncEngInternalBuildPool)
-        image: windows.vs2022.amd64
+        image: 1es-windows-2022
+        os: windows
 
   steps:
   - ${{ if eq(parameters.is1ESPipeline, '') }}:

eng/common/core-templates/steps/get-delegation-sas.yml

@@ -31,7 +31,16 @@ steps:
       # Calculate the expiration of the SAS token and convert to UTC
       $expiry = (Get-Date).AddHours(${{ parameters.expiryInHours }}).ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:ssZ")
 
-      $sas = az storage container generate-sas --account-name ${{ parameters.storageAccount }} --name ${{ parameters.container }} --permissions ${{ parameters.permissions }} --expiry $expiry --auth-mode login --as-user -o tsv
+      # Temporarily work around a helix issue where SAS tokens with / in them will cause incorrect downloads
+      # of correlation payloads. https://github.com/dotnet/dnceng/issues/3484
+      $sas = ""
+      do {
+        $sas = az storage container generate-sas --account-name ${{ parameters.storageAccount }} --name ${{ parameters.container }} --permissions ${{ parameters.permissions }} --expiry $expiry --auth-mode login --as-user -o tsv
+        if ($LASTEXITCODE -ne 0) {
+          Write-Error "Failed to generate SAS token."
+          exit 1
+        }
+      } while($sas.IndexOf('/') -ne -1)
 
       if ($LASTEXITCODE -ne 0) {
         Write-Error "Failed to generate SAS token."

eng/common/cross/build-rootfs.sh

@@ -605,34 +605,34 @@ elif [[ "$__CodeName" == "illumos" ]]; then
     fi
     echo "Building binutils. Please wait.."
     if [[ "$__hasWget" == 1 ]]; then
-        wget -O- https://ftp.gnu.org/gnu/binutils/binutils-2.33.1.tar.bz2 | tar -xjf -
+        wget -O- https://ftp.gnu.org/gnu/binutils/binutils-2.42.tar.xz | tar -xJf -
     else
-        curl -SL https://ftp.gnu.org/gnu/binutils/binutils-2.33.1.tar.bz2 | tar -xjf -
+        curl -SL https://ftp.gnu.org/gnu/binutils/binutils-2.42.tar.xz | tar -xJf -
     fi
     mkdir build-binutils && cd build-binutils
-    ../binutils-2.33.1/configure --prefix="$__RootfsDir" --target="${__illumosArch}-sun-solaris2.10" --program-prefix="${__illumosArch}-illumos-" --with-sysroot="$__RootfsDir"
+    ../binutils-2.42/configure --prefix="$__RootfsDir" --target="${__illumosArch}-sun-solaris2.11" --program-prefix="${__illumosArch}-illumos-" --with-sysroot="$__RootfsDir"
     make -j "$JOBS" && make install && cd ..
     echo "Building gcc. Please wait.."
     if [[ "$__hasWget" == 1 ]]; then
-        wget -O- https://ftp.gnu.org/gnu/gcc/gcc-8.4.0/gcc-8.4.0.tar.xz | tar -xJf -
+        wget -O- https://ftp.gnu.org/gnu/gcc/gcc-13.3.0/gcc-13.3.0.tar.xz | tar -xJf -
     else
-        curl -SL https://ftp.gnu.org/gnu/gcc/gcc-8.4.0/gcc-8.4.0.tar.xz | tar -xJf -
+        curl -SL https://ftp.gnu.org/gnu/gcc/gcc-13.3.0/gcc-13.3.0.tar.xz | tar -xJf -
     fi
     CFLAGS="-fPIC"
     CXXFLAGS="-fPIC"
     CXXFLAGS_FOR_TARGET="-fPIC"
     CFLAGS_FOR_TARGET="-fPIC"
     export CFLAGS CXXFLAGS CXXFLAGS_FOR_TARGET CFLAGS_FOR_TARGET
     mkdir build-gcc && cd build-gcc
-    ../gcc-8.4.0/configure --prefix="$__RootfsDir" --target="${__illumosArch}-sun-solaris2.10" --program-prefix="${__illumosArch}-illumos-" --with-sysroot="$__RootfsDir" --with-gnu-as       \
+    ../gcc-13.3.0/configure --prefix="$__RootfsDir" --target="${__illumosArch}-sun-solaris2.11" --program-prefix="${__illumosArch}-illumos-" --with-sysroot="$__RootfsDir" --with-gnu-as       \
         --with-gnu-ld --disable-nls --disable-libgomp --disable-libquadmath --disable-libssp --disable-libvtv --disable-libcilkrts --disable-libada --disable-libsanitizer \
         --disable-libquadmath-support --disable-shared --enable-tls
     make -j "$JOBS" && make install && cd ..
     BaseUrl=https://pkgsrc.smartos.org
     if [[ "$__UseMirror" == 1 ]]; then
         BaseUrl=https://pkgsrc.smartos.skylime.net
     fi
-    BaseUrl="$BaseUrl/packages/SmartOS/trunk/${__illumosArch}/All"
+    BaseUrl="$BaseUrl/packages/SmartOS/2019Q4/${__illumosArch}/All"
     echo "Downloading manifest"
     if [[ "$__hasWget" == 1 ]]; then
         wget "$BaseUrl"

eng/common/darc-init.ps1

@@ -1,6 +1,6 @@
 param (
     $darcVersion = $null,
-    $versionEndpoint = 'https://maestro.dot.net/api/assets/darc-version?api-version=2019-01-16',
+    $versionEndpoint = 'https://maestro.dot.net/api/assets/darc-version?api-version=2020-02-20',
     $verbosity = 'minimal',
     $toolpath = $null
 )

eng/common/darc-init.sh

@@ -2,7 +2,7 @@
 
 source="${BASH_SOURCE[0]}"
 darcVersion=''
-versionEndpoint='https://maestro.dot.net/api/assets/darc-version?api-version=2019-01-16'
+versionEndpoint='https://maestro.dot.net/api/assets/darc-version?api-version=2020-02-20'
 verbosity='minimal'
 
 while [[ $# > 0 ]]; do

eng/common/internal/Tools.csproj

@@ -4,6 +4,7 @@
   <PropertyGroup>
     <TargetFramework>net472</TargetFramework>
     <AutomaticallyUseReferenceAssemblyPackages>false</AutomaticallyUseReferenceAssemblyPackages>
+    <BuildWithNetFrameworkHostedCompiler>false</BuildWithNetFrameworkHostedCompiler>
   </PropertyGroup>
   <ItemGroup>
     <!-- Clear references, the SDK may add some depending on UsuingToolXxx settings, but we only want to restore the following -->

eng/common/native/init-compiler.sh

@@ -19,19 +19,19 @@ case "$compiler" in
         # clangx.y or clang-x.y
         version="$(echo "$compiler" | tr -d '[:alpha:]-=')"
         majorVersion="${version%%.*}"
-        [ -z "${version##*.*}" ] && minorVersion="${version#*.}"
 
-        if [ -z "$minorVersion" ] && [ -n "$majorVersion" ] && [ "$majorVersion" -le 6 ]; then
-            minorVersion=0;
-        fi
+        # LLVM based on v18 released in early 2024, with two releases per year
+        maxVersion="$((18 + ((($(date +%Y) - 2024) * 12 + $(date +%-m) - 3) / 6)))"
         compiler=clang
         ;;
 
     gcc*|-gcc*|--gcc*)
         # gccx.y or gcc-x.y
         version="$(echo "$compiler" | tr -d '[:alpha:]-=')"
         majorVersion="${version%%.*}"
-        [ -z "${version##*.*}" ] && minorVersion="${version#*.}"
+
+        # GCC based on v14 released in early 2024, with one release per year
+        maxVersion="$((14 + ((($(date +%Y) - 2024) * 12 + $(date +%-m) - 3) / 12)))"
         compiler=gcc
         ;;
 esac
@@ -49,12 +49,10 @@ check_version_exists() {
     desired_version=-1
 
     # Set up the environment to be used for building with the desired compiler.
-    if command -v "$compiler-$1.$2" > /dev/null; then
-        desired_version="-$1.$2"
-    elif command -v "$compiler$1$2" > /dev/null; then
-        desired_version="$1$2"
-    elif command -v "$compiler-$1$2" > /dev/null; then
-        desired_version="-$1$2"
+    if command -v "$compiler-$1" > /dev/null; then
+        desired_version="-$1"
+    elif command -v "$compiler$1" > /dev/null; then
+        desired_version="$1"
     fi
 
     echo "$desired_version"
@@ -75,7 +73,7 @@ set_compiler_version_from_CC() {
     fi
 
     # gcc and clang often display 3 part versions. However, gcc can show only 1 part in some environments.
-    IFS=. read -r majorVersion minorVersion _ <<EOF
+    IFS=. read -r majorVersion _ <<EOF
 $version
 EOF
 }
@@ -84,43 +82,29 @@ if [ -z "$CLR_CC" ]; then
 
     # Set default versions
     if [ -z "$majorVersion" ]; then
-        # note: gcc (all versions) and clang versions higher than 6 do not have minor version in file name, if it is zero.
-        if [ "$compiler" = "clang" ]; then versions="18 17 16 15 14 13 12 11 10 9 8 7 6.0 5.0 4.0 3.9 3.8 3.7 3.6 3.5"
-        elif [ "$compiler" = "gcc" ]; then versions="14 13 12 11 10 9 8 7 6 5 4.9"; fi
-
-        for version in $versions; do
-            _major="${version%%.*}"
-            [ -z "${version##*.*}" ] && _minor="${version#*.}"
-            desired_version="$(check_version_exists "$_major" "$_minor")"
-            if [ "$desired_version" != "-1" ]; then majorVersion="$_major"; break; fi
+        minVersion=8
+        maxVersion="$((maxVersion + 1))" # +1 for headspace
+        i="$maxVersion"
+        while [ "$i" -ge $minVersion ]; do
+            desired_version="$(check_version_exists "$i")"
+            if [ "$desired_version" != "-1" ]; then majorVersion="$i"; break; fi
+            i=$((i - 1))
         done
 
         if [ -z "$majorVersion" ]; then
             if ! command -v "$compiler" > /dev/null; then
-                echo "Error: No usable version of $compiler found."
+                echo "Error: No compatible version of $compiler was found within the range of $minVersion to $maxVersion. Please upgrade your toolchain or specify the compiler explicitly using CLR_CC and CLR_CXX environment variables."
                 exit 1
             fi
 
             CC="$(command -v "$compiler" 2> /dev/null)"
             CXX="$(command -v "$cxxCompiler" 2> /dev/null)"
             set_compiler_version_from_CC
-        else
-            if [ "$compiler" = "clang" ] && [ "$majorVersion" -lt 5 ] && { [ "$build_arch" = "arm" ] || [ "$build_arch" = "armel" ]; }; then
-                # If a major version was provided explicitly, and it was too old, find a newer compiler instead
-                if ! command -v "$compiler" > /dev/null; then
-                    echo "Error: Found clang version $majorVersion which is not supported on arm/armel architectures, and there is no clang in PATH."
-                    exit 1
-                fi
-
-                CC="$(command -v "$compiler" 2> /dev/null)"
-                CXX="$(command -v "$cxxCompiler" 2> /dev/null)"
-                set_compiler_version_from_CC
-            fi
         fi
     else
-        desired_version="$(check_version_exists "$majorVersion" "$minorVersion")"
+        desired_version="$(check_version_exists "$majorVersion")"
         if [ "$desired_version" = "-1" ]; then
-            echo "Error: Could not find specific version of $compiler: $majorVersion $minorVersion."
+            echo "Error: Could not find specific version of $compiler: $majorVersion."
             exit 1
         fi
     fi

eng/common/sdk-task.ps1

@@ -64,7 +64,7 @@ try {
       $GlobalJson.tools | Add-Member -Name "vs" -Value (ConvertFrom-Json "{ `"version`": `"16.5`" }") -MemberType NoteProperty
     }
     if( -not ($GlobalJson.tools.PSObject.Properties.Name -match "xcopy-msbuild" )) {
-      $GlobalJson.tools | Add-Member -Name "xcopy-msbuild" -Value "17.10.0-pre.4.0" -MemberType NoteProperty
+      $GlobalJson.tools | Add-Member -Name "xcopy-msbuild" -Value "17.12.0" -MemberType NoteProperty
     }
     if ($GlobalJson.tools."xcopy-msbuild".Trim() -ine "none") {
         $xcopyMSBuildToolsFolder = InitializeXCopyMSBuild $GlobalJson.tools."xcopy-msbuild" -install $true

eng/common/templates-official/job/job.yml

@@ -1,8 +1,23 @@
+parameters:
+# Sbom related params
+  enableSbom: true
+  runAsPublic: false
+  PackageVersion: 9.0.0
+  BuildDropPath: '$(Build.SourcesDirectory)/artifacts'
+
 jobs:
 - template: /eng/common/core-templates/job/job.yml
   parameters:
     is1ESPipeline: true
 
+    componentGovernanceSteps:
+    - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), eq(parameters.enableSbom, 'true')) }}:
+      - template: /eng/common/templates/steps/generate-sbom.yml
+        parameters:
+          PackageVersion: ${{ parameters.packageVersion }}
+          BuildDropPath: ${{ parameters.buildDropPath }}
+          publishArtifacts: false
+
     # publish artifacts
     # for 1ES managed templates, use the templateContext.output to handle multiple outputs.
     templateContext: