Fixed official builds for Arcade SDK (#5512)

* Added fixes for official builds

* Make .sh files executable

* fix mkl nuget issue

Co-authored-by: Frank Dong <[email protected]>

Author: mstfbl

build/publish.proj

@@ -1,23 +1,28 @@
 <Project>
+
+  <ItemGroup>
+    <PackageReference Include="Microsoft.DotNet.BuildTools" Version="$(BuildToolsPackageVersion)" />
+  </ItemGroup>
+  
   <Import Project="$(MSBuildThisFileDirectory)..\Directory.Build.props" />
 
-  <UsingTask TaskName="ExecWithRetriesForNuGetPush" AssemblyFile="$(ToolsDir)Microsoft.DotNet.Build.Tasks.dll" />
+  <UsingTask TaskName="ExecWithRetriesForNuGetPush" AssemblyFile="$(NuGetPackageRoot)\microsoft.dotnet.buildtools\$(BuildToolsPackageVersion)\lib\Microsoft.DotNet.Build.Tasks.dll" />
 
   <PropertyGroup>
     <PublishSymbolsPackage>Microsoft.SymbolUploader.Build.Task</PublishSymbolsPackage>
     <EnablePublishSymbols Condition="'$(EnablePublishSymbols)'==''" >true</EnablePublishSymbols>
     <NuGetPushTimeoutSeconds Condition="'$(NuGetPushTimeoutSeconds)' == ''">600</NuGetPushTimeoutSeconds>
   </PropertyGroup>
 
-  <Import Project="$(PackagesDir)\$(PublishSymbolsPackage.ToLower())\$(PublishSymbolsPackageVersion)\build\PublishSymbols.targets" />
+  <Import Project="$(NuGetPackageRoot)\$(PublishSymbolsPackage.ToLower())\$(MicrosoftSymbolUploaderBuildTaskVersion)\build\PublishSymbols.targets" />
 
   <Target Name="PublishPackages">
     <Error Condition="'$(NuGetFeedUrl)' == ''" Text="Missing required property NuGetFeedUrl" />
     <Error Condition="'$(NuGetApiKey)' == ''" Text="Missing required property NuGetApiKey" />
 
     <ItemGroup>
-      <NuGetPackages Include="$(PackageOutputPath)**\*.nupkg"
-                     Exclude="$(PackageOutputPath)**\*.symbols.*nupkg" />
+      <NuGetPackages Include="$(ArtifactsDir)packages\**\*.nupkg"
+                     Exclude="$(ArtifactsDir)packages\**\*.snupkg" />
 
       <!--
       IgnorableErrorMessages applies to the "ExectWithRetriesForNuGetPush" task.
@@ -34,7 +39,7 @@
     <Message Text="Pushing ML.NET packages to $(NuGetFeedUrl)" />
 
     <PropertyGroup>
-      <DotnetToolCommand>$(ToolsDir)dotnetcli/dotnet</DotnetToolCommand>
+      <DotnetToolCommand>$(MSBuildThisFileDirectory)..\.dotnet\dotnet</DotnetToolCommand>
       <NuGetPushCommand>$(DotnetToolCommand) nuget push --source $(NuGetFeedUrl) --api-key $(NuGetApiKey) --timeout $(NuGetPushTimeoutSeconds)</NuGetPushCommand>
     </PropertyGroup>
 
@@ -46,7 +51,7 @@
           Condition="'$(EnablePublishSymbols)'=='true'"
           DependsOnTargets="SetupPublishSymbols">
     <Message Text="Attempting to Publish Symbols" />
-    <Error Condition="!Exists('$(PackageOutputPath)')" Text="'PackageOutputPath' folder '$(PackageOutputPath)' does not exist."/>
+    <Error Condition="!Exists('$(ArtifactsDir)packages')" Text="'PackageOutputPath' folder '$(PackageOutputPath)' does not exist."/>
     <Error Condition="'$(SymbolServerPath)'==''" Text="Missing property SymbolServerPath" />
     <Error Condition="'$(SymbolServerPAT)'==''" Text="Missing property SymbolServerPAT" />
     <CallTarget Targets="PublishSymbols" />
@@ -59,11 +64,11 @@
     </PropertyGroup>
 
     <ItemGroup>
-      <SymbolPackagesToPublish Include="$(PackageOutputPath)**\*.symbols.*nupkg" />
+      <SymbolPackagesToPublish Include="$(ArtifactsDir)packages\**\*.snupkg" />
     </ItemGroup>
 
     <Message Importance="High" Text="Publishing @(SymbolPackagesToPublish) to $(SymbolServerPath)"/>
     <Error Condition="'@(SymbolPackagesToPublish)'==''" Text="There are no symbol nuget packages to publish" />
   </Target>
 
-</Project>
+</Project>

build/sign.proj

@@ -23,14 +23,18 @@ phases:
     - agent.os -equals linux
     container: CentosContainer
   steps:
+  - script: ./restore.sh
+    displayName: restore all projects
+  - script: ./build.sh -configuration $(BuildConfig) /p:SkipRIDAgnosticAssets=true -projects $(Build.SourcesDirectory)/src/Redist/Microsoft.ML.DnnImageFeaturizer.ModelRedist/Microsoft.ML.DnnImageFeaturizer.ModelRedist.proj
+    displayName: build redist
   # Only build native assets to avoid conflicts.
-  - script: ./build.sh -buildNative -$(BuildConfig) -skipRIDAgnosticAssets
+  - script: ./build.sh -configuration $(BuildConfig) -projects $(Build.SourcesDirectory)/src/Native/Native.proj /p:TargetArchitecture=x64
     displayName: Build
 
   - task: PublishBuildArtifacts@1
     displayName: Publish Linux package assets
     inputs:
-      pathToPublish: $(Build.SourcesDirectory)/bin/obj/packages
+      pathToPublish: $(Build.SourcesDirectory)/artifacts/pkgassets
       artifactName: PackageAssets
       artifactType: container
 
@@ -46,23 +50,26 @@ phases:
   queue:
     name: Hosted macOS
   steps:
-  # Work around MacOS Homebrew image/environment bug: https://github.com/actions/virtual-environments/issues/1811
   - script: |
       brew uninstall [email protected] |
       brew uninstall [email protected] |
       brew untap local/openssl |
       brew untap local/python2
     displayName: MacOS Homebrew bug Workaround
-  - script: brew update && brew install $(Build.SourcesDirectory)/build/libomp.rb && brew link libomp --force
+  - script: brew update && brew unlink [email protected] && brew install mono-libgdiplus && brew install $(Build.SourcesDirectory)/build/libomp.rb && brew link libomp --force
     displayName: Install build dependencies
+  - script: ./restore.sh
+    displayName: restore all projects
+  - script: ./build.sh -configuration $(BuildConfig) /p:SkipRIDAgnosticAssets=true -projects $(Build.SourcesDirectory)/src/Redist/Microsoft.ML.DnnImageFeaturizer.ModelRedist/Microsoft.ML.DnnImageFeaturizer.ModelRedist.proj
+    displayName: build redist
   # Only build native assets to avoid conflicts.
-  - script: ./build.sh -buildNative -$(BuildConfig) -skipRIDAgnosticAssets
+  - script: ./build.sh -projects $(Build.SourcesDirectory)/src/Native/Native.proj -configuration $(BuildConfig) /p:TargetArchitecture=x64
     displayName: Build
 
   - task: PublishBuildArtifacts@1
     displayName: Publish macOS package assets
     inputs:
-      pathToPublish: $(Build.SourcesDirectory)/bin/obj/packages
+      pathToPublish: $(Build.SourcesDirectory)/artifacts/pkgassets
       artifactName: PackageAssets
       artifactType: container
 
@@ -95,27 +102,26 @@ phases:
     continueOnError: false
     condition: and(succeeded(), in(variables._SignType, 'real', 'test'))
 
+  - script: ./restore.cmd
+    displayName: restore all projects
+  - script: ./build.cmd -configuration $(BuildConfig) /p:SkipRIDAgnosticAssets=true -projects $(Build.SourcesDirectory)/src/Redist/Microsoft.ML.DnnImageFeaturizer.ModelRedist/Microsoft.ML.DnnImageFeaturizer.ModelRedist.proj
+    displayName: build redist
   # Only build native assets to avoid conflicts.
-  - script: ./build.cmd -buildNative -$(BuildConfig) -buildArch=x86 -skipRIDAgnosticAssets
+  - script: ./build.cmd -projects $(Build.SourcesDirectory)/src/Native/Native.proj -configuration $(BuildConfig) /p:TargetArchitecture=x86
     displayName: Build
 
-  - task: MSBuild@1
-    displayName: Sign Windows_x86 Binaries
-    inputs:
-      solution: build/sign.proj
-      msbuildArguments: /p:SignType=$(_SignType)
-      msbuildVersion: 15.0
-    continueOnError: false
+  - script: ./sign.cmd -configuration $(BuildConfig) /p:TargetArchitecture=x86 /p:SignBinaries=true
+    displayName: sign binaries
 
   - task: PublishBuildArtifacts@1
     displayName: Publish Windows_x86 package assets
     inputs:
-      pathToPublish: $(Build.SourcesDirectory)/bin/obj/packages
+      pathToPublish: $(Build.SourcesDirectory)/artifacts/pkgassets
       artifactName: PackageAssets
       artifactType: container
 
   # Terminate all dotnet build processes.
-  - script: $(Build.SourcesDirectory)/Tools/dotnetcli/dotnet.exe build-server shutdown
+  - script: $(Build.SourcesDirectory)/.dotnet/dotnet.exe build-server shutdown
     displayName: Dotnet Server Shutdown
 
 ################################################################################
@@ -148,7 +154,7 @@ phases:
     condition: and(succeeded(), in(variables._SignType, 'real', 'test'))
 
   # Build both native and managed assets.
-  - script: ./build.cmd -$(BuildConfig)
+  - script: ./build.cmd -configuration $(BuildConfig) /p:TargetArchitecture=x64
     displayName: Build
 
   - task: ComponentGovernanceComponentDetection@0
@@ -158,23 +164,18 @@ phases:
       alertWarningLevel: 'High'
 
 
-  - task: MSBuild@1
-    displayName: Sign Windows_x64 Binaries
-    inputs:
-      solution: build/sign.proj
-      msbuildArguments: /p:SignType=$(_SignType)
-      msbuildVersion: 15.0
-    continueOnError: false
+  - script: ./sign.cmd -configuration $(BuildConfig) /p:TargetArchitecture=x64 /p:SignBinaries=true
+    displayName: sign binaries
 
   - task: PublishBuildArtifacts@1
     displayName: Publish Windows_x64 package assets
     inputs:
-      pathToPublish: $(Build.SourcesDirectory)/bin/obj/packages
+      pathToPublish: $(Build.SourcesDirectory)/artifacts/pkgassets
       artifactName: PackageAssets
       artifactType: container
 
   # Terminate all dotnet build processes.
-  - script: $(Build.SourcesDirectory)/Tools/dotnetcli/dotnet.exe build-server shutdown
+  - script: $(Build.SourcesDirectory)/.dotnet/dotnet.exe build-server shutdown
     displayName: Dotnet Server Shutdown
 
 ################################################################################
@@ -220,31 +221,27 @@ phases:
     displayName: Download package assets
     inputs:
       artifactName: PackageAssets
-      downloadPath: $(Build.SourcesDirectory)/bin/obj/packages
+      downloadPath: $(Build.SourcesDirectory)/artifacts/pkgassets
 
   # Workaround https://github.com/Microsoft/vsts-tasks/issues/6739
   - task: CopyFiles@2
     displayName: Copy package assets to correct folder
     inputs:
-      sourceFolder: $(Build.SourcesDirectory)/bin/obj/packages/PackageAssets
-      targetFolder: $(Build.SourcesDirectory)/bin/obj/packages
+      sourceFolder: $(Build.SourcesDirectory)/artifacts/pkgassets/PackageAssets
+      targetFolder: $(Build.SourcesDirectory)/artifacts/pkgassets
 
-  - script: ./build.cmd -buildPackages
-    displayName: Create Packages
+  - script: ./build.cmd -pack -configuration $(BuildConfig)
+    displayName: Build Packages
 
-  - task: MSBuild@1
-    displayName: Sign Packages
-    inputs:
-      solution: build/sign.proj
-      msbuildArguments: /p:SignType=$(_SignType) /p:SignNugetPackages=true
-      msbuildVersion: 15.0
+  - script: ./sign.cmd /p:SignNugetPackages=true
+    displayName: sign packages
     continueOnError: false
 
   - task: NuGetAuthenticate@0
     inputs:
       nuGetServiceConnections: machinelearning-dnceng-public-feed # To allow publishing to a feed of another organization
 
-  - script: Tools\dotnetcli\dotnet msbuild build\publish.proj /t:PublishPackages /p:NuGetFeedUrl=$(_AzureDevopsFeedUrl) /p:NuGetApiKey=AzureArtifacts
+  - script: $(Build.SourcesDirectory)\.dotnet\dotnet.exe msbuild build\publish.proj /t:PublishPackages /p:NuGetFeedUrl=$(_AzureDevopsFeedUrl) /p:NuGetApiKey=AzureArtifacts
     displayName: Publish Packages to AzureDevOps Feed
 
   - task: MSBuild@1
@@ -264,5 +261,5 @@ phases:
     continueOnError: true
 
   # Terminate all dotnet build processes.
-  - script: $(Build.SourcesDirectory)/Tools/dotnetcli/dotnet.exe build-server shutdown
+  - script: $(Build.SourcesDirectory)/.dotnet/dotnet.exe build-server shutdown
     displayName: Dotnet Server Shutdown

build/vsts-ci.yml

@@ -1,6 +1,14 @@
 <Project>
-    <ItemGroup>
-        <!-- <StrongNameSignInfo Include="$(RepoRoot)\Tools\Open.snk" PublicKeyToken="cc7b13ffcd2ddd51" CertificateName="Microsoft400" />
-        <StrongNameSignInfo Include="$(RepoRoot)\Tools\Test.snk" PublicKeyToken="9d77cc7ad39b68eb" CertificateName="Microsoft400" /> -->
-    </ItemGroup
+    <ItemGroup Condition="'$(SignBinaries)' == 'true'">
+        <ItemsToSign Remove="@(ItemsToSign)" />
+        <ItemsToSign Include="$(ArtifactsDir)pkgassets\**\*.dll" />
+        <FileExtensionSignInfo Include="*.dll" CertificateName="Microsoft400" />
+    </ItemGroup>
+    <ItemGroup Condition="'$(SignNugetPackages)' == 'true'">
+        <ItemsToSign Remove="@(ItemsToSign)" />
+        <ItemsToSign Include="$(ArtifactsDir)packages\**\*.nupkg" />
+        <ItemsToSign Include="$(ArtifactsDir)packages\**\*.snupkg" />
+        <FileExtensionSignInfo Include="*.nupkg" CertificateName="NuGet" />
+        <FileExtensionSignInfo Include="*.snupkg" CertificateName="NuGet" />
+    </ItemGroup>
 </Project>

eng/Signing.props

@@ -44,6 +44,7 @@
     <ReportGeneratorVersion>4.3.6</ReportGeneratorVersion>
     <MicrosoftDotNetApiCompatPackageVersion>1.0.0-beta.19225.5</MicrosoftDotNetApiCompatPackageVersion>
     <MicrosoftSourceLinkVersion>1.1.0-beta-20206-02</MicrosoftSourceLinkVersion>
+    <BuildToolsPackageVersion>3.0.0-preview4-04926-01</BuildToolsPackageVersion>
 
     <!-- Test-only Dependencies -->
     <BenchmarkDotNetVersion>0.12.0</BenchmarkDotNetVersion>

eng/Versions.props

@@ -0,0 +1,3 @@
+@echo off
+powershell -ExecutionPolicy ByPass -NoProfile -command "& """%~dp0eng\common\Build.ps1""" -restore -warnAsError 0 %*"
+exit /b %ErrorLevel%

eng/common/SetupNugetSources.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+
+set -e
+
+SOURCE="${BASH_SOURCE[0]}"
+while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink
+  DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
+  SOURCE="$(readlink "$SOURCE")"
+  [[ "$SOURCE" != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located
+done
+DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
+
+"$DIR/eng/common/build.sh" --restore --warnAsError false "$@"

eng/common/cibuild.sh

@@ -0,0 +1,3 @@
+@echo off
+powershell -ExecutionPolicy ByPass -NoProfile -command "& """%~dp0eng\common\Build.ps1""" -sign -warnAsError 0 %*"
+exit /b %ErrorLevel% 

eng/common/cross/arm64/tizen-build-rootfs.sh

@@ -3,6 +3,7 @@
 
   <PropertyGroup>
     <TargetFramework>netstandard2.0</TargetFramework>
+    <IncludeInPackage>Microsoft.Extensions.ML</IncludeInPackage>
     <PackageDescription>An integration package for ML.NET models on scalable web apps and services.</PackageDescription>
   </PropertyGroup>
 

eng/common/cross/arm64/tizen-fetch.sh

@@ -3,6 +3,7 @@
 
   <PropertyGroup>
     <TargetFramework>netstandard2.0</TargetFramework>
+    <IncludeInPackage>Microsoft.ML.AutoML</IncludeInPackage>
     <PackageDescription>ML.NET AutoML: Optimizes an ML pipeline for your dataset, by automatically locating the best feature engineering, model, and hyperparameters</PackageDescription>
   </PropertyGroup>
 

eng/common/cross/armel/tizen-build-rootfs.sh

@@ -3,6 +3,7 @@
 
   <PropertyGroup>
     <TargetFramework>netstandard2.0</TargetFramework>
+    <IncludeInPackage>Microsoft.ML.CodeGenerator</IncludeInPackage>
     <PackageDescription>ML.NET Code Generator</PackageDescription>
   </PropertyGroup>
 

eng/common/cross/armel/tizen-fetch.sh

@@ -3,6 +3,7 @@
 
   <PropertyGroup>
     <TargetFrameworks>netstandard2.0;netcoreapp3.1</TargetFrameworks>
+    <IncludeInPackage>Microsoft.ML.CpuMath</IncludeInPackage>
     <PackageDescription>Microsoft.ML.CpuMath contains optimized math routines for ML.NET.</PackageDescription>
     <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
     <DefineConstants>$(DefineConstants);CPUMATH_INFRASTRUCTURE</DefineConstants>

eng/common/cross/build-android-rootfs.sh

@@ -3,6 +3,7 @@
 
   <PropertyGroup>
     <TargetFramework>netstandard2.0</TargetFramework>
+    <IncludeInPackage>Microsoft.ML.DataView</IncludeInPackage>
     <PackageDescription>Contains the IDataView system which is a set of interfaces and components that provide efficient, compositional processing of schematized data for machine learning and advanced analytics applications.</PackageDescription>
   </PropertyGroup>
 

eng/common/cross/build-rootfs.sh

@@ -3,6 +3,7 @@
 
   <PropertyGroup>
     <TargetFramework>netstandard2.0</TargetFramework>
+    <IncludeInPackage>Microsoft.ML.DnnImageFeaturizer.AlexNet</IncludeInPackage>
     <PackageDescription>ML.NET component for pretrained AlexNet image featurization</PackageDescription>
   </PropertyGroup>