CSHARP-4765: CI script to generate Driver's dev-package (#1169)

Author: sanych-sun

.editorconfig

@@ -187,5 +187,6 @@ indent_size = 2
 # Shell scripts
 [*.sh]
 end_of_line = lf
+indent_size = 2
 [*.{cmd, bat}]
 end_of_line = crlf

.gitignore

@@ -48,6 +48,7 @@ src/packages
 
 # Nupkg artifacts
 *.nupkg
+*.snupkg
 
 # NCrunch artifacts
 *.ncrunch*

evergreen/evergreen.yml

@@ -31,10 +31,10 @@ functions:
     - command: git.get_project
       params:
         directory: mongo-csharp-driver
-    # Applies the subitted patch, if any
+    # Applies the submitted patch, if any
     # Deprecated. Should be removed. But still needed for certain agents (ZAP)
     - command: git.apply_patch
-    # Make an evergreen exapanstion file with dynamic values
+    # Make an evergreen expansion file with dynamic values
     - command: shell.exec
       params:
         working_dir: mongo-csharp-driver
@@ -46,6 +46,11 @@ functions:
               CURRENT_VERSION=latest
            fi
 
+           if [ "${PACKAGE_TARGET}" != "" ]; then
+              PACKAGE_VERSION=$(PACKAGE_TARGET="${PACKAGE_TARGET}" sh ./evergreen/packages-version.sh)
+           fi
+
+           export DOTNET_SDK_PATH="$(pwd)/../.dotnet"
            export DRIVERS_TOOLS="$(pwd)/../drivers-tools"
 
            if [ "Windows_NT" = "$OS" ]; then # Magic variable in cygwin
@@ -68,6 +73,8 @@ functions:
            MONGODB_BINARIES: "$MONGODB_BINARIES"
            UPLOAD_BUCKET: "$UPLOAD_BUCKET"
            PROJECT_DIRECTORY: "$PROJECT_DIRECTORY"
+           PACKAGE_VERSION: "$PACKAGE_VERSION"
+           DOTNET_SDK_PATH: "$DOTNET_SDK_PATH"
            PREPARE_SHELL: |
               set -o errexit
               set -o xtrace
@@ -76,14 +83,16 @@ functions:
               export MONGODB_BINARIES="$MONGODB_BINARIES"
               export UPLOAD_BUCKET="$UPLOAD_BUCKET"
               export PROJECT_DIRECTORY="$PROJECT_DIRECTORY"
-
+              export PACKAGE_VERSION="$PACKAGE_VERSION"
               export TMPDIR="$MONGO_ORCHESTRATION_HOME/db"
-              export PATH="$MONGODB_BINARIES:$PATH"
+              export PATH="$DOTNET_SDK_PATH:$MONGODB_BINARIES:$PATH"
               export PROJECT="${project}"
            EOT
            # See what we've done
            cat expansion.yml
 
+           # Do not output expansion.yml contents after this point
+
            # Add CSFLE variables that shouldn't be output to the logs
            cat <<EOT >> expansion.yml
            PREPARE_CSFLE: |
@@ -97,13 +106,19 @@ functions:
               export FLE_GCP_PRIVATE_KEY=${FLE_GCP_PRIVATE_KEY}
               set -o xtrace # Enable tracing.
            EOT
-           # Do not output expansion.yml contents after this point
 
     # Load the expansion file to make an evergreen variable with the current unique version
     - command: expansions.update
       params:
         file: mongo-csharp-driver/expansion.yml
 
+  install-dotnet:
+    - command: shell.exec
+      params:
+        script: |
+          ${PREPARE_SHELL}
+          OS=${OS} sh ${PROJECT_DIRECTORY}/evergreen/install-dotnet.sh
+
   prepare-resources:
     - command: shell.exec
       params:
@@ -1023,6 +1038,61 @@ functions:
           ${PREPARE_SHELL}
           echo '{"results": [{ "status": "FAIL", "test_file": "Build", "log_raw": "No test-results.json found was created" } ]}' > ${PROJECT_DIRECTORY}/test-results.json
 
+  packages-pack:
+    - command: shell.exec
+      params:
+        working_dir: mongo-csharp-driver
+        script: |
+          ${PREPARE_SHELL}
+          . ./evergreen/packages-pack.sh
+
+  packages-push:
+    - command: shell.exec
+      params:
+        working_dir: mongo-csharp-driver
+        env:
+          PACKAGES_SOURCE: ${PACKAGES_SOURCE}
+          PACKAGES_SOURCE_KEY: ${PACKAGES_SOURCE_KEY}
+        script: |
+          ${PREPARE_SHELL}
+          . ./evergreen/packages-push.sh
+
+  upload-package:
+    - command: s3.put
+      params:
+        aws_key: ${aws_key}
+        aws_secret: ${aws_secret}
+        local_file: ${PROJECT_DIRECTORY}/build/nuget/${PACKAGE_ID}.${PACKAGE_VERSION}.nupkg
+        remote_file: ${UPLOAD_BUCKET}/${revision}/${PACKAGE_ID}.${PACKAGE_VERSION}.nupkg
+        bucket: mciuploads
+        permissions: public-read
+        content_type: ${content_type|application/octet-stream}
+    - command: s3.put
+      params:
+        aws_key: ${aws_key}
+        aws_secret: ${aws_secret}
+        local_file: ${PROJECT_DIRECTORY}/build/nuget/${PACKAGE_ID}.${PACKAGE_VERSION}.snupkg
+        remote_file: ${UPLOAD_BUCKET}/${revision}/${PACKAGE_ID}.${PACKAGE_VERSION}.snupkg
+        bucket: mciuploads
+        permissions: public-read
+        content_type: ${content_type|application/octet-stream}
+
+  download-package:
+    - command: s3.get
+      params:
+        aws_key: ${aws_key}
+        aws_secret: ${aws_secret}
+        local_file: ${PROJECT_DIRECTORY}/build/nuget/${PACKAGE_ID}.${PACKAGE_VERSION}.nupkg
+        remote_file: ${UPLOAD_BUCKET}/${revision}/${PACKAGE_ID}.${PACKAGE_VERSION}.nupkg
+        bucket: mciuploads
+    - command: s3.get
+      params:
+        aws_key: ${aws_key}
+        aws_secret: ${aws_secret}
+        local_file: ${PROJECT_DIRECTORY}/build/nuget/${PACKAGE_ID}.${PACKAGE_VERSION}.snupkg
+        remote_file: ${UPLOAD_BUCKET}/${revision}/${PACKAGE_ID}.${PACKAGE_VERSION}.snupkg
+        bucket: mciuploads
+
 pre:
   - func: fetch-source
   - func: prepare-resources
@@ -1040,7 +1110,6 @@ post:
   - func: cleanup
 
 tasks:
-
     - name: test-net472
       commands:
         - func: bootstrap-mongo-orchestration
@@ -1676,6 +1745,47 @@ tasks:
               export GCPKMS_INSTANCENAME=${GCPKMS_INSTANCENAME}
               GCPKMS_CMD="MONGODB_URI='mongodb://localhost:27017' ./evergreen/run-csfle-gcp-tests.sh" $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/run-command.sh
 
+    - name: packages-create
+      commands:
+      - func: packages-pack
+      - func: upload-package
+        vars:
+          PACKAGE_ID: "MongoDB.Bson"
+      - func: upload-package
+        vars:
+          PACKAGE_ID: "MongoDB.Driver"
+      - func: upload-package
+        vars:
+          PACKAGE_ID: "MongoDB.Driver.Core"
+      - func: upload-package
+        vars:
+          PACKAGE_ID: "MongoDB.Driver.GridFS"
+      - func: upload-package
+        vars:
+          PACKAGE_ID: "mongocsharpdriver"
+
+    - name: packages-push
+      commands:
+        - func: download-package
+          vars:
+            PACKAGE_ID: "MongoDB.Bson"
+        - func: download-package
+          vars:
+            PACKAGE_ID: "MongoDB.Driver"
+        - func: download-package
+          vars:
+            PACKAGE_ID: "MongoDB.Driver.Core"
+        - func: download-package
+          vars:
+            PACKAGE_ID: "MongoDB.Driver.GridFS"
+        - func: download-package
+          vars:
+            PACKAGE_ID: "mongocsharpdriver"
+        - func: packages-push
+          vars:
+            PACKAGES_SOURCE: "https://api.nuget.org/v3/index.json"
+            PACKAGES_SOURCE_KEY: ${nuget_api_key}
+
 axes:
   - id: version
     display_name: MongoDB Version
@@ -1897,6 +2007,19 @@ task_groups:
     tasks:
       - test-csfle-with-gcp-kms
 
+  - name: dev-package
+    setup_task:
+      - func: fetch-source
+        vars:
+          PACKAGE_TARGET: "dev"
+      - func: prepare-resources
+      - func: fix-absolute-paths
+      - func: make-files-executable
+      - func: install-dotnet
+    tasks:
+      - packages-create
+      - packages-push
+
 buildvariants:
 
 - matrix_name: "secure-tests"
@@ -2200,3 +2323,11 @@ buildvariants:
     - name: test-smoke-tests-netcoreapp31
     - name: test-smoke-tests-net50
     - name: test-smoke-tests-net60
+
+- name: dev-package-pack
+  git_tag_only: true
+  display_name: "Dev Package Pack"
+  run_on: ubuntu1804-small
+  tags: ["dev-package"]
+  tasks:
+    - name: dev-package

evergreen/install-dotnet.sh

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+set -o errexit  # Exit the script with error if any of the commands fail
+
+DOTNET_SDK_PATH=./.dotnet
+mkdir -p "$DOTNET_SDK_PATH"
+
+if [[ $OS =~ [Ww]indows.* ]]; then
+  echo "Downloading Windows .NET SDK installer into $DOTNET_SDK_PATH folder..."
+  curl -Lfo "$DOTNET_SDK_PATH"/dotnet-install.ps1 https://dot.net/v1/dotnet-install.ps1
+  echo "Installing .NET LTS SDK..."
+  powershell.exe "$DOTNET_SDK_PATH"/dotnet-install.ps1 -Channel 6.0 -InstallDir "$DOTNET_SDK_PATH" -NoPath
+else
+  echo "Downloading .NET SDK installer into $DOTNET_SDK_PATH folder..."
+  curl -Lfo "$DOTNET_SDK_PATH"/dotnet-install.sh https://dot.net/v1/dotnet-install.sh
+  echo "Installing .NET LTS SDK..."
+  bash "$DOTNET_SDK_PATH"/dotnet-install.sh --channel 6.0 --install-dir "$DOTNET_SDK_PATH" --no-path
+fi

evergreen/packages-pack.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+set -o errexit  # Exit the script with error if any of the commands fail
+
+if [ -z "$PACKAGE_VERSION" ]; then
+  PACKAGE_VERSION=$(sh ./evergreen/packages-version.sh)
+  echo Calculated PACKAGE_VERSION value: "$PACKAGE_VERSION"
+fi
+
+echo Creating nuget package...
+
+dotnet clean ./CSharpDriver.sln
+dotnet pack ./CSharpDriver.sln -o ./build/nuget -c Release -p:Version="$PACKAGE_VERSION" --include-symbols -p:SymbolPackageFormat=snupkg

evergreen/packages-push.sh

@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+set -o errexit  # Exit the script with error if any of the commands fail
+set +o xtrace # Disable tracing.
+
+if [ -z "$PACKAGES_SOURCE" ]; then
+  echo "PACKAGES_SOURCE variable should be set"
+  exit 1
+fi
+
+if [ -z "$PACKAGES_SOURCE_KEY" ]; then
+  echo "PACKAGES_SOURCE_KEY variable should be set"
+  exit 1
+fi
+
+if [ -z "$PACKAGE_VERSION" ]; then
+  echo "PACKAGE_VERSION variable should be set"
+  exit 1
+fi
+
+dotnet nuget push --source "$PACKAGES_SOURCE" --api-key "$PACKAGES_SOURCE_KEY" ./build/nuget/MongoDB.Bson."$PACKAGE_VERSION".nupkg
+dotnet nuget push --source "$PACKAGES_SOURCE" --api-key "$PACKAGES_SOURCE_KEY" ./build/nuget/MongoDB.Driver.Core."$PACKAGE_VERSION".nupkg
+dotnet nuget push --source "$PACKAGES_SOURCE" --api-key "$PACKAGES_SOURCE_KEY" ./build/nuget/MongoDB.Driver."$PACKAGE_VERSION".nupkg
+dotnet nuget push --source "$PACKAGES_SOURCE" --api-key "$PACKAGES_SOURCE_KEY" ./build/nuget/MongoDB.Driver.GridFS."$PACKAGE_VERSION".nupkg
+dotnet nuget push --source "$PACKAGES_SOURCE" --api-key "$PACKAGES_SOURCE_KEY" ./build/nuget/mongocsharpdriver."$PACKAGE_VERSION".nupkg
+
+

evergreen/packages-version.sh

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+set -o errexit  # Exit the script with error if any of the commands fail
+
+if [ -z "$PACKAGE_TARGET" ]; then
+  # Use production release tag if nothing was passed
+  PACKAGE_TARGET="release"
+fi
+
+if [ "${PACKAGE_TARGET}" = "dev" ]; then
+  PACKAGE_VERSION_MATCH="v[0-9]*.[0-9]*.[0-9]*-dev[0-9]*"
+  PACKAGE_VERSION_EXCLUDE=""
+elif [ "${PACKAGE_TARGET}" = "release" ]; then
+  PACKAGE_VERSION_MATCH="v[0-9]*.[0-9]*.[0-9]*"
+  PACKAGE_VERSION_EXCLUDE="*-dev[0-9]*"
+else
+  echo "Unexpected value of PACKAGE_TARGET: ${PACKAGE_TARGET}"
+fi
+
+PACKAGE_VERSION=$(git describe --tags --abbrev=0 --match="${PACKAGE_VERSION_MATCH}" --exclude="${PACKAGE_VERSION_EXCLUDE}")
+PACKAGE_VERSION=$(echo $PACKAGE_VERSION | cut -c 2-)
+echo "$PACKAGE_VERSION"

src/MongoDB.Bson/MongoDB.Bson.csproj

@@ -7,9 +7,11 @@
   <PropertyGroup>
     <AssemblyTitle>MongoDB.Bson</AssemblyTitle>
     <Product>MongoDB.Bson</Product>
+    <PackageId>MongoDB.Bson</PackageId>
     <Description>Official MongoDB supported BSON library. See https://www.mongodb.com/docs/drivers/csharp/ for more details.</Description>
     <PackageDescription>MongoDB's Official Bson Library.</PackageDescription>
     <PackageTags>$(PackageTags);bson</PackageTags>
+    <IsPackable>true</IsPackable>
   </PropertyGroup>
 
   <ItemGroup>

src/MongoDB.Driver.Core/MongoDB.Driver.Core.csproj

@@ -6,8 +6,10 @@
   <PropertyGroup>
     <AssemblyTitle>MongoDB.Driver.Core</AssemblyTitle>
     <Product>MongoDB.Driver.Core</Product>
+    <PackageId>MongoDB.Driver.Core</PackageId>
     <Description>Official MongoDB supported Driver Core library. See https://www.mongodb.com/docs/drivers/csharp/ for more details.</Description>
     <PackageDescription>Core Component of the Official MongoDB .NET Driver.</PackageDescription>
+    <IsPackable>true</IsPackable>
   </PropertyGroup>
 
   <PropertyGroup>

src/MongoDB.Driver.GridFS/MongoDB.Driver.GridFS.csproj

@@ -7,9 +7,11 @@
   <PropertyGroup>
     <AssemblyTitle>MongoDB.Driver.GridFS</AssemblyTitle>
     <Product>MongoDB.Driver.GridFS</Product>
+    <PackageId>MongoDB.Driver.GridFS</PackageId>
     <Description>Official MongoDB supported driver for MongoDB GridFS implementation. See https://www.mongodb.com/docs/drivers/csharp/ for more details.</Description>
     <PackageDescription>GridFS Component of the Official MongoDB .NET Driver.</PackageDescription>
     <PackageTags>$(PackageTags);gridfs</PackageTags>
+    <IsPackable>true</IsPackable>
   </PropertyGroup>
 
   <ItemGroup>

src/MongoDB.Driver.Legacy/MongoDB.Driver.Legacy.csproj

@@ -11,6 +11,7 @@
     <PackageId>mongocsharpdriver</PackageId>
     <PackageDescription>This package contains the legacy driver. The new driver's package name is MongoDB.Driver</PackageDescription>
     <PackageTags></PackageTags>
+    <IsPackable>true</IsPackable>
   </PropertyGroup>
 
   <PropertyGroup>

src/MongoDB.Driver/MongoDB.Driver.csproj

@@ -7,8 +7,10 @@
   <PropertyGroup>
     <AssemblyTitle>MongoDB.Driver</AssemblyTitle>
     <Product>MongoDB.Driver</Product>
+    <PackageId>MongoDB.Driver</PackageId>
     <Description>Official MongoDB supported driver for MongoDB. See https://www.mongodb.com/docs/drivers/csharp/ for more details.</Description>
     <PackageDescription>Official .NET driver for MongoDB.</PackageDescription>
+    <IsPackable>true</IsPackable>
   </PropertyGroup>
 
   <ItemGroup>