RUBY-3352 Backport RUBY-3268 atlas search index helpers to 2.19-stable (#2804)

* RUBY-3268 search index management helpers (#2777)

* RUBY-3268 Index View API for Search Indexes

* documentation, and align method names with the spec

* return the correct value here

* suppress NamespaceNotFound errors for the drop operation

* prose tests for search indexes

* rubocop

* prose tests pass

* first stab at evergreen config for index management specs

* rubocop

* gah, executable permissions

* don't use FLE for full atlas tests

* make sure MONGODB_URI is set

* set the timeout higher for the search index specs

* pass all aggregation options through to the list indexes command

* use the correct implementation for #empty?

* remove unnecessary validation

* bump drivers-evergreen-tools

* RUBY-3324 bump drivers-evergreen-tools to get updated atlas setup/teardown (#2780)

also, expose task_id expansion as environment variable to those scripts

* RUBY-3328 add `execution` expansion to environment for atlas cluster name uniqueness (#2783)

Author: jamis

Diff for: .evergreen/config.yml

@@ -469,6 +469,50 @@ post:
   - func: "delete serverless instance"
 
 task_groups:
+  - name: testatlas_task_group
+    setup_group_can_fail_task: true
+    setup_group_timeout_secs: 1800 # 30 minutes
+    setup_group:
+      - func: fetch source
+      - func: create expansions
+      - command: shell.exec
+        params:
+          shell: "bash"
+          working_dir: "src"
+          script: |
+            ${PREPARE_SHELL}
+
+            DRIVERS_ATLAS_PUBLIC_API_KEY="${DRIVERS_ATLAS_PUBLIC_API_KEY}" \
+              DRIVERS_ATLAS_PRIVATE_API_KEY="${DRIVERS_ATLAS_PRIVATE_API_KEY}" \
+              DRIVERS_ATLAS_GROUP_ID="${DRIVERS_ATLAS_GROUP_ID}" \
+              DRIVERS_ATLAS_LAMBDA_USER="${DRIVERS_ATLAS_LAMBDA_USER}" \
+              DRIVERS_ATLAS_LAMBDA_PASSWORD="${DRIVERS_ATLAS_LAMBDA_PASSWORD}" \
+              LAMBDA_STACK_NAME="dbx-ruby-lambda" \
+              MONGODB_VERSION="7.0" \
+              task_id="${task_id}" \
+              execution="${execution}" \
+              $DRIVERS_TOOLS/.evergreen/atlas/setup-atlas-cluster.sh
+      - command: expansions.update
+        params:
+          file: src/atlas-expansion.yml
+    teardown_group:
+      - command: shell.exec
+        params:
+          shell: "bash"
+          working_dir: "src"
+          script: |
+            ${PREPARE_SHELL}
+
+            DRIVERS_ATLAS_PUBLIC_API_KEY="${DRIVERS_ATLAS_PUBLIC_API_KEY}" \
+              DRIVERS_ATLAS_PRIVATE_API_KEY="${DRIVERS_ATLAS_PRIVATE_API_KEY}" \
+              DRIVERS_ATLAS_GROUP_ID="${DRIVERS_ATLAS_GROUP_ID}" \
+              LAMBDA_STACK_NAME="dbx-ruby-lambda" \
+              task_id="${task_id}" \
+              execution="${execution}" \
+              $DRIVERS_TOOLS/.evergreen/atlas/teardown-atlas-cluster.sh
+    tasks:
+      - test-full-atlas-task
+
   - name: testgcpkms_task_group
     setup_group_can_fail_task: true
     setup_group_timeout_secs: 1800 # 30 minutes
@@ -586,6 +630,16 @@ tasks:
     commands:
       - func: "export AWS auth credentials"
       - func: "run AWS auth tests"
+  - name: "test-full-atlas-task"
+    commands:
+      - command: shell.exec
+        type: test
+        params:
+          working_dir: "src"
+          shell: "bash"
+          script: |
+            ${PREPARE_SHELL}
+            MONGODB_URI="${MONGODB_URI}" .evergreen/run-tests-atlas-full.sh
   - name: "testgcpkms-task"
     commands:
       - command: shell.exec
@@ -1612,6 +1666,14 @@ buildvariants:
       - name: testazurekms_task_group
         batchtime: 20160 # Use a batchtime of 14 days as suggested by the CSFLE test README
 
+  - matrix_name: atlas-full
+    matrix_spec:
+      ruby: "ruby-3.2"
+      os: rhel8
+    display_name: "Atlas (Full)"
+    tasks:
+      - name: testatlas_task_group
+
   - matrix_name: "atlas"
     matrix_spec:
       ruby: ["ruby-3.2", "ruby-3.1", "ruby-3.0", "ruby-2.7", "ruby-2.6", "ruby-2.5", "jruby-9.3", "jruby-9.2"]

Diff for: .evergreen/config/common.yml.erb

@@ -466,6 +466,50 @@ post:
   - func: "delete serverless instance"
 
 task_groups:
+  - name: testatlas_task_group
+    setup_group_can_fail_task: true
+    setup_group_timeout_secs: 1800 # 30 minutes
+    setup_group:
+      - func: fetch source
+      - func: create expansions
+      - command: shell.exec
+        params:
+          shell: "bash"
+          working_dir: "src"
+          script: |
+            ${PREPARE_SHELL}
+
+            DRIVERS_ATLAS_PUBLIC_API_KEY="${DRIVERS_ATLAS_PUBLIC_API_KEY}" \
+              DRIVERS_ATLAS_PRIVATE_API_KEY="${DRIVERS_ATLAS_PRIVATE_API_KEY}" \
+              DRIVERS_ATLAS_GROUP_ID="${DRIVERS_ATLAS_GROUP_ID}" \
+              DRIVERS_ATLAS_LAMBDA_USER="${DRIVERS_ATLAS_LAMBDA_USER}" \
+              DRIVERS_ATLAS_LAMBDA_PASSWORD="${DRIVERS_ATLAS_LAMBDA_PASSWORD}" \
+              LAMBDA_STACK_NAME="dbx-ruby-lambda" \
+              MONGODB_VERSION="7.0" \
+              task_id="${task_id}" \
+              execution="${execution}" \
+              $DRIVERS_TOOLS/.evergreen/atlas/setup-atlas-cluster.sh
+      - command: expansions.update
+        params:
+          file: src/atlas-expansion.yml
+    teardown_group:
+      - command: shell.exec
+        params:
+          shell: "bash"
+          working_dir: "src"
+          script: |
+            ${PREPARE_SHELL}
+
+            DRIVERS_ATLAS_PUBLIC_API_KEY="${DRIVERS_ATLAS_PUBLIC_API_KEY}" \
+              DRIVERS_ATLAS_PRIVATE_API_KEY="${DRIVERS_ATLAS_PRIVATE_API_KEY}" \
+              DRIVERS_ATLAS_GROUP_ID="${DRIVERS_ATLAS_GROUP_ID}" \
+              LAMBDA_STACK_NAME="dbx-ruby-lambda" \
+              task_id="${task_id}" \
+              execution="${execution}" \
+              $DRIVERS_TOOLS/.evergreen/atlas/teardown-atlas-cluster.sh
+    tasks:
+      - test-full-atlas-task
+
   - name: testgcpkms_task_group
     setup_group_can_fail_task: true
     setup_group_timeout_secs: 1800 # 30 minutes
@@ -583,6 +627,16 @@ tasks:
     commands:
       - func: "export AWS auth credentials"
       - func: "run AWS auth tests"
+  - name: "test-full-atlas-task"
+    commands:
+      - command: shell.exec
+        type: test
+        params:
+          working_dir: "src"
+          shell: "bash"
+          script: |
+            ${PREPARE_SHELL}
+            MONGODB_URI="${MONGODB_URI}" .evergreen/run-tests-atlas-full.sh
   - name: "testgcpkms-task"
     commands:
       - command: shell.exec

Diff for: .evergreen/config/standard.yml.erb

@@ -497,6 +497,14 @@ buildvariants:
       - name: testazurekms_task_group
         batchtime: 20160 # Use a batchtime of 14 days as suggested by the CSFLE test README
 
+  - matrix_name: atlas-full
+    matrix_spec:
+      ruby: <%= latest_ruby %>
+      os: rhel8
+    display_name: "Atlas (Full)"
+    tasks:
+      - name: testatlas_task_group
+
   - matrix_name: "atlas"
     matrix_spec:
       ruby: <%= supported_rubies %>

Diff for: .evergreen/run-tests-atlas-full.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+set -ex
+
+. `dirname "$0"`/../spec/shared/shlib/distro.sh
+. `dirname "$0"`/../spec/shared/shlib/set_env.sh
+. `dirname "$0"`/functions.sh
+
+set_env_vars
+set_env_python
+set_env_ruby
+
+bundle_install
+
+ATLAS_URI=$MONGODB_URI \
+  SERVERLESS=1 \
+  EXAMPLE_TIMEOUT=600 \
+  bundle exec rspec -fd spec/integration/search_indexes_prose_spec.rb
+
+test_status=$?
+
+kill_jruby
+
+exit ${test_status}

Diff for: .gitignore

@@ -25,3 +25,8 @@ data/*
 gemfiles/*.gemfile.lock
 .env.private*
 .env
+build
+profile/benchmarking/data
+secrets-export.sh
+secrets-expansion.yml
+atlas-expansion.yml

Diff for: .mod/drivers-evergreen-tools

@@ -56,6 +56,9 @@ Metrics/ModuleLength:
 Metrics/MethodLength:
   Max: 20
 
+Naming/MethodParameterName:
+  AllowedNames: [ id, op ]
+
 RSpec/BeforeAfterAll:
   Enabled: false
 

Diff for: .rubocop.yml

@@ -64,6 +64,7 @@
 require 'mongo/dbref'
 require 'mongo/grid'
 require 'mongo/index'
+require 'mongo/search_index/view'
 require 'mongo/lint'
 require 'mongo/query_cache'
 require 'mongo/server'

Diff for: lib/mongo.rb

@@ -725,13 +725,35 @@ def distinct(field_name, filter = nil, options = {})
     #
     # @option options [ Session ] :session The session to use.
     #
-    # @return [ View::Index ] The index view.
+    # @return [ Index::View ] The index view.
     #
     # @since 2.0.0
     def indexes(options = {})
       Index::View.new(self, options)
     end
 
+    # Get a view of all search indexes for this collection. Can be iterated or
+    # operated on directly. If id or name are given, the iterator will return
+    # only the indicated index. For all other operations, id and name are
+    # ignored.
+    #
+    # @note Only one of id or name may be given; it is an error to specify both,
+    #   although both may be omitted safely.
+    #
+    # @param [ Hash ] options The options to use to configure the view.
+    #
+    # @option options [ String ] :id The id of the specific index to query (optional)
+    # @option options [ String ] :name The name of the specific index to query (optional)
+    # @option options [ Hash ] :aggregate The options hash to pass to the
+    #    aggregate command (optional)
+    #
+    # @return [ SearchIndex::View ] The search index view.
+    #
+    # @since 2.0.0
+    def search_indexes(options = {})
+      SearchIndex::View.new(self, options)
+    end
+
     # Get a pretty printed string inspection for the collection.
     #
     # @example Inspect the collection.

Diff for: lib/mongo/collection.rb

@@ -51,6 +51,9 @@
 require 'mongo/operation/remove_user'
 require 'mongo/operation/create_index'
 require 'mongo/operation/drop_index'
+require 'mongo/operation/create_search_indexes'
+require 'mongo/operation/drop_search_index'
+require 'mongo/operation/update_search_index'
 
 module Mongo
 

Diff for: lib/mongo/operation.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+require 'mongo/operation/create_search_indexes/op_msg'
+
+module Mongo
+  module Operation
+    # A MongoDB createSearchIndexes command operation.
+    #
+    # @api private
+    class CreateSearchIndexes
+      include Specifiable
+      include OpMsgExecutable
+    end
+  end
+end

Diff for: lib/mongo/operation/create_search_indexes.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module Mongo
+  module Operation
+    class CreateSearchIndexes
+      # A MongoDB createSearchIndexes operation sent as an op message.
+      #
+      # @api private
+      class OpMsg < OpMsgBase
+        include ExecutableTransactionLabel
+
+        private
+
+        # Returns the command to send to the database, describing the
+        # desired createSearchIndexes operation.
+        #
+        # @param [ Mongo::Server ] _server the server that will receive the
+        #   command
+        #
+        # @return [ Hash ] the selector
+        def selector(_server)
+          {
+            createSearchIndexes: coll_name,
+            :$db => db_name,
+            indexes: indexes,
+          }
+        end
+      end
+    end
+  end
+end

Diff for: lib/mongo/operation/create_search_indexes/op_msg.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+require 'mongo/operation/drop_search_index/op_msg'
+
+module Mongo
+  module Operation
+    # A MongoDB dropSearchIndex command operation.
+    #
+    # @api private
+    class DropSearchIndex
+      include Specifiable
+      include OpMsgExecutable
+    end
+  end
+end

Diff for: lib/mongo/operation/drop_search_index.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module Mongo
+  module Operation
+    class DropSearchIndex
+      # A MongoDB createSearchIndexes operation sent as an op message.
+      #
+      # @api private
+      class OpMsg < OpMsgBase
+        include ExecutableTransactionLabel
+
+        private
+
+        # Returns the command to send to the database, describing the
+        # desired dropSearchIndex operation.
+        #
+        # @param [ Mongo::Server ] _server the server that will receive the
+        #   command
+        #
+        # @return [ Hash ] the selector
+        def selector(_server)
+          {
+            dropSearchIndex: coll_name,
+            :$db => db_name,
+          }.tap do |sel|
+            sel[:id] = index_id if index_id
+            sel[:name] = index_name if index_name
+          end
+        end
+      end
+    end
+  end
+end

Diff for: lib/mongo/operation/drop_search_index/op_msg.rb

@@ -260,6 +260,13 @@ def index
         spec[INDEX]
       end
 
+      # Get the index id from the spec.
+      #
+      # @return [ String ] The index id.
+      def index_id
+        spec[:index_id]
+      end
+
       # Get the index name from the spec.
       #
       # @example Get the index name.

Diff for: lib/mongo/operation/shared/specifiable.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+require 'mongo/operation/update_search_index/op_msg'
+
+module Mongo
+  module Operation
+    # A MongoDB updateSearchIndex command operation.
+    #
+    # @api private
+    class UpdateSearchIndex
+      include Specifiable
+      include OpMsgExecutable
+    end
+  end
+end