Issue #2926733 by drunken monkey, borisson_: Fixed indexing of leading/trailing whitespace in fulltext tokens on database backend.

Author: drunken-monkey

Diff for: CHANGELOG.txt

@@ -1,5 +1,7 @@
 Search API 1.x, dev (xxxx-xx-xx):
 ---------------------------------
+- #2926733 by drunken monkey, borisson_: Fixed indexing of leading/trailing
+  whitespace in fulltext tokens on database backend.
 - #2922024 by drunken monkey, borisson_: Fixed Stemmer incorrectly processing
   non-English searches.
 - #2931730 by drunken monkey: Adapted tests to changes in drupal_set_message().

Diff for: modules/search_api_db/src/Plugin/search_api/backend/Database.php

@@ -1234,6 +1234,12 @@ protected function indexItem(IndexInterface $index, ItemInterface $item) {
             $word = $token->getText();
             $score = $token->getBoost();
 
+            // In rare cases, tokens with leading or trailing whitespace can
+            // slip through. Since this can lead to errors when such tokens are
+            // part of a primary key (as in this case), we trim such whitespace
+            // here.
+            $word = trim($word);
+
             // Store the first 30 characters of the string as the denormalized
             // value.
             if (Unicode::strlen($denormalized_value) < 30) {

Diff for: modules/search_api_db/tests/src/Kernel/BackendTest.php

@@ -4,8 +4,11 @@
 
 use Drupal\Component\Render\FormattableMarkup;
 use Drupal\search_api\Entity\Server;
+use Drupal\search_api\Plugin\search_api\data_type\value\TextToken;
+use Drupal\search_api\Plugin\search_api\data_type\value\TextValue;
 use Drupal\search_api\Query\QueryInterface;
 use Drupal\search_api\SearchApiException;
+use Drupal\search_api\Utility\Utility;
 use Drupal\search_api_db\Plugin\search_api\backend\Database;
 use Drupal\search_api_db\Tests\DatabaseTestsTrait;
 use Drupal\Tests\search_api\Kernel\BackendTestBase;
@@ -80,6 +83,7 @@ protected function backendSpecificRegressionTests() {
     $this->regressionTest2557291();
     $this->regressionTest2511860();
     $this->regressionTest2846932();
+    $this->regressionTest2926733();
   }
 
   /**
@@ -476,6 +480,38 @@ protected function regressionTest2846932() {
     $index->save();
   }
 
+  /**
+   * Tests indexing of text tokens with leading/trailing whitespace.
+   *
+   * @see https://www.drupal.org/node/2926733
+   */
+  protected function regressionTest2926733() {
+    $index = $this->getIndex();
+    $item_id = $this->getItemIds([1])[0];
+    $fields_helper = \Drupal::getContainer()
+      ->get('search_api.fields_helper');
+    $item = $fields_helper->createItem($index, $item_id);
+    $field = clone $index->getField('body');
+    $value = new TextValue('test');
+    $tokens = [];
+    foreach (['test', ' test', '  test', 'test  ', ' test '] as $token) {
+      $tokens[] = new TextToken($token);
+    }
+    $value->setTokens($tokens);
+    $field->setValues([$value]);
+    $item->setFields([
+      'body' => $field,
+    ]);
+    $item->setFieldsExtracted(TRUE);
+    $index->getServerInstance()->indexItems($index, [$item_id => $item]);
+
+    // Make sure to re-index the proper version of the item to avoid confusing
+    // the other tests.
+    list($datasource_id, $raw_id) = Utility::splitCombinedId($item_id);
+    $index->trackItemsUpdated($datasource_id, [$raw_id]);
+    $this->indexItems($index->id());
+  }
+
   /**
    * {@inheritdoc}
    */