Skip to content

Commit e7b3898

Browse files
millotpmillotp
authored
feat(clients): add optionnal scopes to replaceAllObjects [skip-bc] (#4296)
1 parent 0a370d1 commit e7b3898

File tree

18 files changed

+370
-57
lines changed

18 files changed

+370
-57
lines changed

‎clients/algoliasearch-client-csharp/algoliasearch/Utils/SearchClientExtensions.cs

Lines changed: 13 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -140,11 +140,12 @@ public partial interface ISearchClient
140140
/// <param name="indexName">The index in which to perform the request.</param>
141141
/// <param name="objects">The list of `objects` to store in the given Algolia `indexName`.</param>
142142
/// <param name="batchSize">The size of the chunk of `objects`. The number of `batch` calls will be equal to `length(objects) / batchSize`. Defaults to 1000.</param>
143+
/// <param name="scopes"> The `scopes` to keep from the index. Defaults to ['settings', 'rules', 'synonyms'].</param>
143144
/// <param name="options">Add extra http header or query parameters to Algolia.</param>
144145
/// <param name="cancellationToken">Cancellation Token to cancel the request.</param>
145-
Task<ReplaceAllObjectsResponse> ReplaceAllObjectsAsync<T>(string indexName, IEnumerable<T> objects, int batchSize = 1000, RequestOptions options = null, CancellationToken cancellationToken = default) where T : class;
146+
Task<ReplaceAllObjectsResponse> ReplaceAllObjectsAsync<T>(string indexName, IEnumerable<T> objects, int batchSize = 1000, List<ScopeType> scopes = null, RequestOptions options = null, CancellationToken cancellationToken = default) where T : class;
146147
/// <inheritdoc cref="ReplaceAllObjectsAsync{T}(string, IEnumerable{T}, int, RequestOptions, CancellationToken)"/>
147-
ReplaceAllObjectsResponse ReplaceAllObjects<T>(string indexName, IEnumerable<T> objects, int batchSize = 1000, RequestOptions options = null, CancellationToken cancellationToken = default) where T : class;
148+
ReplaceAllObjectsResponse ReplaceAllObjects<T>(string indexName, IEnumerable<T> objects, int batchSize = 1000, List<ScopeType> scopes = null, RequestOptions options = null, CancellationToken cancellationToken = default) where T : class;
148149

149150
/// <summary>
150151
/// Helper: Chunks the given `objects` list in subset of 1000 elements max in order to make it fit in `batch` requests.
@@ -484,21 +485,26 @@ private static int NextDelay(int retryCount)
484485

485486
/// <inheritdoc/>
486487
public async Task<ReplaceAllObjectsResponse> ReplaceAllObjectsAsync<T>(string indexName, IEnumerable<T> objects,
487-
int batchSize = 1000, RequestOptions options = null, CancellationToken cancellationToken = default) where T : class
488+
int batchSize = 1000, List<ScopeType> scopes = null, RequestOptions options = null, CancellationToken cancellationToken = default) where T : class
488489
{
489490
if (objects == null)
490491
{
491492
throw new ArgumentNullException(nameof(objects));
492493
}
493494

495+
if (scopes == null)
496+
{
497+
scopes = new List<ScopeType> { ScopeType.Settings, ScopeType.Rules, ScopeType.Synonyms };
498+
}
499+
494500
var rnd = new Random();
495501
var tmpIndexName = $"{indexName}_tmp_{rnd.Next(100)}";
496502

497503
try
498504
{
499505
var copyResponse = await OperationIndexAsync(indexName,
500506
new OperationIndexParams(OperationType.Copy, tmpIndexName)
501-
{ Scope = [ScopeType.Settings, ScopeType.Rules, ScopeType.Synonyms] }, options, cancellationToken)
507+
{ Scope = scopes }, options, cancellationToken)
502508
.ConfigureAwait(false);
503509

504510
var batchResponse = await ChunkedBatchAsync(tmpIndexName, objects, Action.AddObject, true, batchSize,
@@ -509,7 +515,7 @@ await WaitForTaskAsync(tmpIndexName, copyResponse.TaskID, requestOptions: option
509515

510516
copyResponse = await OperationIndexAsync(indexName,
511517
new OperationIndexParams(OperationType.Copy, tmpIndexName)
512-
{ Scope = [ScopeType.Settings, ScopeType.Rules, ScopeType.Synonyms] }, options, cancellationToken)
518+
{ Scope = scopes }, options, cancellationToken)
513519
.ConfigureAwait(false);
514520
await WaitForTaskAsync(tmpIndexName, copyResponse.TaskID, requestOptions: options, ct: cancellationToken)
515521
.ConfigureAwait(false);
@@ -537,9 +543,9 @@ await WaitForTaskAsync(tmpIndexName, moveResponse.TaskID, requestOptions: option
537543
}
538544

539545
/// <inheritdoc/>
540-
public ReplaceAllObjectsResponse ReplaceAllObjects<T>(string indexName, IEnumerable<T> objects, int batchSize = 1000,
546+
public ReplaceAllObjectsResponse ReplaceAllObjects<T>(string indexName, IEnumerable<T> objects, int batchSize = 1000, List<ScopeType> scopes = null,
541547
RequestOptions options = null, CancellationToken cancellationToken = default) where T : class =>
542-
AsyncHelper.RunSync(() => ReplaceAllObjectsAsync(indexName, objects, batchSize, options, cancellationToken));
548+
AsyncHelper.RunSync(() => ReplaceAllObjectsAsync(indexName, objects, batchSize, scopes, options, cancellationToken));
543549

544550
/// <inheritdoc/>
545551
public async Task<List<BatchResponse>> ChunkedBatchAsync<T>(string indexName, IEnumerable<T> objects,

‎clients/algoliasearch-client-go/.golangci.yml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,7 @@ linters:
6161

6262
# Deprecated
6363
- execinquery
64+
- exportloopref
6465

6566
issues:
6667
exclude-generated: disable

‎clients/algoliasearch-client-kotlin/client/src/commonMain/kotlin/com/algolia/client/extensions/SearchClient.kt

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -462,12 +462,14 @@ public suspend fun SearchClient.partialUpdateObjects(
462462
* @param indexName The index in which to perform the request.
463463
* @param objects The list of objects to replace.
464464
* @param batchSize The size of the batch. Default is 1000.
465+
* @param scopes The `scopes` to keep from the index. Defaults to ['settings', 'rules', 'synonyms'].
465466
* @return responses from the three-step operations: copy, batch, move.
466467
*/
467468
public suspend fun SearchClient.replaceAllObjects(
468469
indexName: String,
469470
objects: List<JsonObject>,
470471
batchSize: Int = 1000,
472+
scopes: List<ScopeType> = listOf(ScopeType.Settings, ScopeType.Rules, ScopeType.Synonyms),
471473
requestOptions: RequestOptions? = null,
472474
): ReplaceAllObjectsResponse {
473475
val tmpIndexName = "${indexName}_tmp_${Random.nextInt(from = 0, until = 100)}"
@@ -478,7 +480,7 @@ public suspend fun SearchClient.replaceAllObjects(
478480
operationIndexParams = OperationIndexParams(
479481
operation = OperationType.Copy,
480482
destination = tmpIndexName,
481-
scope = listOf(ScopeType.Settings, ScopeType.Rules, ScopeType.Synonyms),
483+
scope = scopes,
482484
),
483485
requestOptions = requestOptions,
484486
)
@@ -499,7 +501,7 @@ public suspend fun SearchClient.replaceAllObjects(
499501
operationIndexParams = OperationIndexParams(
500502
operation = OperationType.Copy,
501503
destination = tmpIndexName,
502-
scope = listOf(ScopeType.Settings, ScopeType.Rules, ScopeType.Synonyms),
504+
scope = scopes,
503505
),
504506
requestOptions = requestOptions,
505507
)

‎clients/algoliasearch-client-scala/src/main/scala/algoliasearch/extension/package.scala

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -353,6 +353,8 @@ package object extension {
353353
* The list of objects to replace.
354354
* @param batchSize
355355
* The size of the batch. Default is 1000.
356+
* @param scopes
357+
* The `scopes` to keep from the index. Defaults to ['settings', 'rules', 'synonyms'].
356358
* @param requestOptions
357359
* Additional request configuration.
358360
* @return
@@ -362,6 +364,7 @@ package object extension {
362364
indexName: String,
363365
objects: Seq[Any],
364366
batchSize: Int = 1000,
367+
scopes: Option[Seq[ScopeType]] = Some(Seq(ScopeType.Settings, ScopeType.Rules, ScopeType.Synonyms)),
365368
requestOptions: Option[RequestOptions] = None
366369
)(implicit ec: ExecutionContext): Future[ReplaceAllObjectsResponse] = {
367370
val tmpIndexName = s"${indexName}_tmp_${scala.util.Random.nextInt(100)}"
@@ -373,7 +376,7 @@ package object extension {
373376
operationIndexParams = OperationIndexParams(
374377
operation = OperationType.Copy,
375378
destination = tmpIndexName,
376-
scope = Some(Seq(ScopeType.Settings, ScopeType.Rules, ScopeType.Synonyms))
379+
scope = scopes
377380
),
378381
requestOptions = requestOptions
379382
)
@@ -394,7 +397,7 @@ package object extension {
394397
operationIndexParams = OperationIndexParams(
395398
operation = OperationType.Copy,
396399
destination = tmpIndexName,
397-
scope = Some(Seq(ScopeType.Settings, ScopeType.Rules, ScopeType.Synonyms))
400+
scope = scopes
398401
),
399402
requestOptions = requestOptions
400403
)

‎clients/algoliasearch-client-swift/Sources/Search/Extra/SearchClientExtension.swift

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -548,13 +548,15 @@ public extension SearchClient {
548548
/// - parameter indexName: The name of the index where to replace the objects
549549
/// - parameter objects: The new objects
550550
/// - parameter batchSize: The maximum number of objects to include in a batch
551+
/// - parameter scopes: The `scopes` to keep from the index. Defaults to ['settings', 'rules', 'synonyms']
551552
/// - parameter requestOptions: The request options
552553
/// - returns: ReplaceAllObjectsResponse
553554
@discardableResult
554555
func replaceAllObjects(
555556
indexName: String,
556557
objects: [some Encodable],
557558
batchSize: Int = 1000,
559+
scopes: [ScopeType] = [.settings, .rules, .synonyms],
558560
requestOptions: RequestOptions? = nil
559561
) async throws -> ReplaceAllObjectsResponse {
560562
let tmpIndexName = "\(indexName)_tmp_\(Int.random(in: 1_000_000 ..< 10_000_000))"
@@ -565,7 +567,7 @@ public extension SearchClient {
565567
operationIndexParams: OperationIndexParams(
566568
operation: .copy,
567569
destination: tmpIndexName,
568-
scope: [.settings, .rules, .synonyms]
570+
scope: scopes
569571
),
570572
requestOptions: requestOptions
571573
)
@@ -584,7 +586,7 @@ public extension SearchClient {
584586
operationIndexParams: OperationIndexParams(
585587
operation: .copy,
586588
destination: tmpIndexName,
587-
scope: [.settings, .rules, .synonyms]
589+
scope: scopes
588590
),
589591
requestOptions: requestOptions
590592
)

‎scripts/cts/runCts.ts

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ import { assertChunkWrapperValid } from './testServer/chunkWrapper.js';
1010
import { startTestServer } from './testServer/index.js';
1111
import { assertValidReplaceAllObjects } from './testServer/replaceAllObjects.js';
1212
import { assertValidReplaceAllObjectsFailed } from './testServer/replaceAllObjectsFailed.js';
13+
import { assertValidReplaceAllObjectsScopes } from './testServer/replaceAllObjectsScopes.js';
1314
import { assertValidTimeouts } from './testServer/timeout.js';
1415
import { assertValidWaitForApiKey } from './testServer/waitFor.js';
1516

@@ -151,6 +152,7 @@ export async function runCts(
151152
assertChunkWrapperValid(languages.length - skip('dart') - skip('scala'));
152153
assertValidReplaceAllObjects(languages.length - skip('dart') - skip('scala'));
153154
assertValidReplaceAllObjectsFailed(languages.length - skip('dart') - skip('scala'));
155+
assertValidReplaceAllObjectsScopes(languages.length - skip('dart') - skip('scala'));
154156
assertValidWaitForApiKey(languages.length - skip('dart') - skip('scala'));
155157
}
156158
if (withBenchmarkServer) {

‎scripts/cts/testServer/index.ts

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ import { chunkWrapperServer } from './chunkWrapper.js';
1313
import { gzipServer } from './gzip.js';
1414
import { replaceAllObjectsServer } from './replaceAllObjects.js';
1515
import { replaceAllObjectsServerFailed } from './replaceAllObjectsFailed.js';
16+
import { replaceAllObjectsScopesServer } from './replaceAllObjectsScopes.js';
1617
import { timeoutServer } from './timeout.js';
1718
import { timeoutServerBis } from './timeoutBis.js';
1819
import { waitForApiKeyServer } from './waitFor.js';
@@ -26,6 +27,7 @@ export async function startTestServer(suites: Record<CTSType, boolean>): Promise
2627
timeoutServerBis(),
2728
replaceAllObjectsServer(),
2829
replaceAllObjectsServerFailed(),
30+
replaceAllObjectsScopesServer(),
2931
chunkWrapperServer(),
3032
waitForApiKeyServer(),
3133
apiKeyServer(),

‎scripts/cts/testServer/replaceAllObjectsScopes.ts

Lines changed: 122 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,122 @@
1+
import type { Server } from 'http';
2+
3+
import { expect } from 'chai';
4+
import type { Express } from 'express';
5+
import express from 'express';
6+
7+
import { setupServer } from './index.js';
8+
9+
const raoState: Record<
10+
string,
11+
{
12+
copyCount: number;
13+
batchCount: number;
14+
waitTaskCount: number;
15+
tmpIndexName: string;
16+
waitingForFinalWaitTask: boolean;
17+
successful: boolean;
18+
}
19+
> = {};
20+
21+
export function assertValidReplaceAllObjectsScopes(expectedCount: number): void {
22+
expect(Object.keys(raoState)).to.have.length(expectedCount);
23+
for (const lang in raoState) {
24+
expect(raoState[lang].successful).to.equal(true);
25+
}
26+
}
27+
28+
function addRoutes(app: Express): void {
29+
app.use(express.urlencoded({ extended: true }));
30+
app.use(
31+
express.json({
32+
type: ['application/json', 'text/plain'], // the js client sends the body as text/plain
33+
}),
34+
);
35+
36+
app.post('/1/indexes/:indexName/operation', (req, res) => {
37+
expect(req.params.indexName).to.match(/^cts_e2e_replace_all_objects_scopes_(.*)$/);
38+
39+
switch (req.body.operation) {
40+
case 'copy': {
41+
expect(req.params.indexName).to.not.include('tmp');
42+
expect(req.body.destination).to.include('tmp');
43+
expect(req.body.scope).to.deep.equal(['settings', 'synonyms']);
44+
45+
const lang = req.params.indexName.replace('cts_e2e_replace_all_objects_scopes_', '');
46+
if (!raoState[lang] || raoState[lang].successful) {
47+
raoState[lang] = {
48+
copyCount: 1,
49+
batchCount: 0,
50+
waitTaskCount: 0,
51+
tmpIndexName: req.body.destination,
52+
waitingForFinalWaitTask: false,
53+
successful: false,
54+
};
55+
} else {
56+
raoState[lang].copyCount++;
57+
}
58+
59+
res.json({ taskID: 123 + raoState[lang].copyCount, updatedAt: '2021-01-01T00:00:00.000Z' });
60+
break;
61+
}
62+
case 'move': {
63+
const lang = req.body.destination.replace('cts_e2e_replace_all_objects_scopes_', '');
64+
expect(raoState).to.include.keys(lang);
65+
expect(raoState[lang]).to.deep.equal({
66+
copyCount: 2,
67+
batchCount: 2,
68+
waitTaskCount: 3,
69+
tmpIndexName: req.params.indexName,
70+
waitingForFinalWaitTask: false,
71+
successful: false,
72+
});
73+
74+
expect(req.body.scope).to.equal(undefined);
75+
76+
raoState[lang].waitingForFinalWaitTask = true;
77+
78+
res.json({ taskID: 777, updatedAt: '2021-01-01T00:00:00.000Z' });
79+
80+
break;
81+
}
82+
default:
83+
res.status(400).json({
84+
message: `invalid operation: ${req.body.operation}, body: ${JSON.stringify(req.body)}`,
85+
});
86+
}
87+
});
88+
89+
app.post('/1/indexes/:indexName/batch', (req, res) => {
90+
const lang = req.params.indexName.match(/^cts_e2e_replace_all_objects_scopes_(.*)_tmp_\d+$/)?.[1] as string;
91+
expect(raoState).to.include.keys(lang);
92+
expect(req.body.requests.every((r) => r.action === 'addObject')).to.equal(true);
93+
94+
raoState[lang].batchCount += req.body.requests.length;
95+
96+
res.json({
97+
taskID: 124 + raoState[lang].batchCount,
98+
objectIDs: req.body.requests.map((r) => r.body.objectID),
99+
});
100+
});
101+
102+
app.get('/1/indexes/:indexName/task/:taskID', (req, res) => {
103+
const lang = req.params.indexName.match(/^cts_e2e_replace_all_objects_scopes_(.*)_tmp_\d+$/)?.[1] as string;
104+
expect(raoState).to.include.keys(lang);
105+
106+
raoState[lang].waitTaskCount++;
107+
if (raoState[lang].waitingForFinalWaitTask) {
108+
expect(req.params.taskID).to.equal('777');
109+
expect(raoState[lang].waitTaskCount).to.equal(4);
110+
111+
raoState[lang].successful = true;
112+
}
113+
114+
res.json({ status: 'published', updatedAt: '2021-01-01T00:00:00.000Z' });
115+
});
116+
}
117+
118+
export function replaceAllObjectsScopesServer(): Promise<Server> {
119+
// this server is used to simulate the responses for the replaceAllObjects method with partial scopes,
120+
// and uses a state machine to determine if the logic is correct.
121+
return setupServer('replaceAllObjectsScopes', 6685, addRoutes);
122+
}

‎specs/search/helpers/replaceAllObjects.yml

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,14 @@ method:
4040
required: false
4141
schema:
4242
type: integer
43+
- in: query
44+
name: scopes
45+
description: List of scopes to kepp in the index. Defaults to `settings`, `synonyms`, and `rules`.
46+
required: false
47+
schema:
48+
type: array
49+
items:
50+
$ref: '../common/enums.yml#/scopeType'
4351
responses:
4452
'200':
4553
description: OK

0 commit comments

Comments
 (0)