Skip to content

Commit 2675046

Browse files
szmarczakszmarczak
authored
Improve the pagination API (#1644)
1 parent e943672 commit 2675046

File tree

6 files changed

+67
-48
lines changed

6 files changed

+67
-48
lines changed

‎readme.md

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -937,7 +937,7 @@ A function that transform [`Response`](#response) into an array of items. This i
937937
Type: `Function`\
938938
Default: [`Link` header logic](source/index.ts)
939939

940-
The function takes three arguments:
940+
The function takes an object with the following properties:
941941
- `response` - The current response object.
942942
- `allItems` - An array of the emitted items.
943943
- `currentItems` - Items from the current response.
@@ -958,7 +958,7 @@ const got = require('got');
958958
offset: 0
959959
},
960960
pagination: {
961-
paginate: (response, allItems, currentItems) => {
961+
paginate: ({response, allItems, currentItems}) => {
962962
const previousSearchParams = response.request.options.searchParams;
963963
const previousOffset = previousSearchParams.get('offset');
964964

@@ -983,18 +983,18 @@ const got = require('got');
983983
###### pagination.filter
984984

985985
Type: `Function`\
986-
Default: `(item, allItems, currentItems) => true`
986+
Default: `({item, allItems, currentItems}) => true`
987987

988988
Checks whether the item should be emitted or not.
989989

990990
###### pagination.shouldContinue
991991

992992
Type: `Function`\
993-
Default: `(item, allItems, currentItems) => true`
993+
Default: `({item, allItems, currentItems}) => true`
994994

995995
Checks whether the pagination should continue.
996996

997-
For example, if you need to stop **before** emitting an entry with some flag, you should use `(item, allItems, currentItems) => !item.flag`. If you want to stop **after** emitting the entry, you should use `(item, allItems, currentItems) => allItems.some(entry => entry.flag)` instead.
997+
For example, if you need to stop **before** emitting an entry with some flag, you should use `({item}) => !item.flag`. If you want to stop **after** emitting the entry, you should use `({item, allItems}) => allItems.some(item => item.flag)` instead.
998998

999999
###### pagination.countLimit
10001000

‎source/as-promise/types.ts

Lines changed: 24 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,19 @@ All parsing methods supported by Got.
1111
*/
1212
export type ResponseType = 'json' | 'buffer' | 'text';
1313

14-
export interface PaginationOptions<T, R> {
14+
export interface PaginateData<BodyType, ElementType> {
15+
response: Response<BodyType>;
16+
allItems: ElementType[];
17+
currentItems: ElementType[];
18+
}
19+
20+
export interface FilterData<ElementType> {
21+
item: ElementType;
22+
allItems: ElementType[];
23+
currentItems: ElementType[];
24+
}
25+
26+
export interface PaginationOptions<ElementType, BodyType> {
1527
/**
1628
All options accepted by `got.paginate()`.
1729
*/
@@ -22,17 +34,17 @@ export interface PaginationOptions<T, R> {
2234
2335
@default response => JSON.parse(response.body)
2436
*/
25-
transform?: (response: Response<R>) => Promise<T[]> | T[];
37+
transform?: (response: Response<BodyType>) => Promise<ElementType[]> | ElementType[];
2638

2739
/**
2840
Checks whether the item should be emitted or not.
2941
30-
@default (item, allItems, currentItems) => true
42+
@default ({item, allItems, currentItems}) => true
3143
*/
32-
filter?: (item: T, allItems: T[], currentItems: T[]) => boolean;
44+
filter?: (data: FilterData<ElementType>) => boolean;
3345

3446
/**
35-
The function takes three arguments:
47+
The function takes an object with the following properties:
3648
- `response` - The current response object.
3749
- `allItems` - An array of the emitted items.
3850
- `currentItems` - Items from the current response.
@@ -76,17 +88,19 @@ export interface PaginationOptions<T, R> {
7688
})();
7789
```
7890
*/
79-
paginate?: (response: Response<R>, allItems: T[], currentItems: T[]) => Options | false;
91+
paginate?: (data: PaginateData<BodyType, ElementType>) => Options | false;
8092

8193
/**
8294
Checks whether the pagination should continue.
8395
84-
For example, if you need to stop **before** emitting an entry with some flag, you should use `(item, allItems, currentItems) => !item.flag`.
85-
If you want to stop **after** emitting the entry, you should use `(item, allItems, currentItems) => allItems.some(entry => entry.flag)` instead.
96+
For example, if you need to stop **before** emitting an entry with some flag, you should use `({item}) => !item.flag`.
97+
98+
If you want to stop **after** emitting the entry, you should use
99+
`({item, allItems}) => allItems.some(item => item.flag)` instead.
86100
87-
@default (item, allItems, currentItems) => true
101+
@default ({item, allItems, currentItems}) => true
88102
*/
89-
shouldContinue?: (item: T, allItems: T[], currentItems: T[]) => boolean;
103+
shouldContinue?: (data: FilterData<ElementType>) => boolean;
90104

91105
/**
92106
The maximum amount of items that should be emitted.

‎source/create.ts

Lines changed: 16 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -225,7 +225,7 @@ const create = (defaults: InstanceDefaults): Got => {
225225
throw new TypeError('`options.pagination` must be implemented');
226226
}
227227

228-
const all: T[] = [];
228+
const allItems: T[] = [];
229229
let {countLimit} = pagination;
230230

231231
let numberOfRequests = 0;
@@ -236,42 +236,46 @@ const create = (defaults: InstanceDefaults): Got => {
236236
}
237237

238238
// @ts-expect-error FIXME!
239-
// TODO: Throw when result is not an instance of Response
239+
// TODO: Throw when response is not an instance of Response
240240
// eslint-disable-next-line no-await-in-loop
241-
const result = (await got(undefined, undefined, normalizedOptions)) as Response;
241+
const response = (await got(undefined, undefined, normalizedOptions)) as Response;
242242

243243
// eslint-disable-next-line no-await-in-loop
244-
const parsed = await pagination.transform(result);
245-
const current: T[] = [];
244+
const parsed = await pagination.transform(response);
245+
const currentItems: T[] = [];
246246

247247
for (const item of parsed) {
248-
if (pagination.filter(item, all, current)) {
249-
if (!pagination.shouldContinue(item, all, current)) {
248+
if (pagination.filter({item, allItems, currentItems})) {
249+
if (!pagination.shouldContinue({item, allItems, currentItems})) {
250250
return;
251251
}
252252

253253
yield item as T;
254254

255255
if (pagination.stackAllItems) {
256-
all.push(item as T);
256+
allItems.push(item as T);
257257
}
258258

259-
current.push(item as T);
259+
currentItems.push(item as T);
260260

261261
if (--countLimit <= 0) {
262262
return;
263263
}
264264
}
265265
}
266266

267-
const optionsToMerge = pagination.paginate(result, all, current);
267+
const optionsToMerge = pagination.paginate({
268+
response,
269+
allItems,
270+
currentItems
271+
});
268272

269273
if (optionsToMerge === false) {
270274
return;
271275
}
272276

273-
if (optionsToMerge === result.request.options) {
274-
normalizedOptions = result.request.options;
277+
if (optionsToMerge === response.request.options) {
278+
normalizedOptions = response.request.options;
275279
} else if (optionsToMerge !== undefined) {
276280
normalizedOptions = normalizeArguments(undefined, optionsToMerge, normalizedOptions);
277281
}

‎source/index.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -77,7 +77,7 @@ const defaults: InstanceDefaults = {
7777

7878
return JSON.parse(response.body as string);
7979
},
80-
paginate: response => {
80+
paginate: ({response}) => {
8181
if (!Reflect.has(response.headers, 'link')) {
8282
return false;
8383
}

‎test/hooks.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1208,7 +1208,7 @@ test('no duplicate hook calls when returning original request options', withServ
12081208
};
12091209

12101210
// Test only two requests, one after another
1211-
const paginate = (response: Response) => requestNumber++ === 0 ? response.request.options : false;
1211+
const paginate = ({response}: {response: Response}) => requestNumber++ === 0 ? response.request.options : false;
12121212

12131213
const instance = got.extend({
12141214
hooks,

‎test/pagination.ts

Lines changed: 20 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -85,11 +85,11 @@ test('filters elements', withServer, async (t, server, got) => {
8585

8686
const result = await got.paginate.all<number>({
8787
pagination: {
88-
filter: (element: number, allItems: number[], currentItems: number[]) => {
88+
filter: ({item, allItems, currentItems}) => {
8989
t.true(Array.isArray(allItems));
9090
t.true(Array.isArray(currentItems));
9191

92-
return element !== 2;
92+
return item !== 2;
9393
}
9494
}
9595
});
@@ -126,7 +126,7 @@ test('custom paginate function', withServer, async (t, server, got) => {
126126

127127
const result = await got.paginate.all<number>({
128128
pagination: {
129-
paginate: response => {
129+
paginate: ({response}) => {
130130
const url = new URL(response.url);
131131

132132
if (url.search === '?page=3') {
@@ -148,13 +148,14 @@ test('custom paginate function using allItems', withServer, async (t, server, go
148148

149149
const result = await got.paginate.all<number>({
150150
pagination: {
151-
paginate: (_response, allItems: number[]) => {
151+
paginate: ({allItems}) => {
152152
if (allItems.length === 2) {
153153
return false;
154154
}
155155

156156
return {path: '/?page=3'};
157-
}
157+
},
158+
stackAllItems: true
158159
}
159160
});
160161

@@ -166,7 +167,7 @@ test('custom paginate function using currentItems', withServer, async (t, server
166167

167168
const result = await got.paginate.all<number>({
168169
pagination: {
169-
paginate: (_response, _allItems: number[], currentItems: number[]) => {
170+
paginate: ({currentItems}) => {
170171
if (currentItems[0] === 3) {
171172
return false;
172173
}
@@ -208,7 +209,7 @@ test('`shouldContinue` works', withServer, async (t, server, got) => {
208209

209210
const options = {
210211
pagination: {
211-
shouldContinue: (_item: unknown, allItems: unknown[], currentItems: unknown[]) => {
212+
shouldContinue: ({allItems, currentItems}: {allItems: number[]; currentItems: number[]}) => {
212213
t.true(Array.isArray(allItems));
213214
t.true(Array.isArray(currentItems));
214215

@@ -357,7 +358,7 @@ test('`hooks` are not duplicated', withServer, async (t, server, got) => {
357358

358359
const result = await got.paginate.all<number>({
359360
pagination: {
360-
paginate: response => {
361+
paginate: ({response}) => {
361362
if ((response.body as string) === '[3]') {
362363
return false; // Stop after page 3
363364
}
@@ -485,21 +486,21 @@ test('`stackAllItems` set to true', withServer, async (t, server, got) => {
485486
const result = await got.paginate.all<number>({
486487
pagination: {
487488
stackAllItems: true,
488-
filter: (_item, allItems, _currentItems) => {
489+
filter: ({allItems}) => {
489490
t.is(allItems.length, itemCount);
490491

491492
return true;
492493
},
493-
shouldContinue: (_item, allItems, _currentItems) => {
494+
shouldContinue: ({allItems}) => {
494495
t.is(allItems.length, itemCount);
495496

496497
return true;
497498
},
498-
paginate: (response, allItems, currentItems) => {
499+
paginate: ({response, allItems, currentItems}) => {
499500
itemCount += 1;
500501
t.is(allItems.length, itemCount);
501502

502-
return got.defaults.options.pagination!.paginate(response, allItems, currentItems);
503+
return got.defaults.options.pagination!.paginate({response, allItems, currentItems});
503504
}
504505
}
505506
});
@@ -513,20 +514,20 @@ test('`stackAllItems` set to false', withServer, async (t, server, got) => {
513514
const result = await got.paginate.all<number>({
514515
pagination: {
515516
stackAllItems: false,
516-
filter: (_item, allItems, _currentItems) => {
517+
filter: ({allItems}) => {
517518
t.is(allItems.length, 0);
518519

519520
return true;
520521
},
521-
shouldContinue: (_item, allItems, _currentItems) => {
522+
shouldContinue: ({allItems}) => {
522523
t.is(allItems.length, 0);
523524

524525
return true;
525526
},
526-
paginate: (response, allItems, currentItems) => {
527+
paginate: ({response, allItems, currentItems}) => {
527528
t.is(allItems.length, 0);
528529

529-
return got.defaults.options.pagination!.paginate(response, allItems, currentItems);
530+
return got.defaults.options.pagination!.paginate({response, allItems, currentItems});
530531
}
531532
}
532533
});
@@ -559,7 +560,7 @@ test('next url in json response', withServer, async (t, server, got) => {
559560
transform: (response: Response<Page>) => {
560561
return [response.body.currentUrl];
561562
},
562-
paginate: (response: Response<Page>) => {
563+
paginate: ({response}) => {
563564
const {next} = response.body;
564565

565566
if (!next) {
@@ -608,7 +609,7 @@ test('pagination using searchParams', withServer, async (t, server, got) => {
608609
transform: (response: Response<Page>) => {
609610
return [response.body.currentUrl];
610611
},
611-
paginate: (response: Response<Page>) => {
612+
paginate: ({response}) => {
612613
const {next} = response.body;
613614
const previousPage = Number(response.request.options.searchParams!.get('page'));
614615

@@ -664,7 +665,7 @@ test('pagination using extended searchParams', withServer, async (t, server, got
664665
transform: (response: Response<Page>) => {
665666
return [response.body.currentUrl];
666667
},
667-
paginate: (response: Response<Page>) => {
668+
paginate: ({response}) => {
668669
const {next} = response.body;
669670
const previousPage = Number(response.request.options.searchParams!.get('page'));
670671

0 commit comments

Comments
 (0)