1- import got from "got"
1+ import got , { Headers , Options } from "got"
22import fileType from "file-type"
33import path from "path"
4- import { IncomingMessage , OutgoingHttpHeaders } from "http"
54import fs from "fs-extra"
65import { createContentDigest } from "./create-content-digest"
76import {
@@ -10,7 +9,8 @@ import {
109 createFilePath ,
1110} from "./filename-utils"
1211
13- import { GatsbyCache } from "gatsby"
12+ import type { IncomingMessage } from "http"
13+ import type { GatsbyCache } from "gatsby"
1414
1515export interface IFetchRemoteFileOptions {
1616 url : string
@@ -19,7 +19,7 @@ export interface IFetchRemoteFileOptions {
1919 htaccess_pass ?: string
2020 htaccess_user ?: string
2121 }
22- httpHeaders ?: OutgoingHttpHeaders
22+ httpHeaders ?: Headers
2323 ext ?: string
2424 name ?: string
2525}
@@ -57,10 +57,10 @@ const INCOMPLETE_RETRY_LIMIT = process.env.GATSBY_INCOMPLETE_RETRY_LIMIT
5757 * @return {Promise<Object> } Resolves with the [http Result Object]{@link https://nodejs.org/api/http.html#http_class_http_serverresponse}
5858 */
5959const requestRemoteNode = (
60- url : got . GotUrl ,
61- headers : OutgoingHttpHeaders ,
60+ url : string | URL ,
61+ headers : Headers ,
6262 tmpFilename : string ,
63- httpOptions : got . GotOptions < string | null > | undefined ,
63+ httpOptions ?: Options ,
6464 attempt : number = 1
6565) : Promise < IncomingMessage > =>
6666 new Promise ( ( resolve , reject ) => {
@@ -71,12 +71,15 @@ const requestRemoteNode = (
7171 const handleTimeout = async ( ) : Promise < void > => {
7272 fsWriteStream . close ( )
7373 fs . removeSync ( tmpFilename )
74+
7475 if ( attempt < STALL_RETRY_LIMIT ) {
7576 // Retry by calling ourself recursively
7677 resolve (
7778 requestRemoteNode ( url , headers , tmpFilename , httpOptions , attempt + 1 )
7879 )
7980 } else {
81+ // TODO move to new Error type
82+ // eslint-disable-next-line prefer-promise-reject-errors
8083 reject ( `Failed to download ${ url } ${ STALL_RETRY_LIMIT } )
8184 }
8285 }
@@ -93,15 +96,21 @@ const requestRemoteNode = (
9396 send : CONNECTION_TIMEOUT , // https://github.com/sindresorhus/got#timeout
9497 } ,
9598 ...httpOptions ,
99+ isStream : true ,
96100 } )
97101
98102 let haveAllBytesBeenWritten = false
103+ // Fixes a bug in latest got where progress.total gets reset when stream ends, even if it wasn't complete.
104+ let totalSize : number | null = null
99105 responseStream . on ( `downloadProgress` , progress => {
100106 if (
101- progress . transferred === progress . total ||
102- progress . total === null ||
103- progress . total === undefined
107+ progress . total != null &&
108+ ( ! totalSize || totalSize < progress . total )
104109 ) {
110+ totalSize = progress . total
111+ }
112+
113+ if ( progress . transferred === totalSize || totalSize === null ) {
105114 haveAllBytesBeenWritten = true
106115 }
107116 } )
@@ -113,29 +122,36 @@ const requestRemoteNode = (
113122 if ( timeout ) {
114123 clearTimeout ( timeout )
115124 }
125+
126+ fsWriteStream . close ( )
116127 fs . removeSync ( tmpFilename )
117128 reject ( error )
118129 } )
119130
120- fsWriteStream . on ( `error` , ( error : any ) => {
131+ fsWriteStream . on ( `error` , ( error : unknown ) => {
121132 if ( timeout ) {
122133 clearTimeout ( timeout )
123134 }
135+
124136 reject ( error )
125137 } )
126138
127139 responseStream . on ( `response` , response => {
128140 resetTimeout ( )
129141
130142 fsWriteStream . on ( `finish` , ( ) => {
143+ if ( timeout ) {
144+ clearTimeout ( timeout )
145+ }
146+
131147 fsWriteStream . close ( )
132148
133149 // We have an incomplete download
134150 if ( ! haveAllBytesBeenWritten ) {
135151 fs . removeSync ( tmpFilename )
136152
137153 if ( attempt < INCOMPLETE_RETRY_LIMIT ) {
138- resolve (
154+ return resolve (
139155 requestRemoteNode (
140156 url ,
141157 headers ,
@@ -145,16 +161,15 @@ const requestRemoteNode = (
145161 )
146162 )
147163 } else {
148- reject (
164+ // TODO move to new Error type
165+ // eslint-disable-next-line prefer-promise-reject-errors
166+ return reject (
149167 `Failed to download ${ url } ${ INCOMPLETE_RETRY_LIMIT }
150168 )
151169 }
152170 }
153171
154- if ( timeout ) {
155- clearTimeout ( timeout )
156- }
157- resolve ( response )
172+ return resolve ( response )
158173 } )
159174 } )
160175 } )
@@ -177,14 +192,11 @@ export async function fetchRemoteFile({
177192 headers [ `If-None-Match` ] = cachedHeaders . etag
178193 }
179194
180- // Add Basic authentication if passed in:
181- // https://github.com/sindresorhus/got/blob/main/documentation/2-options.md#username
182- // The "auth" API isn't particularly extensible, we should define an API that we validate
183- const httpOptions : got . GotOptions < string | null > = { }
195+ // Add htaccess authentication if passed in. This isn't particularly
196+ // extensible. We should define a proper API that we validate.
197+ const httpOptions : Options = { }
184198 if ( auth && ( auth . htaccess_pass || auth . htaccess_user ) ) {
185- // @ts -ignore - We use outdated @types/got typings. Once we update got everywhere we can remove @types/got and have correct typings
186199 httpOptions . username = auth . htaccess_user
187- // @ts -ignore - see above
188200 httpOptions . password = auth . htaccess_pass
189201 }
190202
0 commit comments