Skip to content

Commit 57d4e14

Browse files
PaulWesselseisman
PaulWessel
and
seisman
authored
Simplify Curl operations (#3998)
* COnsolidate the curl calls to a single function Avoid near duplicated code * Update gmt_remote.c * Update gmt_remote.c * Update src/gmt_remote.c Co-authored-by: Dongdong Tian <[email protected]> Co-authored-by: Dongdong Tian <[email protected]>
1 parent 416e03f commit 57d4e14

File tree

1 file changed

+105
-106
lines changed

1 file changed

+105
-106
lines changed

‎src/gmt_remote.c

Lines changed: 105 additions & 106 deletions
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,11 @@ struct sigaction cleanup_action, default_action;
7373
char *file_to_delete_if_ctrl_C;
7474
#endif
7575

76+
struct LOCFILE_FP {
77+
char *file; /* Pointer to file name */
78+
FILE *fp; /* Open file pointer */
79+
};
80+
7681
GMT_LOCAL void gmtremote_delete_file_then_exit (int sig_no) {
7782
/* If we catch a CTRL-C during CURL download we must assume file is corrupted and remove it before exiting */
7883
gmt_M_unused (sig_no);
@@ -457,7 +462,7 @@ GMT_LOCAL int gmtremote_find_and_give_data_attribution (struct GMTAPI_CTRL *API,
457462
return (match);
458463
}
459464

460-
GMT_LOCAL char *gmtremote_lockfile (struct GMTAPI_CTRL *API, char *file) {
465+
GMT_LOCAL char *gmtremote_lockfile (struct GMT_CTRL *GMT, char *file) {
461466
/* Create a dummy file in temp with extension .download and use as a lock file */
462467
char *c = strrchr (file, '/');
463468
char Lfile[PATH_MAX] = {""};
@@ -466,7 +471,7 @@ GMT_LOCAL char *gmtremote_lockfile (struct GMTAPI_CTRL *API, char *file) {
466471
else /* No path, just point to file */
467472
c = file;
468473
if (c[0] == '@') c++; /* Skip any leading @ sign */
469-
sprintf (Lfile, "%s/%s.download", API->tmp_dir, c);
474+
sprintf (Lfile, "%s/%s.download", GMT->parent->tmp_dir, c);
470475
return (strdup (Lfile));
471476
}
472477

@@ -515,68 +520,103 @@ GMT_LOCAL size_t gmtremote_skip_large_files (struct GMT_CTRL *GMT, char * URL, s
515520
return action;
516521
}
517522

518-
/* Deal with hash values of cache/data files */
519-
520-
#define GMT_HASH_TIME_OUT 10L /* Not waiting longer than this to time out on getting the hash file */
521-
522-
GMT_LOCAL int gmtremote_get_url (struct GMT_CTRL *GMT, char *url, char *file, char *orig, unsigned int index) {
523-
bool query = gmt_M_file_is_query (url), turn_off = false;
524-
int curl_err = 0, error = 0;
525-
long time_spent;
526-
char *Lfile = NULL;
527-
FILE *fp = NULL;
523+
CURL * gmtremote_setup_curl (struct GMTAPI_CTRL *API, char *url, char *local_file, struct FtpFile *urlfile, unsigned int time_out) {
524+
/* Single function that sets up an impending CURL operation */
528525
CURL *Curl = NULL;
529-
struct FtpFile urlfile = {NULL, NULL};
530-
struct GMTAPI_CTRL *API = GMT->parent;
531-
time_t begin, end;
532-
533-
if (!query) { /* Only make a filename if not a query */
534-
Lfile = gmtremote_lockfile (API, file);
535-
if ((fp = fopen (Lfile, "w")) == NULL) {
536-
GMT_Report (API, GMT_MSG_ERROR, "Failed to create lock file %s\n", Lfile);
537-
gmt_M_str_free (Lfile);
538-
return 1;
539-
}
540-
gmtlib_file_lock (GMT, fileno(fp)); /* Attempt exclusive lock */
541-
}
542-
543526
if ((Curl = curl_easy_init ()) == NULL) {
544-
GMT_Report (GMT->parent, GMT_MSG_ERROR, "Failed to initiate curl - cannot obtain %s\n", url);
545-
error = 1; goto unlocking1;
527+
GMT_Report (API, GMT_MSG_ERROR, "Failed to initiate curl - cannot obtain %s\n", url);
528+
return NULL;
546529
}
547530
if (curl_easy_setopt (Curl, CURLOPT_SSL_VERIFYPEER, 0L)) { /* Tell libcurl to not verify the peer */
548-
GMT_Report (GMT->parent, GMT_MSG_ERROR, "Failed to set curl option to not verify the peer\n");
549-
error = 1; goto unlocking1;
531+
GMT_Report (API, GMT_MSG_ERROR, "Failed to set curl option to not verify the peer\n");
532+
return NULL;
550533
}
551534
if (curl_easy_setopt (Curl, CURLOPT_FOLLOWLOCATION, 1L)) { /* Tell libcurl to follow 30x redirects */
552-
GMT_Report (GMT->parent, GMT_MSG_ERROR, "Failed to set curl option to follow redirects\n");
553-
error = 1; goto unlocking1;
535+
GMT_Report (API, GMT_MSG_ERROR, "Failed to set curl option to follow redirects\n");
536+
return NULL;
554537
}
555538
if (curl_easy_setopt (Curl, CURLOPT_FAILONERROR, 1L)) { /* Tell libcurl to fail on 4xx responses (e.g. 404) */
556-
GMT_Report (GMT->parent, GMT_MSG_ERROR, "Failed to set curl option to fail for 4xx responses\n");
557-
error = 1; goto unlocking1;
539+
GMT_Report (API, GMT_MSG_ERROR, "Failed to set curl option to fail for 4xx responses\n");
540+
return NULL;
558541
}
559542
if (curl_easy_setopt (Curl, CURLOPT_URL, url)) { /* Set the URL to copy */
560-
GMT_Report (GMT->parent, GMT_MSG_ERROR, "Failed to set curl option to read from %s\n", url);
561-
error = 1; goto unlocking1;
543+
GMT_Report (API, GMT_MSG_ERROR, "Failed to set curl option to read from %s\n", url);
544+
return NULL;
562545
}
563-
if (curl_easy_setopt (Curl, CURLOPT_TIMEOUT, GMT_HASH_TIME_OUT)) { /* Set a max timeout */
564-
GMT_Report (GMT->parent, GMT_MSG_ERROR, "Failed to set curl option to time out after %ld seconds\n", GMT_HASH_TIME_OUT);
565-
error = 1; goto unlocking1;
546+
if (time_out) { /* Set a timeout limit */
547+
if (curl_easy_setopt (Curl, CURLOPT_TIMEOUT, time_out)) {
548+
GMT_Report (API, GMT_MSG_ERROR, "Failed to set curl option to time out after %d seconds\n", time_out);
549+
return NULL;
550+
}
566551
}
567-
urlfile.filename = file; /* Set pointer to local filename */
552+
urlfile->filename = local_file; /* Set pointer to local filename */
568553
/* Define our callback to get called when there's data to be written */
569554
if (curl_easy_setopt (Curl, CURLOPT_WRITEFUNCTION, gmtremote_fwrite_callback)) {
570-
GMT_Report (GMT->parent, GMT_MSG_ERROR, "Failed to set curl output callback function\n");
571-
error = 1; goto unlocking1;
555+
GMT_Report (API, GMT_MSG_ERROR, "Failed to set curl output callback function\n");
556+
return NULL;
572557
}
573-
/* Set a pointer to our struct to pass to the callback */
574-
if (curl_easy_setopt (Curl, CURLOPT_WRITEDATA, &urlfile)) {
575-
GMT_Report (GMT->parent, GMT_MSG_ERROR, "Failed to set curl option to write to %s\n", file);
576-
error = 1; goto unlocking1;
558+
/* Set a pointer to our struct that will be passed to the callback function */
559+
if (curl_easy_setopt (Curl, CURLOPT_WRITEDATA, urlfile)) {
560+
GMT_Report (API, GMT_MSG_ERROR, "Failed to set curl option to write to %s\n", local_file);
561+
return NULL;
562+
}
563+
564+
return Curl; /* Happily return the Curl pointer */
565+
}
566+
567+
struct LOCFILE_FP *gmtremote_lock_on (struct GMT_CTRL *GMT, char *file) {
568+
/* Creates filename for lock and activates the lock */
569+
struct LOCFILE_FP *P = gmt_M_memory (GMT, NULL, 1, struct LOCFILE_FP);
570+
P->file = gmtremote_lockfile (GMT, file);
571+
if ((P->fp = fopen (P->file, "w")) == NULL) {
572+
GMT_Report (GMT->parent, GMT_MSG_ERROR, "Failed to create lock file %s\n", P->file);
573+
gmt_M_str_free (P->file);
574+
gmt_M_free (GMT, P);
575+
return NULL;
576+
}
577+
gmtlib_file_lock (GMT, fileno(P->fp)); /* Attempt exclusive lock */
578+
return P;
579+
}
580+
581+
void gmtremote_lock_off (struct GMT_CTRL *GMT, struct LOCFILE_FP **P) {
582+
/* Deactivates the lock on the file */
583+
gmtlib_file_unlock (GMT, fileno((*P)->fp));
584+
fclose ((*P)->fp);
585+
gmt_remove_file (GMT, (*P)->file);
586+
gmt_M_str_free ((*P)->file);
587+
gmt_M_free (GMT, *P);
588+
}
589+
590+
/* Deal with hash values of cache/data files */
591+
592+
#define GMT_HASH_TIME_OUT 10L /* Not waiting longer than this to time out on getting the hash file */
593+
594+
GMT_LOCAL int gmtremote_get_url (struct GMT_CTRL *GMT, char *url, char *file, char *orig, unsigned int index) {
595+
bool turn_ctrl_C_off = false;
596+
int curl_err = 0, error = 0;
597+
long time_spent;
598+
CURL *Curl = NULL;
599+
struct LOCFILE_FP *LF = NULL;
600+
struct FtpFile urlfile = {NULL, NULL};
601+
struct GMTAPI_CTRL *API = GMT->parent;
602+
time_t begin, end;
603+
604+
if (GMT->current.setting.auto_download == GMT_NO_DOWNLOAD) { /* Not allowed to use remote copying */
605+
GMT_Report (GMT->parent, GMT_MSG_ERROR, "Remote download is currently deactivated\n");
606+
return 1;
577607
}
608+
if (GMT->current.io.internet_error) return 1; /* Not able to use remote copying in this session */
609+
610+
/* Make a lock */
611+
if ((LF = gmtremote_lock_on (GMT, file)) == NULL)
612+
return 1;
613+
614+
/* Initialize the curl session */
615+
if ((Curl = gmtremote_setup_curl (API, url, file, &urlfile, GMT_HASH_TIME_OUT)) == NULL)
616+
goto unlocking1;
617+
578618
GMT_Report (GMT->parent, GMT_MSG_INFORMATION, "Downloading file %s ...\n", url);
579-
gmtremote_turn_on_ctrl_C_check (file); turn_off = true;
619+
gmtremote_turn_on_ctrl_C_check (file); turn_ctrl_C_off = true;
580620
begin = time (NULL);
581621
if ((curl_err = curl_easy_perform (Curl))) { /* Failed, give error message */
582622
end = time (NULL);
@@ -607,14 +647,11 @@ GMT_LOCAL int gmtremote_get_url (struct GMT_CTRL *GMT, char *url, char *file, ch
607647

608648
unlocking1:
609649

610-
if (!query) { /* Remove lock file after successful download */
611-
gmtlib_file_unlock (GMT, fileno(fp));
612-
fclose(fp);
613-
gmt_remove_file (GMT, Lfile);
614-
gmt_M_str_free (Lfile);
615-
}
650+
/* Remove lock file after successful download */
651+
gmtremote_lock_off (GMT, &LF);
652+
653+
if (turn_ctrl_C_off) gmtremote_turn_off_ctrl_C_check ();
616654

617-
if (turn_off) gmtremote_turn_off_ctrl_C_check ();
618655
return error;
619656
}
620657

@@ -1114,20 +1151,19 @@ int gmtlib_file_is_jpeg2000_tile (struct GMTAPI_CTRL *API, char *file) {
11141151
}
11151152

11161153
int gmt_download_file (struct GMT_CTRL *GMT, const char *name, char *url, char *localfile, bool be_fussy) {
1117-
bool query = gmt_M_file_is_query (url), turn_off = false;
1154+
bool query = gmt_M_file_is_query (url), turn_ctrl_C_off = false;
11181155
int curl_err, error = 0;
11191156
size_t fsize;
1120-
char *Lfile = NULL;
11211157
CURL *Curl = NULL;
1122-
FILE *fp = NULL;
11231158
struct FtpFile urlfile = {NULL, NULL};
1159+
struct LOCFILE_FP *LF = NULL;
11241160
struct GMTAPI_CTRL *API = GMT->parent;
11251161

11261162
if (GMT->current.setting.auto_download == GMT_NO_DOWNLOAD) { /* Not allowed to use remote copying */
11271163
GMT_Report (GMT->parent, GMT_MSG_ERROR, "Remote download is currently deactivated\n");
11281164
return 1;
11291165
}
1130-
if (GMT->current.io.internet_error) return 1; /* Not able to use remote copying in this session */
1166+
if (GMT->current.io.internet_error) return 1; /* Not able to use remote copying in this session */
11311167

11321168
/* Check if the file is too big for our current limit */
11331169

@@ -1140,52 +1176,19 @@ int gmt_download_file (struct GMT_CTRL *GMT, const char *name, char *url, char *
11401176

11411177
/* Here we will try to download a file */
11421178

1143-
if (!query) { /* Only make a filename if not a query */
1144-
Lfile = gmtremote_lockfile (API, (char *)name);
1145-
if ((fp = fopen (Lfile, "w")) == NULL) {
1146-
GMT_Report (API, GMT_MSG_ERROR, "Failed to create lock file %s\n", Lfile);
1147-
gmt_M_str_free (Lfile);
1148-
return 1;
1149-
}
1150-
gmtlib_file_lock (GMT, fileno(fp)); /* Attempt exclusive lock */
1151-
}
1152-
1153-
if ((Curl = curl_easy_init ()) == NULL) {
1154-
GMT_Report (API, GMT_MSG_ERROR, "Failed to initiate curl\n");
1155-
error = 1; goto unlocking2;
1156-
}
1157-
if (curl_easy_setopt (Curl, CURLOPT_SSL_VERIFYPEER, 0L)) { /* Tell libcurl to not verify the peer */
1158-
GMT_Report (API, GMT_MSG_ERROR, "Failed to set curl option to not verify the peer\n");
1159-
error = 1; goto unlocking2;
1160-
}
1161-
if (curl_easy_setopt (Curl, CURLOPT_FOLLOWLOCATION, 1L)) { /* Tell libcurl to follow 30x redirects */
1162-
GMT_Report (API, GMT_MSG_ERROR, "Failed to set curl option to follow redirects\n");
1163-
error = 1; goto unlocking2;
1164-
}
1165-
if (curl_easy_setopt (Curl, CURLOPT_FAILONERROR, 1L)) { /* Tell libcurl to fail on 4xx responses (e.g. 404) */
1166-
error = 1; goto unlocking2;
1167-
}
1179+
/* Only make a lock if not a query */
1180+
if (!query && (LF = gmtremote_lock_on (GMT, (char *)name)) == NULL)
1181+
return 1;
11681182

1169-
if (curl_easy_setopt (Curl, CURLOPT_URL, url)) { /* Set the URL to copy */
1170-
GMT_Report (API, GMT_MSG_ERROR, "Failed to set curl option to read from %s\n", url);
1171-
error = 1; goto unlocking2;
1172-
}
1173-
urlfile.filename = localfile; /* Set pointer to local filename */
1174-
/* Define our callback to get called when there's data to be written */
1175-
if (curl_easy_setopt (Curl, CURLOPT_WRITEFUNCTION, gmtremote_fwrite_callback)) {
1176-
GMT_Report (API, GMT_MSG_ERROR, "Failed to set curl output callback function\n");
1177-
error = 1; goto unlocking2;
1178-
}
1179-
/* Set a pointer to our struct to pass to the callback */
1180-
if (curl_easy_setopt (Curl, CURLOPT_WRITEDATA, &urlfile)) {
1181-
GMT_Report (API, GMT_MSG_ERROR, "Failed to set curl option to write to %s\n", localfile);
1182-
error = 1; goto unlocking2;
1183-
}
1183+
/* Initialize the curl session */
1184+
if ((Curl = gmtremote_setup_curl (API, url, localfile, &urlfile, 0)) == NULL)
1185+
goto unlocking2;
11841186

11851187
gmtremote_find_and_give_data_attribution (API, name);
11861188

11871189
GMT_Report (API, GMT_MSG_INFORMATION, "Downloading file %s ...\n", url);
1188-
gmtremote_turn_on_ctrl_C_check (localfile); turn_off = true;
1190+
gmtremote_turn_on_ctrl_C_check (localfile);
1191+
turn_ctrl_C_off = true;
11891192
if ((curl_err = curl_easy_perform (Curl))) { /* Failed, give error message */
11901193
if (be_fussy || !(curl_err == CURLE_REMOTE_FILE_NOT_FOUND || curl_err == CURLE_HTTP_RETURNED_ERROR)) { /* Unexpected failure - want to bitch about it */
11911194
GMT_Report (API, GMT_MSG_ERROR, "Libcurl Error: %s\n", curl_easy_strerror (curl_err));
@@ -1206,14 +1209,10 @@ int gmt_download_file (struct GMT_CTRL *GMT, const char *name, char *url, char *
12061209

12071210
unlocking2:
12081211

1209-
if (!query) { /* Remove lock file after successful download */
1210-
gmtlib_file_unlock (GMT, fileno(fp));
1211-
fclose(fp);
1212-
gmt_remove_file (GMT, Lfile);
1213-
gmt_M_str_free (Lfile);
1214-
}
1212+
if (!query) /* Remove lock file after successful download (unless query) */
1213+
gmtremote_lock_off (GMT, &LF);
12151214

1216-
if (turn_off) gmtremote_turn_off_ctrl_C_check ();
1215+
if (turn_ctrl_C_off) gmtremote_turn_off_ctrl_C_check ();
12171216

12181217
if (error == 0) error = gmtremote_convert_jp2_to_nc (API, localfile);
12191218

0 commit comments

Comments
 (0)