Skip to content

Let libcurl downloads be managed via lockfiles #3735

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
PaulWessel merged 6 commits into from
Jul 29, 2020
Merged

Let libcurl downloads be managed via lockfiles #3735

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
6bb71f9
Try to use filelock for downloads
PaulWessel Jul 26, 2020
280edb0
Update gmt_remote.c
PaulWessel Jul 26, 2020
9a39d8a
Merge branch 'master' into curl-flock
PaulWessel Jul 26, 2020
2df8f03
Delete a failing gmt_data_server.txt file so it can be regenerated
PaulWessel Jul 26, 2020
718e483
Merge branch 'master' into curl-flock
PaulWessel Jul 28, 2020
b291ab2
Merge branch 'master' into curl-flock
PaulWessel Jul 29, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 10 additions & 10 deletions src/gmt_init.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -425,7 +425,7 @@ static struct GMT_HASH keys_hashnode[GMT_N_KEYS];
#include <Windows.h>

/*! . */
GMT_LOCAL bool gmtinit_file_lock (struct GMT_CTRL *GMT, int fd) {
bool gmtlib_file_lock (struct GMT_CTRL *GMT, int fd) {
OVERLAPPED over = { 0 };
HANDLE hand = (HANDLE)_get_osfhandle(fd);
if (!LockFileEx(hand, LOCKFILE_EXCLUSIVE_LOCK, 0, 1, 0, &over)) /* Will block until exclusive lock is acquired */
Expand All @@ -437,7 +437,7 @@ GMT_LOCAL bool gmtinit_file_lock (struct GMT_CTRL *GMT, int fd) {
}

/*! . */
GMT_LOCAL bool gmtinit_file_unlock (struct GMT_CTRL *GMT, int fd) {
bool gmtlib_file_unlock (struct GMT_CTRL *GMT, int fd) {
HANDLE hand = (HANDLE)_get_osfhandle(fd);
if (!UnlockFile(hand, 0, 0, 0, 1))
{
Expand All @@ -449,7 +449,7 @@ GMT_LOCAL bool gmtinit_file_unlock (struct GMT_CTRL *GMT, int fd) {

#elif defined (HAVE_FCNTL_H_) /* Use POSIX fcntl */
/*! . */
GMT_LOCAL bool gmtinit_file_lock (struct GMT_CTRL *GMT, int fd) {
bool gmtlib_file_lock (struct GMT_CTRL *GMT, int fd) {
int status;
struct flock lock;
lock.l_type = F_WRLCK; /* Lock for exclusive reading/writing */
Expand All @@ -466,7 +466,7 @@ GMT_LOCAL bool gmtinit_file_lock (struct GMT_CTRL *GMT, int fd) {
}

/*! . */
GMT_LOCAL bool gmtinit_file_unlock (struct GMT_CTRL *GMT, int fd) {
bool gmtlib_file_unlock (struct GMT_CTRL *GMT, int fd) {
int status;
struct flock lock;
lock.l_type = F_UNLCK; /* Release lock and close file */
Expand All @@ -484,13 +484,13 @@ GMT_LOCAL bool gmtinit_file_unlock (struct GMT_CTRL *GMT, int fd) {

#else /* Not Windows and fcntl not available */
/*! . */
GMT_LOCAL bool gmtinit_file_lock (struct GMT_CTRL *GMT, int fd) {
bool gmtlib_file_lock (struct GMT_CTRL *GMT, int fd) {
GMT_Report (GMT->parent, GMT_MSG_DEBUG, "File locking not supported.\n");
return false;
}

/*! . */
GMT_LOCAL bool gmtinit_file_unlock (struct GMT_CTRL *GMT, int fd) {
bool gmtlib_file_unlock (struct GMT_CTRL *GMT, int fd) {
return false;
}
#endif
Expand Down Expand Up @@ -2880,7 +2880,7 @@ GMT_LOCAL int gmtinit_get_history (struct GMT_CTRL *GMT) {
}

/* When we get here the file exists */
gmtinit_file_lock (GMT, fileno(fp));
gmtlib_file_lock (GMT, fileno(fp));
/* Format of GMT gmt.history is as follow:
* BEGIN GMT <version> This is the start of parsable section
* OPT ARG
Expand Down Expand Up @@ -2919,7 +2919,7 @@ GMT_LOCAL int gmtinit_get_history (struct GMT_CTRL *GMT) {
}

/* Close the file */
gmtinit_file_unlock (GMT, fileno(fp));
gmtlib_file_unlock (GMT, fileno(fp));
fclose (fp);

GMT_Report (GMT->parent, GMT_MSG_DEBUG, "Exit: gmtinit_get_history\n");
Expand Down Expand Up @@ -2975,7 +2975,7 @@ GMT_LOCAL int gmtinit_put_history (struct GMT_CTRL *GMT) {
if ((fp = fopen (hfile, "w")) == NULL) return (-1); /* Not OK to be unsuccessful in creating this file */

/* When we get here the file is open */
if (!gmtinit_file_lock (GMT, fileno(fp)))
if (!gmtlib_file_lock (GMT, fileno(fp)))
GMT_Report (GMT->parent, GMT_MSG_WARNING, "Directory %s is not locked for exclusive access. Multiple gmt processes running at once could corrupt history file.\n", hfile);

fprintf (fp, "# GMT %d Session common arguments shelf\n", GMT_MAJOR_VERSION);
Expand All @@ -2991,7 +2991,7 @@ GMT_LOCAL int gmtinit_put_history (struct GMT_CTRL *GMT) {
fprintf (fp, "END\n");

/* Close the file */
gmtinit_file_unlock (GMT, fileno(fp));
gmtlib_file_unlock (GMT, fileno(fp));
fclose (fp);

return (GMT_NOERROR);
Expand Down
2 changes: 2 additions & 0 deletions src/gmt_internals.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,8 @@ struct GMT_XINGS {
EXTERN_MSC char *dlerror (void);
#endif

EXTERN_MSC bool gmtlib_file_lock (struct GMT_CTRL *GMT, int fd);
EXTERN_MSC bool gmtlib_file_unlock (struct GMT_CTRL *GMT, int fd);
EXTERN_MSC int gmtlib_file_is_jpeg2000_tile (struct GMTAPI_CTRL *API, char *file);
EXTERN_MSC int gmtlib_download_remote_file (struct GMT_CTRL *GMT, const char* file_name, char *path, int k_data, unsigned int mode);
EXTERN_MSC int gmtlib_get_serverfile_index (struct GMTAPI_CTRL *API, const char *file);
Expand Down
47 changes: 46 additions & 1 deletion src/gmt_remote.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -197,12 +197,16 @@ GMT_LOCAL struct GMT_DATA_INFO *gmtremote_data_load (struct GMTAPI_CTRL *API, in
if (fgets (line, GMT_LEN256, fp) == NULL) { /* Try to get first record */
fclose (fp);
GMT_Report (API, GMT_MSG_ERROR, "Read error first record in file %s\n", file);
GMT_Report (API, GMT_MSG_ERROR, "Deleting %s so it can get regenerated - please try again\n", file);
gmt_remove_file (GMT, file);
return NULL;
}
*n = atoi (line); /* Number of non-commented records to follow */
if (*n <= 0 || *n > GMT_BIG_CHUNK) { /* Probably not a good value */
fclose (fp);
GMT_Report (API, GMT_MSG_ERROR, "Bad record counter in file %s\n", file);
GMT_Report (API, GMT_MSG_ERROR, "Deleting %s so it can get regenerated - please try again\n", file);
gmt_remove_file (GMT, file);
return NULL;
}
if (fgets (line, GMT_LEN256, fp) == NULL) { /* Try to get second record */
Expand Down Expand Up @@ -453,7 +457,20 @@ GMT_LOCAL int gmtremote_find_and_give_data_attribution (struct GMTAPI_CTRL *API,
return (match);
}

GMT_LOCAL size_t gmtremote_skip_large_files (struct GMT_CTRL *GMT, char* URL, size_t limit) {
GMT_LOCAL char *gmtremote_lockfile (struct GMTAPI_CTRL *API, char *file) {
/* Create a dummy file in temp with extension .download and use as a lock file */
char *c = strrchr (file, '/');
char Lfile[PATH_MAX] = {""};
if (c) /* Found the last slash, skip it */
c++;
else /* No path, just point to file */
c = file;
if (c[0] == '@') c++; /* Skip any leading @ sign */
sprintf (Lfile, "%s/%s.download", API->tmp_dir, c);
return (strdup (Lfile));
}

GMT_LOCAL size_t gmtremote_skip_large_files (struct GMT_CTRL *GMT, char * URL, size_t limit) {
/* Get the remote file's size and if too large we refuse to download */
CURL *curl = NULL;
CURLcode res;
Expand Down Expand Up @@ -505,10 +522,20 @@ GMT_LOCAL size_t gmtremote_skip_large_files (struct GMT_CTRL *GMT, char* URL, si
GMT_LOCAL int gmtremote_get_url (struct GMT_CTRL *GMT, char *url, char *file, char *orig, unsigned int index) {
int curl_err = 0;
long time_spent;
char *Lfile = NULL;
FILE *fp = NULL;
CURL *Curl = NULL;
struct FtpFile urlfile = {NULL, NULL};
struct GMTAPI_CTRL *API = GMT->parent;
time_t begin, end;

Lfile = gmtremote_lockfile (API, file);
if ((fp = fopen (Lfile, "w")) == NULL) {
GMT_Report (API, GMT_MSG_ERROR, "Failed to create lock file %s\n", Lfile);
return 1;
}
gmtlib_file_lock (GMT, fileno(fp)); /* Attempt exclusive lock */

if ((Curl = curl_easy_init ()) == NULL) {
GMT_Report (GMT->parent, GMT_MSG_ERROR, "Failed to initiate curl - cannot obtain %s\n", url);
return 1;
Expand Down Expand Up @@ -573,6 +600,10 @@ GMT_LOCAL int gmtremote_get_url (struct GMT_CTRL *GMT, char *url, char *file, ch
curl_easy_cleanup (Curl);
if (urlfile.fp) /* close the local file */
fclose (urlfile.fp);
gmtlib_file_unlock (GMT, fileno(fp));
gmt_remove_file (GMT, Lfile);
gmt_M_str_free (Lfile);

gmtremote_turn_off_ctrl_C_check ();
return 0;
}
Expand Down Expand Up @@ -1075,7 +1106,9 @@ int gmtlib_file_is_jpeg2000_tile (struct GMTAPI_CTRL *API, char *file) {
int gmt_download_file (struct GMT_CTRL *GMT, const char *name, char *url, char *localfile, bool be_fussy) {
int curl_err, error;
size_t fsize;
char *Lfile = NULL;
CURL *Curl = NULL;
FILE *fp = NULL;
struct FtpFile urlfile = {NULL, NULL};
struct GMTAPI_CTRL *API = GMT->parent;

Expand All @@ -1096,6 +1129,13 @@ int gmt_download_file (struct GMT_CTRL *GMT, const char *name, char *url, char *

/* Here we will try to download a file */

Lfile = gmtremote_lockfile (API, (char *)name);
if ((fp = fopen (Lfile, "w")) == NULL) {
GMT_Report (API, GMT_MSG_ERROR, "Failed to create lock file %s\n", Lfile);
return 1;
}
gmtlib_file_lock (GMT, fileno(fp)); /* Attempt exclusive lock */

if ((Curl = curl_easy_init ()) == NULL) {
GMT_Report (API, GMT_MSG_ERROR, "Failed to initiate curl\n");
return 1;
Expand Down Expand Up @@ -1149,6 +1189,11 @@ int gmt_download_file (struct GMT_CTRL *GMT, const char *name, char *url, char *
curl_easy_cleanup (Curl);
if (urlfile.fp) /* close the local file */
fclose (urlfile.fp);

gmtlib_file_unlock (GMT, fileno(fp));
gmt_remove_file (GMT, Lfile);
gmt_M_str_free (Lfile);

gmtremote_turn_off_ctrl_C_check ();

error = gmtremote_convert_jp2_to_nc (API, localfile);
Expand Down