diff options
Diffstat (limited to 'lib/libalpm/dload.c')
-rw-r--r-- | lib/libalpm/dload.c | 533 |
1 files changed, 234 insertions, 299 deletions
diff --git a/lib/libalpm/dload.c b/lib/libalpm/dload.c index a98d84ac..46925a99 100644 --- a/lib/libalpm/dload.c +++ b/lib/libalpm/dload.c @@ -29,14 +29,9 @@ #include <sys/types.h> #include <sys/stat.h> #include <signal.h> -/* the following two are needed for FreeBSD's libfetch */ -#include <limits.h> /* PATH_MAX */ -#if defined(HAVE_SYS_PARAM_H) -#include <sys/param.h> /* MAXHOSTNAMELEN */ -#endif -#ifdef HAVE_LIBFETCH -#include <fetch.h> +#ifdef HAVE_LIBCURL +#include <curl/curl.h> #endif /* libalpm */ @@ -47,112 +42,185 @@ #include "util.h" #include "handle.h" -static char *get_filename(const char *url) { +#ifdef HAVE_LIBCURL +static double prevprogress; /* last download amount */ +#endif + +static const char *get_filename(const char *url) +{ char *filename = strrchr(url, '/'); if(filename != NULL) { filename++; } - return(filename); + return filename; } -#ifdef HAVE_LIBFETCH -static char *get_destfile(const char *path, const char *filename) { - char *destfile; - /* len = localpath len + filename len + null */ - size_t len = strlen(path) + strlen(filename) + 1; - CALLOC(destfile, len, sizeof(char), RET_ERR(PM_ERR_MEMORY, NULL)); - snprintf(destfile, len, "%s%s", path, filename); +#ifdef HAVE_LIBCURL +static char *get_fullpath(const char *path, const char *filename, + const char *suffix) +{ + char *filepath; + /* len = localpath len + filename len + suffix len + null */ + size_t len = strlen(path) + strlen(filename) + strlen(suffix) + 1; + CALLOC(filepath, len, sizeof(char), return NULL); + snprintf(filepath, len, "%s%s%s", path, filename, suffix); - return(destfile); + return filepath; } -static char *get_tempfile(const char *path, const char *filename) { - char *tempfile; - /* len = localpath len + filename len + '.part' len + null */ - size_t len = strlen(path) + strlen(filename) + 6; - CALLOC(tempfile, len, sizeof(char), RET_ERR(PM_ERR_MEMORY, NULL)); - snprintf(tempfile, len, "%s%s.part", path, filename); +#define check_stop() if(dload_interrupted) { ret = -1; goto cleanup; } +enum sighandlers { OLD = 0, NEW = 1 }; - return(tempfile); +static int dload_interrupted; +static void inthandler(int UNUSED signum) +{ + dload_interrupted = 1; } -static const char *gethost(struct url *fileurl) +static int curl_progress(void *file, double dltotal, double dlnow, + double UNUSED ultotal, double UNUSED ulnow) { - const char *host = _("disk"); - if(strcmp(SCHEME_FILE, fileurl->scheme) != 0) { - host = fileurl->host; + struct fileinfo *dlfile = (struct fileinfo *)file; + double current_size, total_size; + + /* SIGINT sent, abort by alerting curl */ + if(dload_interrupted) { + return 1; + } + + /* none of what follows matters if the front end has no callback */ + if(dlfile->handle->dlcb == NULL) { + return 0; + } + + current_size = dlfile->initial_size + dlnow; + total_size = dlfile->initial_size + dltotal; + + if(DOUBLE_EQ(dltotal, 0) || DOUBLE_EQ(prevprogress, total_size)) { + return 0; + } + + /* initialize the progress bar here to avoid displaying it when + * a repo is up to date and nothing gets downloaded */ + if(DOUBLE_EQ(prevprogress, 0)) { + dlfile->handle->dlcb(dlfile->filename, 0, (long)dltotal); } - return(host); + + dlfile->handle->dlcb(dlfile->filename, (long)current_size, (long)total_size); + + prevprogress = current_size; + + return 0; } -int dload_interrupted; -static void inthandler(int signum) +static int curl_gethost(const char *url, char *buffer) { - dload_interrupted = 1; + size_t hostlen; + char *p; + + if(strncmp(url, "file://", 7) == 0) { + strcpy(buffer, _("disk")); + } else { + p = strstr(url, "//"); + if(!p) { + return 1; + } + p += 2; /* jump over the found // */ + hostlen = strcspn(p, "/"); + if(hostlen > 255) { + /* buffer overflow imminent */ + return 1; + } + snprintf(buffer, hostlen + 1, "%s", p); + } + + return 0; } -#define check_stop() if(dload_interrupted) { ret = -1; goto cleanup; } -enum sighandlers { OLD = 0, NEW = 1 }; +static int utimes_long(const char *path, long time) +{ + if(time != -1) { + struct timeval tv[2]; + memset(&tv, 0, sizeof(tv)); + tv[0].tv_sec = tv[1].tv_sec = time; + return utimes(path, tv); + } + return 0; +} -static int download_internal(const char *url, const char *localpath, - int force) { + +static int curl_download_internal(pmhandle_t *handle, + const char *url, const char *localpath, + int force, int allow_resume, int errors_ok) +{ + int ret = -1; FILE *localf = NULL; + const char *useragent; + const char *open_mode = "wb"; + char *destfile, *tempfile; + /* RFC1123 states applications should support this length */ + char hostname[256]; + char error_buffer[CURL_ERROR_SIZE]; struct stat st; - int ret = 0; - off_t dl_thisfile = 0; - ssize_t nread = 0; - char *tempfile, *destfile, *filename; + long timecond, remote_time; + double remote_size, bytes_dl; struct sigaction sig_pipe[2], sig_int[2]; - - off_t local_size = 0; - time_t local_time = 0; - - struct url *fileurl; - struct url_stat ust; - fetchIO *dlf = NULL; - - char buffer[PM_DLBUF_LEN]; - - filename = get_filename(url); - if(!filename) { - _alpm_log(PM_LOG_ERROR, _("url '%s' is invalid\n"), url); - RET_ERR(PM_ERR_SERVER_BAD_URL, -1); + struct fileinfo dlfile; + + dlfile.handle = handle; + dlfile.initial_size = 0.0; + dlfile.filename = get_filename(url); + if(!dlfile.filename || curl_gethost(url, hostname) != 0) { + _alpm_log(handle, PM_LOG_ERROR, _("url '%s' is invalid\n"), url); + RET_ERR(handle, PM_ERR_SERVER_BAD_URL, -1); } - fileurl = fetchParseURL(url); - if(!fileurl) { - _alpm_log(PM_LOG_ERROR, _("url '%s' is invalid\n"), url); - RET_ERR(PM_ERR_LIBFETCH, -1); + destfile = get_fullpath(localpath, dlfile.filename, ""); + tempfile = get_fullpath(localpath, dlfile.filename, ".part"); + if(!destfile || !tempfile) { + goto cleanup; } - destfile = get_destfile(localpath, filename); - tempfile = get_tempfile(localpath, filename); - - if(stat(tempfile, &st) == 0 && S_ISREG(st.st_mode) && st.st_size > 0) { - _alpm_log(PM_LOG_DEBUG, "tempfile found, attempting continuation\n"); - local_time = fileurl->last_modified = st.st_mtime; - local_size = fileurl->offset = (off_t)st.st_size; - dl_thisfile = st.st_size; - localf = fopen(tempfile, "ab"); - } else if(!force && stat(destfile, &st) == 0 && S_ISREG(st.st_mode) && st.st_size > 0) { - _alpm_log(PM_LOG_DEBUG, "destfile found, using mtime only\n"); - local_time = fileurl->last_modified = st.st_mtime; - local_size = /* no fu->off here */ (off_t)st.st_size; - } else { - _alpm_log(PM_LOG_DEBUG, "no file found matching criteria, starting from scratch\n"); + error_buffer[0] = '\0'; + + /* the curl_easy handle is initialized with the alpm handle, so we only need + * to reset the curl handle set parameters for each time it's used. */ + curl_easy_reset(handle->curl); + curl_easy_setopt(handle->curl, CURLOPT_URL, url); + curl_easy_setopt(handle->curl, CURLOPT_FAILONERROR, 1L); + curl_easy_setopt(handle->curl, CURLOPT_ERRORBUFFER, error_buffer); + curl_easy_setopt(handle->curl, CURLOPT_CONNECTTIMEOUT, 10L); + curl_easy_setopt(handle->curl, CURLOPT_FILETIME, 1L); + curl_easy_setopt(handle->curl, CURLOPT_NOPROGRESS, 0L); + curl_easy_setopt(handle->curl, CURLOPT_FOLLOWLOCATION, 1L); + curl_easy_setopt(handle->curl, CURLOPT_PROGRESSFUNCTION, curl_progress); + curl_easy_setopt(handle->curl, CURLOPT_PROGRESSDATA, (void *)&dlfile); + curl_easy_setopt(handle->curl, CURLOPT_LOW_SPEED_LIMIT, 1024L); + curl_easy_setopt(handle->curl, CURLOPT_LOW_SPEED_TIME, 10L); + + useragent = getenv("HTTP_USER_AGENT"); + if(useragent != NULL) { + curl_easy_setopt(handle->curl, CURLOPT_USERAGENT, useragent); } - /* pass the raw filename for passing to the callback function */ - _alpm_log(PM_LOG_DEBUG, "using '%s' for download progress\n", filename); + if(!allow_resume && !force && stat(destfile, &st) == 0) { + /* start from scratch, but only download if our local is out of date. */ + curl_easy_setopt(handle->curl, CURLOPT_TIMECONDITION, CURL_TIMECOND_IFMODSINCE); + curl_easy_setopt(handle->curl, CURLOPT_TIMEVALUE, (long)st.st_mtime); + } else if(stat(tempfile, &st) == 0 && allow_resume) { + /* a previous partial download exists, resume from end of file. */ + open_mode = "ab"; + curl_easy_setopt(handle->curl, CURLOPT_RESUME_FROM, (long)st.st_size); + _alpm_log(handle, PM_LOG_DEBUG, "tempfile found, attempting continuation"); + dlfile.initial_size = (double)st.st_size; + } - /* print proxy info for debug purposes */ - _alpm_log(PM_LOG_DEBUG, "HTTP_PROXY: %s\n", getenv("HTTP_PROXY")); - _alpm_log(PM_LOG_DEBUG, "http_proxy: %s\n", getenv("http_proxy")); - _alpm_log(PM_LOG_DEBUG, "FTP_PROXY: %s\n", getenv("FTP_PROXY")); - _alpm_log(PM_LOG_DEBUG, "ftp_proxy: %s\n", getenv("ftp_proxy")); + localf = fopen(tempfile, open_mode); + if(localf == NULL) { + goto cleanup; + } - /* 10s timeout */ - fetchTimeout = 10; + curl_easy_setopt(handle->curl, CURLOPT_WRITEDATA, localf); /* ignore any SIGPIPE signals- these may occur if our FTP socket dies or * something along those lines. Store the old signal handler first. */ @@ -169,168 +237,67 @@ static int download_internal(const char *url, const char *localpath, sigaction(SIGINT, NULL, &sig_int[OLD]); sigaction(SIGINT, &sig_int[NEW], NULL); - /* NOTE: libfetch does not reset the error code, be sure to do it before - * calls into the library */ - - /* TODO: if we call fetchStat() and get a redirect (disabling automagic - * redirect following), we should repeat the file locator stuff and get a new - * filename rather than only base if off the first URL, and then verify - * get_filename() didn't return ''. Of course, libfetch might not even allow - * us to even get that URL...FS#22645. This would allow us to download things - * without totally puking like - * http://www.archlinux.org/packages/community/x86_64/exim/download/ */ - - /* find out the remote size *and* mtime in one go. there is a lot of - * trouble in trying to do both size and "if-modified-since" logic in a - * non-stat request, so avoid it. */ - fetchLastErrCode = 0; - if(fetchStat(fileurl, &ust, "") == -1) { - pm_errno = PM_ERR_LIBFETCH; - _alpm_log(PM_LOG_ERROR, _("failed retrieving file '%s' from %s : %s\n"), - filename, gethost(fileurl), fetchLastErrString); - ret = -1; - goto cleanup; - } - check_stop(); - - _alpm_log(PM_LOG_DEBUG, "ust.mtime: %ld local_time: %ld compare: %ld\n", - ust.mtime, local_time, local_time - ust.mtime); - _alpm_log(PM_LOG_DEBUG, "ust.size: %jd local_size: %jd compare: %jd\n", - (intmax_t)ust.size, (intmax_t)local_size, (intmax_t)(local_size - ust.size)); - if(!force && ust.mtime && ust.mtime == local_time - && ust.size && ust.size == local_size) { - /* the remote time and size values agreed with what we have, so move on - * because there is nothing more to do. */ - _alpm_log(PM_LOG_DEBUG, "files are identical, skipping %s\n", filename); - ret = 1; - goto cleanup; - } - if(!ust.mtime || ust.mtime != local_time) { - _alpm_log(PM_LOG_DEBUG, "mtimes were different or unavailable, downloading %s from beginning\n", filename); - fileurl->offset = 0; - } + /* Progress 0 - initialize */ + prevprogress = 0; - fetchLastErrCode = 0; - dlf = fetchGet(fileurl, ""); - check_stop(); + /* perform transfer */ + handle->curlerr = curl_easy_perform(handle->curl); - if(fetchLastErrCode != 0 || dlf == NULL) { - pm_errno = PM_ERR_LIBFETCH; - _alpm_log(PM_LOG_ERROR, _("failed retrieving file '%s' from %s : %s\n"), - filename, gethost(fileurl), fetchLastErrString); - ret = -1; + /* was it a success? */ + if(handle->curlerr == CURLE_ABORTED_BY_CALLBACK) { goto cleanup; - } else { - _alpm_log(PM_LOG_DEBUG, "connected to %s successfully\n", fileurl->host); - } - - if(localf && fileurl->offset == 0) { - _alpm_log(PM_LOG_WARNING, _("resuming download of %s not possible; starting over\n"), filename); - fclose(localf); - localf = NULL; - } else if(fileurl->offset) { - _alpm_log(PM_LOG_DEBUG, "resuming download at position %jd\n", (intmax_t)fileurl->offset); - } - - - if(localf == NULL) { - _alpm_rmrf(tempfile); - fileurl->offset = (off_t)0; - dl_thisfile = 0; - localf = fopen(tempfile, "wb"); - if(localf == NULL) { /* still null? */ - pm_errno = PM_ERR_RETRIEVE; - _alpm_log(PM_LOG_ERROR, _("error writing to file '%s': %s\n"), - tempfile, strerror(errno)); - ret = -1; - goto cleanup; + } else if(handle->curlerr != CURLE_OK) { + if(!errors_ok) { + handle->pm_errno = PM_ERR_LIBCURL; + _alpm_log(handle, PM_LOG_ERROR, _("failed retrieving file '%s' from %s : %s\n"), + dlfile.filename, hostname, error_buffer); + } else { + _alpm_log(handle, PM_LOG_DEBUG, "failed retrieving file '%s' from %s : %s\n", + dlfile.filename, hostname, error_buffer); } + unlink(tempfile); + goto cleanup; } - /* Progress 0 - initialize */ - if(handle->dlcb) { - handle->dlcb(filename, 0, ust.size); - } - - while((nread = fetchIO_read(dlf, buffer, PM_DLBUF_LEN)) > 0) { - check_stop(); - size_t nwritten = 0; - nwritten = fwrite(buffer, 1, (size_t)nread, localf); - if((nwritten != (size_t)nread) || ferror(localf)) { - pm_errno = PM_ERR_RETRIEVE; - _alpm_log(PM_LOG_ERROR, _("error writing to file '%s': %s\n"), - tempfile, strerror(errno)); - ret = -1; - goto cleanup; - } - dl_thisfile += nread; - - if(handle->dlcb) { - handle->dlcb(filename, dl_thisfile, ust.size); - } - } + /* retrieve info about the state of the transfer */ + curl_easy_getinfo(handle->curl, CURLINFO_FILETIME, &remote_time); + curl_easy_getinfo(handle->curl, CURLINFO_CONTENT_LENGTH_DOWNLOAD, &remote_size); + curl_easy_getinfo(handle->curl, CURLINFO_SIZE_DOWNLOAD, &bytes_dl); + curl_easy_getinfo(handle->curl, CURLINFO_CONDITION_UNMET, &timecond); - /* did the transfer complete normally? */ - if (nread == -1) { - /* not PM_ERR_LIBFETCH here because libfetch error string might be empty */ - pm_errno = PM_ERR_RETRIEVE; - _alpm_log(PM_LOG_ERROR, _("failed retrieving file '%s' from %s\n"), - filename, gethost(fileurl)); - ret = -1; + /* time condition was met and we didn't download anything. we need to + * clean up the 0 byte .part file that's left behind. */ + if(timecond == 1 && DOUBLE_EQ(bytes_dl, 0)) { + ret = 1; + unlink(tempfile); goto cleanup; } - if (ust.size != -1 && dl_thisfile < ust.size) { - pm_errno = PM_ERR_RETRIEVE; - _alpm_log(PM_LOG_ERROR, _("%s appears to be truncated: %jd/%jd bytes\n"), - filename, (intmax_t)dl_thisfile, (intmax_t)ust.size); - ret = -1; + /* remote_size isn't necessarily the full size of the file, just what the + * server reported as remaining to download. compare it to what curl reported + * as actually being transferred during curl_easy_perform() */ + if(!DOUBLE_EQ(remote_size, -1) && !DOUBLE_EQ(bytes_dl, -1) && + !DOUBLE_EQ(bytes_dl, remote_size)) { + handle->pm_errno = PM_ERR_RETRIEVE; + _alpm_log(handle, PM_LOG_ERROR, _("%s appears to be truncated: %jd/%jd bytes\n"), + dlfile.filename, (intmax_t)bytes_dl, (intmax_t)remote_size); goto cleanup; } - /* probably safer to close the file descriptors now before renaming the file, - * for example to make sure the buffers are flushed. - */ - fclose(localf); - localf = NULL; - fetchIO_close(dlf); - dlf = NULL; - - /* set the times on the file to the same as that of the remote file */ - if(ust.mtime) { - struct timeval tv[2]; - memset(&tv, 0, sizeof(tv)); - tv[0].tv_sec = ust.atime; - tv[1].tv_sec = ust.mtime; - utimes(tempfile, tv); - } - if(rename(tempfile, destfile)) { - _alpm_log(PM_LOG_ERROR, _("could not rename %s to %s (%s)\n"), - tempfile, destfile, strerror(errno)); - ret = -1; - } ret = 0; cleanup: - FREE(tempfile); - FREE(destfile); if(localf != NULL) { - /* if we still had a local file open, we got interrupted. set the mtimes on - * the file accordingly. */ - fflush(localf); - if(ust.mtime) { - struct timeval tv[2]; - memset(&tv, 0, sizeof(tv)); - tv[0].tv_sec = ust.atime; - tv[1].tv_sec = ust.mtime; - futimes(fileno(localf), tv); - } fclose(localf); + utimes_long(tempfile, remote_time); } - if(dlf != NULL) { - fetchIO_close(dlf); + + if(ret == 0) { + rename(tempfile, destfile); } - fetchFreeURL(fileurl); + + FREE(tempfile); + FREE(destfile); /* restore the old signal handlers */ sigaction(SIGINT, &sig_int[OLD], NULL); @@ -340,106 +307,74 @@ cleanup: raise(SIGINT); } - return(ret); + return ret; } #endif -static int download(const char *url, const char *localpath, - int force) { +int _alpm_download(pmhandle_t *handle, const char *url, const char *localpath, + int force, int allow_resume, int errors_ok) +{ if(handle->fetchcb == NULL) { -#ifdef HAVE_LIBFETCH - return(download_internal(url, localpath, force)); +#ifdef HAVE_LIBCURL + return curl_download_internal(handle, url, localpath, + force, allow_resume, errors_ok); #else - RET_ERR(PM_ERR_EXTERNAL_DOWNLOAD, -1); + RET_ERR(handle, PM_ERR_EXTERNAL_DOWNLOAD, -1); #endif } else { int ret = handle->fetchcb(url, localpath, force); - if(ret == -1) { - RET_ERR(PM_ERR_EXTERNAL_DOWNLOAD, -1); - } - return(ret); - } -} - -/* - * Download a single file - * - servers must be a list of urls WITHOUT trailing slashes. - * - * RETURN: 0 for successful download - * 1 if the files are identical - * -1 on error - */ -int _alpm_download_single_file(const char *filename, - alpm_list_t *servers, const char *localpath, - int force) -{ - alpm_list_t *i; - int ret = -1; - - ASSERT(servers != NULL, RET_ERR(PM_ERR_SERVER_NONE, -1)); - - for(i = servers; i; i = i->next) { - const char *server = i->data; - char *fileurl = NULL; - size_t len; - - /* print server + filename into a buffer */ - len = strlen(server) + strlen(filename) + 2; - CALLOC(fileurl, len, sizeof(char), RET_ERR(PM_ERR_MEMORY, -1)); - snprintf(fileurl, len, "%s/%s", server, filename); - - ret = download(fileurl, localpath, force); - FREE(fileurl); - if(ret != -1) { - break; - } - } - - return(ret); -} - -int _alpm_download_files(alpm_list_t *files, - alpm_list_t *servers, const char *localpath) -{ - int ret = 0; - alpm_list_t *lp; - - for(lp = files; lp; lp = lp->next) { - char *filename = lp->data; - if(_alpm_download_single_file(filename, servers, - localpath, 0) == -1) { - ret++; + if(ret == -1 && !errors_ok) { + RET_ERR(handle, PM_ERR_EXTERNAL_DOWNLOAD, -1); } + return ret; } - - return(ret); } /** Fetch a remote pkg. */ -char SYMEXPORT *alpm_fetch_pkgurl(const char *url) +char SYMEXPORT *alpm_fetch_pkgurl(pmhandle_t *handle, const char *url) { - char *filename, *filepath; - const char *cachedir; + char *filepath; + const char *filename, *cachedir; int ret; - ALPM_LOG_FUNC; - filename = get_filename(url); /* find a valid cache dir to download to */ - cachedir = _alpm_filecache_setup(); + cachedir = _alpm_filecache_setup(handle); /* download the file */ - ret = download(url, cachedir, 0); + ret = _alpm_download(handle, url, cachedir, 0, 1, 0); if(ret == -1) { - _alpm_log(PM_LOG_WARNING, _("failed to download %s\n"), url); - return(NULL); + _alpm_log(handle, PM_LOG_WARNING, _("failed to download %s\n"), url); + return NULL; + } + _alpm_log(handle, PM_LOG_DEBUG, "successfully downloaded %s\n", url); + + /* attempt to download the signature */ + if(ret == 0 && (handle->sigverify == PM_PGP_VERIFY_ALWAYS || + handle->sigverify == PM_PGP_VERIFY_OPTIONAL)) { + char *sig_url; + size_t len; + int errors_ok = (handle->sigverify == PM_PGP_VERIFY_OPTIONAL); + + len = strlen(url) + 5; + CALLOC(sig_url, len, sizeof(char), RET_ERR(handle, PM_ERR_MEMORY, NULL)); + snprintf(sig_url, len, "%s.sig", url); + + ret = _alpm_download(handle, sig_url, cachedir, 1, 0, errors_ok); + if(ret == -1 && !errors_ok) { + _alpm_log(handle, PM_LOG_WARNING, _("failed to download %s\n"), sig_url); + /* Warn now, but don't return NULL. We will fail later during package + * load time. */ + } else if(ret == 0) { + _alpm_log(handle, PM_LOG_DEBUG, "successfully downloaded %s\n", sig_url); + } + FREE(sig_url); } - _alpm_log(PM_LOG_DEBUG, "successfully downloaded %s\n", url); /* we should be able to find the file the second time around */ - filepath = _alpm_filecache_find(filename); - return(filepath); + filepath = _alpm_filecache_find(handle, filename); + return filepath; } /* vim: set ts=2 sw=2 noet: */ |