NetSurf
|
implementation of fetching of data from http and https schemes. More...
#include "utils/inet.h"
#include <assert.h>
#include <errno.h>
#include <inttypes.h>
#include <stdbool.h>
#include <string.h>
#include <strings.h>
#include <time.h>
#include <sys/stat.h>
#include <libwapcaplet/libwapcaplet.h>
#include <nsutils/time.h>
#include "utils/corestrings.h"
#include "utils/hashmap.h"
#include "utils/nsoption.h"
#include "utils/log.h"
#include "utils/messages.h"
#include "utils/utils.h"
#include "utils/ring.h"
#include "utils/useragent.h"
#include "utils/file.h"
#include "netsurf/fetch.h"
#include "netsurf/misc.h"
#include "desktop/gui_internal.h"
#include "content/fetch.h"
#include "content/fetchers.h"
#include "content/fetchers/curl.h"
#include "content/urldb.h"
Go to the source code of this file.
Data Structures | |
struct | cert_info |
SSL certificate info. More... | |
struct | curl_fetch_info |
Information for a single fetch. More... | |
struct | cache_handle |
curl handle cache entry More... | |
Macros | |
#define | UPDATES_PER_SECOND 2 |
maximum number of progress notifications per second More... | |
#define | CIPHER_SUITES |
The ciphersuites the browser is prepared to use for TLS1.3. More... | |
#define | CIPHER_LIST |
The ciphersuites the browser is prepared to use for TLS<1.3. More... | |
#define | CIPHER_LIST_LEGACY |
The legacy cipher suites the browser is prepared to use for TLS<1.3. More... | |
#define | NSCURLOPT_PROGRESS_FUNCTION CURLOPT_PROGRESSFUNCTION |
#define | NSCURLOPT_PROGRESS_DATA CURLOPT_PROGRESSDATA |
#define | NSCURL_PROGRESS_T double |
#define | NSCURL_POSTDATA_T struct curl_httppost |
#define | NSCURL_POSTDATA_CURLOPT CURLOPT_HTTPPOST |
#define | NSCURL_POSTDATA_FREE(x) curl_formfree(x) |
#define | APPEND(list, value) |
#define | SETOPT(option, value) |
#define | SETOPT(option, value) |
#define | UPDATE_DELAY_MS (1000 / UPDATES_PER_SECOND) |
#define | SKIP_ST(o) for (i = (o); i < (int) size && (data[i] == ' ' || data[i] == '\t'); i++) |
#define | SETOPT(option, value) |
Typedefs | |
typedef char | X509 |
Functions | |
static void | X509_free (X509 *cert) |
static uint32_t | curl_fetch_ssl_key_hash (void *key) |
static bool | curl_fetch_ssl_key_eq (void *key1, void *key2) |
static void * | curl_fetch_ssl_value_alloc (void *key) |
static void | curl_fetch_ssl_value_destroy (void *value) |
static bool | fetch_curl_initialise (lwc_string *scheme) |
Initialise a cURL fetcher. More... | |
static void | fetch_curl_finalise (lwc_string *scheme) |
Finalise a cURL fetcher. More... | |
static bool | fetch_curl_can_fetch (const nsurl *url) |
Check if this fetcher can fetch a url. More... | |
static struct fetch_postdata * | fetch_curl_alloc_postdata (const char *post_urlenc, const struct fetch_multipart_data *post_multipart) |
allocate postdata More... | |
static void | fetch_curl_free_postdata (struct fetch_postdata *postdata) |
free postdata More... | |
static struct curl_fetch_info * | fetch_alloc (void) |
construct a new fetch structure More... | |
static void * | fetch_curl_setup (struct fetch *parent_fetch, nsurl *url, bool only_2xx, bool downgrade_tls, const char *post_urlenc, const struct fetch_multipart_data *post_multipart, const char **headers) |
Start fetching data for the given URL. More... | |
static void | fetch_curl_report_certs_upstream (struct curl_fetch_info *f) |
Report the certificate information in the fetch to the users. More... | |
static struct curl_httppost * | fetch_curl_postdata_convert (CURL *chandle, const struct fetch_multipart_data *control) |
Convert a list of struct fetch_multipart_data to a list of struct curl_httppost for libcurl. More... | |
static CURLcode | fetch_curl_set_postdata (struct curl_fetch_info *f) |
Setup multipart post data. More... | |
static CURLcode | fetch_curl_set_options (struct curl_fetch_info *f) |
Set options specific for a fetch. More... | |
static bool | fetch_curl_initiate_fetch (struct curl_fetch_info *fetch, CURL *handle) |
Initiate a fetch from the queue. More... | |
static CURL * | fetch_curl_get_handle (lwc_string *host) |
Find a CURL handle to use to dispatch a job. More... | |
static bool | fetch_curl_start (void *vfetch) |
Dispatch a single job. More... | |
static void | fetch_curl_cache_handle (CURL *handle, lwc_string *host) |
Cache a CURL handle for the provided host (if wanted) More... | |
static void | fetch_curl_stop (struct curl_fetch_info *f) |
Clean up the provided fetch object and free it. More... | |
static void | fetch_curl_abort (void *vf) |
Abort a fetch. More... | |
static void | fetch_curl_free (void *vf) |
Free a fetch structure and associated resources. More... | |
static bool | fetch_curl_process_headers (struct curl_fetch_info *f) |
Find the status code and content type and inform the caller. More... | |
static void | fetch_curl_done (CURL *curl_handle, CURLcode result) |
Handle a completed fetch (CURLMSG_DONE from curl_multi_info_read()). More... | |
static void | fetch_curl_poll (lwc_string *scheme_ignored) |
Do some work on current fetches. More... | |
static int | fetch_curl_progress (void *clientp, NSCURL_PROGRESS_T dltotal, NSCURL_PROGRESS_T dlnow, NSCURL_PROGRESS_T ultotal, NSCURL_PROGRESS_T ulnow) |
Callback function for fetch progress. More... | |
static int | fetch_curl_debug (CURL *handle, curl_infotype type, char *data, size_t size, void *userptr) |
Format curl debug for nslog. More... | |
static curl_socket_t | fetch_curl_socket_open (void *clientp, curlsocktype purpose, struct curl_sockaddr *address) |
static int | fetch_curl_socket_close (void *clientp, curl_socket_t item) |
static size_t | fetch_curl_data (char *data, size_t size, size_t nmemb, void *_f) |
Callback function for cURL. More... | |
static size_t | fetch_curl_header (char *data, size_t size, size_t nmemb, void *_f) |
Callback function for headers. More... | |
static int | fetch_curl_fdset (lwc_string *scheme, fd_set *read_set, fd_set *write_set, fd_set *error_set) |
nserror | fetch_curl_register (void) |
Register curl scheme handler. More... | |
Variables | |
static hashmap_parameters_t | curl_fetch_ssl_hashmap_parameters |
static hashmap_t * | curl_fetch_ssl_hashmap = NULL |
CURLM * | fetch_curl_multi |
Global cURL multi handle. More... | |
static CURL * | fetch_blank_curl |
Curl handle with default options set; not used for transfers. More... | |
static struct cache_handle * | curl_handle_ring = 0 |
Ring of cached handles. More... | |
static int | curl_fetchers_registered = 0 |
Count of how many schemes the curl fetcher is handling. More... | |
static bool | curl_with_openssl |
Flag for runtime detection of openssl usage. More... | |
static char | fetch_error_buffer [CURL_ERROR_SIZE] |
Error buffer for cURL. More... | |
static char | fetch_proxy_userpwd [100] |
Proxy authentication details. More... | |
static bool | inside_curl = false |
Interlock to prevent initiation during callbacks. More... | |
implementation of fetching of data from http and https schemes.
This implementation uses libcurl's 'multi' interface.
The CURL handles are cached in the curl_handle_ring.
Definition in file curl.c.
#define APPEND | ( | list, | |
value | |||
) |
#define CIPHER_LIST |
The ciphersuites the browser is prepared to use for TLS<1.3.
#define CIPHER_LIST_LEGACY |
#define CIPHER_SUITES |
#define SETOPT | ( | option, | |
value | |||
) |
#define SETOPT | ( | option, | |
value | |||
) |
#define SETOPT | ( | option, | |
value | |||
) |
#define SKIP_ST | ( | o | ) | for (i = (o); i < (int) size && (data[i] == ' ' || data[i] == '\t'); i++) |
#define UPDATE_DELAY_MS (1000 / UPDATES_PER_SECOND) |
#define UPDATES_PER_SECOND 2 |
|
static |
Definition at line 144 of file curl.c.
References nsurl_get_component(), NSURL_HOST, and NSURL_PORT.
|
static |
Definition at line 124 of file curl.c.
References nsurl_get_component(), NSURL_HOST, and NSURL_PORT.
|
static |
Definition at line 177 of file curl.c.
References cert_chain_alloc(), and NSERROR_OK.
|
static |
Definition at line 189 of file curl.c.
References cert_chain_free(), NSERROR_OK, and NSLOG.
|
static |
construct a new fetch structure
Definition at line 419 of file curl.c.
References fetch(), fetch::host, fetch::http_code, and fetch::url.
Referenced by fetch_curl_setup().
|
static |
Abort a fetch.
Definition at line 1413 of file curl.c.
References curl_fetch_info::abort, curl_fetch_info::curl_handle, fetch_curl_stop(), fetch_free(), curl_fetch_info::fetch_handle, fetch_remove_from_queues(), inside_curl, NSLOG, nsurl_access(), and curl_fetch_info::url.
Referenced by fetch_curl_register().
|
static |
allocate postdata
Definition at line 367 of file curl.c.
References fetch_postdata::data, fetch_multipart_data_clone(), FETCH_POSTDATA_MULTIPART, FETCH_POSTDATA_NONE, FETCH_POSTDATA_URLENC, fetch_postdata::multipart, fetch_postdata::type, and fetch_postdata::urlenc.
Referenced by fetch_curl_setup().
|
static |
Cache a CURL handle for the provided host (if wanted)
Definition at line 1336 of file curl.c.
References curl_handle_ring, cache_handle::handle, cache_handle::host, nsoption_int, cache_handle::r_next, RING_FINDBYLWCHOST, RING_GETSIZE, and RING_INSERT.
Referenced by fetch_curl_stop().
|
static |
Check if this fetcher can fetch a url.
url | The url to check. |
Definition at line 356 of file curl.c.
References nsurl_has_component(), and NSURL_HOST.
Referenced by fetch_curl_register().
|
static |
Callback function for cURL.
Definition at line 1816 of file curl.c.
References curl_fetch_info::abort, fetch_msg::buf, curl_fetch_info::curl_handle, fetch_msg::data, fetch_curl_process_headers(), FETCH_DATA, curl_fetch_info::fetch_handle, fetch_send_callback(), fetch_set_http_code(), curl_fetch_info::had_headers, fetch_msg::header_or_data, curl_fetch_info::http_code, fetch_msg::len, curl_fetch_info::stopped, and fetch_msg::type.
Referenced by fetch_curl_register().
|
static |
Format curl debug for nslog.
Definition at line 1772 of file curl.c.
Referenced by fetch_curl_register().
|
static |
Handle a completed fetch (CURLMSG_DONE from curl_multi_info_read()).
curl_handle | curl easy handle of fetch |
result | The result code of the completed fetch. |
Definition at line 1535 of file curl.c.
References curl_fetch_info::abort, curl_fetch_info::curl_handle, fetch_msg::data, fetch_msg::error, FETCH_CERT_ERR, fetch_curl_process_headers(), fetch_curl_report_certs_upstream(), fetch_curl_stop(), FETCH_ERROR, FETCH_FINISHED, fetch_free(), curl_fetch_info::fetch_handle, fetch_send_callback(), FETCH_SSL_ERR, FETCH_TIMEDOUT, curl_fetch_info::had_headers, NSLOG, nsurl_access(), result, curl_fetch_info::sent_ssl_chain, curl_fetch_info::stopped, fetch_msg::type, and curl_fetch_info::url.
Referenced by fetch_curl_poll().
|
static |
Definition at line 1946 of file curl.c.
References fetch_curl_multi.
Referenced by fetch_curl_register().
|
static |
Finalise a cURL fetcher.
scheme | The scheme to finalise. |
Definition at line 312 of file curl.c.
References curl_fetch_ssl_hashmap, curl_fetchers_registered, curl_handle_ring, fetch_blank_curl, fetch_curl_multi, cache_handle::handle, hashmap_destroy(), cache_handle::host, NSLOG, and RING_REMOVE.
Referenced by fetch_curl_register().
|
static |
Free a fetch structure and associated resources.
Definition at line 1437 of file curl.c.
References cert_info::cert, curl_fetch_info::cert_data, curl_fetch_info::cookie_string, curl_fetch_info::curl_handle, curl_fetch_info::curl_postdata, fetch_curl_free_postdata(), curl_fetch_info::headers, curl_fetch_info::host, curl_fetch_info::location, MAX_CERT_DEPTH, NSCURL_POSTDATA_FREE, nsurl_unref(), curl_fetch_info::postdata, curl_fetch_info::realm, curl_fetch_info::url, and X509_free().
Referenced by fetch_curl_register().
|
static |
free postdata
Definition at line 398 of file curl.c.
References fetch_postdata::data, fetch_multipart_data_destroy(), FETCH_POSTDATA_MULTIPART, FETCH_POSTDATA_NONE, FETCH_POSTDATA_URLENC, fetch_postdata::multipart, fetch_postdata::type, and fetch_postdata::urlenc.
Referenced by fetch_curl_free(), and fetch_curl_setup().
|
static |
Find a CURL handle to use to dispatch a job.
Definition at line 1302 of file curl.c.
References curl_handle_ring, fetch_blank_curl, cache_handle::handle, cache_handle::host, RING_FINDBYLWCHOST, and RING_REMOVE.
Referenced by fetch_curl_start().
|
static |
Callback function for headers.
See RFC 2616 4.2.
Definition at line 1864 of file curl.c.
References curl_fetch_info::abort, fetch_msg::buf, curl_fetch_info::content_length, fetch_msg::data, fetch_curl_report_certs_upstream(), curl_fetch_info::fetch_handle, FETCH_HEADER, fetch_send_callback(), fetch_set_cookie(), fetch_msg::header_or_data, fetch_msg::len, curl_fetch_info::location, NSLOG, curl_fetch_info::realm, curl_fetch_info::sent_ssl_chain, SKIP_ST, curl_fetch_info::stopped, and fetch_msg::type.
Referenced by fetch_curl_register().
|
static |
Initialise a cURL fetcher.
Definition at line 298 of file curl.c.
References curl_fetchers_registered, and NSLOG.
Referenced by fetch_curl_register().
|
static |
Initiate a fetch from the queue.
fetch | fetch to use to fetch content. |
handle | CURL handle to be used to fetch the content. |
Definition at line 1274 of file curl.c.
References fetch_curl_multi, fetch_curl_set_options(), and NSLOG.
Referenced by fetch_curl_start().
|
static |
Do some work on current fetches.
Must be called regularly to make progress on fetches.
Definition at line 1644 of file curl.c.
References fetch_curl_done(), fetch_curl_multi, inside_curl, NSLOG, and nsoption_bool.
Referenced by fetch_curl_register().
|
static |
Convert a list of struct fetch_multipart_data to a list of struct curl_httppost for libcurl.
Definition at line 1044 of file curl.c.
References gui_file_table::basename, netsurf_table::fetch, fetch_multipart_data::file, netsurf_table::file, guit, gui_fetch_table::mimetype, fetch_multipart_data::name, fetch_multipart_data::next, NSERROR_OK, NSLOG, fetch_multipart_data::rawfile, and fetch_multipart_data::value.
Referenced by fetch_curl_set_postdata().
|
static |
Find the status code and content type and inform the caller.
Return true if the fetch is being aborted.
Definition at line 1472 of file curl.c.
References curl_fetch_info::abort, fetch_msg::auth, curl_fetch_info::curl_handle, fetch_msg::data, fetch_msg::error, FETCH_AUTH, FETCH_ERROR, curl_fetch_info::fetch_handle, FETCH_NOTMODIFIED, FETCH_POSTDATA_NONE, FETCH_REDIRECT, fetch_send_callback(), fetch_set_http_code(), curl_fetch_info::had_headers, curl_fetch_info::http_code, curl_fetch_info::location, messages_get(), NSLOG, nsurl_access(), curl_fetch_info::only_2xx, curl_fetch_info::postdata, fetch_msg::realm, curl_fetch_info::realm, fetch_msg::redirect, fetch_msg::type, fetch_postdata::type, and curl_fetch_info::url.
Referenced by fetch_curl_data(), and fetch_curl_done().
|
static |
Callback function for fetch progress.
< Progress buffer for cURL
Definition at line 1724 of file curl.c.
References curl_fetch_info::abort, fetch_msg::data, curl_fetch_info::fetch_handle, FETCH_PROGRESS, fetch_send_callback(), human_friendly_bytesize(), curl_fetch_info::last_progress_update, messages_get(), fetch_msg::progress, fetch_msg::type, and UPDATE_DELAY_MS.
Referenced by fetch_curl_register().
nserror fetch_curl_register | ( | void | ) |
Register curl scheme handler.
Definition at line 1965 of file curl.c.
References CIPHER_LIST, CIPHER_SUITES, curl_fetch_ssl_hashmap, curl_fetch_ssl_hashmap_parameters, curl_with_openssl, fetch_blank_curl, fetch_curl_abort(), fetch_curl_can_fetch(), fetch_curl_data(), fetch_curl_debug(), fetch_curl_fdset(), fetch_curl_finalise(), fetch_curl_free(), fetch_curl_header(), fetch_curl_initialise(), fetch_curl_multi, fetch_curl_poll(), fetch_curl_progress(), fetch_curl_setup(), fetch_curl_socket_close(), fetch_curl_socket_open(), fetch_curl_start(), fetch_error_buffer, fetcher_add(), hashmap_create(), fetcher_operation_table::initialise, NSCURLOPT_PROGRESS_FUNCTION, NSERROR_INIT_FAILED, NSERROR_NOMEM, NSERROR_OK, NSLOG, nsoption_bool, nsoption_charp, nsoption_int, nsoption_uint, SETOPT, and user_agent_string().
Referenced by fetcher_init().
|
static |
Report the certificate information in the fetch to the users.
Definition at line 848 of file curl.c.
References fetch_msg::chain, curl_fetch_ssl_hashmap, fetch_msg::data, FETCH_CERTS, curl_fetch_info::fetch_handle, fetch_send_callback(), hashmap_lookup(), curl_fetch_info::sent_ssl_chain, fetch_msg::type, and curl_fetch_info::url.
Referenced by fetch_curl_done(), and fetch_curl_header().
|
static |
Set options specific for a fetch.
f | The fetch to set options on. |
Definition at line 1167 of file curl.c.
References CIPHER_LIST, CIPHER_LIST_LEGACY, curl_fetch_info::cookie_string, curl_with_openssl, curl_fetch_info::downgrade_tls, fetch_curl_set_postdata(), fetch_proxy_userpwd, curl_fetch_info::headers, NSCURLOPT_PROGRESS_DATA, nsoption_bool, nsoption_charp, nsoption_int, nsurl_access(), OPTION_HTTP_PROXY_AUTH_BASIC, OPTION_HTTP_PROXY_AUTH_NONE, SETOPT, curl_fetch_info::url, urldb_get_auth_details(), urldb_get_cert_permissions(), and urldb_get_cookie().
Referenced by fetch_curl_initiate_fetch().
|
static |
Setup multipart post data.
Definition at line 1123 of file curl.c.
References curl_fetch_info::curl_handle, curl_fetch_info::curl_postdata, fetch_postdata::data, fetch_curl_postdata_convert(), FETCH_POSTDATA_MULTIPART, FETCH_POSTDATA_NONE, FETCH_POSTDATA_URLENC, fetch_postdata::multipart, NSCURL_POSTDATA_CURLOPT, curl_fetch_info::postdata, SETOPT, fetch_postdata::type, and fetch_postdata::urlenc.
Referenced by fetch_curl_set_options().
|
static |
Start fetching data for the given URL.
The function returns immediately. The fetch may be queued for later processing.
A pointer to an opaque struct curl_fetch_info is returned, which can be passed to fetch_abort() to abort the fetch at any time. Returns 0 if memory is exhausted (or some other fatal error occurred).
The caller must supply a callback function which is called when anything interesting happens. The callback function is first called with msg FETCH_HEADER, with the header in data, then one or more times with FETCH_DATA with some data for the url, and finally with FETCH_FINISHED. Alternatively, FETCH_ERROR indicates an error occurred: data contains an error message. FETCH_REDIRECT may replace the FETCH_HEADER, FETCH_DATA, FETCH_FINISHED sequence if the server sends a replacement URL.
Some private data can be passed as the last parameter to fetch_start, and callbacks will contain this.
Definition at line 474 of file curl.c.
References APPEND, fetch(), fetch_alloc(), fetch_curl_alloc_postdata(), fetch_curl_free_postdata(), fetch::host, NSLOG, nsoption_bool, nsoption_charp, nsurl_access(), nsurl_get_component(), NSURL_HOST, nsurl_ref(), nsurl_unref(), and fetch::url.
Referenced by fetch_curl_register().
|
static |
Definition at line 1806 of file curl.c.
References netsurf_table::fetch, guit, and gui_fetch_table::socket_close.
Referenced by fetch_curl_register().
|
static |
Definition at line 1795 of file curl.c.
References netsurf_table::fetch, guit, and gui_fetch_table::socket_open.
Referenced by fetch_curl_register().
|
static |
Dispatch a single job.
Definition at line 1322 of file curl.c.
References fetch_curl_get_handle(), fetch_curl_initiate_fetch(), fetch::host, inside_curl, and NSLOG.
Referenced by fetch_curl_register().
|
static |
Clean up the provided fetch object and free it.
Will prod the queue afterwards to allow pending requests to be initiated.
Definition at line 1389 of file curl.c.
References curl_fetch_info::curl_handle, fetch_curl_cache_handle(), fetch_curl_multi, curl_fetch_info::fetch_handle, fetch_remove_from_queues(), curl_fetch_info::host, NSLOG, nsurl_access(), and curl_fetch_info::url.
Referenced by fetch_curl_abort(), and fetch_curl_done().
|
static |
Definition at line 113 of file curl.c.
Referenced by fetch_curl_free().
|
static |
Definition at line 206 of file curl.c.
Referenced by fetch_curl_finalise(), fetch_curl_register(), and fetch_curl_report_certs_upstream().
|
static |
Definition at line 197 of file curl.c.
Referenced by fetch_curl_register().
|
static |
Count of how many schemes the curl fetcher is handling.
Definition at line 280 of file curl.c.
Referenced by fetch_curl_finalise(), and fetch_curl_initialise().
|
static |
Ring of cached handles.
Definition at line 277 of file curl.c.
Referenced by fetch_curl_cache_handle(), fetch_curl_finalise(), and fetch_curl_get_handle().
|
static |
Flag for runtime detection of openssl usage.
Definition at line 283 of file curl.c.
Referenced by fetch_curl_register(), and fetch_curl_set_options().
|
static |
Curl handle with default options set; not used for transfers.
Definition at line 274 of file curl.c.
Referenced by fetch_curl_finalise(), fetch_curl_get_handle(), and fetch_curl_register().
CURLM* fetch_curl_multi |
Global cURL multi handle.
Definition at line 271 of file curl.c.
Referenced by fetch_curl_fdset(), fetch_curl_finalise(), fetch_curl_initiate_fetch(), fetch_curl_poll(), fetch_curl_register(), and fetch_curl_stop().
|
static |
|
static |
Proxy authentication details.
Definition at line 289 of file curl.c.
Referenced by fetch_curl_set_options().
|
static |
Interlock to prevent initiation during callbacks.
Definition at line 292 of file curl.c.
Referenced by fetch_curl_abort(), fetch_curl_poll(), and fetch_curl_start().