implementation of fetching of data from http and https schemes. More...

#include "utils/inet.h"
#include <assert.h>
#include <errno.h>
#include <inttypes.h>
#include <stdbool.h>
#include <string.h>
#include <strings.h>
#include <time.h>
#include <sys/stat.h>
#include <libwapcaplet/libwapcaplet.h>
#include <nsutils/time.h>
#include "utils/corestrings.h"
#include "utils/hashmap.h"
#include "utils/nsoption.h"
#include "utils/log.h"
#include "utils/messages.h"
#include "utils/utils.h"
#include "utils/ring.h"
#include "utils/useragent.h"
#include "utils/file.h"
#include "netsurf/fetch.h"
#include "netsurf/misc.h"
#include "desktop/gui_internal.h"
#include "content/fetch.h"
#include "content/fetchers.h"
#include "content/fetchers/curl.h"
#include "content/urldb.h"

Include dependency graph for curl.c:

Data Structures
struct	cert_info
	SSL certificate info. More...

struct	curl_fetch_info
	Information for a single fetch. More...

struct	cache_handle
	curl handle cache entry More...

Macros
#define	UPDATES_PER_SECOND 2
	maximum number of progress notifications per second More...

#define	CIPHER_SUITES
	The ciphersuites the browser is prepared to use for TLS1.3. More...

#define	CIPHER_LIST
	The ciphersuites the browser is prepared to use for TLS<1.3. More...

#define	CIPHER_LIST_LEGACY
	The legacy cipher suites the browser is prepared to use for TLS<1.3. More...

#define	NSCURLOPT_PROGRESS_FUNCTION CURLOPT_PROGRESSFUNCTION

#define	NSCURLOPT_PROGRESS_DATA CURLOPT_PROGRESSDATA

#define	NSCURL_PROGRESS_T double

#define	NSCURL_POSTDATA_T struct curl_httppost

#define	NSCURL_POSTDATA_CURLOPT CURLOPT_HTTPPOST

#define	NSCURL_POSTDATA_FREE(x) curl_formfree(x)

#define	APPEND(list, value)

#define	SETOPT(option, value)

#define	SETOPT(option, value)

#define	UPDATE_DELAY_MS (1000 / UPDATES_PER_SECOND)

#define	SKIP_ST(o) for (i = (o); i < (int) size && (data[i] == ' ' \|\| data[i] == '\t'); i++)

#define	SETOPT(option, value)

Typedefs
typedef char	X509

Functions
static void	X509_free (X509 *cert)

static uint32_t	curl_fetch_ssl_key_hash (void *key)

static bool	curl_fetch_ssl_key_eq (void key1, void key2)

static void *	curl_fetch_ssl_value_alloc (void *key)

static void	curl_fetch_ssl_value_destroy (void *value)

static bool	fetch_curl_initialise (lwc_string *scheme)
	Initialise a cURL fetcher. More...

static void	fetch_curl_finalise (lwc_string *scheme)
	Finalise a cURL fetcher. More...

static bool	fetch_curl_can_fetch (const nsurl *url)
	Check if this fetcher can fetch a url. More...

static struct fetch_postdata *	fetch_curl_alloc_postdata (const char post_urlenc, const struct fetch_multipart_data post_multipart)
	allocate postdata More...

static void	fetch_curl_free_postdata (struct fetch_postdata *postdata)
	free postdata More...

static struct curl_fetch_info *	fetch_alloc (void)
	construct a new fetch structure More...

static void *	fetch_curl_setup (struct fetch parent_fetch, nsurl url, bool only_2xx, bool downgrade_tls, const char post_urlenc, const struct fetch_multipart_data post_multipart, const char **headers)
	Start fetching data for the given URL. More...

static void	fetch_curl_report_certs_upstream (struct curl_fetch_info *f)
	Report the certificate information in the fetch to the users. More...

static struct curl_httppost *	fetch_curl_postdata_convert (CURL chandle, const struct fetch_multipart_data control)
	Convert a list of struct fetch_multipart_data to a list of struct curl_httppost for libcurl. More...

static CURLcode	fetch_curl_set_postdata (struct curl_fetch_info *f)
	Setup multipart post data. More...

static CURLcode	fetch_curl_set_options (struct curl_fetch_info *f)
	Set options specific for a fetch. More...

static bool	fetch_curl_initiate_fetch (struct curl_fetch_info fetch, CURL handle)
	Initiate a fetch from the queue. More...

static CURL *	fetch_curl_get_handle (lwc_string *host)
	Find a CURL handle to use to dispatch a job. More...

static bool	fetch_curl_start (void *vfetch)
	Dispatch a single job. More...

static void	fetch_curl_cache_handle (CURL handle, lwc_string host)
	Cache a CURL handle for the provided host (if wanted) More...

static void	fetch_curl_stop (struct curl_fetch_info *f)
	Clean up the provided fetch object and free it. More...

static void	fetch_curl_abort (void *vf)
	Abort a fetch. More...

static void	fetch_curl_free (void *vf)
	Free a fetch structure and associated resources. More...

static bool	fetch_curl_process_headers (struct curl_fetch_info *f)
	Find the status code and content type and inform the caller. More...

static void	fetch_curl_done (CURL *curl_handle, CURLcode result)
	Handle a completed fetch (CURLMSG_DONE from curl_multi_info_read()). More...

static void	fetch_curl_poll (lwc_string *scheme_ignored)
	Do some work on current fetches. More...

static int	fetch_curl_progress (void *clientp, NSCURL_PROGRESS_T dltotal, NSCURL_PROGRESS_T dlnow, NSCURL_PROGRESS_T ultotal, NSCURL_PROGRESS_T ulnow)
	Callback function for fetch progress. More...

static int	fetch_curl_debug (CURL handle, curl_infotype type, char data, size_t size, void *userptr)
	Format curl debug for nslog. More...

static curl_socket_t	fetch_curl_socket_open (void clientp, curlsocktype purpose, struct curl_sockaddr address)

static int	fetch_curl_socket_close (void *clientp, curl_socket_t item)

static size_t	fetch_curl_data (char data, size_t size, size_t nmemb, void _f)
	Callback function for cURL. More...

static size_t	fetch_curl_header (char data, size_t size, size_t nmemb, void _f)
	Callback function for headers. More...

static int	fetch_curl_fdset (lwc_string scheme, fd_set read_set, fd_set write_set, fd_set error_set)

nserror	fetch_curl_register (void)
	Register curl scheme handler. More...

Variables
static hashmap_parameters_t	curl_fetch_ssl_hashmap_parameters

static hashmap_t *	curl_fetch_ssl_hashmap = NULL

CURLM *	fetch_curl_multi
	Global cURL multi handle. More...

static CURL *	fetch_blank_curl
	Curl handle with default options set; not used for transfers. More...

static struct cache_handle *	curl_handle_ring = 0
	Ring of cached handles. More...

static int	curl_fetchers_registered = 0
	Count of how many schemes the curl fetcher is handling. More...

static bool	curl_with_openssl
	Flag for runtime detection of openssl usage. More...

static char	fetch_error_buffer [CURL_ERROR_SIZE]
	Error buffer for cURL. More...

static char	fetch_proxy_userpwd [100]
	Proxy authentication details. More...

static bool	inside_curl = false
	Interlock to prevent initiation during callbacks. More...

Detailed Description

implementation of fetching of data from http and https schemes.

This implementation uses libcurl's 'multi' interface.

The CURL handles are cached in the curl_handle_ring.

Definition in file curl.c.

Macro Definition Documentation

◆ APPEND

#define APPEND	(	list,
		value
	)

Value:

        slist = curl_slist_append(list, value);         \
        if (slist == NULL)                              \
                goto failed;                            \
        list = slist;

◆ CIPHER_LIST

#define CIPHER_LIST

Value:

        /* disable everything */                                \
        "-ALL:"                                                 \
        /* enable TLSv1.2 ECDHE AES GCM suites */               \
        "EECDH+AESGCM+TLSv1.2:"                                 \
        /* enable ECDHE CHACHA20/POLY1305 suites */             \
        "EECDH+CHACHA20:"                                       \
        /* Sort above by strength */                            \
        "@STRENGTH:"                                            \
        /* enable ECDHE (auth=RSA, mac=SHA1) AES CBC suites */  \
        "EECDH+aRSA+AES+SHA1"

The ciphersuites the browser is prepared to use for TLS<1.3.

Definition at line 80 of file curl.c.

◆ CIPHER_LIST_LEGACY

#define CIPHER_LIST_LEGACY

Value:

        /* as above */                                          \
        CIPHER_LIST":"                                          \
        /* enable (non-PFS) RSA AES GCM suites */               \
        "RSA+AESGCM:"                                           \
        /* enable (non-PFS) RSA (mac=SHA1) AES CBC suites */    \
        "RSA+AES+SHA1"

The legacy cipher suites the browser is prepared to use for TLS<1.3.

Definition at line 95 of file curl.c.

◆ CIPHER_SUITES

#define CIPHER_SUITES

Value:

        "TLS_AES_256_GCM_SHA384:"                               \
        "TLS_CHACHA20_POLY1305_SHA256:"                         \
        "TLS_AES_128_GCM_SHA256"

The ciphersuites the browser is prepared to use for TLS1.3.

Definition at line 72 of file curl.c.

◆ NSCURL_POSTDATA_CURLOPT

#define NSCURL_POSTDATA_CURLOPT CURLOPT_HTTPPOST

Definition at line 230 of file curl.c.

◆ NSCURL_POSTDATA_FREE

#define NSCURL_POSTDATA_FREE ( x ) curl_formfree(x)

Definition at line 231 of file curl.c.

◆ NSCURL_POSTDATA_T

#define NSCURL_POSTDATA_T struct curl_httppost

Definition at line 229 of file curl.c.

◆ NSCURL_PROGRESS_T

#define NSCURL_PROGRESS_T double

Definition at line 221 of file curl.c.

◆ NSCURLOPT_PROGRESS_DATA

#define NSCURLOPT_PROGRESS_DATA CURLOPT_PROGRESSDATA

Definition at line 220 of file curl.c.

◆ NSCURLOPT_PROGRESS_FUNCTION

#define NSCURLOPT_PROGRESS_FUNCTION CURLOPT_PROGRESSFUNCTION

Definition at line 219 of file curl.c.

◆ SETOPT [1/3]

#define SETOPT	(	option,
		value
	)

Value:

        { \
        code = curl_easy_setopt(f->curl_handle, option, value); \
        if (code != CURLE_OK)                                   \
                return code;                                    \
        }

◆ SETOPT [2/3]

#define SETOPT	(	option,
		value
	)

Value:

        { \
        code = curl_easy_setopt(f->curl_handle, option, value); \
        if (code != CURLE_OK)                                   \
                return code;                                    \
        }

◆ SETOPT [3/3]

#define SETOPT	(	option,
		value
	)

Value:

        code = curl_easy_setopt(fetch_blank_curl, option, value);       \
        if (code != CURLE_OK) {                                         \
                NSLOG(netsurf, ERROR, "attempting curl_easy_setopt(%s, ...)", #option); \
                goto curl_easy_setopt_failed;                           \
        }

◆ SKIP_ST

#define SKIP_ST ( o ) for (i = (o); i < (int) size && (data[i] == ' ' || data[i] == '\t'); i++)

◆ UPDATE_DELAY_MS

#define UPDATE_DELAY_MS (1000 / UPDATES_PER_SECOND)

◆ UPDATES_PER_SECOND

#define UPDATES_PER_SECOND 2

maximum number of progress notifications per second

Definition at line 67 of file curl.c.

Typedef Documentation

◆ X509

typedef char X509

Definition at line 111 of file curl.c.

Function Documentation

◆ curl_fetch_ssl_key_eq()

static bool curl_fetch_ssl_key_eq	(	void *	key1,
		void *	key2
	)

static

Definition at line 144 of file curl.c.

References nsurl_get_component(), NSURL_HOST, and NSURL_PORT.

Here is the call graph for this function:

◆ curl_fetch_ssl_key_hash()

static uint32_t curl_fetch_ssl_key_hash ( void * key )

static

Definition at line 124 of file curl.c.

References nsurl_get_component(), NSURL_HOST, and NSURL_PORT.

Here is the call graph for this function:

◆ curl_fetch_ssl_value_alloc()

static void * curl_fetch_ssl_value_alloc ( void * key )

static

Definition at line 177 of file curl.c.

References cert_chain_alloc(), and NSERROR_OK.

Here is the call graph for this function:

◆ curl_fetch_ssl_value_destroy()

static void curl_fetch_ssl_value_destroy ( void * value )

static

Definition at line 189 of file curl.c.

References cert_chain_free(), NSERROR_OK, and NSLOG.

Here is the call graph for this function:

◆ fetch_alloc()

static struct curl_fetch_info * fetch_alloc ( void )

static

construct a new fetch structure

Definition at line 419 of file curl.c.

References fetch(), fetch::host, fetch::http_code, and fetch::url.

Referenced by fetch_curl_setup().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ fetch_curl_abort()

static void fetch_curl_abort ( void * vf )

static

Abort a fetch.

Definition at line 1413 of file curl.c.

References curl_fetch_info::abort, curl_fetch_info::curl_handle, fetch_curl_stop(), fetch_free(), curl_fetch_info::fetch_handle, fetch_remove_from_queues(), inside_curl, NSLOG, nsurl_access(), and curl_fetch_info::url.

Referenced by fetch_curl_register().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ fetch_curl_alloc_postdata()

static struct fetch_postdata * fetch_curl_alloc_postdata	(	const char *	post_urlenc,
		const struct fetch_multipart_data *	post_multipart
	)

static

allocate postdata

Definition at line 367 of file curl.c.

References fetch_postdata::data, fetch_multipart_data_clone(), FETCH_POSTDATA_MULTIPART, FETCH_POSTDATA_NONE, FETCH_POSTDATA_URLENC, fetch_postdata::multipart, fetch_postdata::type, and fetch_postdata::urlenc.

Referenced by fetch_curl_setup().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ fetch_curl_cache_handle()

static void fetch_curl_cache_handle	(	CURL *	handle,
		lwc_string *	host
	)

static

Cache a CURL handle for the provided host (if wanted)

Definition at line 1336 of file curl.c.

References curl_handle_ring, cache_handle::handle, cache_handle::host, nsoption_int, cache_handle::r_next, RING_FINDBYLWCHOST, RING_GETSIZE, and RING_INSERT.

Referenced by fetch_curl_stop().

Here is the caller graph for this function:

◆ fetch_curl_can_fetch()

static bool fetch_curl_can_fetch ( const nsurl * url )

static

Check if this fetcher can fetch a url.

Parameters

url	The url to check.

Returns: true if the fetcher supports the url else false.

Definition at line 356 of file curl.c.

References nsurl_has_component(), and NSURL_HOST.

Referenced by fetch_curl_register().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ fetch_curl_data()

static size_t fetch_curl_data	(	char *	data,
		size_t	size,
		size_t	nmemb,
		void *	_f
	)

static

Callback function for cURL.

Definition at line 1816 of file curl.c.

References curl_fetch_info::abort, fetch_msg::buf, curl_fetch_info::curl_handle, fetch_msg::data, fetch_curl_process_headers(), FETCH_DATA, curl_fetch_info::fetch_handle, fetch_send_callback(), fetch_set_http_code(), curl_fetch_info::had_headers, fetch_msg::header_or_data, curl_fetch_info::http_code, fetch_msg::len, curl_fetch_info::stopped, and fetch_msg::type.

Referenced by fetch_curl_register().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ fetch_curl_debug()

static int fetch_curl_debug	(	CURL *	handle,
		curl_infotype	type,
		char *	data,
		size_t	size,
		void *	userptr
	)

static

Format curl debug for nslog.

Definition at line 1772 of file curl.c.

References NSLOG, and type.

Referenced by fetch_curl_register().

Here is the caller graph for this function:

◆ fetch_curl_done()

static void fetch_curl_done	(	CURL *	curl_handle,
		CURLcode	result
	)

static

Handle a completed fetch (CURLMSG_DONE from curl_multi_info_read()).

Parameters

curl_handle	curl easy handle of fetch
result	The result code of the completed fetch.

Definition at line 1535 of file curl.c.

References curl_fetch_info::abort, curl_fetch_info::curl_handle, fetch_msg::data, fetch_msg::error, FETCH_CERT_ERR, fetch_curl_process_headers(), fetch_curl_report_certs_upstream(), fetch_curl_stop(), FETCH_ERROR, FETCH_FINISHED, fetch_free(), curl_fetch_info::fetch_handle, fetch_send_callback(), FETCH_SSL_ERR, FETCH_TIMEDOUT, curl_fetch_info::had_headers, NSLOG, nsurl_access(), result, curl_fetch_info::sent_ssl_chain, curl_fetch_info::stopped, fetch_msg::type, and curl_fetch_info::url.

Referenced by fetch_curl_poll().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ fetch_curl_fdset()

static int fetch_curl_fdset	(	lwc_string *	scheme,
		fd_set *	read_set,
		fd_set *	write_set,
		fd_set *	error_set
	)

static

Definition at line 1946 of file curl.c.

References fetch_curl_multi.

Referenced by fetch_curl_register().

Here is the caller graph for this function:

◆ fetch_curl_finalise()

static void fetch_curl_finalise ( lwc_string * scheme )

static

Finalise a cURL fetcher.

Parameters

scheme The scheme to finalise.

Definition at line 312 of file curl.c.

References curl_fetch_ssl_hashmap, curl_fetchers_registered, curl_handle_ring, fetch_blank_curl, fetch_curl_multi, cache_handle::handle, hashmap_destroy(), cache_handle::host, NSLOG, and RING_REMOVE.

Referenced by fetch_curl_register().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ fetch_curl_free()

static void fetch_curl_free ( void * vf )

static

Free a fetch structure and associated resources.

Definition at line 1437 of file curl.c.

References cert_info::cert, curl_fetch_info::cert_data, curl_fetch_info::cookie_string, curl_fetch_info::curl_handle, curl_fetch_info::curl_postdata, fetch_curl_free_postdata(), curl_fetch_info::headers, curl_fetch_info::host, curl_fetch_info::location, MAX_CERT_DEPTH, NSCURL_POSTDATA_FREE, nsurl_unref(), curl_fetch_info::postdata, curl_fetch_info::realm, curl_fetch_info::url, and X509_free().

Referenced by fetch_curl_register().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ fetch_curl_free_postdata()

static void fetch_curl_free_postdata ( struct fetch_postdata * postdata )

static

free postdata

Definition at line 398 of file curl.c.

References fetch_postdata::data, fetch_multipart_data_destroy(), FETCH_POSTDATA_MULTIPART, FETCH_POSTDATA_NONE, FETCH_POSTDATA_URLENC, fetch_postdata::multipart, fetch_postdata::type, and fetch_postdata::urlenc.

Referenced by fetch_curl_free(), and fetch_curl_setup().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ fetch_curl_get_handle()

static CURL * fetch_curl_get_handle ( lwc_string * host )

static

Find a CURL handle to use to dispatch a job.

Definition at line 1302 of file curl.c.

References curl_handle_ring, fetch_blank_curl, cache_handle::handle, cache_handle::host, RING_FINDBYLWCHOST, and RING_REMOVE.

Referenced by fetch_curl_start().

Here is the caller graph for this function:

◆ fetch_curl_header()

static size_t fetch_curl_header	(	char *	data,
		size_t	size,
		size_t	nmemb,
		void *	_f
	)

static

Callback function for headers.

See RFC 2616 4.2.

Definition at line 1864 of file curl.c.

References curl_fetch_info::abort, fetch_msg::buf, curl_fetch_info::content_length, fetch_msg::data, fetch_curl_report_certs_upstream(), curl_fetch_info::fetch_handle, FETCH_HEADER, fetch_send_callback(), fetch_set_cookie(), fetch_msg::header_or_data, fetch_msg::len, curl_fetch_info::location, NSLOG, curl_fetch_info::realm, curl_fetch_info::sent_ssl_chain, SKIP_ST, curl_fetch_info::stopped, and fetch_msg::type.

Referenced by fetch_curl_register().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ fetch_curl_initialise()

static bool fetch_curl_initialise ( lwc_string * scheme )

static

Initialise a cURL fetcher.

Definition at line 298 of file curl.c.

References curl_fetchers_registered, and NSLOG.

Referenced by fetch_curl_register().

Here is the caller graph for this function:

◆ fetch_curl_initiate_fetch()

static bool fetch_curl_initiate_fetch	(	struct curl_fetch_info *	fetch,
		CURL *	handle
	)

static

Initiate a fetch from the queue.

Parameters

fetch	fetch to use to fetch content.
handle	CURL handle to be used to fetch the content.

Returns: true if the fetch was successfully initiated else false.

Definition at line 1274 of file curl.c.

References fetch_curl_multi, fetch_curl_set_options(), and NSLOG.

Referenced by fetch_curl_start().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ fetch_curl_poll()

static void fetch_curl_poll ( lwc_string * scheme_ignored )

static

Do some work on current fetches.

Must be called regularly to make progress on fetches.

Definition at line 1644 of file curl.c.

References fetch_curl_done(), fetch_curl_multi, inside_curl, NSLOG, and nsoption_bool.

Referenced by fetch_curl_register().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ fetch_curl_postdata_convert()

static struct curl_httppost * fetch_curl_postdata_convert	(	CURL *	chandle,
		const struct fetch_multipart_data *	control
	)

static

Convert a list of struct fetch_multipart_data to a list of struct curl_httppost for libcurl.

Definition at line 1044 of file curl.c.

References gui_file_table::basename, netsurf_table::fetch, fetch_multipart_data::file, netsurf_table::file, guit, gui_fetch_table::mimetype, fetch_multipart_data::name, fetch_multipart_data::next, NSERROR_OK, NSLOG, fetch_multipart_data::rawfile, and fetch_multipart_data::value.

Referenced by fetch_curl_set_postdata().

Here is the caller graph for this function:

◆ fetch_curl_process_headers()

static bool fetch_curl_process_headers ( struct curl_fetch_info * f )

static

Find the status code and content type and inform the caller.

Return true if the fetch is being aborted.

Definition at line 1472 of file curl.c.

References curl_fetch_info::abort, fetch_msg::auth, curl_fetch_info::curl_handle, fetch_msg::data, fetch_msg::error, FETCH_AUTH, FETCH_ERROR, curl_fetch_info::fetch_handle, FETCH_NOTMODIFIED, FETCH_POSTDATA_NONE, FETCH_REDIRECT, fetch_send_callback(), fetch_set_http_code(), curl_fetch_info::had_headers, curl_fetch_info::http_code, curl_fetch_info::location, messages_get(), NSLOG, nsurl_access(), curl_fetch_info::only_2xx, curl_fetch_info::postdata, fetch_msg::realm, curl_fetch_info::realm, fetch_msg::redirect, fetch_msg::type, fetch_postdata::type, and curl_fetch_info::url.

Referenced by fetch_curl_data(), and fetch_curl_done().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ fetch_curl_progress()

static int fetch_curl_progress	(	void *	clientp,
		NSCURL_PROGRESS_T	dltotal,
		NSCURL_PROGRESS_T	dlnow,
		NSCURL_PROGRESS_T	ultotal,
		NSCURL_PROGRESS_T	ulnow
	)

static

Callback function for fetch progress.

< Progress buffer for cURL

Definition at line 1724 of file curl.c.

References curl_fetch_info::abort, fetch_msg::data, curl_fetch_info::fetch_handle, FETCH_PROGRESS, fetch_send_callback(), human_friendly_bytesize(), curl_fetch_info::last_progress_update, messages_get(), fetch_msg::progress, fetch_msg::type, and UPDATE_DELAY_MS.

Referenced by fetch_curl_register().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ fetch_curl_register()

nserror fetch_curl_register ( void )

Register curl scheme handler.

Returns: NSERROR_OK on successful registration or error code on failure.

Definition at line 1965 of file curl.c.

References CIPHER_LIST, CIPHER_SUITES, curl_fetch_ssl_hashmap, curl_fetch_ssl_hashmap_parameters, curl_with_openssl, fetch_blank_curl, fetch_curl_abort(), fetch_curl_can_fetch(), fetch_curl_data(), fetch_curl_debug(), fetch_curl_fdset(), fetch_curl_finalise(), fetch_curl_free(), fetch_curl_header(), fetch_curl_initialise(), fetch_curl_multi, fetch_curl_poll(), fetch_curl_progress(), fetch_curl_setup(), fetch_curl_socket_close(), fetch_curl_socket_open(), fetch_curl_start(), fetch_error_buffer, fetcher_add(), hashmap_create(), fetcher_operation_table::initialise, NSCURLOPT_PROGRESS_FUNCTION, NSERROR_INIT_FAILED, NSERROR_NOMEM, NSERROR_OK, NSLOG, nsoption_bool, nsoption_charp, nsoption_int, nsoption_uint, SETOPT, and user_agent_string().

Referenced by fetcher_init().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ fetch_curl_report_certs_upstream()

static void fetch_curl_report_certs_upstream ( struct curl_fetch_info * f )

static

Report the certificate information in the fetch to the users.

Definition at line 848 of file curl.c.

References fetch_msg::chain, curl_fetch_ssl_hashmap, fetch_msg::data, FETCH_CERTS, curl_fetch_info::fetch_handle, fetch_send_callback(), hashmap_lookup(), curl_fetch_info::sent_ssl_chain, fetch_msg::type, and curl_fetch_info::url.

Referenced by fetch_curl_done(), and fetch_curl_header().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ fetch_curl_set_options()

static CURLcode fetch_curl_set_options ( struct curl_fetch_info * f )

static

Set options specific for a fetch.

Parameters

f	The fetch to set options on.

Returns: A curl result code.

Definition at line 1167 of file curl.c.

References CIPHER_LIST, CIPHER_LIST_LEGACY, curl_fetch_info::cookie_string, curl_with_openssl, curl_fetch_info::downgrade_tls, fetch_curl_set_postdata(), fetch_proxy_userpwd, curl_fetch_info::headers, NSCURLOPT_PROGRESS_DATA, nsoption_bool, nsoption_charp, nsoption_int, nsurl_access(), OPTION_HTTP_PROXY_AUTH_BASIC, OPTION_HTTP_PROXY_AUTH_NONE, SETOPT, curl_fetch_info::url, urldb_get_auth_details(), urldb_get_cert_permissions(), and urldb_get_cookie().

Referenced by fetch_curl_initiate_fetch().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ fetch_curl_set_postdata()

static CURLcode fetch_curl_set_postdata ( struct curl_fetch_info * f )

static

Setup multipart post data.

Definition at line 1123 of file curl.c.

References curl_fetch_info::curl_handle, curl_fetch_info::curl_postdata, fetch_postdata::data, fetch_curl_postdata_convert(), FETCH_POSTDATA_MULTIPART, FETCH_POSTDATA_NONE, FETCH_POSTDATA_URLENC, fetch_postdata::multipart, NSCURL_POSTDATA_CURLOPT, curl_fetch_info::postdata, SETOPT, fetch_postdata::type, and fetch_postdata::urlenc.

Referenced by fetch_curl_set_options().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ fetch_curl_setup()

static void * fetch_curl_setup	(	struct fetch *	parent_fetch,
		nsurl *	url,
		bool	only_2xx,
		bool	downgrade_tls,
		const char *	post_urlenc,
		const struct fetch_multipart_data *	post_multipart,
		const char **	headers
	)

static

Start fetching data for the given URL.

The function returns immediately. The fetch may be queued for later processing.

A pointer to an opaque struct curl_fetch_info is returned, which can be passed to fetch_abort() to abort the fetch at any time. Returns 0 if memory is exhausted (or some other fatal error occurred).

The caller must supply a callback function which is called when anything interesting happens. The callback function is first called with msg FETCH_HEADER, with the header in data, then one or more times with FETCH_DATA with some data for the url, and finally with FETCH_FINISHED. Alternatively, FETCH_ERROR indicates an error occurred: data contains an error message. FETCH_REDIRECT may replace the FETCH_HEADER, FETCH_DATA, FETCH_FINISHED sequence if the server sends a replacement URL.

Some private data can be passed as the last parameter to fetch_start, and callbacks will contain this.

Definition at line 474 of file curl.c.

References APPEND, fetch(), fetch_alloc(), fetch_curl_alloc_postdata(), fetch_curl_free_postdata(), fetch::host, NSLOG, nsoption_bool, nsoption_charp, nsurl_access(), nsurl_get_component(), NSURL_HOST, nsurl_ref(), nsurl_unref(), and fetch::url.

Referenced by fetch_curl_register().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ fetch_curl_socket_close()

static int fetch_curl_socket_close	(	void *	clientp,
		curl_socket_t	item
	)

static

Definition at line 1806 of file curl.c.

References netsurf_table::fetch, guit, and gui_fetch_table::socket_close.

Referenced by fetch_curl_register().

Here is the caller graph for this function:

◆ fetch_curl_socket_open()

static curl_socket_t fetch_curl_socket_open	(	void *	clientp,
		curlsocktype	purpose,
		struct curl_sockaddr *	address
	)

static

Definition at line 1795 of file curl.c.

References netsurf_table::fetch, guit, and gui_fetch_table::socket_open.

Referenced by fetch_curl_register().

Here is the caller graph for this function:

◆ fetch_curl_start()

static bool fetch_curl_start ( void * vfetch )

static

Dispatch a single job.

Definition at line 1322 of file curl.c.

References fetch_curl_get_handle(), fetch_curl_initiate_fetch(), fetch::host, inside_curl, and NSLOG.

Referenced by fetch_curl_register().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ fetch_curl_stop()

static void fetch_curl_stop ( struct curl_fetch_info * f )

static

Clean up the provided fetch object and free it.

Will prod the queue afterwards to allow pending requests to be initiated.

Definition at line 1389 of file curl.c.

References curl_fetch_info::curl_handle, fetch_curl_cache_handle(), fetch_curl_multi, curl_fetch_info::fetch_handle, fetch_remove_from_queues(), curl_fetch_info::host, NSLOG, nsurl_access(), and curl_fetch_info::url.

Referenced by fetch_curl_abort(), and fetch_curl_done().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ X509_free()

static void X509_free ( X509 * cert )

static

Definition at line 113 of file curl.c.

Referenced by fetch_curl_free().

Here is the caller graph for this function:

Variable Documentation

◆ curl_fetch_ssl_hashmap

hashmap_t* curl_fetch_ssl_hashmap = NULL

static

Definition at line 206 of file curl.c.

Referenced by fetch_curl_finalise(), fetch_curl_register(), and fetch_curl_report_certs_upstream().

◆ curl_fetch_ssl_hashmap_parameters

hashmap_parameters_t curl_fetch_ssl_hashmap_parameters

static

Initial value:

= {
        .key_clone = (hashmap_key_clone_t)nsurl_ref,
        .key_destroy = (hashmap_key_destroy_t)nsurl_unref,
        .key_eq = curl_fetch_ssl_key_eq,
        .key_hash = curl_fetch_ssl_key_hash,
        .value_alloc = curl_fetch_ssl_value_alloc,
        .value_destroy = curl_fetch_ssl_value_destroy,
}

Definition at line 197 of file curl.c.

Referenced by fetch_curl_register().

◆ curl_fetchers_registered

int curl_fetchers_registered = 0

static

Count of how many schemes the curl fetcher is handling.

Definition at line 280 of file curl.c.

Referenced by fetch_curl_finalise(), and fetch_curl_initialise().

◆ curl_handle_ring

struct cache_handle* curl_handle_ring = 0

static

Ring of cached handles.

Definition at line 277 of file curl.c.

Referenced by fetch_curl_cache_handle(), fetch_curl_finalise(), and fetch_curl_get_handle().

◆ curl_with_openssl

bool curl_with_openssl

static

Flag for runtime detection of openssl usage.

Definition at line 283 of file curl.c.

Referenced by fetch_curl_register(), and fetch_curl_set_options().

◆ fetch_blank_curl

CURL* fetch_blank_curl

static

Curl handle with default options set; not used for transfers.

Definition at line 274 of file curl.c.

Referenced by fetch_curl_finalise(), fetch_curl_get_handle(), and fetch_curl_register().

◆ fetch_curl_multi

CURLM* fetch_curl_multi

Global cURL multi handle.

Definition at line 271 of file curl.c.

Referenced by fetch_curl_fdset(), fetch_curl_finalise(), fetch_curl_initiate_fetch(), fetch_curl_poll(), fetch_curl_register(), and fetch_curl_stop().

◆ fetch_error_buffer

char fetch_error_buffer[CURL_ERROR_SIZE]

static

Error buffer for cURL.

Definition at line 286 of file curl.c.

Referenced by fetch_curl_register().

◆ fetch_proxy_userpwd

char fetch_proxy_userpwd[100]

static

Proxy authentication details.

Definition at line 289 of file curl.c.

Referenced by fetch_curl_set_options().

◆ inside_curl

bool inside_curl = false

static

Interlock to prevent initiation during callbacks.

Definition at line 292 of file curl.c.

Referenced by fetch_curl_abort(), fetch_curl_poll(), and fetch_curl_start().

Data Structures

Macros

Typedefs

Functions

Variables

Detailed Description

Macro Definition Documentation

◆ APPEND

◆ CIPHER_LIST

◆ CIPHER_LIST_LEGACY

◆ CIPHER_SUITES

◆ NSCURL_POSTDATA_CURLOPT

◆ NSCURL_POSTDATA_FREE

◆ NSCURL_POSTDATA_T

◆ NSCURL_PROGRESS_T

◆ NSCURLOPT_PROGRESS_DATA

◆ NSCURLOPT_PROGRESS_FUNCTION

◆ SETOPT [1/3]

◆ SETOPT [2/3]

◆ SETOPT [3/3]

◆ SKIP_ST

◆ UPDATE_DELAY_MS

◆ UPDATES_PER_SECOND

Typedef Documentation

◆ X509

Function Documentation

◆ curl_fetch_ssl_key_eq()

◆ curl_fetch_ssl_key_hash()

◆ curl_fetch_ssl_value_alloc()

◆ curl_fetch_ssl_value_destroy()

◆ fetch_alloc()

◆ fetch_curl_abort()

◆ fetch_curl_alloc_postdata()

◆ fetch_curl_cache_handle()

◆ fetch_curl_can_fetch()

◆ fetch_curl_data()

◆ fetch_curl_debug()

◆ fetch_curl_done()

◆ fetch_curl_fdset()

◆ fetch_curl_finalise()

◆ fetch_curl_free()

◆ fetch_curl_free_postdata()

◆ fetch_curl_get_handle()

◆ fetch_curl_header()

◆ fetch_curl_initialise()

◆ fetch_curl_initiate_fetch()

◆ fetch_curl_poll()

◆ fetch_curl_postdata_convert()

◆ fetch_curl_process_headers()

◆ fetch_curl_progress()

◆ fetch_curl_register()

◆ fetch_curl_report_certs_upstream()

◆ fetch_curl_set_options()

◆ fetch_curl_set_postdata()

◆ fetch_curl_setup()

◆ fetch_curl_socket_close()

◆ fetch_curl_socket_open()

◆ fetch_curl_start()

◆ fetch_curl_stop()

◆ X509_free()

Variable Documentation

◆ curl_fetch_ssl_hashmap

◆ curl_fetch_ssl_hashmap_parameters

◆ curl_fetchers_registered

◆ curl_handle_ring

◆ curl_with_openssl

◆ fetch_blank_curl

◆ fetch_curl_multi

◆ fetch_error_buffer

◆ fetch_proxy_userpwd

◆ inside_curl