NetSurf
curl.c
Go to the documentation of this file.
1/*
2 * Copyright 2006-2019 Daniel Silverstone <dsilvers@digital-scurf.org>
3 * Copyright 2010-2018 Vincent Sanders <vince@netsurf-browser.org>
4 * Copyright 2007 James Bursa <bursa@users.sourceforge.net>
5 *
6 * This file is part of NetSurf.
7 *
8 * NetSurf is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License as published by
10 * the Free Software Foundation; version 2 of the License.
11 *
12 * NetSurf is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 * GNU General Public License for more details.
16 *
17 * You should have received a copy of the GNU General Public License
18 * along with this program. If not, see <http://www.gnu.org/licenses/>.
19 */
20
21/**
22 * \file
23 * implementation of fetching of data from http and https schemes.
24 *
25 * This implementation uses libcurl's 'multi' interface.
26 *
27 * The CURL handles are cached in the curl_handle_ring.
28 */
29
30/* must come first to ensure winsock2.h vs windows.h ordering issues */
31#include "utils/inet.h"
32
33#include <assert.h>
34#include <errno.h>
35#include <inttypes.h>
36#include <stdbool.h>
37#include <string.h>
38#include <strings.h>
39#include <time.h>
40#include <sys/stat.h>
41
42#include <libwapcaplet/libwapcaplet.h>
43#include <nsutils/time.h>
44
45#include "utils/corestrings.h"
46#include "utils/hashmap.h"
47#include "utils/nsoption.h"
48#include "utils/log.h"
49#include "utils/messages.h"
50#include "utils/utils.h"
51#include "utils/ring.h"
52#include "utils/useragent.h"
53#include "utils/file.h"
54#include "utils/string.h"
55#include "netsurf/fetch.h"
56#include "netsurf/misc.h"
58
59#include "content/fetch.h"
60#include "content/fetchers.h"
62#include "content/urldb.h"
63
64/**
65 * maximum number of progress notifications per second
66 */
67#define UPDATES_PER_SECOND 2
68
69/**
70 * The ciphersuites the browser is prepared to use for TLS1.3
71 */
72#define CIPHER_SUITES \
73 "TLS_AES_256_GCM_SHA384:" \
74 "TLS_CHACHA20_POLY1305_SHA256:" \
75 "TLS_AES_128_GCM_SHA256"
76
77/**
78 * The ciphersuites the browser is prepared to use for TLS<1.3
79 */
80#define CIPHER_LIST \
81 /* disable everything */ \
82 "-ALL:" \
83 /* enable TLSv1.2 ECDHE AES GCM suites */ \
84 "EECDH+AESGCM+TLSv1.2:" \
85 /* enable ECDHE CHACHA20/POLY1305 suites */ \
86 "EECDH+CHACHA20:" \
87 /* Sort above by strength */ \
88 "@STRENGTH:" \
89 /* enable ECDHE (auth=RSA, mac=SHA1) AES CBC suites */ \
90 "EECDH+aRSA+AES+SHA1"
91
92/**
93 * The legacy cipher suites the browser is prepared to use for TLS<1.3
94 */
95#define CIPHER_LIST_LEGACY \
96 /* as above */ \
97 CIPHER_LIST":" \
98 /* enable (non-PFS) RSA AES GCM suites */ \
99 "RSA+AESGCM:" \
100 /* enable (non-PFS) RSA (mac=SHA1) AES CBC suites */ \
101 "RSA+AES+SHA1"
102
103/* Open SSL compatability for certificate handling */
104#ifdef WITH_OPENSSL
105
106#include <openssl/ssl.h>
107#include <openssl/x509v3.h>
108
109#else /* WITH_OPENSSL */
110
111typedef char X509;
112
113static void X509_free(X509 *cert)
114{
115 free(cert);
116}
117
118#endif /* WITH_OPENSSL */
119
120/* SSL certificate chain cache */
121
122/* We're only interested in the hostname and port */
123static uint32_t
125{
126 nsurl *url = key;
127 lwc_string *hostname = nsurl_get_component(url, NSURL_HOST);
128 lwc_string *port = nsurl_get_component(url, NSURL_PORT);
129 uint32_t hash;
130
131 if (port == NULL)
132 port = lwc_string_ref(corestring_lwc_443);
133
134 hash = lwc_string_hash_value(hostname) ^ lwc_string_hash_value(port);
135
136 lwc_string_unref(hostname);
137 lwc_string_unref(port);
138
139 return hash;
140}
141
142/* We only compare the hostname and port */
143static bool
144curl_fetch_ssl_key_eq(void *key1, void *key2)
145{
146 nsurl *url1 = key1;
147 nsurl *url2 = key2;
148 lwc_string *hostname1 = nsurl_get_component(url1, NSURL_HOST);
149 lwc_string *hostname2 = nsurl_get_component(url2, NSURL_HOST);
150 lwc_string *port1 = nsurl_get_component(url1, NSURL_PORT);
151 lwc_string *port2 = nsurl_get_component(url2, NSURL_PORT);
152 bool iseq = false;
153
154 if (port1 == NULL)
155 port1 = lwc_string_ref(corestring_lwc_443);
156 if (port2 == NULL)
157 port2 = lwc_string_ref(corestring_lwc_443);
158
159 if (lwc_string_isequal(hostname1, hostname2, &iseq) != lwc_error_ok ||
160 iseq == false)
161 goto out;
162
163 iseq = false;
164 if (lwc_string_isequal(port1, port2, &iseq) != lwc_error_ok)
165 goto out;
166
167out:
168 lwc_string_unref(hostname1);
169 lwc_string_unref(hostname2);
170 lwc_string_unref(port1);
171 lwc_string_unref(port2);
172
173 return iseq;
174}
175
176static void *
178{
179 struct cert_chain *out;
180
181 if (cert_chain_alloc(0, &out) != NSERROR_OK) {
182 return NULL;
183 }
184
185 return out;
186}
187
188static void
190{
191 struct cert_chain *chain = value;
192 if (cert_chain_free(chain) != NSERROR_OK) {
193 NSLOG(netsurf, WARNING, "Problem freeing SSL certificate chain");
194 }
195}
196
199 .key_destroy = (hashmap_key_destroy_t)nsurl_unref,
200 .key_eq = curl_fetch_ssl_key_eq,
201 .key_hash = curl_fetch_ssl_key_hash,
202 .value_alloc = curl_fetch_ssl_value_alloc,
203 .value_destroy = curl_fetch_ssl_value_destroy,
204};
205
207
208/** SSL certificate info */
209struct cert_info {
210 X509 *cert; /**< Pointer to certificate */
211 long err; /**< OpenSSL error code */
212};
213
214#if LIBCURL_VERSION_NUM >= 0x072000 /* 7.32.0 depricated CURLOPT_PROGRESSFUNCTION*/
215#define NSCURLOPT_PROGRESS_FUNCTION CURLOPT_XFERINFOFUNCTION
216#define NSCURLOPT_PROGRESS_DATA CURLOPT_XFERINFODATA
217#define NSCURL_PROGRESS_T curl_off_t
218#else
219#define NSCURLOPT_PROGRESS_FUNCTION CURLOPT_PROGRESSFUNCTION
220#define NSCURLOPT_PROGRESS_DATA CURLOPT_PROGRESSDATA
221#define NSCURL_PROGRESS_T double
222#endif
223
224#if LIBCURL_VERSION_NUM >= 0x073800 /* 7.56.0 depricated curl_formadd */
225#define NSCURL_POSTDATA_T curl_mime
226#define NSCURL_POSTDATA_CURLOPT CURLOPT_MIMEPOST
227#define NSCURL_POSTDATA_FREE(x) curl_mime_free(x)
228#else
229#define NSCURL_POSTDATA_T struct curl_httppost
230#define NSCURL_POSTDATA_CURLOPT CURLOPT_HTTPPOST
231#define NSCURL_POSTDATA_FREE(x) curl_formfree(x)
232#endif
233
234/** Information for a single fetch. */
236 struct fetch *fetch_handle; /**< The fetch handle we're parented by. */
237 CURL * curl_handle; /**< cURL handle if being fetched, or 0. */
238 bool sent_ssl_chain; /**< Have we tried to send the SSL chain */
239 bool had_headers; /**< Headers have been processed. */
240 bool abort; /**< Abort requested. */
241 bool stopped; /**< Download stopped on purpose. */
242 bool only_2xx; /**< Only HTTP 2xx responses acceptable. */
243 bool downgrade_tls; /**< Downgrade to TLS 1.2 */
244 nsurl *url; /**< URL of this fetch. */
245 lwc_string *host; /**< The hostname of this fetch. */
246 struct curl_slist *headers; /**< List of request headers. */
247 char *location; /**< Response Location header, or 0. */
248 unsigned long content_length; /**< Response Content-Length, or 0. */
249 char *cookie_string; /**< Cookie string for this fetch */
250 char *realm; /**< HTTP Auth Realm */
251 struct fetch_postdata *postdata; /**< POST data */
252 NSCURL_POSTDATA_T *curl_postdata; /**< POST data in curl representation */
253
254 long http_code; /**< HTTP result code from cURL. */
255
256 uint64_t last_progress_update; /**< Time of last progress update */
257 int cert_depth; /**< deepest certificate in use */
258 struct cert_info cert_data[MAX_CERT_DEPTH]; /**< HTTPS certificate data */
259};
260
261/** curl handle cache entry */
263 CURL *handle; /**< The cached cURL handle */
264 lwc_string *host; /**< The host for which this handle is cached */
265
266 struct cache_handle *r_prev; /**< Previous cached handle in ring. */
267 struct cache_handle *r_next; /**< Next cached handle in ring. */
268};
269
270/** Global cURL multi handle. */
272
273/** Curl handle with default options set; not used for transfers. */
274static CURL *fetch_blank_curl;
275
276/** Ring of cached handles */
278
279/** Count of how many schemes the curl fetcher is handling */
281
282/** Flag for runtime detection of openssl usage */
284
285/** Error buffer for cURL. */
286static char fetch_error_buffer[CURL_ERROR_SIZE];
287
288/** Proxy authentication details. */
289static char fetch_proxy_userpwd[100];
290
291/** Interlock to prevent initiation during callbacks */
292static bool inside_curl = false;
293
294
295/**
296 * Initialise a cURL fetcher.
297 */
298static bool fetch_curl_initialise(lwc_string *scheme)
299{
300 NSLOG(netsurf, INFO, "Initialise cURL fetcher for %s",
301 lwc_string_data(scheme));
303 return true; /* Always succeeds */
304}
305
306
307/**
308 * Finalise a cURL fetcher.
309 *
310 * \param scheme The scheme to finalise.
311 */
312static void fetch_curl_finalise(lwc_string *scheme)
313{
314 struct cache_handle *h;
315
317 NSLOG(netsurf, INFO, "Finalise cURL fetcher %s",
318 lwc_string_data(scheme));
319 if (curl_fetchers_registered == 0) {
320 CURLMcode codem;
321 /* All the fetchers have been finalised. */
322 NSLOG(netsurf, INFO,
323 "All cURL fetchers finalised, closing down cURL");
324
325 curl_easy_cleanup(fetch_blank_curl);
326
327 codem = curl_multi_cleanup(fetch_curl_multi);
328 if (codem != CURLM_OK)
329 NSLOG(netsurf, INFO,
330 "curl_multi_cleanup failed: ignoring");
331
332 curl_global_cleanup();
333
334 NSLOG(netsurf, DEBUG, "Cleaning up SSL cert chain hashmap");
337 }
338
339 /* Free anything remaining in the cached curl handle ring */
340 while (curl_handle_ring != NULL) {
343 lwc_string_unref(h->host);
344 curl_easy_cleanup(h->handle);
345 free(h);
346 }
347}
348
349
350/**
351 * Check if this fetcher can fetch a url.
352 *
353 * \param url The url to check.
354 * \return true if the fetcher supports the url else false.
355 */
356static bool fetch_curl_can_fetch(const nsurl *url)
357{
358 return nsurl_has_component(url, NSURL_HOST);
359}
360
361
362
363/**
364 * allocate postdata
365 */
366static struct fetch_postdata *
367fetch_curl_alloc_postdata(const char *post_urlenc,
368 const struct fetch_multipart_data *post_multipart)
369{
370 struct fetch_postdata *postdata;
371 postdata = calloc(1, sizeof(struct fetch_postdata));
372 if (postdata != NULL) {
373
374 if (post_urlenc) {
375 postdata->type = FETCH_POSTDATA_URLENC;
376 postdata->data.urlenc = strdup(post_urlenc);
377 if (postdata->data.urlenc == NULL) {
378 free(postdata);
379 postdata = NULL;
380 }
381 } else if (post_multipart) {
382 postdata->type = FETCH_POSTDATA_MULTIPART;
383 postdata->data.multipart = fetch_multipart_data_clone(post_multipart);
384 if (postdata->data.multipart == NULL) {
385 free(postdata);
386 postdata = NULL;
387 }
388 } else {
389 postdata->type = FETCH_POSTDATA_NONE;
390 }
391 }
392 return postdata;
393}
394
395/**
396 * free postdata
397 */
398static void fetch_curl_free_postdata(struct fetch_postdata *postdata)
399{
400 if (postdata != NULL) {
401 switch (postdata->type) {
403 break;
405 free(postdata->data.urlenc);
406 break;
409 break;
410 }
411
412 free(postdata);
413 }
414}
415
416/**
417 *construct a new fetch structure
418 */
419static struct curl_fetch_info *fetch_alloc(void)
420{
421 struct curl_fetch_info *fetch;
422 fetch = malloc(sizeof (*fetch));
423 if (fetch == NULL)
424 return NULL;
425
426 fetch->curl_handle = NULL;
427 fetch->sent_ssl_chain = false;
428 fetch->had_headers = false;
429 fetch->abort = false;
430 fetch->stopped = false;
431 fetch->only_2xx = false;
432 fetch->downgrade_tls = false;
433 fetch->headers = NULL;
434 fetch->url = NULL;
435 fetch->host = NULL;
436 fetch->location = NULL;
437 fetch->content_length = 0;
438 fetch->http_code = 0;
439 fetch->cookie_string = NULL;
440 fetch->realm = NULL;
441 fetch->last_progress_update = 0;
442 fetch->postdata = NULL;
443 fetch->curl_postdata = NULL;
444
445 /* Clear certificate chain data */
446 memset(fetch->cert_data, 0, sizeof(fetch->cert_data));
447 fetch->cert_depth = -1;
448
449 return fetch;
450}
451
452/**
453 * Start fetching data for the given URL.
454 *
455 * The function returns immediately. The fetch may be queued for later
456 * processing.
457 *
458 * A pointer to an opaque struct curl_fetch_info is returned, which can be
459 * passed to fetch_abort() to abort the fetch at any time. Returns 0 if memory
460 * is exhausted (or some other fatal error occurred).
461 *
462 * The caller must supply a callback function which is called when anything
463 * interesting happens. The callback function is first called with msg
464 * FETCH_HEADER, with the header in data, then one or more times
465 * with FETCH_DATA with some data for the url, and finally with
466 * FETCH_FINISHED. Alternatively, FETCH_ERROR indicates an error occurred:
467 * data contains an error message. FETCH_REDIRECT may replace the FETCH_HEADER,
468 * FETCH_DATA, FETCH_FINISHED sequence if the server sends a replacement URL.
469 *
470 * Some private data can be passed as the last parameter to fetch_start, and
471 * callbacks will contain this.
472 */
473static void *
474fetch_curl_setup(struct fetch *parent_fetch,
475 nsurl *url,
476 bool only_2xx,
477 bool downgrade_tls,
478 const char *post_urlenc,
479 const struct fetch_multipart_data *post_multipart,
480 const char **headers)
481{
482 struct curl_fetch_info *fetch;
483 struct curl_slist *slist;
484 int i;
485
486 fetch = fetch_alloc();
487 if (fetch == NULL)
488 return NULL;
489
490 NSLOG(netsurf, INFO, "fetch %p, url '%s'", fetch, nsurl_access(url));
491
492 fetch->only_2xx = only_2xx;
493 fetch->downgrade_tls = downgrade_tls;
494 fetch->fetch_handle = parent_fetch;
495 fetch->url = nsurl_ref(url);
497 if (fetch->host == NULL) {
498 goto failed;
499 }
500 fetch->postdata = fetch_curl_alloc_postdata(post_urlenc, post_multipart);
501 if (fetch->postdata == NULL) {
502 goto failed;
503 }
504
505#define APPEND(list, value) \
506 slist = curl_slist_append(list, value); \
507 if (slist == NULL) \
508 goto failed; \
509 list = slist;
510
511 /* remove curl default headers */
512 APPEND(fetch->headers, "Pragma:");
513
514 /* when doing a POST libcurl sends Expect: 100-continue" by default
515 * which fails with lighttpd, so disable it (see bug 1429054) */
516 APPEND(fetch->headers, "Expect:");
517
518 if ((nsoption_charp(accept_language) != NULL) &&
519 (nsoption_charp(accept_language)[0] != '\0')) {
520 char s[80];
521 snprintf(s, sizeof s, "Accept-Language: %s, *;q=0.1",
522 nsoption_charp(accept_language));
523 s[sizeof s - 1] = 0;
524 APPEND(fetch->headers, s);
525 }
526
527 if (nsoption_charp(accept_charset) != NULL &&
528 nsoption_charp(accept_charset)[0] != '\0') {
529 char s[80];
530 snprintf(s, sizeof s, "Accept-Charset: %s, *;q=0.1",
531 nsoption_charp(accept_charset));
532 s[sizeof s - 1] = 0;
533 APPEND(fetch->headers, s);
534 }
535
536 if (nsoption_bool(do_not_track) == true) {
537 APPEND(fetch->headers, "DNT: 1");
538 }
539
540 /* And add any headers specified by the caller */
541 for (i = 0; headers[i] != NULL; i++) {
542 APPEND(fetch->headers, headers[i]);
543 }
544
545 return fetch;
546
547#undef APPEND
548
549failed:
550 if (fetch->host != NULL)
551 lwc_string_unref(fetch->host);
552
555 curl_slist_free_all(fetch->headers);
556 free(fetch);
557 return NULL;
558}
559
560
561#ifdef WITH_OPENSSL
562
563/**
564 * Retrieve the ssl cert chain for the fetch, creating a blank one if needed
565 */
566static struct cert_chain *
567fetch_curl_get_cached_chain(struct curl_fetch_info *f)
568{
569 struct cert_chain *chain;
570
572 if (chain == NULL) {
574 }
575
576 return chain;
577}
578
579/**
580 * Report the certificate information in the fetch to the users
581 */
582static void
583fetch_curl_store_certs_in_cache(struct curl_fetch_info *f)
584{
585 size_t depth;
586 BIO *mem;
587 BUF_MEM *buf[MAX_CERT_DEPTH];
588 struct cert_chain chain, *cached_chain;
589 struct cert_info *certs;
590
591 memset(&chain, 0, sizeof(chain));
592
593 certs = f->cert_data;
594 chain.depth = f->cert_depth + 1; /* 0 indexed certificate depth */
595
596 for (depth = 0; depth < chain.depth; depth++) {
597 if (certs[depth].cert == NULL) {
598 /* This certificate is missing, skip it */
599 chain.certs[depth].err = SSL_CERT_ERR_CERT_MISSING;
600 continue;
601 }
602
603 /* error code (if any) */
604 switch (certs[depth].err) {
605 case X509_V_OK:
606 chain.certs[depth].err = SSL_CERT_ERR_OK;
607 break;
608
609 case X509_V_ERR_UNABLE_TO_GET_ISSUER_CERT:
610 /* fallthrough */
611 case X509_V_ERR_UNABLE_TO_DECODE_ISSUER_PUBLIC_KEY:
612 chain.certs[depth].err = SSL_CERT_ERR_BAD_ISSUER;
613 break;
614
615 case X509_V_ERR_UNABLE_TO_DECRYPT_CERT_SIGNATURE:
616 /* fallthrough */
617 case X509_V_ERR_UNABLE_TO_DECRYPT_CRL_SIGNATURE:
618 /* fallthrough */
619 case X509_V_ERR_CERT_SIGNATURE_FAILURE:
620 /* fallthrough */
621 case X509_V_ERR_CRL_SIGNATURE_FAILURE:
622 chain.certs[depth].err = SSL_CERT_ERR_BAD_SIG;
623 break;
624
625 case X509_V_ERR_CERT_NOT_YET_VALID:
626 /* fallthrough */
627 case X509_V_ERR_CRL_NOT_YET_VALID:
628 chain.certs[depth].err = SSL_CERT_ERR_TOO_YOUNG;
629 break;
630
631 case X509_V_ERR_CERT_HAS_EXPIRED:
632 /* fallthrough */
633 case X509_V_ERR_CRL_HAS_EXPIRED:
634 chain.certs[depth].err = SSL_CERT_ERR_TOO_OLD;
635 break;
636
637 case X509_V_ERR_DEPTH_ZERO_SELF_SIGNED_CERT:
638 chain.certs[depth].err = SSL_CERT_ERR_SELF_SIGNED;
639 break;
640
641 case X509_V_ERR_SELF_SIGNED_CERT_IN_CHAIN:
643 break;
644
645 case X509_V_ERR_CERT_REVOKED:
646 chain.certs[depth].err = SSL_CERT_ERR_REVOKED;
647 break;
648
649 case X509_V_ERR_HOSTNAME_MISMATCH:
651 break;
652
653 default:
654 chain.certs[depth].err = SSL_CERT_ERR_UNKNOWN;
655 break;
656 }
657
658 /*
659 * get certificate in Distinguished Encoding Rules (DER) format.
660 */
661 mem = BIO_new(BIO_s_mem());
662 i2d_X509_bio(mem, certs[depth].cert);
663 BIO_get_mem_ptr(mem, &buf[depth]);
664 (void) BIO_set_close(mem, BIO_NOCLOSE);
665 BIO_free(mem);
666
667 chain.certs[depth].der = (uint8_t *)buf[depth]->data;
668 chain.certs[depth].der_length = buf[depth]->length;
669 }
670
671 /* Now dup that chain into the cache */
672 cached_chain = fetch_curl_get_cached_chain(f);
673 if (cert_chain_dup_into(&chain, cached_chain) != NSERROR_OK) {
674 /* Something went wrong storing the chain, give up */
676 }
677
678 /* release the openssl memory buffer */
679 for (depth = 0; depth < chain.depth; depth++) {
680 if (chain.certs[depth].err == SSL_CERT_ERR_CERT_MISSING) {
681 continue;
682 }
683 if (buf[depth] != NULL) {
684 BUF_MEM_free(buf[depth]);
685 }
686 }
687}
688
689/**
690 * OpenSSL Certificate verification callback
691 *
692 * Called for each certificate in a chain being verified. OpenSSL
693 * calls this in deepest first order from the certificate authority to
694 * the peer certificate at position 0.
695 *
696 * Each certificate is stored in the fetch context the first time it
697 * is presented. If an error is encountered it is only returned for
698 * the peer certificate at position 0 allowing the enumeration of the
699 * entire chain not stopping early at the depth of the erroring
700 * certificate.
701 *
702 * \param verify_ok 0 if the caller has already determined the chain
703 * has errors else 1
704 * \param x509_ctx certificate context being verified
705 * \return 1 to indicate verification should continue and 0 to indicate
706 * verification should stop.
707 */
708static int
709fetch_curl_verify_callback(int verify_ok, X509_STORE_CTX *x509_ctx)
710{
711 int depth;
712 struct curl_fetch_info *fetch;
713
714 depth = X509_STORE_CTX_get_error_depth(x509_ctx);
715 fetch = X509_STORE_CTX_get_app_data(x509_ctx);
716
717 /* certificate chain is excessively deep so fail verification */
718 if (depth >= MAX_CERT_DEPTH) {
719 X509_STORE_CTX_set_error(x509_ctx,
720 X509_V_ERR_CERT_CHAIN_TOO_LONG);
721 return 0;
722 }
723
724 /* record the max depth */
725 if (depth > fetch->cert_depth) {
726 fetch->cert_depth = depth;
727 }
728
729 /* save the certificate by incrementing the reference count and
730 * keeping a pointer.
731 */
732 if (!fetch->cert_data[depth].cert) {
733 fetch->cert_data[depth].cert = X509_STORE_CTX_get_current_cert(x509_ctx);
734 X509_up_ref(fetch->cert_data[depth].cert);
735 fetch->cert_data[depth].err = X509_STORE_CTX_get_error(x509_ctx);
736 }
737
738 /* allow certificate chain to be completed */
739 if (depth > 0) {
740 verify_ok = 1;
741 } else {
742 /* search for deeper certificates in the chain with errors */
743 for (depth = fetch->cert_depth; depth > 0; depth--) {
744 if (fetch->cert_data[depth].err != 0) {
745 /* error in previous certificate so fail verification */
746 verify_ok = 0;
747 X509_STORE_CTX_set_error(x509_ctx, fetch->cert_data[depth].err);
748 }
749 }
750 }
751
752 return verify_ok;
753}
754
755
756/**
757 * OpenSSL certificate chain verification callback
758 *
759 * Verifies certificate chain by calling standard implementation after
760 * setting up context for the certificate callback.
761 *
762 * \param x509_ctx The certificate store to validate
763 * \param parm The fetch context.
764 * \return 1 to indicate verification success and 0 to indicate verification failure.
765 */
766static int fetch_curl_cert_verify_callback(X509_STORE_CTX *x509_ctx, void *parm)
767{
768 struct curl_fetch_info *f = (struct curl_fetch_info *) parm;
769 int ok;
770 X509_VERIFY_PARAM *vparam;
771
772 /* Configure the verification parameters to include hostname */
773 vparam = X509_STORE_CTX_get0_param(x509_ctx);
774 X509_VERIFY_PARAM_set_hostflags(vparam, X509_CHECK_FLAG_NO_PARTIAL_WILDCARDS);
775
776 ok = X509_VERIFY_PARAM_set1_host(vparam,
777 lwc_string_data(f->host),
778 lwc_string_length(f->host));
779
780 /* Store fetch struct in context for verify callback */
781 if (ok) {
782 ok = X509_STORE_CTX_set_app_data(x509_ctx, parm);
783 }
784
785 /* verify the certificate chain using standard call */
786 if (ok) {
787 ok = X509_verify_cert(x509_ctx);
788 }
789
790 fetch_curl_store_certs_in_cache(f);
791
792 return ok;
793}
794
795
796/**
797 * cURL SSL setup callback
798 *
799 * \param curl_handle The curl handle to perform the ssl operation on.
800 * \param _sslctx The ssl context.
801 * \param parm The callback context.
802 * \return A curl result code.
803 */
804static CURLcode
805fetch_curl_sslctxfun(CURL *curl_handle, void *_sslctx, void *parm)
806{
807 struct curl_fetch_info *f = (struct curl_fetch_info *) parm;
808 SSL_CTX *sslctx = _sslctx;
809 long options = SSL_OP_ALL | SSL_OP_NO_SSLv2 | SSL_OP_NO_SSLv3 |
810 SSL_OP_NO_TLSv1 | SSL_OP_NO_TLSv1_1;
811
812 /* set verify callback for each certificate in chain */
813 SSL_CTX_set_verify(sslctx, SSL_VERIFY_PEER, fetch_curl_verify_callback);
814
815 /* set callback used to verify certificate chain */
816 SSL_CTX_set_cert_verify_callback(sslctx,
817 fetch_curl_cert_verify_callback,
818 parm);
819
820 if (f->downgrade_tls) {
821 /* Disable TLS 1.3 if the server can't cope with it */
822#ifdef SSL_OP_NO_TLSv1_3
823 options |= SSL_OP_NO_TLSv1_3;
824#endif
825#ifdef SSL_MODE_SEND_FALLBACK_SCSV
826 /* Ensure server rejects the connection if downgraded too far */
827 SSL_CTX_set_mode(sslctx, SSL_MODE_SEND_FALLBACK_SCSV);
828#endif
829 }
830
831 SSL_CTX_set_options(sslctx, options);
832
833#ifdef SSL_OP_NO_TICKET
834 SSL_CTX_clear_options(sslctx, SSL_OP_NO_TICKET);
835#endif
836
837 return CURLE_OK;
838}
839
840
841#endif /* WITH_OPENSSL */
842
843
844/**
845 * Report the certificate information in the fetch to the users
846 */
847static void
849{
850 fetch_msg msg;
851 struct cert_chain *chain;
852
854
855 if (chain != NULL) {
856 msg.type = FETCH_CERTS;
857 msg.data.chain = chain;
858
860 }
861
862 f->sent_ssl_chain = true;
863}
864
865#if LIBCURL_VERSION_NUM >= 0x073800 /* 7.56.0 depricated curl_formadd */
866
867/**
868 * curl mime data context
869 */
870struct curl_mime_ctx {
871 char *buffer;
872 curl_off_t size;
873 curl_off_t position;
874};
875
876static size_t mime_data_read_callback(char *buffer, size_t size, size_t nitems, void *arg)
877{
878 struct curl_mime_ctx *mctx = (struct curl_mime_ctx *) arg;
879 curl_off_t sz = mctx->size - mctx->position;
880
881 nitems *= size;
882 if(sz > (curl_off_t)nitems) {
883 sz = nitems;
884 }
885 if(sz) {
886 memcpy(buffer, mctx->buffer + mctx->position, sz);
887 }
888 mctx->position += sz;
889 return sz;
890}
891
892static int mime_data_seek_callback(void *arg, curl_off_t offset, int origin)
893{
894 struct curl_mime_ctx *mctx = (struct curl_mime_ctx *) arg;
895
896 switch(origin) {
897 case SEEK_END:
898 offset += mctx->size;
899 break;
900 case SEEK_CUR:
901 offset += mctx->position;
902 break;
903 }
904
905 if(offset < 0) {
906 return CURL_SEEKFUNC_FAIL;
907 }
908 mctx->position = offset;
909 return CURL_SEEKFUNC_OK;
910}
911
912static void mime_data_free_callback(void *arg)
913{
914 struct curl_mime_ctx *mctx = (struct curl_mime_ctx *) arg;
915 free(mctx);
916}
917
918/**
919 * Convert a POST data list to a libcurl curl_mime.
920 *
921 * \param chandle curl fetch handle.
922 * \param multipart limked list of struct ::fetch_multipart forming post data.
923 */
924static curl_mime *
925fetch_curl_postdata_convert(CURL *chandle,
926 const struct fetch_multipart_data *multipart)
927{
928 curl_mime *cmime;
929 curl_mimepart *part;
930 CURLcode code = CURLE_OK;
931 size_t value_len;
932
933 cmime = curl_mime_init(chandle);
934 if (cmime == NULL) {
935 NSLOG(netsurf, WARNING, "postdata conversion failed to curl mime context");
936 return NULL;
937 }
938
939 /* iterate post data */
940 for (; multipart != NULL; multipart = multipart->next) {
941 part = curl_mime_addpart(cmime);
942 if (part == NULL) {
943 goto convert_failed;
944 }
945
946 code = curl_mime_name(part, multipart->name);
947 if (code != CURLE_OK) {
948 goto convert_failed;
949 }
950
951 value_len = strlen(multipart->value);
952
953 if (multipart->file && value_len==0) {
954 /* file entries with no filename require special handling */
955 code=curl_mime_data(part, multipart->value, value_len);
956 if (code != CURLE_OK) {
957 goto convert_failed;
958 }
959
960 code = curl_mime_filename(part, "");
961 if (code != CURLE_OK) {
962 goto convert_failed;
963 }
964
965 code = curl_mime_type(part, "application/octet-stream");
966 if (code != CURLE_OK) {
967 goto convert_failed;
968 }
969
970 } else if(multipart->file) {
971 /* file entry */
972 nserror ret;
973 char *leafname = NULL;
974 char *mimetype = NULL;
975
976 code = curl_mime_filedata(part, multipart->rawfile);
977 if (code != CURLE_OK) {
978 goto convert_failed;
979 }
980
981 ret = guit->file->basename(multipart->value, &leafname, NULL);
982 if (ret != NSERROR_OK) {
983 goto convert_failed;
984 }
985 code = curl_mime_filename(part, leafname);
986 free(leafname);
987 if (code != CURLE_OK) {
988 goto convert_failed;
989 }
990
991 mimetype = guit->fetch->mimetype(multipart->value);
992 if (mimetype == NULL) {
993 mimetype=strdup("text/plain");
994 }
995 if (mimetype == NULL) {
996 goto convert_failed;
997 }
998 code = curl_mime_type(part, mimetype);
999 free(mimetype);
1000 if (code != CURLE_OK) {
1001 goto convert_failed;
1002 }
1003
1004 } else {
1005 /* make the curl mime reference the existing multipart
1006 * data which requires use of a callback and context.
1007 */
1008 struct curl_mime_ctx *cb_ctx;
1009 cb_ctx = malloc(sizeof(struct curl_mime_ctx));
1010 if (cb_ctx == NULL) {
1011 goto convert_failed;
1012 }
1013 cb_ctx->buffer = multipart->value;
1014 cb_ctx->size = value_len;
1015 cb_ctx->position = 0;
1016 code = curl_mime_data_cb(part,
1017 value_len,
1018 mime_data_read_callback,
1019 mime_data_seek_callback,
1020 mime_data_free_callback,
1021 cb_ctx);
1022 if (code != CURLE_OK) {
1023 free(cb_ctx);
1024 goto convert_failed;
1025 }
1026 }
1027 }
1028
1029 return cmime;
1030
1031convert_failed:
1032 NSLOG(netsurf, WARNING, "postdata conversion failed with curl code: %d", code);
1033 curl_mime_free(cmime);
1034 return NULL;
1035}
1036
1037#else /* LIBCURL_VERSION_NUM >= 0x073800 */
1038
1039/**
1040 * Convert a list of struct ::fetch_multipart_data to a list of
1041 * struct curl_httppost for libcurl.
1042 */
1043static struct curl_httppost *
1045 const struct fetch_multipart_data *control)
1046{
1047 struct curl_httppost *post = NULL, *last = NULL;
1048 CURLFORMcode code;
1049 nserror ret;
1050
1051 for (; control; control = control->next) {
1052 if (control->file) {
1053 char *leafname = NULL;
1054 ret = guit->file->basename(control->value, &leafname, NULL);
1055 if (ret != NSERROR_OK) {
1056 continue;
1057 }
1058
1059 /* We have to special case filenames of "", so curl
1060 * a) actually attempts the fetch and
1061 * b) doesn't attempt to open the file ""
1062 */
1063 if (control->value[0] == '\0') {
1064 /* dummy buffer - needs to be static so
1065 * pointer's still valid when we go out
1066 * of scope (not that libcurl should be
1067 * attempting to access it, of course).
1068 */
1069 static char buf;
1070
1071 code = curl_formadd(&post, &last,
1072 CURLFORM_COPYNAME, control->name,
1073 CURLFORM_BUFFER, control->value,
1074 /* needed, as basename("") == "." */
1075 CURLFORM_FILENAME, "",
1076 CURLFORM_BUFFERPTR, &buf,
1077 CURLFORM_BUFFERLENGTH, 0,
1078 CURLFORM_CONTENTTYPE,
1079 "application/octet-stream",
1080 CURLFORM_END);
1081 if (code != CURL_FORMADD_OK)
1082 NSLOG(netsurf, INFO,
1083 "curl_formadd: %d (%s)", code,
1084 control->name);
1085 } else {
1086 char *mimetype = guit->fetch->mimetype(control->value);
1087 code = curl_formadd(&post, &last,
1088 CURLFORM_COPYNAME, control->name,
1089 CURLFORM_FILE, control->rawfile,
1090 CURLFORM_FILENAME, leafname,
1091 CURLFORM_CONTENTTYPE,
1092 (mimetype != 0 ? mimetype : "text/plain"),
1093 CURLFORM_END);
1094 if (code != CURL_FORMADD_OK)
1095 NSLOG(netsurf, INFO,
1096 "curl_formadd: %d (%s=%s)",
1097 code,
1098 control->name,
1099 control->value);
1100 free(mimetype);
1101 }
1102 free(leafname);
1103 } else {
1104 code = curl_formadd(&post, &last,
1105 CURLFORM_COPYNAME, control->name,
1106 CURLFORM_COPYCONTENTS, control->value,
1107 CURLFORM_END);
1108 if (code != CURL_FORMADD_OK)
1109 NSLOG(netsurf, INFO,
1110 "curl_formadd: %d (%s=%s)", code,
1111 control->name, control->value);
1112 }
1113 }
1114
1115 return post;
1116}
1117
1118#endif /* LIBCURL_VERSION_NUM >= 0x073800 */
1119
1120/**
1121 * Setup multipart post data
1122 */
1124{
1125 CURLcode code = CURLE_OK;
1126
1127#undef SETOPT
1128#define SETOPT(option, value) { \
1129 code = curl_easy_setopt(f->curl_handle, option, value); \
1130 if (code != CURLE_OK) \
1131 return code; \
1132 }
1133
1134 switch (f->postdata->type) {
1136 SETOPT(CURLOPT_POSTFIELDS, NULL);
1138 SETOPT(CURLOPT_HTTPGET, 1L);
1139 break;
1140
1143 SETOPT(CURLOPT_HTTPGET, 0L);
1144 SETOPT(CURLOPT_POSTFIELDS, f->postdata->data.urlenc);
1145 break;
1146
1148 SETOPT(CURLOPT_POSTFIELDS, NULL);
1149 SETOPT(CURLOPT_HTTPGET, 0L);
1150 if (f->curl_postdata == NULL) {
1151 f->curl_postdata =
1153 f->postdata->data.multipart);
1154 }
1156 break;
1157 }
1158 return code;
1159}
1160
1161/**
1162 * Set options specific for a fetch.
1163 *
1164 * \param f The fetch to set options on.
1165 * \return A curl result code.
1166 */
1167static CURLcode fetch_curl_set_options(struct curl_fetch_info *f)
1168{
1169 CURLcode code;
1170 const char *auth;
1171
1172#undef SETOPT
1173#define SETOPT(option, value) { \
1174 code = curl_easy_setopt(f->curl_handle, option, value); \
1175 if (code != CURLE_OK) \
1176 return code; \
1177 }
1178
1179 SETOPT(CURLOPT_URL, nsurl_access(f->url));
1180 SETOPT(CURLOPT_PRIVATE, f);
1181 SETOPT(CURLOPT_WRITEDATA, f);
1182 SETOPT(CURLOPT_WRITEHEADER, f);
1184 SETOPT(CURLOPT_HTTPHEADER, f->headers);
1185 code = fetch_curl_set_postdata(f);
1186 if (code != CURLE_OK) {
1187 return code;
1188 }
1189
1190 f->cookie_string = urldb_get_cookie(f->url, true);
1191 if (f->cookie_string) {
1192 SETOPT(CURLOPT_COOKIE, f->cookie_string);
1193 } else {
1194 SETOPT(CURLOPT_COOKIE, NULL);
1195 }
1196
1197 if ((auth = urldb_get_auth_details(f->url, NULL)) != NULL) {
1198 SETOPT(CURLOPT_HTTPAUTH, CURLAUTH_BASIC);
1199 SETOPT(CURLOPT_USERPWD, auth);
1200 } else {
1201 SETOPT(CURLOPT_USERPWD, NULL);
1202 }
1203
1204 /* set up proxy options */
1205 if (nsoption_bool(http_proxy) &&
1206 (nsoption_charp(http_proxy_host) != NULL) &&
1207 (strncmp(nsurl_access(f->url), "file:", 5) != 0)) {
1208 SETOPT(CURLOPT_PROXY, nsoption_charp(http_proxy_host));
1209 SETOPT(CURLOPT_PROXYPORT, (long) nsoption_int(http_proxy_port));
1210
1211#if LIBCURL_VERSION_NUM >= 0x071304
1212 /* Added in 7.19.4 */
1213 /* setup the omission list */
1214 SETOPT(CURLOPT_NOPROXY, nsoption_charp(http_proxy_noproxy));
1215#endif
1216
1217 if (nsoption_int(http_proxy_auth) != OPTION_HTTP_PROXY_AUTH_NONE) {
1218 SETOPT(CURLOPT_PROXYAUTH,
1219 nsoption_int(http_proxy_auth) ==
1221 (long) CURLAUTH_BASIC :
1222 (long) CURLAUTH_NTLM);
1223 snprintf(fetch_proxy_userpwd,
1224 sizeof fetch_proxy_userpwd,
1225 "%s:%s",
1226 nsoption_charp(http_proxy_auth_user),
1227 nsoption_charp(http_proxy_auth_pass));
1228 SETOPT(CURLOPT_PROXYUSERPWD, fetch_proxy_userpwd);
1229 }
1230 } else {
1231 SETOPT(CURLOPT_PROXY, NULL);
1232 }
1233
1234
1235 if (curl_with_openssl) {
1236 SETOPT(CURLOPT_SSL_CIPHER_LIST,
1238 }
1239
1240 /* Force-enable SSL session ID caching, as some distros are odd. */
1241 SETOPT(CURLOPT_SSL_SESSIONID_CACHE, 1);
1242
1244 /* Disable certificate verification */
1245 SETOPT(CURLOPT_SSL_VERIFYPEER, 0L);
1246 SETOPT(CURLOPT_SSL_VERIFYHOST, 0L);
1247 if (curl_with_openssl) {
1248 SETOPT(CURLOPT_SSL_CTX_FUNCTION, NULL);
1249 SETOPT(CURLOPT_SSL_CTX_DATA, NULL);
1250 }
1251 } else {
1252 /* do verification */
1253 SETOPT(CURLOPT_SSL_VERIFYPEER, 1L);
1254 SETOPT(CURLOPT_SSL_VERIFYHOST, 2L);
1255#ifdef WITH_OPENSSL
1256 if (curl_with_openssl) {
1257 SETOPT(CURLOPT_SSL_CTX_FUNCTION, fetch_curl_sslctxfun);
1258 SETOPT(CURLOPT_SSL_CTX_DATA, f);
1259 }
1260#endif
1261 }
1262
1263 return CURLE_OK;
1264}
1265
1266/**
1267 * Initiate a fetch from the queue.
1268 *
1269 * \param fetch fetch to use to fetch content.
1270 * \param handle CURL handle to be used to fetch the content.
1271 * \return true if the fetch was successfully initiated else false.
1272 */
1273static bool
1275{
1276 CURLcode code;
1277 CURLMcode codem;
1278
1279 fetch->curl_handle = handle;
1280
1281 /* Initialise the handle */
1283 if (code != CURLE_OK) {
1284 fetch->curl_handle = 0;
1285 /* The handle maybe went bad, eat it */
1286 NSLOG(netsurf, WARNING, "cURL handle maybe went bad, retry later");
1287 curl_easy_cleanup(handle);
1288 return false;
1289 }
1290
1291 /* add to the global curl multi handle */
1292 codem = curl_multi_add_handle(fetch_curl_multi, fetch->curl_handle);
1293 assert(codem == CURLM_OK || codem == CURLM_CALL_MULTI_PERFORM);
1294
1295 return true;
1296}
1297
1298
1299/**
1300 * Find a CURL handle to use to dispatch a job
1301 */
1302static CURL *fetch_curl_get_handle(lwc_string *host)
1303{
1304 struct cache_handle *h;
1305 CURL *ret;
1307 if (h) {
1308 ret = h->handle;
1309 lwc_string_unref(h->host);
1311 free(h);
1312 } else {
1313 ret = curl_easy_duphandle(fetch_blank_curl);
1314 }
1315 return ret;
1316}
1317
1318
1319/**
1320 * Dispatch a single job
1321 */
1322static bool fetch_curl_start(void *vfetch)
1323{
1324 struct curl_fetch_info *fetch = (struct curl_fetch_info*)vfetch;
1325 if (inside_curl) {
1326 NSLOG(netsurf, DEBUG, "Deferring fetch because we're inside cURL");
1327 return false;
1328 }
1331}
1332
1333/**
1334 * Cache a CURL handle for the provided host (if wanted)
1335 */
1336static void fetch_curl_cache_handle(CURL *handle, lwc_string *host)
1337{
1338#if LIBCURL_VERSION_NUM >= 0x071e00
1339 /* 7.30.0 or later has its own connection caching; suppress ours */
1340 curl_easy_cleanup(handle);
1341 return;
1342#else
1343 struct cache_handle *h = 0;
1344 int c;
1346 if (h) {
1347 /* Already have a handle cached for this hostname */
1348 curl_easy_cleanup(handle);
1349 return;
1350 }
1351 /* We do not have a handle cached, first up determine if the cache is full */
1353 if (c >= nsoption_int(max_cached_fetch_handles)) {
1354 /* Cache is full, so, we rotate the ring by one and
1355 * replace the oldest handle with this one. We do this
1356 * without freeing/allocating memory (except the
1357 * hostname) and without removing the entry from the
1358 * ring and then re-inserting it, in order to be as
1359 * efficient as we can.
1360 */
1361 if (curl_handle_ring != NULL) {
1362 h = curl_handle_ring;
1364 curl_easy_cleanup(h->handle);
1365 h->handle = handle;
1366 lwc_string_unref(h->host);
1367 h->host = lwc_string_ref(host);
1368 } else {
1369 /* Actually, we don't want to cache any handles */
1370 curl_easy_cleanup(handle);
1371 }
1372
1373 return;
1374 }
1375 /* The table isn't full yet, so make a shiny new handle to add to the ring */
1376 h = (struct cache_handle*)malloc(sizeof(struct cache_handle));
1377 h->handle = handle;
1378 h->host = lwc_string_ref(host);
1380#endif
1381}
1382
1383
1384/**
1385 * Clean up the provided fetch object and free it.
1386 *
1387 * Will prod the queue afterwards to allow pending requests to be initiated.
1388 */
1389static void fetch_curl_stop(struct curl_fetch_info *f)
1390{
1391 CURLMcode codem;
1392
1393 assert(f);
1394 NSLOG(netsurf, INFO, "fetch %p, url '%s'", f, nsurl_access(f->url));
1395
1396 if (f->curl_handle) {
1397 /* remove from curl multi handle */
1398 codem = curl_multi_remove_handle(fetch_curl_multi,
1399 f->curl_handle);
1400 assert(codem == CURLM_OK);
1401 /* Put this curl handle into the cache if wanted. */
1403 f->curl_handle = 0;
1404 }
1405
1407}
1408
1409
1410/**
1411 * Abort a fetch.
1412 */
1413static void fetch_curl_abort(void *vf)
1414{
1415 struct curl_fetch_info *f = (struct curl_fetch_info *)vf;
1416 assert(f);
1417 NSLOG(netsurf, INFO, "fetch %p, url '%s'", f, nsurl_access(f->url));
1418 if (f->curl_handle) {
1419 if (inside_curl) {
1420 NSLOG(netsurf, DEBUG, "Deferring cleanup");
1421 f->abort = true;
1422 } else {
1423 NSLOG(netsurf, DEBUG, "Immediate abort");
1424 fetch_curl_stop(f);
1426 }
1427 } else {
1430 }
1431}
1432
1433
1434/**
1435 * Free a fetch structure and associated resources.
1436 */
1437static void fetch_curl_free(void *vf)
1438{
1439 struct curl_fetch_info *f = (struct curl_fetch_info *)vf;
1440 int i;
1441
1442 if (f->curl_handle) {
1443 curl_easy_cleanup(f->curl_handle);
1444 }
1445 nsurl_unref(f->url);
1446 lwc_string_unref(f->host);
1447 free(f->location);
1448 free(f->cookie_string);
1449 free(f->realm);
1450 if (f->headers) {
1451 curl_slist_free_all(f->headers);
1452 }
1455
1456 /* free certificate data */
1457 for (i = 0; i < MAX_CERT_DEPTH; i++) {
1458 if (f->cert_data[i].cert != NULL) {
1459 X509_free(f->cert_data[i].cert);
1460 }
1461 }
1462
1463 free(f);
1464}
1465
1466
1467/**
1468 * Find the status code and content type and inform the caller.
1469 *
1470 * Return true if the fetch is being aborted.
1471 */
1473{
1474 long http_code;
1475 CURLcode code;
1476 fetch_msg msg;
1477
1478 f->had_headers = true;
1479
1480 if (!f->http_code) {
1481 code = curl_easy_getinfo(f->curl_handle, CURLINFO_HTTP_CODE,
1482 &f->http_code);
1484 assert(code == CURLE_OK);
1485 }
1486 http_code = f->http_code;
1487 NSLOG(netsurf, INFO, "HTTP status code %li", http_code);
1488
1489 if ((http_code == 304) && (f->postdata->type==FETCH_POSTDATA_NONE)) {
1490 /* Not Modified && GET request */
1491 msg.type = FETCH_NOTMODIFIED;
1493 return true;
1494 }
1495
1496 /* handle HTTP redirects (3xx response codes) */
1497 if (300 <= http_code && http_code < 400 && f->location != 0) {
1498 NSLOG(netsurf, INFO, "FETCH_REDIRECT, '%s'", f->location);
1499 msg.type = FETCH_REDIRECT;
1500 msg.data.redirect = f->location;
1502 return true;
1503 }
1504
1505 /* handle HTTP 401 (Authentication errors) */
1506 if (http_code == 401) {
1507 msg.type = FETCH_AUTH;
1508 msg.data.auth.realm = f->realm;
1510 return true;
1511 }
1512
1513 /* handle HTTP errors (non 2xx response codes) */
1514 if (f->only_2xx && strncmp(nsurl_access(f->url), "http", 4) == 0 &&
1515 (http_code < 200 || 299 < http_code)) {
1516 msg.type = FETCH_ERROR;
1517 msg.data.error = messages_get("Not2xx");
1519 return true;
1520 }
1521
1522 if (f->abort)
1523 return true;
1524
1525 return false;
1526}
1527
1528
1529/**
1530 * Handle a completed fetch (CURLMSG_DONE from curl_multi_info_read()).
1531 *
1532 * \param curl_handle curl easy handle of fetch
1533 * \param result The result code of the completed fetch.
1534 */
1535static void fetch_curl_done(CURL *curl_handle, CURLcode result)
1536{
1537 bool finished = false;
1538 bool error = false;
1539 bool cert = false;
1540 bool abort_fetch;
1541 struct curl_fetch_info *f;
1542 char **_hideous_hack = (char **) (void *) &f;
1543 CURLcode code;
1544
1545 /* find the structure associated with this fetch */
1546 /* For some reason, cURL thinks CURLINFO_PRIVATE should be a string?! */
1547 code = curl_easy_getinfo(curl_handle, CURLINFO_PRIVATE, _hideous_hack);
1548 assert(code == CURLE_OK);
1549
1550 abort_fetch = f->abort;
1551 NSLOG(netsurf, INFO, "done %s", nsurl_access(f->url));
1552
1553 if ((abort_fetch == false) &&
1554 (result == CURLE_OK ||
1555 ((result == CURLE_WRITE_ERROR) && (f->stopped == false)))) {
1556 /* fetch completed normally or the server fed us a junk gzip
1557 * stream (usually in the form of garbage at the end of the
1558 * stream). Curl will have fed us all but the last chunk of
1559 * decoded data, which is sad as, if we'd received the last
1560 * chunk, too, we'd be able to render the whole object.
1561 * As is, we'll just have to accept that the end of the
1562 * object will be truncated in this case and leave it to
1563 * the content handlers to cope.
1564 */
1565 if (f->stopped ||
1567 ; /* redirect with no body or similar */
1568 } else {
1569 finished = true;
1570 }
1571 } else if (result == CURLE_PARTIAL_FILE) {
1572 /* CURLE_PARTIAL_FILE occurs if the received body of a
1573 * response is smaller than that specified in the
1574 * Content-Length header.
1575 */
1577 ; /* redirect with partial body, or similar */
1578 else {
1579 finished = true;
1580 }
1581 } else if (result == CURLE_WRITE_ERROR && f->stopped) {
1582 /* CURLE_WRITE_ERROR occurs when fetch_curl_data
1583 * returns 0, which we use to abort intentionally
1584 */
1585 ;
1586 } else if (result == CURLE_SSL_PEER_CERTIFICATE ||
1587 result == CURLE_SSL_CACERT) {
1588 /* Some kind of failure has occurred. If we don't know
1589 * what happened, we'll have reported unknown errors up
1590 * to the user already via the certificate chain error fields.
1591 */
1592 cert = true;
1593 } else {
1594 NSLOG(netsurf, INFO, "Unknown cURL response code %d", result);
1595 error = true;
1596 }
1597
1598 fetch_curl_stop(f);
1599
1600 if (f->sent_ssl_chain == false) {
1602 }
1603
1604 if (abort_fetch) {
1605 ; /* fetch was aborted: no callback */
1606 } else if (finished) {
1607 fetch_msg msg;
1608 msg.type = FETCH_FINISHED;
1610 } else if (cert) {
1611 /* user needs to validate certificate with issue */
1612 fetch_msg msg;
1613 msg.type = FETCH_CERT_ERR;
1615 } else if (error) {
1616 fetch_msg msg;
1617 switch (result) {
1618 case CURLE_SSL_CONNECT_ERROR:
1619 msg.type = FETCH_SSL_ERR;
1620 break;
1621
1622 case CURLE_OPERATION_TIMEDOUT:
1623 msg.type = FETCH_TIMEDOUT;
1624 msg.data.error = curl_easy_strerror(result);
1625 break;
1626
1627 default:
1628 msg.type = FETCH_ERROR;
1629 msg.data.error = curl_easy_strerror(result);
1630 }
1631
1633 }
1634
1636}
1637
1638
1639/**
1640 * Do some work on current fetches.
1641 *
1642 * Must be called regularly to make progress on fetches.
1643 */
1644static void fetch_curl_poll(lwc_string *scheme_ignored)
1645{
1646 int running, queue;
1647 CURLMcode codem;
1648 CURLMsg *curl_msg;
1649
1650 if (nsoption_bool(suppress_curl_debug) == false) {
1651 fd_set read_fd_set, write_fd_set, exc_fd_set;
1652 int max_fd = -1;
1653 int i;
1654
1655 FD_ZERO(&read_fd_set);
1656 FD_ZERO(&write_fd_set);
1657 FD_ZERO(&exc_fd_set);
1658
1659 codem = curl_multi_fdset(fetch_curl_multi,
1660 &read_fd_set, &write_fd_set,
1661 &exc_fd_set, &max_fd);
1662 assert(codem == CURLM_OK);
1663
1664 NSLOG(netsurf, DEEPDEBUG,
1665 "Curl file descriptor states (maxfd=%i):", max_fd);
1666 for (i = 0; i <= max_fd; i++) {
1667 bool read = false;
1668 bool write = false;
1669 bool error = false;
1670
1671 if (FD_ISSET(i, &read_fd_set)) {
1672 read = true;
1673 }
1674 if (FD_ISSET(i, &write_fd_set)) {
1675 write = true;
1676 }
1677 if (FD_ISSET(i, &exc_fd_set)) {
1678 error = true;
1679 }
1680 if (read || write || error) {
1681 NSLOG(netsurf, DEEPDEBUG, " fd %i: %s %s %s", i,
1682 read ? "read" : " ",
1683 write ? "write" : " ",
1684 error ? "error" : " ");
1685 }
1686 }
1687 }
1688
1689 /* do any possible work on the current fetches */
1690 inside_curl = true;
1691 do {
1692 codem = curl_multi_perform(fetch_curl_multi, &running);
1693 if (codem != CURLM_OK && codem != CURLM_CALL_MULTI_PERFORM) {
1694 NSLOG(netsurf, WARNING,
1695 "curl_multi_perform: %i %s",
1696 codem, curl_multi_strerror(codem));
1697 return;
1698 }
1699 } while (codem == CURLM_CALL_MULTI_PERFORM);
1700
1701 /* process curl results */
1702 curl_msg = curl_multi_info_read(fetch_curl_multi, &queue);
1703 while (curl_msg) {
1704 switch (curl_msg->msg) {
1705 case CURLMSG_DONE:
1706 fetch_curl_done(curl_msg->easy_handle,
1707 curl_msg->data.result);
1708 break;
1709 default:
1710 break;
1711 }
1712 curl_msg = curl_multi_info_read(fetch_curl_multi, &queue);
1713 }
1714 inside_curl = false;
1715}
1716
1717
1718
1719
1720/**
1721 * Callback function for fetch progress.
1722 */
1723static int
1725 NSCURL_PROGRESS_T dltotal,
1726 NSCURL_PROGRESS_T dlnow,
1727 NSCURL_PROGRESS_T ultotal,
1728 NSCURL_PROGRESS_T ulnow)
1729{
1730 static char fetch_progress_buffer[256]; /**< Progress buffer for cURL */
1731 struct curl_fetch_info *f = (struct curl_fetch_info *) clientp;
1732 uint64_t time_now_ms;
1733 fetch_msg msg;
1734
1735 if (f->abort) {
1736 return 0;
1737 }
1738
1739 msg.type = FETCH_PROGRESS;
1740 msg.data.progress = fetch_progress_buffer;
1741
1742 /* Rate limit each fetch's progress notifications */
1743 nsu_getmonotonic_ms(&time_now_ms);
1744#define UPDATE_DELAY_MS (1000 / UPDATES_PER_SECOND)
1745 if (time_now_ms - f->last_progress_update < UPDATE_DELAY_MS) {
1746 return 0;
1747 }
1748#undef UPDATE_DELAY_MS
1749 f->last_progress_update = time_now_ms;
1750
1751 if (dltotal > 0) {
1752 snprintf(fetch_progress_buffer, 255,
1753 messages_get("Progress"),
1755 human_friendly_bytesize(dltotal));
1757 } else {
1758 snprintf(fetch_progress_buffer, 255,
1759 messages_get("ProgressU"),
1762 }
1763
1764 return 0;
1765}
1766
1767
1768/**
1769 * Format curl debug for nslog
1770 */
1771static int
1772fetch_curl_debug(CURL *handle,
1773 curl_infotype type,
1774 char *data,
1775 size_t size,
1776 void *userptr)
1777{
1778 static const char s_infotype[CURLINFO_END][3] = {
1779 "* ", "< ", "> ", "{ ", "} ", "{ ", "} "
1780 };
1781 switch(type) {
1782 case CURLINFO_TEXT:
1783 case CURLINFO_HEADER_OUT:
1784 case CURLINFO_HEADER_IN:
1785 NSLOG(fetch, DEBUG, "%s%.*s", s_infotype[type], (int)size - 1, data);
1786 break;
1787
1788 default:
1789 break;
1790 }
1791 return 0;
1792}
1793
1794
1795static curl_socket_t fetch_curl_socket_open(void *clientp,
1796 curlsocktype purpose, struct curl_sockaddr *address)
1797{
1798 (void) clientp;
1799 (void) purpose;
1800
1801 return (curl_socket_t) guit->fetch->socket_open(
1802 address->family, address->socktype,
1803 address->protocol);
1804}
1805
1806static int fetch_curl_socket_close(void *clientp, curl_socket_t item)
1807{
1808 (void) clientp;
1809
1810 return guit->fetch->socket_close((int) item);
1811}
1812
1813/**
1814 * Callback function for cURL.
1815 */
1816static size_t fetch_curl_data(char *data, size_t size, size_t nmemb, void *_f)
1817{
1818 struct curl_fetch_info *f = _f;
1819 CURLcode code;
1820 fetch_msg msg;
1821
1822 /* ensure we only have to get this information once */
1823 if (!f->http_code) {
1824 code = curl_easy_getinfo(f->curl_handle, CURLINFO_HTTP_CODE,
1825 &f->http_code);
1827 assert(code == CURLE_OK);
1828 }
1829
1830 /* ignore body if this is a 401 reply by skipping it and reset
1831 * the HTTP response code to enable follow up fetches.
1832 */
1833 if (f->http_code == 401) {
1834 f->http_code = 0;
1835 return size * nmemb;
1836 }
1837
1838 if (f->abort || (!f->had_headers && fetch_curl_process_headers(f))) {
1839 f->stopped = true;
1840 return 0;
1841 }
1842
1843 /* send data to the caller */
1844 msg.type = FETCH_DATA;
1845 msg.data.header_or_data.buf = (const uint8_t *) data;
1846 msg.data.header_or_data.len = size * nmemb;
1848
1849 if (f->abort) {
1850 f->stopped = true;
1851 return 0;
1852 }
1853
1854 return size * nmemb;
1855}
1856
1857
1858/**
1859 * Callback function for headers.
1860 *
1861 * See RFC 2616 4.2.
1862 */
1863static size_t
1864fetch_curl_header(char *data, size_t size, size_t nmemb, void *_f)
1865{
1866 struct curl_fetch_info *f = _f;
1867 int i;
1868 fetch_msg msg;
1869 size *= nmemb;
1870
1871 if (f->abort) {
1872 f->stopped = true;
1873 return 0;
1874 }
1875
1876 if (f->sent_ssl_chain == false) {
1878 }
1879
1880 msg.type = FETCH_HEADER;
1881 msg.data.header_or_data.buf = (const uint8_t *) data;
1882 msg.data.header_or_data.len = size;
1884
1885#define SKIP_ST(o) for (i = (o); i < (int) size && (data[i] == ' ' || data[i] == '\t'); i++)
1886
1887 if (12 < size && strncasecmp(data, "Location:", 9) == 0) {
1888 /* extract Location header */
1889 free(f->location);
1890 f->location = malloc(size);
1891 if (!f->location) {
1892 NSLOG(netsurf, INFO, "malloc failed");
1893 return size;
1894 }
1895 SKIP_ST(9);
1896 strncpy(f->location, data + i, size - i);
1897 f->location[size - i] = '\0';
1898 for (i = size - i - 1; i >= 0 &&
1899 (f->location[i] == ' ' ||
1900 f->location[i] == '\t' ||
1901 f->location[i] == '\r' ||
1902 f->location[i] == '\n'); i--)
1903 f->location[i] = '\0';
1904 } else if (15 < size && strncasecmp(data, "Content-Length:", 15) == 0) {
1905 /* extract Content-Length header */
1906 SKIP_ST(15);
1907 if (i < (int)size && '0' <= data[i] && data[i] <= '9')
1908 f->content_length = atol(data + i);
1909 } else if (17 < size && strncasecmp(data, "WWW-Authenticate:", 17) == 0) {
1910 /* extract the first Realm from WWW-Authenticate header */
1911 SKIP_ST(17);
1912
1913 while (i < (int) size - 5 &&
1914 strncasecmp(data + i, "realm", 5))
1915 i++;
1916 while (i < (int) size - 1 && data[++i] != '"')
1917 /* */;
1918 i++;
1919
1920 if (i < (int) size) {
1921 size_t end = i;
1922
1923 while (end < size && data[end] != '"')
1924 ++end;
1925
1926 if (end < size) {
1927 free(f->realm);
1928 f->realm = malloc(end - i + 1);
1929 if (f->realm != NULL) {
1930 strncpy(f->realm, data + i, end - i);
1931 f->realm[end - i] = '\0';
1932 }
1933 }
1934 }
1935 } else if (11 < size && strncasecmp(data, "Set-Cookie:", 11) == 0) {
1936 /* extract Set-Cookie header */
1937 SKIP_ST(11);
1938
1939 fetch_set_cookie(f->fetch_handle, &data[i]);
1940 }
1941
1942 return size;
1943#undef SKIP_ST
1944}
1945
1946static int fetch_curl_fdset(lwc_string *scheme, fd_set *read_set,
1947 fd_set *write_set, fd_set *error_set)
1948{
1949 CURLMcode code;
1950 int maxfd = -1;
1951
1952 code = curl_multi_fdset(fetch_curl_multi,
1953 read_set,
1954 write_set,
1955 error_set,
1956 &maxfd);
1957 assert(code == CURLM_OK);
1958
1959 return maxfd;
1960}
1961
1962
1963
1964/* exported function documented in content/fetchers/curl.h */
1966{
1967 CURLcode code;
1968 curl_version_info_data *data;
1969 int i;
1970 lwc_string *scheme;
1971 const struct fetcher_operation_table fetcher_ops = {
1973 .acceptable = fetch_curl_can_fetch,
1974 .setup = fetch_curl_setup,
1975 .start = fetch_curl_start,
1976 .abort = fetch_curl_abort,
1977 .free = fetch_curl_free,
1978 .poll = fetch_curl_poll,
1979 .fdset = fetch_curl_fdset,
1980 .finalise = fetch_curl_finalise
1981 };
1982
1983#if LIBCURL_VERSION_NUM >= 0x073800
1984 /* version 7.56.0 can select which SSL backend to use */
1985 CURLsslset setres;
1986
1987 setres = curl_global_sslset(CURLSSLBACKEND_OPENSSL, NULL, NULL);
1988 if (setres == CURLSSLSET_OK) {
1989 curl_with_openssl = true;
1990 } else {
1991 curl_with_openssl = false;
1992 }
1993#endif
1994
1995 NSLOG(netsurf, INFO, "curl_version %s", curl_version());
1996
1997 code = curl_global_init(CURL_GLOBAL_ALL);
1998 if (code != CURLE_OK) {
1999 NSLOG(netsurf, INFO, "curl_global_init failed.");
2000 return NSERROR_INIT_FAILED;
2001 }
2002
2003 fetch_curl_multi = curl_multi_init();
2004 if (!fetch_curl_multi) {
2005 NSLOG(netsurf, INFO, "curl_multi_init failed.");
2006 return NSERROR_INIT_FAILED;
2007 }
2008
2009#if LIBCURL_VERSION_NUM >= 0x071e00
2010 /* built against 7.30.0 or later: configure caching */
2011 {
2012 CURLMcode mcode;
2013 int maxconnects = nsoption_int(max_fetchers) +
2014 nsoption_int(max_cached_fetch_handles);
2015
2016#undef SETOPT
2017#define SETOPT(option, value) \
2018 mcode = curl_multi_setopt(fetch_curl_multi, option, value); \
2019 if (mcode != CURLM_OK) { \
2020 NSLOG(netsurf, ERROR, "attempting curl_multi_setopt(%s, ...)", #option); \
2021 goto curl_multi_setopt_failed; \
2022 }
2023
2024 SETOPT(CURLMOPT_MAXCONNECTS, maxconnects);
2025 SETOPT(CURLMOPT_MAX_TOTAL_CONNECTIONS, maxconnects);
2026 SETOPT(CURLMOPT_MAX_HOST_CONNECTIONS, nsoption_int(max_fetchers_per_host));
2027 }
2028#endif
2029
2030 /* Create a curl easy handle with the options that are common to all
2031 * fetches.
2032 */
2033 fetch_blank_curl = curl_easy_init();
2034 if (!fetch_blank_curl) {
2035 NSLOG(netsurf, INFO, "curl_easy_init failed");
2036 return NSERROR_INIT_FAILED;
2037 }
2038
2039#undef SETOPT
2040#define SETOPT(option, value) \
2041 code = curl_easy_setopt(fetch_blank_curl, option, value); \
2042 if (code != CURLE_OK) { \
2043 NSLOG(netsurf, ERROR, "attempting curl_easy_setopt(%s, ...)", #option); \
2044 goto curl_easy_setopt_failed; \
2045 }
2046
2047 SETOPT(CURLOPT_ERRORBUFFER, fetch_error_buffer);
2048 SETOPT(CURLOPT_DEBUGFUNCTION, fetch_curl_debug);
2049 if (nsoption_bool(suppress_curl_debug)) {
2050 SETOPT(CURLOPT_VERBOSE, 0);
2051 } else {
2052 SETOPT(CURLOPT_VERBOSE, 1);
2053 }
2054
2055 /* Currently we explode if curl uses HTTP2, so force 1.1. */
2056 SETOPT(CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_1);
2057
2058 SETOPT(CURLOPT_WRITEFUNCTION, fetch_curl_data);
2059 SETOPT(CURLOPT_HEADERFUNCTION, fetch_curl_header);
2061 SETOPT(CURLOPT_NOPROGRESS, 0);
2062 SETOPT(CURLOPT_USERAGENT, user_agent_string());
2063 SETOPT(CURLOPT_ENCODING, "gzip");
2064 SETOPT(CURLOPT_LOW_SPEED_LIMIT, 1L);
2065 SETOPT(CURLOPT_LOW_SPEED_TIME, 180L);
2066 SETOPT(CURLOPT_NOSIGNAL, 1L);
2067 SETOPT(CURLOPT_CONNECTTIMEOUT, nsoption_uint(curl_fetch_timeout));
2068 SETOPT(CURLOPT_OPENSOCKETFUNCTION, fetch_curl_socket_open);
2069 SETOPT(CURLOPT_CLOSESOCKETFUNCTION, fetch_curl_socket_close);
2070
2071 if (nsoption_charp(ca_bundle) &&
2072 strcmp(nsoption_charp(ca_bundle), "")) {
2073 NSLOG(netsurf, INFO, "ca_bundle: '%s'",
2074 nsoption_charp(ca_bundle));
2075 SETOPT(CURLOPT_CAINFO, nsoption_charp(ca_bundle));
2076 }
2077 if (nsoption_charp(ca_path) && strcmp(nsoption_charp(ca_path), "")) {
2078 NSLOG(netsurf, INFO, "ca_path: '%s'", nsoption_charp(ca_path));
2079 SETOPT(CURLOPT_CAPATH, nsoption_charp(ca_path));
2080 }
2081
2082#if LIBCURL_VERSION_NUM < 0x073800
2083 /*
2084 * before 7.56.0 Detect openssl from whether the SSL CTX
2085 * function API works
2086 */
2087 code = curl_easy_setopt(fetch_blank_curl, CURLOPT_SSL_CTX_FUNCTION, NULL);
2088 if (code != CURLE_OK) {
2089 curl_with_openssl = false;
2090 } else {
2091 curl_with_openssl = true;
2092 }
2093#endif
2094
2095 if (curl_with_openssl) {
2096 /* only set the cipher list with openssl otherwise the
2097 * fetch fails with "Unknown cipher in list"
2098 */
2099#if LIBCURL_VERSION_NUM >= 0x073d00
2100 /* Need libcurl 7.61.0 or later built against OpenSSL with
2101 * TLS1.3 support */
2102 code = curl_easy_setopt(fetch_blank_curl,
2103 CURLOPT_TLS13_CIPHERS, CIPHER_SUITES);
2104 if (code != CURLE_OK && code != CURLE_NOT_BUILT_IN)
2105 goto curl_easy_setopt_failed;
2106#endif
2107 SETOPT(CURLOPT_SSL_CIPHER_LIST, CIPHER_LIST);
2108 }
2109
2110 NSLOG(netsurf, INFO, "cURL %slinked against openssl",
2111 curl_with_openssl ? "" : "not ");
2112
2113 /* cURL initialised okay, register the fetchers */
2114
2115 data = curl_version_info(CURLVERSION_NOW);
2116
2118 if (curl_fetch_ssl_hashmap == NULL) {
2119 NSLOG(netsurf, CRITICAL, "Unable to initialise SSL certificate hashmap");
2120 return NSERROR_NOMEM;
2121 }
2122
2123 for (i = 0; data->protocols[i]; i++) {
2124 if (strcmp(data->protocols[i], "http") == 0) {
2125 scheme = lwc_string_ref(corestring_lwc_http);
2126
2127 } else if (strcmp(data->protocols[i], "https") == 0) {
2128 scheme = lwc_string_ref(corestring_lwc_https);
2129
2130 } else {
2131 /* Ignore non-http(s) protocols */
2132 continue;
2133 }
2134
2135 if (fetcher_add(scheme, &fetcher_ops) != NSERROR_OK) {
2136 NSLOG(netsurf, INFO,
2137 "Unable to register cURL fetcher for %s",
2138 data->protocols[i]);
2139 }
2140 }
2141
2142 return NSERROR_OK;
2143
2144curl_easy_setopt_failed:
2145 NSLOG(netsurf, INFO, "curl_easy_setopt failed.");
2146 return NSERROR_INIT_FAILED;
2147
2148#if LIBCURL_VERSION_NUM >= 0x071e00
2149curl_multi_setopt_failed:
2150 NSLOG(netsurf, INFO, "curl_multi_setopt failed.");
2151 return NSERROR_INIT_FAILED;
2152#endif
2153}
STATIC char result[100]
Definition: arexx.c:77
static osspriteop_area * buffer
The buffer characteristics.
Definition: buffer.c:55
void fetch_set_http_code(struct fetch *fetch, long http_code)
set the http code of a fetch
Definition: fetch.c:794
void fetch_set_cookie(struct fetch *fetch, const char *data)
set cookie data on a fetch
Definition: fetch.c:803
nserror fetcher_add(lwc_string *scheme, const struct fetcher_operation_table *ops)
Register a fetcher for a scheme.
Definition: fetch.c:357
void fetch_send_callback(const fetch_msg *msg, struct fetch *fetch)
send message to fetch
Definition: fetch.c:757
void fetch_free(struct fetch *f)
Free a fetch structure and associated resources.
Definition: fetch.c:548
struct fetch_multipart_data * fetch_multipart_data_clone(const struct fetch_multipart_data *list)
Clone a linked list of fetch_multipart_data.
Definition: fetch.c:620
void fetch_remove_from_queues(struct fetch *fetch)
remove a queued fetch
Definition: fetch.c:767
void fetch_multipart_data_destroy(struct fetch_multipart_data *list)
Free a linked list of fetch_multipart_data.
Definition: fetch.c:701
Fetching of data from a URL (interface).
@ FETCH_POSTDATA_URLENC
Definition: fetch.h:101
@ FETCH_POSTDATA_NONE
Definition: fetch.h:100
@ FETCH_POSTDATA_MULTIPART
Definition: fetch.h:102
@ FETCH_REDIRECT
Definition: fetch.h:49
@ FETCH_DATA
Definition: fetch.h:44
@ FETCH_AUTH
Definition: fetch.h:51
@ FETCH_CERTS
Definition: fetch.h:42
@ FETCH_HEADER
Definition: fetch.h:43
@ FETCH_PROGRESS
Definition: fetch.h:41
@ FETCH_NOTMODIFIED
Definition: fetch.h:50
@ FETCH_FINISHED
Definition: fetch.h:46
@ FETCH_SSL_ERR
Definition: fetch.h:53
@ FETCH_ERROR
Definition: fetch.h:48
@ FETCH_TIMEDOUT
Definition: fetch.h:47
@ FETCH_CERT_ERR
Definition: fetch.h:52
Useful interned string pointers (interface).
#define CIPHER_SUITES
The ciphersuites the browser is prepared to use for TLS1.3.
Definition: curl.c:72
static char fetch_proxy_userpwd[100]
Proxy authentication details.
Definition: curl.c:289
static struct curl_fetch_info * fetch_alloc(void)
construct a new fetch structure
Definition: curl.c:419
static bool fetch_curl_process_headers(struct curl_fetch_info *f)
Find the status code and content type and inform the caller.
Definition: curl.c:1472
CURLM * fetch_curl_multi
Global cURL multi handle.
Definition: curl.c:271
static void curl_fetch_ssl_value_destroy(void *value)
Definition: curl.c:189
static size_t fetch_curl_header(char *data, size_t size, size_t nmemb, void *_f)
Callback function for headers.
Definition: curl.c:1864
static bool inside_curl
Interlock to prevent initiation during callbacks.
Definition: curl.c:292
#define NSCURL_PROGRESS_T
Definition: curl.c:221
static CURLcode fetch_curl_set_options(struct curl_fetch_info *f)
Set options specific for a fetch.
Definition: curl.c:1167
static hashmap_t * curl_fetch_ssl_hashmap
Definition: curl.c:206
static void * curl_fetch_ssl_value_alloc(void *key)
Definition: curl.c:177
#define NSCURL_POSTDATA_T
Definition: curl.c:229
#define UPDATE_DELAY_MS
static char fetch_error_buffer[CURL_ERROR_SIZE]
Error buffer for cURL.
Definition: curl.c:286
static void fetch_curl_free(void *vf)
Free a fetch structure and associated resources.
Definition: curl.c:1437
#define APPEND(list, value)
#define CIPHER_LIST
The ciphersuites the browser is prepared to use for TLS<1.3.
Definition: curl.c:80
static bool curl_with_openssl
Flag for runtime detection of openssl usage.
Definition: curl.c:283
static bool fetch_curl_initiate_fetch(struct curl_fetch_info *fetch, CURL *handle)
Initiate a fetch from the queue.
Definition: curl.c:1274
static int fetch_curl_socket_close(void *clientp, curl_socket_t item)
Definition: curl.c:1806
static CURL * fetch_curl_get_handle(lwc_string *host)
Find a CURL handle to use to dispatch a job.
Definition: curl.c:1302
char X509
Definition: curl.c:111
static void fetch_curl_cache_handle(CURL *handle, lwc_string *host)
Cache a CURL handle for the provided host (if wanted)
Definition: curl.c:1336
static void X509_free(X509 *cert)
Definition: curl.c:113
static void fetch_curl_free_postdata(struct fetch_postdata *postdata)
free postdata
Definition: curl.c:398
static int fetch_curl_debug(CURL *handle, curl_infotype type, char *data, size_t size, void *userptr)
Format curl debug for nslog.
Definition: curl.c:1772
static struct fetch_postdata * fetch_curl_alloc_postdata(const char *post_urlenc, const struct fetch_multipart_data *post_multipart)
allocate postdata
Definition: curl.c:367
#define NSCURLOPT_PROGRESS_DATA
Definition: curl.c:220
#define SKIP_ST(o)
static void fetch_curl_abort(void *vf)
Abort a fetch.
Definition: curl.c:1413
static struct curl_httppost * fetch_curl_postdata_convert(CURL *chandle, const struct fetch_multipart_data *control)
Convert a list of struct fetch_multipart_data to a list of struct curl_httppost for libcurl.
Definition: curl.c:1044
static hashmap_parameters_t curl_fetch_ssl_hashmap_parameters
Definition: curl.c:197
#define NSCURLOPT_PROGRESS_FUNCTION
Definition: curl.c:219
static bool fetch_curl_start(void *vfetch)
Dispatch a single job.
Definition: curl.c:1322
static void fetch_curl_report_certs_upstream(struct curl_fetch_info *f)
Report the certificate information in the fetch to the users.
Definition: curl.c:848
static void * fetch_curl_setup(struct fetch *parent_fetch, nsurl *url, bool only_2xx, bool downgrade_tls, const char *post_urlenc, const struct fetch_multipart_data *post_multipart, const char **headers)
Start fetching data for the given URL.
Definition: curl.c:474
static CURL * fetch_blank_curl
Curl handle with default options set; not used for transfers.
Definition: curl.c:274
static size_t fetch_curl_data(char *data, size_t size, size_t nmemb, void *_f)
Callback function for cURL.
Definition: curl.c:1816
static int curl_fetchers_registered
Count of how many schemes the curl fetcher is handling.
Definition: curl.c:280
nserror fetch_curl_register(void)
Register curl scheme handler.
Definition: curl.c:1965
static void fetch_curl_stop(struct curl_fetch_info *f)
Clean up the provided fetch object and free it.
Definition: curl.c:1389
static int fetch_curl_fdset(lwc_string *scheme, fd_set *read_set, fd_set *write_set, fd_set *error_set)
Definition: curl.c:1946
#define NSCURL_POSTDATA_FREE(x)
Definition: curl.c:231
static void fetch_curl_finalise(lwc_string *scheme)
Finalise a cURL fetcher.
Definition: curl.c:312
static bool fetch_curl_can_fetch(const nsurl *url)
Check if this fetcher can fetch a url.
Definition: curl.c:356
#define CIPHER_LIST_LEGACY
The legacy cipher suites the browser is prepared to use for TLS<1.3.
Definition: curl.c:95
static CURLcode fetch_curl_set_postdata(struct curl_fetch_info *f)
Setup multipart post data.
Definition: curl.c:1123
static int fetch_curl_progress(void *clientp, NSCURL_PROGRESS_T dltotal, NSCURL_PROGRESS_T dlnow, NSCURL_PROGRESS_T ultotal, NSCURL_PROGRESS_T ulnow)
Callback function for fetch progress.
Definition: curl.c:1724
static bool curl_fetch_ssl_key_eq(void *key1, void *key2)
Definition: curl.c:144
static bool fetch_curl_initialise(lwc_string *scheme)
Initialise a cURL fetcher.
Definition: curl.c:298
static struct cache_handle * curl_handle_ring
Ring of cached handles.
Definition: curl.c:277
static void fetch_curl_poll(lwc_string *scheme_ignored)
Do some work on current fetches.
Definition: curl.c:1644
#define SETOPT(option, value)
static void fetch_curl_done(CURL *curl_handle, CURLcode result)
Handle a completed fetch (CURLMSG_DONE from curl_multi_info_read()).
Definition: curl.c:1535
#define NSCURL_POSTDATA_CURLOPT
Definition: curl.c:230
static uint32_t curl_fetch_ssl_key_hash(void *key)
Definition: curl.c:124
static curl_socket_t fetch_curl_socket_open(void *clientp, curlsocktype purpose, struct curl_sockaddr *address)
Definition: curl.c:1795
Fetching of data from a URL (Registration).
char options[PATH_MAX]
Definition: gui.c:91
nserror
Enumeration of error codes.
Definition: errors.h:29
@ NSERROR_INIT_FAILED
Initialisation failed.
Definition: errors.h:38
@ NSERROR_NOMEM
Memory exhaustion.
Definition: errors.h:32
@ NSERROR_OK
No error.
Definition: errors.h:30
Interface for fetchers factory.
const char * type
Definition: filetype.cpp:44
static nserror fetch(nsurl *url, enum backing_store_flags bsflags, uint8_t **data_out, size_t *datalen_out)
Retrieve an object from the backing store.
struct netsurf_table * guit
The global interface table.
Definition: gui_factory.c:50
Interface to core interface table.
bool hashmap_remove(hashmap_t *hashmap, void *key)
Remove an entry from the hashmap.
Definition: hashmap.c:206
hashmap_t * hashmap_create(hashmap_parameters_t *params)
Create a hashmap.
Definition: hashmap.c:67
void * hashmap_lookup(hashmap_t *hashmap, void *key)
Look up a key in a hashmap.
Definition: hashmap.c:113
void * hashmap_insert(hashmap_t *hashmap, void *key)
Create an entry in a hashmap.
Definition: hashmap.c:131
void hashmap_destroy(hashmap_t *hashmap)
Destroy a hashmap.
Definition: hashmap.c:91
void(* hashmap_key_destroy_t)(void *)
Key destructor function type.
Definition: hashmap.h:42
void *(* hashmap_key_clone_t)(void *)
Key cloning function type.
Definition: hashmap.h:37
Interface to platform-specific fetcher operations.
Interface to platform-specific miscellaneous browser operation table.
internet structures and defines
Netsurf additional integer type formatting macros.
#define NSLOG(catname, level, logmsg, args...)
Definition: log.h:116
const char * messages_get(const char *key)
Fast lookup of a message by key from the standard Messages hash.
Definition: messages.c:241
Localised message support (interface).
void nsurl_unref(nsurl *url)
Drop a reference to a NetSurf URL object.
const char * nsurl_access(const nsurl *url)
Access a NetSurf URL object as a string.
bool nsurl_has_component(const nsurl *url, nsurl_component part)
Enquire about the existence of componenets in a given URL.
nsurl * nsurl_ref(nsurl *url)
Increment the reference count to a NetSurf URL object.
lwc_string * nsurl_get_component(const nsurl *url, nsurl_component part)
Get part of a URL as a lwc_string, from a NetSurf URL object.
@ NSURL_HOST
Definition: nsurl.h:49
@ NSURL_PORT
Definition: nsurl.h:50
struct nsurl nsurl
NetSurf URL object.
Definition: nsurl.h:31
Ring list structure.
#define RING_GETSIZE(ringtype, ring, sizevar)
Measure the size of a ring and put it in the supplied variable.
Definition: ring.h:86
#define RING_FINDBYLWCHOST(ring, element, lwc_hostname)
Find the element (by hostname) in the given ring, leave it in the provided element variable.
Definition: ring.h:69
#define RING_REMOVE(ring, element)
Remove the given element from the specified ring.
Definition: ring.h:53
#define RING_INSERT(ring, element)
Insert the given item into the specified ring.
Definition: ring.h:40
nserror cert_chain_alloc(size_t depth, struct cert_chain **chain_out)
create new certificate chain
Definition: ssl_certs.c:41
@ SSL_CERT_ERR_TOO_YOUNG
This certificate is not yet valid.
Definition: ssl_certs.h:41
@ SSL_CERT_ERR_BAD_SIG
Bad signature on this certificate.
Definition: ssl_certs.h:40
@ SSL_CERT_ERR_BAD_ISSUER
Bad issuer.
Definition: ssl_certs.h:39
@ SSL_CERT_ERR_SELF_SIGNED
This certificate (or the chain) is self signed.
Definition: ssl_certs.h:43
@ SSL_CERT_ERR_OK
Nothing wrong with this certificate.
Definition: ssl_certs.h:37
@ SSL_CERT_ERR_HOSTNAME_MISMATCH
This certificate host did not match the server.
Definition: ssl_certs.h:46
@ SSL_CERT_ERR_TOO_OLD
This certificate is no longer valid.
Definition: ssl_certs.h:42
@ SSL_CERT_ERR_REVOKED
This certificate has been revoked.
Definition: ssl_certs.h:45
@ SSL_CERT_ERR_CHAIN_SELF_SIGNED
This certificate chain is self signed.
Definition: ssl_certs.h:44
@ SSL_CERT_ERR_CERT_MISSING
This certificate was missing from the chain, its data is useless.
Definition: ssl_certs.h:47
@ SSL_CERT_ERR_UNKNOWN
Unknown error.
Definition: ssl_certs.h:38
nserror cert_chain_dup_into(const struct cert_chain *src, struct cert_chain *dst)
duplicate a certificate chain into an existing chain
Definition: ssl_certs.c:64
nserror cert_chain_free(struct cert_chain *chain)
free a certificate chain
Definition: ssl_certs.c:317
#define MAX_CERT_DEPTH
maximum number of X509 certificates in chain for TLS connection
Definition: ssl_certs.h:54
Interface to utility string handling.
char * human_friendly_bytesize(unsigned long long int bytesize)
Create a human readable representation of a size in bytes.
Definition: utils.c:209
curl handle cache entry
Definition: curl.c:262
CURL * handle
The cached cURL handle.
Definition: curl.c:263
lwc_string * host
The host for which this handle is cached.
Definition: curl.c:264
struct cache_handle * r_next
Next cached handle in ring.
Definition: curl.c:267
struct cache_handle * r_prev
Previous cached handle in ring.
Definition: curl.c:266
X509 certificate chain.
Definition: ssl_certs.h:59
struct cert_chain::@57 certs[MAX_CERT_DEPTH]
size_t depth
the number of certificates in the chain
Definition: ssl_certs.h:63
uint8_t * der
data in Distinguished Encoding Rules (DER) format
Definition: ssl_certs.h:73
size_t der_length
DER length.
Definition: ssl_certs.h:78
ssl_cert_err err
Whatever is wrong with this certificate.
Definition: ssl_certs.h:68
SSL certificate info.
Definition: curl.c:209
X509 * cert
Pointer to certificate.
Definition: curl.c:210
long err
OpenSSL error code.
Definition: curl.c:211
Information for a single fetch.
Definition: curl.c:235
struct fetch_postdata * postdata
POST data.
Definition: curl.c:251
bool downgrade_tls
Downgrade to TLS 1.2.
Definition: curl.c:243
char * realm
HTTP Auth Realm.
Definition: curl.c:250
bool abort
Abort requested.
Definition: curl.c:240
bool stopped
Download stopped on purpose.
Definition: curl.c:241
bool sent_ssl_chain
Have we tried to send the SSL chain.
Definition: curl.c:238
lwc_string * host
The hostname of this fetch.
Definition: curl.c:245
int cert_depth
deepest certificate in use
Definition: curl.c:257
struct curl_slist * headers
List of request headers.
Definition: curl.c:246
long http_code
HTTP result code from cURL.
Definition: curl.c:254
uint64_t last_progress_update
Time of last progress update.
Definition: curl.c:256
nsurl * url
URL of this fetch.
Definition: curl.c:244
char * cookie_string
Cookie string for this fetch.
Definition: curl.c:249
unsigned long content_length
Response Content-Length, or 0.
Definition: curl.c:248
bool only_2xx
Only HTTP 2xx responses acceptable.
Definition: curl.c:242
CURL * curl_handle
cURL handle if being fetched, or 0.
Definition: curl.c:237
struct fetch * fetch_handle
The fetch handle we're parented by.
Definition: curl.c:236
NSCURL_POSTDATA_T * curl_postdata
POST data in curl representation.
Definition: curl.c:252
char * location
Response Location header, or 0.
Definition: curl.c:247
bool had_headers
Headers have been processed.
Definition: curl.c:239
struct cert_info cert_data[MAX_CERT_DEPTH]
HTTPS certificate data.
Definition: curl.c:258
Fetcher message data.
Definition: fetch.h:72
const uint8_t * buf
Definition: fetch.h:79
fetch_msg_type type
Definition: fetch.h:73
struct fetch_msg::@118::@119 header_or_data
const char * realm
Definition: fetch.h:89
const struct cert_chain * chain
Definition: fetch.h:92
size_t len
Definition: fetch.h:80
struct fetch_msg::@118::@120 auth
union fetch_msg::@118 data
const char * progress
Definition: fetch.h:76
const char * redirect
Definition: fetch.h:86
const char * error
Definition: fetch.h:83
Fetch POST multipart data.
Definition: fetch.h:109
char * rawfile
Raw filename if file is true.
Definition: fetch.h:115
char * value
Item value.
Definition: fetch.h:113
struct fetch_multipart_data * next
Next in linked list.
Definition: fetch.h:110
char * name
Name of item.
Definition: fetch.h:112
bool file
Item is a file.
Definition: fetch.h:116
fetch POST data
Definition: fetch.h:122
struct fetch_multipart_data * multipart
Multipart post data if type is FETCH_POSTDATA_MULTIPART.
Definition: fetch.h:128
union fetch_postdata::@121 data
char * urlenc
Url encoded POST string if type is FETCH_POSTDATA_URLENC.
Definition: fetch.h:126
fetch_postdata_type type
Definition: fetch.h:123
Information for a single fetch.
Definition: fetch.c:89
long http_code
HTTP response code, or 0.
Definition: fetch.c:96
nsurl * url
URL.
Definition: fetch.c:91
lwc_string * host
Host part of URL, interned.
Definition: fetch.c:95
Fetcher operations API.
Definition: fetchers.h:49
bool(* initialise)(lwc_string *scheme)
The initialiser for the fetcher.
Definition: fetchers.h:55
char *(* mimetype)(const char *ro_path)
Find a MIME type for a local file.
Definition: fetch.h:100
int(* socket_close)(int socket)
Close a socket.
Definition: fetch.h:118
int(* socket_open)(int domain, int type, int protocol)
Open a socket.
Definition: fetch.h:110
nserror(* basename)(const char *path, char **str, size_t *size)
Get the basename of a file.
Definition: file.h:84
Parameters for hashmaps.
Definition: hashmap.h:77
hashmap_key_clone_t key_clone
A function which when called will clone a key and give ownership of the returned object to the hashma...
Definition: hashmap.h:82
The content of a hashmap.
Definition: hashmap.c:43
struct gui_file_table * file
File table.
Definition: gui_table.h:104
struct gui_fetch_table * fetch
Fetcher table.
Definition: gui_table.h:94
Interface to time operations.
bool urldb_get_cert_permissions(nsurl *url)
Retrieve certificate verification permissions from database.
Definition: urldb.c:3480
const char * urldb_get_auth_details(nsurl *url, const char *realm)
Look up authentication details in database.
Definition: urldb.c:3405
char * urldb_get_cookie(nsurl *url, bool include_http_only)
Retrieve cookies for an URL.
Definition: urldb.c:3997
Unified URL information database internal interface.
const char * user_agent_string(void)
Retrieve the core user agent for this release.
Definition: useragent.c:79
Default operations table for files.
Option reading and saving interface.
#define nsoption_charp(OPTION)
Get the value of a string option.
Definition: nsoption.h:331
#define nsoption_int(OPTION)
Get the value of an integer option.
Definition: nsoption.h:313
#define nsoption_uint(OPTION)
Get the value of an unsigned integer option.
Definition: nsoption.h:322
@ OPTION_HTTP_PROXY_AUTH_NONE
Definition: nsoption.h:86
@ OPTION_HTTP_PROXY_AUTH_BASIC
Definition: nsoption.h:87
#define nsoption_bool(OPTION)
Get the value of a boolean option.
Definition: nsoption.h:304
Interface to a number of general purpose functionality.