38#include <nsutils/time.h>
39#include <nsutils/base64.h>
136#define INVALID_AGE -1
311 "Created user %p (%p, %p, %p)", u, h, (
void *) cb, pw);
330 assert(user->
next == NULL);
331 assert(user->
prev == NULL);
351 assert(user != NULL);
352 assert(
object != NULL);
353 assert(object->
users != NULL);
355 assert((user->
prev != NULL) || (object->
users == user));
357 if (user == object->
users)
358 object->users = user->
next;
362 if (user->
next != NULL)
368 if (object->
users == NULL) {
369 object->last_used = time(NULL);
372 NSLOG(
llcache, DEBUG,
"Removing user %p from %p", user,
object);
390 user =
object->users;
391 while (user != NULL) {
398 next_user = user->
next;
451 if (post_clone == NULL)
457 if (orig->
type == LLCACHE_POST_URL_ENCODED) {
497 char **name,
char **value)
500 const uint8_t *colon;
503 while (data[0] ==
' ' || data[0] ==
'\t' ||
504 data[0] ==
'\r' || data[0] ==
'\n') {
509 colon = (
const uint8_t *) strchr((
const char *) data,
':');
512 colon = data + strlen((
const char *)data);
515 while ((colon > data) &&
516 (colon[-1] ==
' ' || colon[-1] ==
'\t' ||
517 colon[-1] ==
'\r' || colon[-1] ==
'\n')) {
520 n =
strndup((
const char *) data, colon - data);
533 while (colon > data && (colon[-1] ==
' ' ||
534 colon[-1] ==
'\t' || colon[-1] ==
'\r' ||
538 n =
strndup((
const char *) data, colon - data);
543 while (*colon !=
':') {
550 }
while (*colon ==
' ' || *colon ==
'\t' ||
551 *colon ==
'\r' || *colon ==
'\n');
554 while (len > 0 && (data[len - 1] ==
' ' ||
555 data[len - 1] ==
'\t' ||
556 data[len - 1] ==
'\r' ||
557 data[len - 1] ==
'\n')) {
561 v =
strndup((
const char *) colon, len - (colon - data));
628 name_len = strlen(name);
632 if (strcasecmp(name,
"Age") == 0) {
634 if (
'0' <= *value && *value <=
'9') {
635 object->cache.age = atoi(value);
642 if (strcasecmp(name,
"Date") == 0) {
649 "Processing Date header value \"%s\" returned %d",
652 }
else if (strcasecmp(name,
"ETag") == 0) {
655 object->cache.etag = strdup(value);
658 "No memory to duplicate ETag");
665 if (strcasecmp(name,
"Expires") == 0) {
672 "Processing Expires header value \"%s\" returned %d",
674 object->cache.expires = (time_t)0x7fffffff;
680 if (strcasecmp(name,
"Cache-Control") == 0) {
683 }
else if (strcasecmp(name,
"Last-Modified") == 0) {
703 object->num_headers--;
709 object->headers = NULL;
735 const uint8_t *data,
size_t len)
754 if (strncmp((
const char *) data,
"HTTP/",
SLEN(
"HTTP/")) == 0) {
755 time_t req_time =
object->cache.req_time;
760 object->cache.req_time = req_time;
767 object->cache.res_time = time(NULL);
792 temp = realloc(object->
headers,
800 object->headers = temp;
802 object->headers[
object->num_headers].
name = name;
803 object->headers[
object->num_headers].value = value;
805 object->num_headers++;
822 lwc_string *ref_scheme;
835 if (scheme == NULL) {
840 if (ref_scheme == NULL) {
842 lwc_string_unref(scheme);
856 if (lwc_string_isequal(scheme, ref_scheme,
857 &match) != lwc_error_ok) {
860 if (lwc_string_isequal(scheme, corestring_lwc_https,
861 &match1) != lwc_error_ok) {
864 if (lwc_string_isequal(ref_scheme, corestring_lwc_http,
865 &match2) != lwc_error_ok) {
868 if (match ==
true || (match1 ==
true && match2 ==
true)) {
869 const size_t len =
SLEN(
"Referer: ") +
872 header = malloc(len);
873 if (header == NULL) {
876 snprintf(header, len,
"Referer: %s",
879 *header_out = header;
885 lwc_string_unref(scheme);
886 lwc_string_unref(ref_scheme);
905 const char *urlenc = NULL;
907 char **headers = NULL;
913 urlenc =
object->fetch.post->data.urlenc;
915 multipart =
object->fetch.post->data.multipart;
920 headers = malloc(4 *
sizeof(
char *));
921 if (headers == NULL) {
927 const size_t len =
SLEN(
"If-None-Match: ") +
930 headers[header_idx] = malloc(len);
931 if (headers[header_idx] == NULL) {
936 snprintf(headers[header_idx], len,
"If-None-Match: %s",
945 const size_t len =
SLEN(
"If-Modified-Since: ") + 29 + 1;
947 headers[header_idx] = malloc(len);
948 if (headers[header_idx] == NULL) {
949 while (--header_idx >= 0)
950 free(headers[header_idx]);
955 snprintf(headers[header_idx], len,
"If-Modified-Since: %s",
969 headers[header_idx] = NULL;
973 object->cache.req_time = time(NULL);
974 object->cache.fin_time =
object->cache.req_time;
991 (
const char **)headers,
995 while (--header_idx >= 0) {
996 free(headers[header_idx]);
1021 uint32_t redirect_count,
bool hsts_in_use)
1024 nsurl *referer_clone = NULL;
1027 NSLOG(
llcache, DEBUG,
"Starting fetch for %p",
object);
1035 if (referer != NULL)
1038 object->fetch.flags = flags;
1039 object->fetch.referer = referer_clone;
1040 object->fetch.post = post_clone;
1041 object->fetch.redirect_count = redirect_count;
1043 object->fetch.hsts_in_use = hsts_in_use;
1062 NSLOG(
llcache, DEBUG,
"Destroying object %p, %s",
object,
1079 object->fetch.fetch = NULL;
1098 for (i = 0; i <
object->num_headers; i++) {
1119 object->prev = NULL;
1120 object->next = *list;
1123 (*list)->
prev = object;
1138 int current_age, freshness_lifetime;
1139 time_t now = time(NULL);
1142 current_age =
max(0, (
cd->res_time -
cd->date));
1144 current_age +=
cd->res_time -
cd->req_time + now -
cd->res_time;
1148 freshness_lifetime =
cd->max_age;
1149 }
else if (
cd->expires != 0) {
1150 freshness_lifetime =
cd->expires -
cd->date;
1151 }
else if (
cd->last_modified != 0) {
1152 freshness_lifetime = (now -
cd->last_modified) / 10;
1154 freshness_lifetime = 0;
1157 NSLOG(
llcache, DEBUG,
"%d:%d", freshness_lifetime, current_age);
1160 (freshness_lifetime > current_age)) {
1167 return freshness_lifetime - current_age;
1181 int remaining_lifetime;
1186 NSLOG(
llcache, DEBUG,
"%p: (%d > 0 || %d != %d)",
object,
1197 ((remaining_lifetime > 0) ||
1269 if (
object == *list)
1270 *list =
object->
next;
1274 if (object->
next != NULL)
1324 size_t *datasize_out)
1332 size_t cert_chain_depth;
1334 if (object->
chain != NULL) {
1335 cert_chain_depth =
object->chain->depth;
1337 cert_chain_depth = 0;
1342 allocsize += 10 + 1;
1344 allocsize += 10 + 1;
1346 allocsize += 10 + 1;
1348 allocsize += 10 + 1;
1350 for (hloop = 0 ; hloop <
object->num_headers ; hloop++) {
1351 allocsize += strlen(object->
headers[hloop].
name) + 1;
1358 allocsize += 10 + 1;
1360 for (hloop = 0; hloop < cert_chain_depth; hloop++) {
1361 allocsize += 10 + 1;
1362 allocsize += 4 * ((
object->chain->certs[hloop].der_length + 2) / 3);
1365 data = malloc(allocsize);
1371 datasize = allocsize;
1379 if (use > datasize) {
1432 for (hloop = 0 ; hloop <
object->num_headers ; hloop++) {
1433 use = snprintf(op, datasize,
1448 use = snprintf(op, datasize,
"%" PRIsizet, cert_chain_depth);
1459 for (hloop = 0; hloop < cert_chain_depth; hloop++) {
1463 use = snprintf(op, datasize,
"%d",
1476 size_t output_length = datasize;
1477 res = nsu_base64_encode(
1482 if (res != NSUERROR_OK) {
1485 use = output_length;
1497 NSLOG(
llcache, DEBUG,
"Filled buffer with %d spare", datasize);
1500 *datasize_out = allocsize - datasize;
1535 uint8_t *metadata = NULL;
1536 size_t metadatalen = 0;
1537 size_t remaining = 0;
1543 size_t source_length;
1544 time_t request_time;
1545 time_t response_time;
1546 time_t completion_time;
1549 size_t ssl_cert_count = 0;
1572 remaining = metadatalen;
1576 ln = (
char *)metadata;
1577 lnsize = strlen(ln);
1578 remaining -= lnsize + 1;
1596 NSLOG(
llcache, INFO,
"Got metadata for %s instead of %s",
1611 lnsize = strlen(ln);
1612 remaining -= lnsize + 1;
1614 if ((lnsize < 1) || (sscanf(ln,
"%" PRIsizet, &source_length) != 1)) {
1623 lnsize = strlen(ln);
1624 remaining -= lnsize + 1;
1634 lnsize = strlen(ln);
1635 remaining -= lnsize + 1;
1645 lnsize = strlen(ln);
1646 remaining -= lnsize + 1;
1656 lnsize = strlen(ln);
1657 remaining -= lnsize + 1;
1659 if ((lnsize < 1) || (sscanf(ln,
"%" PRIsizet, &num_headers) != 1)) {
1665 for (hloop = 0 ; hloop < num_headers; hloop++) {
1668 lnsize = strlen(ln);
1669 remaining -= lnsize + 1;
1678 if (remaining == 0) {
1679 goto skip_ssl_certificates;
1685 lnsize = strlen(ln);
1686 remaining -= lnsize + 1;
1688 if ((lnsize < 1) || (sscanf(ln,
"%" PRIsizet, &ssl_cert_count) != 1)) {
1693 if (ssl_cert_count == 0) {
1694 goto skip_ssl_certificates;
1707 for (hloop = 0; hloop < ssl_cert_count; hloop++) {
1714 lnsize = strlen(ln);
1715 remaining -= lnsize + 1;
1716 if ((lnsize < 1) || (sscanf(ln,
"%d", &errcode) != 1)) {
1731 lnsize = strlen(ln);
1732 remaining -= lnsize + 1;
1734 nsures = nsu_base64_decode_alloc((
const uint8_t *)ln,
1738 if (nsures != NSUERROR_OK) {
1745skip_ssl_certificates:
1749 object->source_len = source_length;
1752 object->source_alloc = metadatalen;
1754 object->cache.req_time = request_time;
1755 object->cache.res_time = response_time;
1756 object->cache.fin_time = completion_time;
1758 object->chain = chain;
1767 "metadata error on line %d error code %d\n",
1785 bool persistable =
false;
1790 if ((lwc_string_isequal(scheme, corestring_lwc_http,
1791 &match) == lwc_error_ok &&
1793 (lwc_string_isequal(scheme, corestring_lwc_https,
1794 &match) == lwc_error_ok &&
1799 lwc_string_unref(scheme);
1813 bool cachable =
false;
1818 if ((lwc_string_isequal(scheme, corestring_lwc_http,
1819 &match) == lwc_error_ok &&
1821 (lwc_string_isequal(scheme, corestring_lwc_https,
1822 &match) == lwc_error_ok &&
1824 (lwc_string_isequal(scheme, corestring_lwc_data,
1825 &match) == lwc_error_ok &&
1827 (lwc_string_isequal(scheme, corestring_lwc_resource,
1828 &match) == lwc_error_ok &&
1830 (lwc_string_isequal(scheme, corestring_lwc_file,
1831 &match) == lwc_error_ok &&
1836 lwc_string_unref(scheme);
1857 uint32_t redirect_count)
1860 nsurl *referer_clone = NULL;
1869 object->cache.req_time = time(NULL);
1870 object->cache.fin_time =
object->cache.req_time;
1885 if (referer != NULL) {
1889 object->fetch.flags = flags;
1890 object->fetch.referer = referer_clone;
1891 object->fetch.post = post_clone;
1892 object->fetch.redirect_count = redirect_count;
1896 object->fetch.fetch = NULL;
1918 uint32_t redirect_count,
1926 "Searching cache for %s flags:%"PRIx32
" referer:%s post:%p",
1934 if ((newest == NULL ||
1945 if (newest == NULL) {
1946 NSLOG(
llcache, DEBUG,
"No viable object found in llcache");
1955 NSLOG(
llcache, DEBUG,
"retrieved object from persistent store");
1994 NSLOG(
llcache, DEBUG,
"Persistent retrieval failed for %p", newest);
2003 }
else if (newest != NULL) {
2015 NSLOG(
llcache, DEBUG,
"Found candidate %p (%p)", obj, newest);
2030 redirect_count, hsts_in_use);
2045 NSLOG(
llcache, DEBUG,
"Persistent retrieval failed for %p", newest);
2063 redirect_count, hsts_in_use);
2094 uint32_t redirect_count,
2100 nsurl *defragmented_url;
2101 bool uncachable =
false;
2116 }
else if (post != NULL) {
2133 redirect_count, hsts_in_use);
2144 flags, referer, post, redirect_count,
2174 assert(user->
next == NULL);
2175 assert(user->
prev == NULL);
2176 assert(user->
handle != NULL);
2181 user->
next =
object->users;
2183 if (object->
users != NULL)
2184 object->users->
prev = user;
2185 object->users = user;
2187 NSLOG(
llcache, DEBUG,
"Adding user %p to %p", user,
object);
2203 lwc_string *scheme = NULL;
2208 if (lwc_string_caseless_isequal(scheme, corestring_lwc_http,
2209 &match) != lwc_error_ok || match ==
false) {
2211 if (lwc_string_caseless_isequal(scheme, corestring_lwc_https,
2212 &match) == lwc_error_ok && match) {
2217 *hsts_in_use =
false;
2219 lwc_string_unref(scheme);
2223 lwc_string_unref(scheme);
2234 *hsts_in_use =
false;
2249 lwc_string *scheme = NULL;
2253 if (lwc_string_caseless_isequal(scheme, corestring_lwc_https,
2254 &match) != lwc_error_ok || match ==
false) {
2256 lwc_string_unref(scheme);
2259 lwc_string_unref(scheme);
2266 for (i = 0; i <
object->num_headers; i++) {
2267 if (strcasecmp(
"Strict-Transport-Security",
2294 nsurl *url, *hsts_url;
2296 lwc_string *object_scheme;
2297 bool match, hsts_in_use;
2304 object->fetch.fetch = NULL;
2315#define REDIRECT_LIMIT 10
2325#undef REDIRECT_LIMIT
2342 event.data.redirect.from =
object->url;
2343 event.data.redirect.to = hsts_url;
2359 if ((lwc_string_isequal(object_scheme, corestring_lwc_resource,
2360 &match) == lwc_error_ok && match ==
false) &&
2361 (lwc_string_isequal(object_scheme, corestring_lwc_about,
2362 &match) == lwc_error_ok && match ==
false)) {
2364 if ((lwc_string_isequal(object_scheme, corestring_lwc_file,
2365 &match) == lwc_error_ok && match ==
true) ||
2366 (lwc_string_isequal(object_scheme, corestring_lwc_about,
2367 &match) == lwc_error_ok && match ==
true) ||
2368 (lwc_string_isequal(object_scheme, corestring_lwc_resource,
2369 &match) == lwc_error_ok && match ==
true)) {
2370 lwc_string_unref(object_scheme);
2371 lwc_string_unref(scheme);
2377 lwc_string_unref(scheme);
2378 lwc_string_unref(object_scheme);
2386 if (http_code == 301 || http_code == 302 || http_code == 303) {
2389 }
else if (http_code != 307 || post != NULL) {
2399 hsts_in_use, &dest);
2408 for (user = object->
users; user != NULL; user = next) {
2416 *replacement = dest;
2430 object->cache.date = time(NULL);
2454 for (user = object->
users; user != NULL; user = next) {
2462 object->candidate->candidate_count--;
2472 object->candidate->cache.no_cache =
2481 *replacement = object;
2486 object->fetch.fetch = NULL;
2511 const uint8_t *data,
2531 if ((http_code != 200 && http_code != 203) ||
2541 object->candidate->candidate_count--;
2542 object->candidate = NULL;
2550 const size_t new_len =
object->source_len + len + 64 * 1024;
2551 uint8_t *temp = realloc(object->
source_data, new_len);
2555 object->source_data = temp;
2556 object->source_alloc = new_len;
2561 object->source_len += len;
2581 object->fetch.fetch = NULL;
2599 object->fetch.tried_with_auth =
false;
2608 event.data.error.msg = realm;
2614 object->fetch.tried_with_auth =
true;
2632 object->fetch.fetch = NULL;
2638 object->fetch.tainted_tls =
true;
2684 object->fetch.fetch = NULL;
2690 object->fetch.tainted_tls =
true;
2712 object->fetch.tried_with_tls_downgrade =
true;
2734 object->fetch.retries_remaining--;
2740 object->fetch.fetch = NULL;
2744 object->candidate->candidate_count--;
2745 object->candidate = NULL;
2753 event.data.error.msg = NULL;
2780 int remaining_lifetime;
2782#define MAX_PERSIST_PER_RUN 128
2808 lst[lst_len] = object;
2822 *lst_len_out = lst_len;
2825#undef MAX_PERSIST_PER_RUN
2840 size_t *written_out,
2841 unsigned long *elapsed)
2845 size_t metadatasize;
2846 uint64_t startms = 0;
2847 uint64_t endms = 1000;
2849 nsu_getmonotonic_ms(&startms);
2882 nsu_getmonotonic_ms(&endms);
2886 *written_out =
object->source_len + metadatasize;
2891 *elapsed = endms - startms;
2896 if (*elapsed == 0) {
2917 uint64_t total_bandwidth;
2923 if (total_bandwidth < llcache->minimum_bandwidth) {
2946 unsigned long write_limit;
2949 unsigned long elapsed;
2951 size_t total_written = 0;
2952 unsigned long total_elapsed = 1;
2953 unsigned long total_bandwidth = 0;
2957 NSLOG(
llcache, DEBUG,
"Unable to construct candidate list for persistent writeout");
2964 for (idx = 0; idx < lst_count; idx++) {
2971 total_written += written;
2972 total_elapsed += elapsed;
2973 total_bandwidth = (total_written * 1000) / total_elapsed;
2976 "Wrote %"PRIsizet" bytes in %lums bw:%lu %s",
2977 written, elapsed, (written * 1000) / elapsed,
2989 if (total_bandwidth < llcache->minimum_bandwidth) {
3012 }
else if (total_written > write_limit) {
3031 if (idx == lst_count) {
3033 if (total_written > 0) {
3044 "writeout size:%"PRIsizet" time:%lu bandwidth:%lubytes/s",
3045 total_written, total_elapsed, total_bandwidth);
3065 NSLOG(
llcache, CRITICAL,
"Callback happened after llcache finalisation");
3073 switch (msg->
type) {
3089 object->candidate->candidate_count--;
3090 object->candidate = NULL;
3116 object->fetch.fetch = NULL;
3122 if (temp != NULL || object->
source_len == 0) {
3123 object->source_data = temp;
3124 object->source_alloc =
object->source_len;
3130 object->cache.fin_time = time(NULL);
3148 object->fetch.fetch = NULL;
3152 object->candidate->candidate_count--;
3153 object->candidate = NULL;
3181 if (object->
chain != NULL) {
3183 object->chain = NULL;
3190 "Unable to duplicate cert chain into cache: %s",
3208 object->candidate->candidate_count--;
3209 object->candidate = NULL;
3220 object->candidate->candidate_count--;
3221 object->candidate = NULL;
3232 object->candidate->candidate_count--;
3233 object->candidate = NULL;
3252 object->fetch.fetch = NULL;
3276 assert(handle->
object != NULL);
3279 if (user->
handle == handle)
3297 while (list != NULL) {
3304 return list != NULL;
3318 bool emitted_notify =
false;
3343 for (user = object->
users; user != NULL; user = next_user) {
3379 if (handle->
state != objstate) {
3380 if (emitted_notify ==
false) {
3382 "Notifying users of %p",
3384 emitted_notify =
true;
3388 "User %p state: %d Object state: %d",
3400 if (object->
chain != NULL) {
3402 event.data.chain =
object->chain;
3403 error = handle->
cb(handle, &event, handle->
pw);
3409 next_user = user->
next;
3424 next_user = user->
next;
3441 error = handle->
cb(handle, &event, handle->
pw);
3444 next_user = user->
next;
3459 next_user = user->
next;
3472 size_t orig_handle_read;
3476 event.data.data.buf =
3477 object->source_data + handle->
bytes;
3478 event.data.data.len =
3479 object->source_len - handle->
bytes;
3488 orig_handle_read = 0;
3489 handle->
bytes =
object->source_len = 0;
3491 orig_handle_read = handle->
bytes;
3492 handle->
bytes =
object->source_len;
3496 error = handle->
cb(handle, &event, handle->
pw);
3498 next_user = user->
next;
3508 handle->
bytes = orig_handle_read;
3513 next_user = user->
next;
3530 error = handle->
cb(handle, &event, handle->
pw);
3532 next_user = user->
next;
3547 next_user = user->
next;
3559 next_user = user->
next;
3605 if (newobj->
headers == NULL) {
3617 if (nh->
name == NULL || nh->
value == NULL) {
3624 if (object->
chain != NULL) {
3645static inline uint32_t
3651 tot =
sizeof(*object);
3655 tot +=
object->source_len;
3660 for (hdrc = 0; hdrc <
object->num_headers; hdrc++) {
3691 object = object->
next) {
3696 object = object->
next) {
3728 uint32_t llcache_size = 0;
3729 int remaining_lifetime;
3748 if ((object->
users == NULL) &&
3751 NSLOG(
llcache, DEBUG,
"Discarding uncachable object with no users (%p) %s",
3772 if ((object->
users == NULL) &&
3775 (remaining_lifetime <= 0)) {
3777 NSLOG(
llcache, DEBUG,
"discarding stale cacheable object with no "
3778 "users or pending fetches (%p) %s",
3800 if (limit < llcache_size) {
3809 ((limit < llcache_size) && (
object != NULL));
3812 if ((object->
users == NULL) &&
3818 object->source_data = NULL;
3820 llcache_size -=
object->source_len;
3823 "Freeing source data for %p len:%"PRIsizet,
3833 ((limit < llcache_size) && (
object != NULL));
3836 if ((object->
users == NULL) &&
3842 "discarding backed object len:%"PRIsizet" age:%ld (%p) %s",
3863 ((limit < llcache_size) && (
object != NULL));
3867 if ((object->
users == NULL) &&
3872 "discarding fresh object len:%"PRIsizet" age:%ld (%p) %s",
3878 llcache_size -=
object->source_len +
sizeof(*object);
3886 NSLOG(
llcache, DEBUG,
"Size: %"PRIu32
" (limit: %"PRIu32
")", llcache_size, limit);
3907 "llcache initialising with a limit of %"PRIu32
" bytes",
3919 uint64_t total_bandwidth = 0;
3932 for (user = object->
users; user != NULL; user = next_user) {
3933 next_user = user->
next;
3935 if (user->
handle != NULL)
3942 object->fetch.fetch = NULL;
3953 for (user = object->
users; user != NULL; user = next_user) {
3954 next_user = user->
next;
3956 if (user->
handle != NULL)
3963 object->fetch.fetch = NULL;
3977 "Backing store wrote %"PRIu64" bytes in %"PRIu64" ms ""(average %"PRIu64" bytes/second)",
4025 hsts_in_use, &
object);
4064 assert(user != NULL);
4103 bool all_alone =
true;
4106 if (user->
prev != NULL)
4108 if (user->
next != NULL)
4111 if (all_alone ==
false) {
4122 if (newuser == NULL) {
4147 object->fetch.fetch = NULL;
4166 if (user->
prev != NULL || user->
next != NULL)
4184 if ((handle->
object != NULL) &&
4220 for (i = 0; i <
object->num_headers; i++) {
4221 if (strcasecmp(key, object->
headers[i].
name) == 0)
static struct bitmap snapshot
Low-level source data cache backing store interface.
@ BACKING_STORE_NONE
no special processing
@ BACKING_STORE_META
data is metadata
uint32_t http_cache_control_max_age(http_cache_control *cc)
Get the value of a cache control's max-age.
bool http_cache_control_has_max_age(http_cache_control *cc)
Determine if a valid max-age directive is present.
nserror http_parse_cache_control(const char *header_value, http_cache_control **result)
Parse an HTTP Cache-Control header value.
void http_cache_control_destroy(http_cache_control *victim)
Destroy a cache_control object.
bool http_cache_control_no_cache(http_cache_control *cc)
Get the value of a cache control's no-cache flag.
bool http_cache_control_no_store(http_cache_control *cc)
Get the value of a cache control's no-store flag.
char * strndup(const char *s, size_t n)
Duplicate up to n characters of a string.
bool fetch_can_fetch(const nsurl *url)
Check if a URL's scheme can be fetched.
long fetch_http_code(struct fetch *fetch)
Get the HTTP response code.
nserror fetch_start(nsurl *url, nsurl *referer, fetch_callback callback, void *p, bool only_2xx, const char *post_urlenc, const struct fetch_multipart_data *post_multipart, bool verifiable, bool downgrade_tls, const char *headers[], struct fetch **fetch_out)
Start fetching data for the given URL.
void fetch_abort(struct fetch *f)
Abort a fetch.
struct fetch_multipart_data * fetch_multipart_data_clone(const struct fetch_multipart_data *list)
Clone a linked list of fetch_multipart_data.
void fetch_multipart_data_destroy(struct fetch_multipart_data *list)
Free a linked list of fetch_multipart_data.
Fetching of data from a URL (interface).
Useful interned string pointers (interface).
nserror
Enumeration of error codes.
@ NSERROR_TIMEOUT
Operation timed out.
@ NSERROR_NOT_FOUND
Requested item not found.
@ NSERROR_BAD_URL
Bad URL.
@ NSERROR_BAD_AUTH
Fetch needs authentication data.
@ NSERROR_BAD_CERTS
Fetch needs certificate chain check.
@ NSERROR_NO_FETCH_HANDLER
No fetch handler for URL scheme.
@ NSERROR_NEED_DATA
More data needed.
@ NSERROR_BAD_REDIRECT
Fetch encountered a bad redirect.
@ NSERROR_UNKNOWN
Unknown error - DO NOT USE.
@ NSERROR_INVALID
Invalid data.
@ NSERROR_NOMEM
Memory exhaustion.
struct netsurf_table * guit
The global interface table.
Interface to core interface table.
HTTP header parsing functions.
Interface to platform-specific miscellaneous browser operation table.
Netsurf additional integer type formatting macros.
#define PRIsizet
c99 standard printf formatting for size_t type
static nserror build_candidate_list(struct llcache_object ***lst_out, int *lst_len_out)
Construct a sorted list of objects available for writeout operation.
static nserror llcache_object_refetch(llcache_object *object)
(Re)fetch an object
nserror llcache_initialise(const struct llcache_parameters *prm)
Initialise the low-level cache.
void llcache_clean(bool purge)
Cause the low-level cache to attempt to perform cleanup.
static nserror llcache_post_data_clone(const llcache_post_data *orig, llcache_post_data **clone)
Clone a POST data object.
static nserror llcache_object_new(nsurl *url, llcache_object **result)
Create a new low-level cache object.
static void llcache_persist(void *p)
Possibly write objects data to backing store.
static nserror llcache_fetch_parse_cache_control(llcache_object *object, char *value)
parse cache control header value
static void llcache_fetch_callback(const fetch_msg *msg, void *p)
Handler for fetch events.
static nserror llcache_object_fetch(llcache_object *object, uint32_t flags, nsurl *referer, const llcache_post_data *post, uint32_t redirect_count, bool hsts_in_use)
Kick-off a fetch for an object.
static bool llcache__scheme_is_cachable(const nsurl *url)
Check whether a scheme is cachable.
nserror llcache_handle_clone(llcache_handle *handle, llcache_handle **result)
Clone a low-level cache handle, producing a new handle to the same fetch/content.
static nserror llcache_object_retrieve_from_cache(nsurl *url, uint32_t flags, nsurl *referer, const llcache_post_data *post, uint32_t redirect_count, bool hsts_in_use, llcache_object **result)
Retrieve a potentially cached object.
static nserror llcache_object_clone_cache_data(llcache_object *source, llcache_object *destination, bool deep)
Clone an object's cache data.
static nserror llcache_object_user_new(llcache_handle_callback cb, void *pw, llcache_object_user **user)
Create a new object user.
static nserror llcache_object_fetch_persistent(llcache_object *object, uint32_t flags, nsurl *referer, const llcache_post_data *post, uint32_t redirect_count)
Attempt to retrieve an object from persistent storage.
static nserror llcache_fetch_ssl_error(llcache_object *object)
Handle a TLS connection setup failure.
static nserror llcache_fetch_cert_error(llcache_object *object)
Handle a TLS certificate verification failure.
static nserror llcache_serialise_metadata(llcache_object *object, uint8_t **data_out, size_t *datasize_out)
Generate a serialised version of an object's metadata.
static void llcache_destroy_headers(llcache_object *object)
Destroy headers.
static nserror llcache_object_user_destroy(llcache_object_user *user)
Destroy an object user.
static nserror llcache_fetch_timeout(llcache_object *object)
handle time out while trying to fetch.
static nserror llcache_fetch_process_data(llcache_object *object, const uint8_t *data, size_t len)
Process a chunk of fetched data.
static nserror llcache_object_snapshot(llcache_object *object, llcache_object **snapshot)
Make a snapshot of the current state of an llcache_object.
static void llcache_persist_slowcheck(void *p)
Check for overall write performance.
static nserror get_referer_header(nsurl *url, nsurl *referer, char **header_out)
construct a Referer header appropriate for the request
static struct llcache_s * llcache
low level cache state
nserror llcache_handle_abort(llcache_handle *handle)
Abort a low-level fetch, informing all users of this action.
static bool llcache__scheme_is_persistable(const nsurl *url)
Check whether a scheme is persistable.
static nserror llcache_fetch_auth(llcache_object *object, const char *realm)
Handle an authentication request.
static nserror llcache_retrieve_persisted_data(llcache_object *object)
Retrieve source data for an object from persistent store if necessary.
static uint32_t total_object_size(llcache_object *object)
total ram usage of object
#define MAX_PERSIST_PER_RUN
llcache_fetch_state
State of a low-level cache object fetch.
@ LLCACHE_FETCH_HEADERS
Fetching headers.
@ LLCACHE_FETCH_INIT
Initial state, before fetch.
@ LLCACHE_FETCH_DATA
Fetching object data.
@ LLCACHE_FETCH_COMPLETE
Fetch completed.
static llcache_object_user * llcache_object_find_user(const llcache_handle *handle)
Find a user of a low-level cache object.
static nserror llcache_fetch_split_header(const uint8_t *data, size_t len, char **name, char **value)
Split a fetch header into name and value.
static nserror llcache_send_event_to_users(llcache_object *object, llcache_event *event)
Iterate the users of an object, calling their callbacks.
static nserror llcache_object_retrieve(nsurl *url, uint32_t flags, nsurl *referer, const llcache_post_data *post, uint32_t redirect_count, bool hsts_in_use, llcache_object **result)
Retrieve an object from the cache, fetching it if necessary.
void llcache_finalise(void)
Finalise the low-level cache.
static nserror llcache_fetch_redirect(llcache_object *object, const char *target, llcache_object **replacement)
Handle FETCH_REDIRECT event.
static nserror llcache_object_add_to_list(llcache_object *object, llcache_object **list)
Add a low-level cache object to a cache list.
bool llcache_handle_references_same_object(const llcache_handle *a, const llcache_handle *b)
Determine if the same underlying object is referenced by the given handles.
nsurl * llcache_handle_get_url(const llcache_handle *handle)
Retrieve the post-redirect URL of a low-level cache object.
llcache_validate
Validation control.
@ LLCACHE_VALIDATE_ALWAYS
Always revalidate.
@ LLCACHE_VALIDATE_FRESH
Only revalidate if not fresh.
@ LLCACHE_VALIDATE_ONCE
Revalidate once only.
static nserror llcache_hsts_transform_url(nsurl *url, nsurl **result, bool *hsts_in_use)
Transform a request-URI based on HSTS policy.
const uint8_t * llcache_handle_get_source_data(const llcache_handle *handle, size_t *size)
Retrieve source data of a low-level cache object.
static void llcache_users_not_caught_up(void)
Ask for llcache_catch_up_all_users to be scheduled ASAP to pump the user state machines.
static nserror llcache_fetch_process_header(llcache_object *object, const uint8_t *data, size_t len)
Process a fetch header.
struct llcache_object_user llcache_object_user
Low-level cache object user record.
static nserror llcache_object_notify_users(llcache_object *object)
Notify users of an object's current state.
const char * llcache_handle_get_header(const llcache_handle *handle, const char *key)
Retrieve a header value associated with a low-level cache object.
static bool llcache_object_in_list(const llcache_object *object, const llcache_object *list)
Determine if a low-level cache object resides in a given list.
static nserror llcache_process_metadata(llcache_object *object)
Deserialisation of an object's metadata.
nserror llcache_handle_change_callback(llcache_handle *handle, llcache_handle_callback cb, void *pw)
Change the callback associated with a low-level cache handle.
static nserror llcache_hsts_update_policy(llcache_object *object)
Update HSTS policy for target domain.
static nserror llcache_object_remove_from_list(llcache_object *object, llcache_object **list)
Remove a low-level cache object from a cache list.
nserror llcache_handle_retrieve(nsurl *url, uint32_t flags, nsurl *referer, const llcache_post_data *post, llcache_handle_callback cb, void *pw, llcache_handle **result)
Retrieve a handle for a low-level cache object.
static int llcache_object_rfc2616_remaining_lifetime(const llcache_cache_control *cd)
Determine the remaining lifetime of a cache object using the.
nserror llcache_handle_force_stream(llcache_handle *handle)
Force a low-level cache handle into streaming mode.
static nserror llcache_object_destroy(llcache_object *object)
Destroy a low-level cache object.
static nserror write_backing_store(struct llcache_object *object, size_t *written_out, unsigned long *elapsed)
Write an object to the backing store.
static void llcache_invalidate_cache_control_data(llcache_object *object)
Invalidate cache control data.
static nserror llcache_fetch_notmodified(llcache_object *object, llcache_object **replacement)
Handle FETCH_NOTMODIFIED event.
nserror llcache_handle_invalidate_cache_data(llcache_handle *handle)
Invalidate cache data for a low-level cache object.
static nserror llcache_object_remove_user(llcache_object *object, llcache_object_user *user)
Remove a user from a low-level cache object.
static nserror llcache_object_cache_update(llcache_object *object)
Update an object's cache state.
static nserror llcache_fetch_header_cache_control(llcache_object *object, char *name, char *value)
Update cache control from appropriate header.
static nserror llcache_object_add_user(llcache_object *object, llcache_object_user *user)
Add a user to a low-level cache object.
static void llcache_catch_up_all_users(void *ignored)
Catch up the cache users with state changes from fetchers.
static bool llcache_object_is_fresh(const llcache_object *object)
Determine if an object is still fresh.
#define INVALID_AGE
Cache control value for invalid age.
llcache_store_state
Current status of an object's data.
@ LLCACHE_STATE_RAM
source data is stored in RAM only
@ LLCACHE_STATE_DISC
source data is stored on disc
nserror llcache_handle_release(llcache_handle *handle)
Release a low-level cache handle.
nserror(* llcache_handle_callback)(llcache_handle *handle, const llcache_event *event, void *pw)
Client callback for low-level cache events.
@ LLCACHE_RETRIEVE_STREAM_DATA
@ LLCACHE_RETRIEVE_NO_ERROR_PAGES
Stream data (implies that object is not cacheable)
@ LLCACHE_RETRIEVE_VERIFIABLE
Requested URL was verified.
@ LLCACHE_RETRIEVE_FORCE_FETCH
Force a new fetch.
@ LLCACHE_EVENT_DONE
Finished fetching data.
@ LLCACHE_EVENT_ERROR
An error occurred during fetch.
@ LLCACHE_EVENT_GOT_CERTS
SSL certificates arrived.
@ LLCACHE_EVENT_REDIRECT
Fetch URL redirect occured.
@ LLCACHE_EVENT_PROGRESS
Fetch progress update.
@ LLCACHE_EVENT_HAD_HEADERS
Received all headers.
@ LLCACHE_EVENT_HAD_DATA
Received some data.
#define NSLOG(catname, level, logmsg, args...)
const char * messages_get_errorcode(nserror code)
lookup of a message by errorcode from the standard Messages hash.
const char * messages_get(const char *key)
Fast lookup of a message by key from the standard Messages hash.
Localised message support (interface).
NetSurf URL handling (interface).
bool nsurl_compare(const nsurl *url1, const nsurl *url2, nsurl_component parts)
Compare two URLs.
nserror nsurl_create(const char *const url_s, nsurl **url)
Create a NetSurf URL object from a URL string.
nserror nsurl_defragment(const nsurl *url, nsurl **no_frag)
Create a NetSurf URL object without a fragment from a NetSurf URL.
void nsurl_unref(nsurl *url)
Drop a reference to a NetSurf URL object.
const char * nsurl_access(const nsurl *url)
Access a NetSurf URL object as a string.
size_t nsurl_length(const nsurl *url)
Find the length of a NetSurf URL object's URL, as returned by nsurl_access.
bool nsurl_has_component(const nsurl *url, nsurl_component part)
Enquire about the existence of componenets in a given URL.
nserror nsurl_replace_scheme(const nsurl *url, lwc_string *scheme, nsurl **new_url)
Create a NetSurf URL object, with scheme replaced.
nsurl * nsurl_ref(nsurl *url)
Increment the reference count to a NetSurf URL object.
lwc_string * nsurl_get_component(const nsurl *url, nsurl_component part)
Get part of a URL as a lwc_string, from a NetSurf URL object.
nserror nsurl_join(const nsurl *base, const char *rel, nsurl **joined)
Join a base url to a relative link part, creating a new NetSurf URL object.
struct nsurl nsurl
NetSurf URL object.
nserror cert_chain_alloc(size_t depth, struct cert_chain **chain_out)
create new certificate chain
ssl_cert_err
ssl certificate error status
@ SSL_CERT_ERR_OK
Nothing wrong with this certificate.
@ SSL_CERT_ERR_UNKNOWN
Unknown error.
#define SSL_CERT_ERR_MAX_KNOWN
Always the max known ssl certificate error type.
nserror cert_chain_dup(const struct cert_chain *src, struct cert_chain **dst_out)
duplicate a certificate chain
nserror cert_chain_free(struct cert_chain *chain)
free a certificate chain
#define MAX_CERT_DEPTH
maximum number of X509 certificates in chain for TLS connection
size_t cert_chain_size(const struct cert_chain *chain)
total number of data bytes in a chain
Interface to utility string handling.
struct cert_chain::@57 certs[MAX_CERT_DEPTH]
uint8_t * der
data in Distinguished Encoding Rules (DER) format
size_t der_length
DER length.
ssl_cert_err err
Whatever is wrong with this certificate.
struct fetch_msg::@118::@119 header_or_data
const struct cert_chain * chain
struct fetch_msg::@118::@120 auth
union fetch_msg::@118 data
Fetch POST multipart data.
Information for a single fetch.
nserror(* initialise)(const struct llcache_store_parameters *parameters)
Initialise the backing store.
nserror(* store)(struct nsurl *url, enum backing_store_flags flags, uint8_t *data, const size_t datalen)
Place an object in the backing store.
nserror(* finalise)(void)
Finalise the backing store.
nserror(* invalidate)(struct nsurl *url)
Invalidate a source object from the backing store.
nserror(* release)(struct nsurl *url, enum backing_store_flags flags)
release a previously fetched or stored memory object.
nserror(* fetch)(struct nsurl *url, enum backing_store_flags flags, uint8_t **data, size_t *datalen)
Retrieve an object from the backing store.
nserror(* schedule)(int t, void(*callback)(void *p), void *p)
Schedule a callback.
Representation of a Cache-Control.
time_t date
Date: response header.
int max_age
Max-Age Cache-control parameter.
time_t res_time
Time of response.
int age
Age: response header.
time_t fin_time
Time of request completion.
time_t expires
Expires: response header.
llcache_validate no_cache
No-Cache Cache-control parameter.
char * etag
Etag: response header.
time_t last_modified
Last-Modified: response header.
time_t req_time
Time of request.
Low-level cache object fetch context.
bool tried_with_auth
Whether we've tried with auth.
bool hsts_in_use
Whether HSTS applies to this fetch.
bool tainted_tls
Whether the TLS transport is tainted.
struct fetch * fetch
Fetch handle for this object.
llcache_post_data * post
POST data, or NULL for GET.
llcache_fetch_state state
Current state of object fetch.
uint32_t flags
Fetch flags.
uint32_t retries_remaining
Number of times to retry on timeout.
bool tried_with_tls_downgrade
Whether we've tried TLS 1.2.
nsurl * referer
Referring URL, or NULL if none.
uint32_t redirect_count
Count of redirects followed.
Handle to low-level cache object.
llcache_handle_callback cb
Client callback.
size_t bytes
Last reported byte count.
llcache_object * object
Pointer to associated object.
llcache_fetch_state state
Last known state of object fetch.
Low-level cache object user record.
bool iterator_target
This is the iterator target.
struct llcache_object_user * next
Next in list.
bool queued_for_delete
This user is queued for deletion.
llcache_handle * handle
Handle data for client.
struct llcache_object_user * prev
Previous in list.
llcache_object * prev
Previous in list.
llcache_object * candidate
Object to use, if fetch determines that it is still fresh.
size_t source_len
Byte length of source data.
nsurl * url
Post-redirect URL for object.
llcache_cache_control cache
Cache control data for object.
time_t last_used
time the last user was removed from the object
llcache_object_user * users
List of users.
uint8_t * source_data
Source data for object.
uint32_t candidate_count
Count of objects this is a candidate for.
llcache_fetch_ctx fetch
Fetch context for object.
struct cert_chain * chain
Certificate chain from the fetch.
llcache_header * headers
Fetch headers.
llcache_store_state store_state
where the data for the object is stored
llcache_object * next
Next in list.
size_t num_headers
Number of fetch headers.
size_t source_alloc
Allocated size of source buffer.
Parameters to configure the low level cache.
struct llcache_store_parameters store
int minimum_lifetime
The minimum lifetime to consider sending objects to backing store.
size_t limit
The target upper bound for the RAM cache size.
size_t maximum_bandwidth
The maximum bandwidth to allow the backing store to use in bytes/second.
uint32_t fetch_attempts
The number of fetches to attempt when timing out.
size_t minimum_bandwidth
The minimum bandwidth to allow the backing store to use in bytes/second.
unsigned long time_quantum
The time quantum over which to calculate the bandwidth values.
POST data object for low-level cache requests.
struct fetch_multipart_data * multipart
Multipart data.
enum llcache_post_data::@122 type
Type of POST data.
union llcache_post_data::@123 data
POST data content.
char * urlenc
URL encoded data.
Core llcache control context.
llcache_object * cached_objects
Head of the low-level cached object list.
uint32_t fetch_attempts
The number of fetch attempts we make when timing out.
bool all_caught_up
Whether or not our users are caught up.
size_t maximum_bandwidth
The maximum bandwidth to allow the backing store to use in bytes/second.
llcache_object * uncached_objects
Head of the low-level uncached object list.
uint32_t limit
The target upper bound for the RAM cache size.
int minimum_lifetime
The minimum lifetime to consider sending objects to backing store.
size_t minimum_bandwidth
The minimum bandwidth to allow the backing store to use in bytes/second.
unsigned long time_quantum
The time over which to apply the bandwidth calculations in ms.
uint64_t total_elapsed
Total number of milliseconds taken to write to backing store.
uint64_t total_written
Total number of bytes written to backing store.
struct gui_misc_table * misc
Browser table.
struct gui_llcache_table * llcache
Low level cache table.
int nsc_sntimet(char *str, size_t size, time_t *timep)
Write the time in seconds since epoch to a buffer.
nserror nsc_strntimet(const char *str, size_t size, time_t *timep)
Converts a date string to a number of seconds since epoch.
nserror nsc_snptimet(const char *str, size_t size, time_t *timep)
Parse time in seconds since epoc.
const char * rfc1123_date(time_t t)
Create an RFC 1123 compliant date string from a Unix timestamp.
Interface to time operations.
bool urldb_get_hsts_enabled(struct nsurl *url)
Determine if HSTS policy is enabled for an URL.
bool urldb_set_hsts_policy(struct nsurl *url, const char *header)
Set HSTS policy for an URL.
const char * urldb_get_auth_details(nsurl *url, const char *realm)
Look up authentication details in database.
Unified URL information database internal interface.
Option reading and saving interface.
#define nsoption_bool(OPTION)
Get the value of a boolean option.
iconv_t cd
Iconv conversion descriptor.
Interface to a number of general purpose functionality.
#define SLEN(x)
Calculate length of constant C string.
static nserror line(const struct redraw_context *ctx, const plot_style_t *style, const struct rect *line)
Plots a line.