NetSurf
llcache.c
Go to the documentation of this file.
1/*
2 * Copyright 2009 John-Mark Bell <jmb@netsurf-browser.org>
3 *
4 * This file is part of NetSurf, http://www.netsurf-browser.org/
5 *
6 * NetSurf is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation; version 2 of the License.
9 *
10 * NetSurf is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
14 *
15 * You should have received a copy of the GNU General Public License
16 * along with this program. If not, see <http://www.gnu.org/licenses/>.
17 */
18
19/**
20 * \file
21 *
22 * Low-level resource cache implementation
23 *
24 * This is the implementation of the low level cache. This cache
25 * stores source objects in memory and may use a persistent backing
26 * store to extend their lifetime.
27 *
28 * \todo fix writeout conditions and ordering.
29 *
30 * \todo instrument and (auto)tune
31 *
32 */
33
34#include <stdlib.h>
35#include <stdint.h>
36#include <string.h>
37#include <strings.h>
38#include <nsutils/time.h>
39#include <nsutils/base64.h>
40
41#include "netsurf/inttypes.h"
42#include "utils/config.h"
43#include "utils/corestrings.h"
44#include "utils/log.h"
45#include "utils/messages.h"
46#include "utils/nsurl.h"
47#include "utils/utils.h"
48#include "utils/time.h"
49#include "utils/http.h"
50#include "utils/nsoption.h"
51#include "netsurf/misc.h"
53
54#include "content/fetch.h"
56#include "content/urldb.h"
57
58/**
59 * State of a low-level cache object fetch.
60 */
61typedef enum {
62 LLCACHE_FETCH_INIT, /**< Initial state, before fetch */
63 LLCACHE_FETCH_HEADERS, /**< Fetching headers */
64 LLCACHE_FETCH_DATA, /**< Fetching object data */
65 LLCACHE_FETCH_COMPLETE /**< Fetch completed */
67
68/**
69 * Type of low-level cache object.
70 */
72
73/**
74 * Handle to low-level cache object.
75 */
77 llcache_object *object; /**< Pointer to associated object */
78
79 llcache_handle_callback cb; /**< Client callback */
80 void *pw; /**< Client data */
81
82 llcache_fetch_state state; /**< Last known state of object fetch */
83 size_t bytes; /**< Last reported byte count */
84};
85
86/**
87 * Low-level cache object user record.
88 */
89typedef struct llcache_object_user {
90 llcache_handle *handle; /**< Handle data for client */
91
92 bool iterator_target; /**< This is the iterator target */
93 bool queued_for_delete; /**< This user is queued for deletion */
94
95 struct llcache_object_user *prev; /**< Previous in list */
96 struct llcache_object_user *next; /**< Next in list */
98
99/**
100 * Low-level cache object fetch context.
101 */
102typedef struct {
103 uint32_t flags; /**< Fetch flags */
104 nsurl *referer; /**< Referring URL, or NULL if none */
105 llcache_post_data *post; /**< POST data, or NULL for GET */
106
107 struct fetch *fetch; /**< Fetch handle for this object */
108
109 llcache_fetch_state state; /**< Current state of object fetch */
110
111 uint32_t redirect_count; /**< Count of redirects followed */
112
113 uint32_t retries_remaining; /**< Number of times to retry on timeout */
114
115 bool hsts_in_use; /**< Whether HSTS applies to this fetch */
116
117 bool tried_with_auth; /**< Whether we've tried with auth */
118
119 bool tried_with_tls_downgrade; /**< Whether we've tried TLS 1.2 */
120
121 bool tainted_tls; /**< Whether the TLS transport is tainted */
123
124/**
125 * Validation control.
126 */
127typedef enum {
128 LLCACHE_VALIDATE_FRESH, /**< Only revalidate if not fresh */
129 LLCACHE_VALIDATE_ALWAYS, /**< Always revalidate */
130 LLCACHE_VALIDATE_ONCE /**< Revalidate once only */
132
133/**
134 * Cache control value for invalid age.
135 */
136#define INVALID_AGE -1
137
138/** Cache control data */
139typedef struct {
140 time_t req_time; /**< Time of request */
141 time_t res_time; /**< Time of response */
142 time_t fin_time; /**< Time of request completion */
143 time_t date; /**< Date: response header */
144 time_t expires; /**< Expires: response header */
145 int age; /**< Age: response header */
146 int max_age; /**< Max-Age Cache-control parameter */
147 llcache_validate no_cache; /**< No-Cache Cache-control parameter */
148 char *etag; /**< Etag: response header */
149 time_t last_modified; /**< Last-Modified: response header */
151
152/** Representation of a fetch header */
153typedef struct {
154 char *name; /**< Header name */
155 char *value; /**< Header value */
157
158/** Current status of an object's data */
159typedef enum {
160 LLCACHE_STATE_RAM = 0, /**< source data is stored in RAM only */
161 LLCACHE_STATE_DISC, /**< source data is stored on disc */
163
164/**
165 * Low-level cache object
166 *
167 * \todo Consider whether a list is a sane container.
168 */
170 llcache_object *prev; /**< Previous in list */
171 llcache_object *next; /**< Next in list */
172
173 nsurl *url; /**< Post-redirect URL for object */
174
175 /** \todo We need a generic dynamic buffer object */
176 uint8_t *source_data; /**< Source data for object */
177 size_t source_len; /**< Byte length of source data */
178 size_t source_alloc; /**< Allocated size of source buffer */
179
180 struct cert_chain *chain; /**< Certificate chain from the fetch */
181
182 llcache_store_state store_state; /**< where the data for the object is stored */
183
184 llcache_object_user *users; /**< List of users */
185
186 llcache_fetch_ctx fetch; /**< Fetch context for object */
187
188 llcache_cache_control cache; /**< Cache control data for object */
189 llcache_object *candidate; /**< Object to use, if fetch determines
190 * that it is still fresh
191 */
192 uint32_t candidate_count; /**< Count of objects this is a
193 * candidate for
194 */
195
196 llcache_header *headers; /**< Fetch headers */
197 size_t num_headers; /**< Number of fetch headers */
198
199 /* Instrumentation. These elements are strictly for information
200 * to improve the cache performance and to provide performance
201 * metrics. The values are non-authoritative and must not be used to
202 * determine object lifetime etc.
203 */
204 time_t last_used; /**< time the last user was removed from the object */
205};
206
207/**
208 * Core llcache control context.
209 */
210struct llcache_s {
211 /** Head of the low-level cached object list */
213
214 /** Head of the low-level uncached object list */
216
217 /** The target upper bound for the RAM cache size */
218 uint32_t limit;
219
220 /** The number of fetch attempts we make when timing out */
222
223 /** Whether or not our users are caught up */
225
226
227 /* backing store elements */
228
229
230 /**
231 * The minimum lifetime to consider sending objects to backing
232 * store.
233 */
235
236 /**
237 * The time over which to apply the bandwidth calculations in ms
238 */
239 unsigned long time_quantum;
240
241 /**
242 * The minimum bandwidth to allow the backing store to use in
243 * bytes/second. Below this the backing store will be
244 * disabled.
245 */
247
248 /**
249 * The maximum bandwidth to allow the backing store to use in
250 * bytes/second
251 */
253
254 /**
255 * Total number of bytes written to backing store.
256 */
258
259 /**
260 * Total number of milliseconds taken to write to backing store.
261 */
263
264};
265
266/** low level cache state */
267static struct llcache_s *llcache = NULL;
268
269/* forward referenced callback function */
270static void llcache_fetch_callback(const fetch_msg *msg, void *p);
271
272/* forward referenced catch up function */
273static void llcache_users_not_caught_up(void);
274
275
276/******************************************************************************
277 * Low-level cache internals *
278 ******************************************************************************/
279
280/**
281 * Create a new object user.
282 *
283 * \param cb Callback routine.
284 * \param pw Private data for callback.
285 * \param user Pointer to location to receive result.
286 * \return NSERROR_OK on success, appropriate error otherwise
287 */
289 llcache_object_user **user)
290{
293
294 h = calloc(1, sizeof(llcache_handle));
295 if (h == NULL) {
296 return NSERROR_NOMEM;
297 }
298
299 u = calloc(1, sizeof(llcache_object_user));
300 if (u == NULL) {
301 free(h);
302 return NSERROR_NOMEM;
303 }
304
305 h->cb = cb;
306 h->pw = pw;
307
308 u->handle = h;
309
310 NSLOG(llcache, DEBUG,
311 "Created user %p (%p, %p, %p)", u, h, (void *) cb, pw);
312
313 *user = u;
314
315 return NSERROR_OK;
316}
317
318/**
319 * Destroy an object user
320 *
321 * \param user User to destroy
322 * \return NSERROR_OK on success, appropriate error otherwise
323 *
324 * \pre User is not attached to an object
325 */
327{
328 NSLOG(llcache, DEBUG, "Destroyed user %p", user);
329
330 assert(user->next == NULL);
331 assert(user->prev == NULL);
332
333 if (user->handle != NULL)
334 free(user->handle);
335
336 free(user);
337
338 return NSERROR_OK;
339}
340
341/**
342 * Remove a user from a low-level cache object
343 *
344 * \param object Object to remove user from
345 * \param user User to remove
346 * \return NSERROR_OK.
347 */
350{
351 assert(user != NULL);
352 assert(object != NULL);
353 assert(object->users != NULL);
354 assert(user->handle == NULL || user->handle->object == object);
355 assert((user->prev != NULL) || (object->users == user));
356
357 if (user == object->users)
358 object->users = user->next;
359 else
360 user->prev->next = user->next;
361
362 if (user->next != NULL)
363 user->next->prev = user->prev;
364
365 user->next = user->prev = NULL;
366
367 /* record the time the last user was removed from the object */
368 if (object->users == NULL) {
369 object->last_used = time(NULL);
370 }
371
372 NSLOG(llcache, DEBUG, "Removing user %p from %p", user, object);
373
374 return NSERROR_OK;
375}
376
377/**
378 * Iterate the users of an object, calling their callbacks.
379 *
380 * \param object The object to iterate
381 * \param event The event to pass to the callback.
382 * \return NSERROR_OK on success, appropriate error otherwise.
383 */
385 llcache_event *event)
386{
387 nserror error = NSERROR_OK;
388 llcache_object_user *user, *next_user;
389
390 user = object->users;
391 while (user != NULL) {
392 bool was_target = user->iterator_target;
393 user->iterator_target = true;
394
395 error = user->handle->cb(user->handle, event,
396 user->handle->pw);
397
398 next_user = user->next;
399
400 user->iterator_target = was_target;
401
402 if (user->queued_for_delete && !was_target) {
403 llcache_object_remove_user(object, user);
405 }
406
407 if (error != NSERROR_OK)
408 break;
409
410 user = next_user;
411 }
412
413 return error;
414}
415
416/**
417 * Create a new low-level cache object
418 *
419 * \param url URL of object to create
420 * \param result Pointer to location to receive result
421 * \return NSERROR_OK on success, appropriate error otherwise
422 */
424{
425 llcache_object *obj = calloc(1, sizeof(llcache_object));
426 if (obj == NULL)
427 return NSERROR_NOMEM;
428
429 NSLOG(llcache, DEBUG, "Created object %p (%s)", obj, nsurl_access(url));
430
431 obj->url = nsurl_ref(url);
432
433 *result = obj;
434
435 return NSERROR_OK;
436}
437
438/**
439 * Clone a POST data object
440 *
441 * \param orig Object to clone
442 * \param clone Pointer to location to receive clone
443 * \return NSERROR_OK on success, appropriate error otherwise
444 */
446 llcache_post_data **clone)
447{
448 llcache_post_data *post_clone;
449
450 post_clone = calloc(1, sizeof(llcache_post_data));
451 if (post_clone == NULL)
452 return NSERROR_NOMEM;
453
454 post_clone->type = orig->type;
455
456 /* Deep-copy the type-specific data */
457 if (orig->type == LLCACHE_POST_URL_ENCODED) {
458 post_clone->data.urlenc = strdup(orig->data.urlenc);
459 if (post_clone->data.urlenc == NULL) {
460 free(post_clone);
461
462 return NSERROR_NOMEM;
463 }
464 } else {
466 orig->data.multipart);
467 if (post_clone->data.multipart == NULL) {
468 free(post_clone);
469
470 return NSERROR_NOMEM;
471 }
472 }
473
474 *clone = post_clone;
475
476 return NSERROR_OK;
477}
478
479/**
480 * Split a fetch header into name and value
481 *
482 * HTTP header splitting according to grammar defined in RFC7230 section 3.2
483 * https://tools.ietf.org/html/rfc7230#section-3.2
484 *
485 * This implementation is non conformant in that it:
486 * - includes carrige return and newline in whitespace (3.2.3)
487 * - allows whitespace before and after the field-name token (3.2.4)
488 * - does not handle obsolete line folding (3.2.4)
489 *
490 * \param data Header string
491 * \param len Byte length of header
492 * \param name Pointer to location to receive header name
493 * \param value Pointer to location to receive header value
494 * \return NSERROR_OK on success, appropriate error otherwise
495 */
496static nserror llcache_fetch_split_header(const uint8_t *data, size_t len,
497 char **name, char **value)
498{
499 char *n, *v;
500 const uint8_t *colon;
501
502 /* Strip leading whitespace from name */
503 while (data[0] == ' ' || data[0] == '\t' ||
504 data[0] == '\r' || data[0] == '\n') {
505 data++;
506 }
507
508 /* Find colon */
509 colon = (const uint8_t *) strchr((const char *) data, ':');
510 if (colon == NULL) {
511 /* Failed, assume a key with no value */
512 colon = data + strlen((const char *)data);
513
514 /* Strip trailing whitespace from name */
515 while ((colon > data) &&
516 (colon[-1] == ' ' || colon[-1] == '\t' ||
517 colon[-1] == '\r' || colon[-1] == '\n')) {
518 colon--;
519 }
520 n = strndup((const char *) data, colon - data);
521 if (n == NULL)
522 return NSERROR_NOMEM;
523
524 v = strdup("");
525 if (v == NULL) {
526 free(n);
527 return NSERROR_NOMEM;
528 }
529 } else {
530 /* Split header into name & value */
531
532 /* Strip trailing whitespace from name */
533 while (colon > data && (colon[-1] == ' ' ||
534 colon[-1] == '\t' || colon[-1] == '\r' ||
535 colon[-1] == '\n'))
536 colon--;
537
538 n = strndup((const char *) data, colon - data);
539 if (n == NULL)
540 return NSERROR_NOMEM;
541
542 /* Find colon again */
543 while (*colon != ':') {
544 colon++;
545 }
546
547 /* Skip over colon and any subsequent whitespace */
548 do {
549 colon++;
550 } while (*colon == ' ' || *colon == '\t' ||
551 *colon == '\r' || *colon == '\n');
552
553 /* Strip trailing whitespace from value */
554 while (len > 0 && (data[len - 1] == ' ' ||
555 data[len - 1] == '\t' ||
556 data[len - 1] == '\r' ||
557 data[len - 1] == '\n')) {
558 len--;
559 }
560
561 v = strndup((const char *) colon, len - (colon - data));
562 if (v == NULL) {
563 free(n);
564 return NSERROR_NOMEM;
565 }
566 }
567
568 *name = n;
569 *value = v;
570
571 return NSERROR_OK;
572}
573
574/**
575 * parse cache control header value
576 *
577 * \param object Object to parse header for
578 * \param value header value
579 * \return NSERROR_OK on success, appropriate error otherwise
580 */
581static nserror
583{
585 nserror error;
586
587 error = http_parse_cache_control(value, &cc);
588 if (error != NSERROR_OK) {
589 /* Ignore parse errors */
590 return NSERROR_OK;
591 }
592
595 /**
596 * \todo When we get a disk cache we should
597 * distinguish between these two.
598 */
599 object->cache.no_cache = LLCACHE_VALIDATE_ALWAYS;
600 }
601
603 object->cache.max_age = http_cache_control_max_age(cc);
604 }
605
607
608 return NSERROR_OK;
609}
610
611/**
612 * Update cache control from appropriate header
613 *
614 * \param object Object to parse header for
615 * \param name header name
616 * \param value header value
617 * \return NSERROR_OK on success, appropriate error otherwise
618 */
619static nserror
621 char *name,
622 char *value)
623{
624 nserror res;
625 size_t name_len;
626
627 /* Parse cache headers to populate cache control data */
628 name_len = strlen(name);
629
630 switch (name_len) {
631 case 3:
632 if (strcasecmp(name, "Age") == 0) {
633 /* extract Age header */
634 if ('0' <= *value && *value <= '9') {
635 object->cache.age = atoi(value);
636 }
637
638 }
639 break;
640
641 case 4:
642 if (strcasecmp(name, "Date") == 0) {
643 /* extract Date header */
644 res = nsc_strntimet(value,
645 strlen(value),
646 &object->cache.date);
647 if (res != NSERROR_OK) {
648 NSLOG(llcache, INFO,
649 "Processing Date header value \"%s\" returned %d",
650 value, res);
651 }
652 } else if (strcasecmp(name, "ETag") == 0) {
653 /* extract ETag header */
654 free(object->cache.etag);
655 object->cache.etag = strdup(value);
656 if (object->cache.etag == NULL) {
657 NSLOG(llcache, INFO,
658 "No memory to duplicate ETag");
659 return NSERROR_NOMEM;
660 }
661 }
662 break;
663
664 case 7:
665 if (strcasecmp(name, "Expires") == 0) {
666 /* process Expires header value */
667 res = nsc_strntimet(value,
668 strlen(value),
669 &object->cache.expires);
670 if (res != NSERROR_OK) {
671 NSLOG(llcache, INFO,
672 "Processing Expires header value \"%s\" returned %d",
673 value, res);
674 object->cache.expires = (time_t)0x7fffffff;
675 }
676 }
677 break;
678
679 case 13:
680 if (strcasecmp(name, "Cache-Control") == 0) {
681 /* parse Cache-Control header value */
683 } else if (strcasecmp(name, "Last-Modified") == 0) {
684 /* parse Last-Modified header value */
685 nsc_strntimet(value,
686 strlen(value),
687 &object->cache.last_modified);
688 }
689 break;
690 }
691
692 return NSERROR_OK;
693}
694
695/**
696 * Destroy headers.
697 *
698 * \param object The object to destroy headers within.
699 */
700static inline void llcache_destroy_headers(llcache_object *object)
701{
702 while (object->num_headers > 0) {
703 object->num_headers--;
704
705 free(object->headers[object->num_headers].name);
706 free(object->headers[object->num_headers].value);
707 }
708 free(object->headers);
709 object->headers = NULL;
710}
711
712/**
713 * Invalidate cache control data.
714 *
715 * \param object The object to invalidate cache control for.
716 */
718{
719 free(object->cache.etag);
720 memset(&(object->cache), 0, sizeof(llcache_cache_control));
721
722 object->cache.age = INVALID_AGE;
723 object->cache.max_age = INVALID_AGE;
724}
725
726/**
727 * Process a fetch header
728 *
729 * \param object Object being fetched
730 * \param data Header string
731 * \param len Byte length of header
732 * \return NSERROR_OK on success, appropriate error otherwise
733 */
735 const uint8_t *data, size_t len)
736{
737 nserror res;
738 char *name, *value;
739 llcache_header *temp;
740
741 /**
742 * \note The headers for multiple HTTP responses may be
743 * delivered to us if the fetch layer receives a 401 response
744 * for which it has authentication credentials. This will
745 * result in a silent re-request after which we'll receive the
746 * actual response headers for the object we want to fetch
747 * (assuming that the credentials were correct of course)
748 *
749 * Therefore, if the header is an HTTP response start marker, then we
750 * must discard any headers we've read so far, reset the cache data
751 * that we might have computed, and start again.
752 */
753 /** \todo Properly parse the response line */
754 if (strncmp((const char *) data, "HTTP/", SLEN("HTTP/")) == 0) {
755 time_t req_time = object->cache.req_time;
756
758
759 /* Restore request time, so we compute object's age correctly */
760 object->cache.req_time = req_time;
761
763 }
764
765 /* Set fetch response time if not already set */
766 if (object->cache.res_time == 0) {
767 object->cache.res_time = time(NULL);
768 }
769
770 /* Parse header into name-value pair */
771 res = llcache_fetch_split_header(data, len, &name, &value);
772 if (res != NSERROR_OK) {
773 return res;
774 }
775
776 /* deal with empty header */
777 if (name[0] == 0) {
778 free(name);
779 free(value);
780 return NSERROR_OK;
781 }
782
783 /* update cache control data from header */
784 res = llcache_fetch_header_cache_control(object, name, value);
785 if (res != NSERROR_OK) {
786 free(name);
787 free(value);
788 return res;
789 }
790
791 /* Append header data to the object's headers array */
792 temp = realloc(object->headers,
793 (object->num_headers + 1) * sizeof(llcache_header));
794 if (temp == NULL) {
795 free(name);
796 free(value);
797 return NSERROR_NOMEM;
798 }
799
800 object->headers = temp;
801
802 object->headers[object->num_headers].name = name;
803 object->headers[object->num_headers].value = value;
804
805 object->num_headers++;
806
807 return NSERROR_OK;
808}
809
810/**
811 * construct a Referer header appropriate for the request
812 *
813 * \param url The url being navigated to
814 * \param referer The referring url
815 * \param header_out A pointer to receive the header. The buffer must
816 * be freed by the caller.
817 * \return NSERROR_OK and \a header_out updated on success else error code
818 */
819static nserror get_referer_header(nsurl *url, nsurl *referer, char **header_out)
820{
822 lwc_string *ref_scheme;
823 lwc_string *scheme;
824 bool match;
825 bool match1;
826 bool match2;
827 char *header;
828
829 /* Determine whether to send the Referer header */
830 if (!nsoption_bool(send_referer)) {
831 return NSERROR_INVALID;
832 }
833
834 scheme = nsurl_get_component(url, NSURL_SCHEME);
835 if (scheme == NULL) {
836 return NSERROR_BAD_URL;
837 }
838
839 ref_scheme = nsurl_get_component(referer, NSURL_SCHEME);
840 if (ref_scheme == NULL) {
841 /* referer has no scheme so no header */
842 lwc_string_unref(scheme);
843 return NSERROR_INVALID;
844 }
845
846 /* User permits us to send the header
847 * Only send it if:
848 * 1) The fetch and referer schemes match
849 * or 2) The fetch is https and the referer is http
850 *
851 * This ensures that referer information is only sent
852 * across schemes in the special case of an https
853 * request from a page served over http. The inverse
854 * (https -> http) should not send the referer (15.1.3)
855 */
856 if (lwc_string_isequal(scheme, ref_scheme,
857 &match) != lwc_error_ok) {
858 match = false;
859 }
860 if (lwc_string_isequal(scheme, corestring_lwc_https,
861 &match1) != lwc_error_ok) {
862 match1 = false;
863 }
864 if (lwc_string_isequal(ref_scheme, corestring_lwc_http,
865 &match2) != lwc_error_ok) {
866 match2 = false;
867 }
868 if (match == true || (match1 == true && match2 == true)) {
869 const size_t len = SLEN("Referer: ") +
870 nsurl_length(referer) + 1;
871
872 header = malloc(len);
873 if (header == NULL) {
874 res = NSERROR_NOMEM;
875 } else {
876 snprintf(header, len, "Referer: %s",
877 nsurl_access(referer));
878
879 *header_out = header;
880 res = NSERROR_OK;
881 }
882 }
883
884
885 lwc_string_unref(scheme);
886 lwc_string_unref(ref_scheme);
887
888 return res;
889}
890
891/**
892 * (Re)fetch an object
893 *
894 * Sets up headers and attempts to start an actual fetch from the
895 * fetchers system updating the llcache object with the new fetch on
896 * successful start.
897 *
898 * \pre The fetch parameters in object->fetch must be populated
899 *
900 * \param object Object to refetch
901 * \return NSERROR_OK on success, appropriate error otherwise
902 */
904{
905 const char *urlenc = NULL;
906 struct fetch_multipart_data *multipart = NULL;
907 char **headers = NULL;
908 int header_idx = 0;
909 nserror res;
910
911 if (object->fetch.post != NULL) {
912 if (object->fetch.post->type == LLCACHE_POST_URL_ENCODED) {
913 urlenc = object->fetch.post->data.urlenc;
914 } else {
915 multipart = object->fetch.post->data.multipart;
916 }
917 }
918
919 /* Generate headers */
920 headers = malloc(4 * sizeof(char *));
921 if (headers == NULL) {
922 return NSERROR_NOMEM;
923 }
924
925 /* cache-control header for etag */
926 if (object->cache.etag != NULL) {
927 const size_t len = SLEN("If-None-Match: ") +
928 strlen(object->cache.etag) + 1;
929
930 headers[header_idx] = malloc(len);
931 if (headers[header_idx] == NULL) {
932 free(headers);
933 return NSERROR_NOMEM;
934 }
935
936 snprintf(headers[header_idx], len, "If-None-Match: %s",
937 object->cache.etag);
938
939 header_idx++;
940 }
941
942 /* cache-control header for modification time */
943 if (object->cache.last_modified != 0) {
944 /* Maximum length of an RFC 1123 date is 29 bytes */
945 const size_t len = SLEN("If-Modified-Since: ") + 29 + 1;
946
947 headers[header_idx] = malloc(len);
948 if (headers[header_idx] == NULL) {
949 while (--header_idx >= 0)
950 free(headers[header_idx]);
951 free(headers);
952 return NSERROR_NOMEM;
953 }
954
955 snprintf(headers[header_idx], len, "If-Modified-Since: %s",
957
958 header_idx++;
959 }
960
961 /* Referer header */
962 if (object->fetch.referer != NULL) {
963 if (get_referer_header(object->url,
964 object->fetch.referer,
965 &headers[header_idx]) == NSERROR_OK) {
966 header_idx++;
967 }
968 }
969 headers[header_idx] = NULL;
970
971 /* Reset cache control data */
973 object->cache.req_time = time(NULL);
974 object->cache.fin_time = object->cache.req_time;
975
976 /* Reset fetch state */
977 object->fetch.state = LLCACHE_FETCH_INIT;
978
979 NSLOG(llcache, DEBUG, "Re-fetching %p", object);
980
981 /* Kick off fetch */
982 res = fetch_start(object->url,
983 object->fetch.referer,
985 object,
987 urlenc,
988 multipart,
991 (const char **)headers,
992 &object->fetch.fetch);
993
994 /* Clean up cache-control headers */
995 while (--header_idx >= 0) {
996 free(headers[header_idx]);
997 }
998 free(headers);
999
1000 return res;
1001}
1002
1003/**
1004 * Kick-off a fetch for an object
1005 *
1006 * \pre object::url must contain the URL to fetch
1007 * \pre If there is a freshness validation candidate,
1008 * object::candidate and object::cache must be filled in
1009 * \pre There must not be a fetch in progress for \a object
1010 *
1011 * \param object Object to fetch
1012 * \param flags Fetch flags
1013 * \param referer Referring URL, or NULL for none
1014 * \param post POST data, or NULL for GET
1015 * \param redirect_count Number of redirects followed so far
1016 * \param hsts_in_use Whether HSTS applies to this fetch
1017 * \return NSERROR_OK on success, appropriate error otherwise
1018 */
1019static nserror llcache_object_fetch(llcache_object *object, uint32_t flags,
1020 nsurl *referer, const llcache_post_data *post,
1021 uint32_t redirect_count, bool hsts_in_use)
1022{
1023 nserror error;
1024 nsurl *referer_clone = NULL;
1025 llcache_post_data *post_clone = NULL;
1026
1027 NSLOG(llcache, DEBUG, "Starting fetch for %p", object);
1028
1029 if (post != NULL) {
1030 error = llcache_post_data_clone(post, &post_clone);
1031 if (error != NSERROR_OK)
1032 return error;
1033 }
1034
1035 if (referer != NULL)
1036 referer_clone = nsurl_ref(referer);
1037
1038 object->fetch.flags = flags;
1039 object->fetch.referer = referer_clone;
1040 object->fetch.post = post_clone;
1041 object->fetch.redirect_count = redirect_count;
1042 object->fetch.retries_remaining = llcache->fetch_attempts;
1043 object->fetch.hsts_in_use = hsts_in_use;
1044
1045 return llcache_object_refetch(object);
1046}
1047
1048/**
1049 * Destroy a low-level cache object
1050 *
1051 * \pre Object is detached from cache list
1052 * \pre Object has no users
1053 * \pre Object is not a candidate (i.e. object::candidate_count == 0)
1054 *
1055 * \param object Object to destroy
1056 * \return NSERROR_OK on success, appropriate error otherwise
1057 */
1059{
1060 size_t i;
1061
1062 NSLOG(llcache, DEBUG, "Destroying object %p, %s", object,
1063 nsurl_access(object->url));
1064
1065 cert_chain_free(object->chain);
1066
1067 if (object->source_data != NULL) {
1068 if (object->store_state == LLCACHE_STATE_DISC) {
1070 } else {
1071 free(object->source_data);
1072 }
1073 }
1074
1075 nsurl_unref(object->url);
1076
1077 if (object->fetch.fetch != NULL) {
1078 fetch_abort(object->fetch.fetch);
1079 object->fetch.fetch = NULL;
1080 }
1081
1082 if (object->fetch.referer != NULL)
1083 nsurl_unref(object->fetch.referer);
1084
1085 if (object->fetch.post != NULL) {
1086 if (object->fetch.post->type == LLCACHE_POST_URL_ENCODED) {
1087 free(object->fetch.post->data.urlenc);
1088 } else {
1090 object->fetch.post->data.multipart);
1091 }
1092
1093 free(object->fetch.post);
1094 }
1095
1096 free(object->cache.etag);
1097
1098 for (i = 0; i < object->num_headers; i++) {
1099 free(object->headers[i].name);
1100 free(object->headers[i].value);
1101 }
1102 free(object->headers);
1103
1104 free(object);
1105
1106 return NSERROR_OK;
1107}
1108
1109/**
1110 * Add a low-level cache object to a cache list
1111 *
1112 * \param object Object to add
1113 * \param list List to add to
1114 * \return NSERROR_OK
1115 */
1117 llcache_object **list)
1118{
1119 object->prev = NULL;
1120 object->next = *list;
1121
1122 if (*list != NULL)
1123 (*list)->prev = object;
1124 *list = object;
1125
1126 return NSERROR_OK;
1127}
1128
1129/**
1130 * Determine the remaining lifetime of a cache object using the
1131 *
1132 * \param cd cache control data.
1133 * \return The length of time remaining for the object or 0 if expired.
1134 */
1135static int
1137{
1138 int current_age, freshness_lifetime;
1139 time_t now = time(NULL);
1140
1141 /* Calculate staleness of cached object as per RFC 2616 13.2.3/13.2.4 */
1142 current_age = max(0, (cd->res_time - cd->date));
1143 current_age = max(current_age, (cd->age == INVALID_AGE) ? 0 : cd->age);
1144 current_age += cd->res_time - cd->req_time + now - cd->res_time;
1145
1146 /* Determine freshness lifetime of this object */
1147 if (cd->max_age != INVALID_AGE) {
1148 freshness_lifetime = cd->max_age;
1149 } else if (cd->expires != 0) {
1150 freshness_lifetime = cd->expires - cd->date;
1151 } else if (cd->last_modified != 0) {
1152 freshness_lifetime = (now - cd->last_modified) / 10;
1153 } else {
1154 freshness_lifetime = 0;
1155 }
1156
1157 NSLOG(llcache, DEBUG, "%d:%d", freshness_lifetime, current_age);
1158
1159 if ((cd->no_cache == LLCACHE_VALIDATE_FRESH) &&
1160 (freshness_lifetime > current_age)) {
1161 /* object was not forbidden from being returned from
1162 * the cache unvalidated (i.e. the response contained
1163 * a no-cache directive)
1164 *
1165 * The object current age is within the freshness lifetime.
1166 */
1167 return freshness_lifetime - current_age;
1168 }
1169
1170 return 0; /* object has no remaining lifetime */
1171}
1172
1173/**
1174 * Determine if an object is still fresh
1175 *
1176 * \param object Object to consider
1177 * \return True if object is still fresh, false otherwise
1178 */
1179static bool llcache_object_is_fresh(const llcache_object *object)
1180{
1181 int remaining_lifetime;
1182 const llcache_cache_control *cd = &object->cache;
1183
1184 remaining_lifetime = llcache_object_rfc2616_remaining_lifetime(cd);
1185
1186 NSLOG(llcache, DEBUG, "%p: (%d > 0 || %d != %d)", object,
1187 remaining_lifetime,
1189
1190 /* The object is fresh if:
1191 * - it was not forbidden from being returned from the cache
1192 * unvalidated.
1193 *
1194 * - it has remaining lifetime or still being fetched.
1195 */
1196 return ((cd->no_cache == LLCACHE_VALIDATE_FRESH) &&
1197 ((remaining_lifetime > 0) ||
1198 (object->fetch.state != LLCACHE_FETCH_COMPLETE)));
1199}
1200
1201/**
1202 * Clone an object's cache data
1203 *
1204 * \post If \a deep is false, then any pointers in \a source will be set to NULL
1205 *
1206 * \param source Source object containing cache data to clone
1207 * \param destination Destination object to clone cache data into
1208 * \param deep Whether to deep-copy the data or not
1209 * \return NSERROR_OK on success, appropriate error otherwise
1210 */
1212 llcache_object *destination, bool deep)
1213{
1214 /* ETag must be first, as it can fail when deep cloning */
1215 if (source->cache.etag != NULL) {
1216 char *etag = source->cache.etag;
1217
1218 if (deep) {
1219 /* Copy the etag */
1220 etag = strdup(source->cache.etag);
1221 if (etag == NULL)
1222 return NSERROR_NOMEM;
1223 } else {
1224 /* Destination takes ownership */
1225 source->cache.etag = NULL;
1226 }
1227
1228 if (destination->cache.etag != NULL)
1229 free(destination->cache.etag);
1230
1231 destination->cache.etag = etag;
1232 }
1233
1234 destination->cache.req_time = source->cache.req_time;
1235 destination->cache.res_time = source->cache.res_time;
1236 destination->cache.fin_time = source->cache.fin_time;
1237
1238 if (source->cache.date != 0)
1239 destination->cache.date = source->cache.date;
1240
1241 if (source->cache.expires != 0)
1242 destination->cache.expires = source->cache.expires;
1243
1244 if (source->cache.age != INVALID_AGE)
1245 destination->cache.age = source->cache.age;
1246
1247 if (source->cache.max_age != INVALID_AGE)
1248 destination->cache.max_age = source->cache.max_age;
1249
1250 if (source->cache.no_cache != LLCACHE_VALIDATE_FRESH)
1251 destination->cache.no_cache = source->cache.no_cache;
1252
1253 if (source->cache.last_modified != 0)
1254 destination->cache.last_modified = source->cache.last_modified;
1255
1256 return NSERROR_OK;
1257}
1258
1259/**
1260 * Remove a low-level cache object from a cache list
1261 *
1262 * \param object Object to remove
1263 * \param list List to remove from
1264 * \return NSERROR_OK
1265 */
1266static nserror
1268{
1269 if (object == *list)
1270 *list = object->next;
1271 else
1272 object->prev->next = object->next;
1273
1274 if (object->next != NULL)
1275 object->next->prev = object->prev;
1276
1277 return NSERROR_OK;
1278}
1279
1280/**
1281 * Retrieve source data for an object from persistent store if necessary.
1282 *
1283 * If an object's source data has been placed in the persistent store
1284 * and there is no in-memory copy, then attempt to retrieve the source
1285 * data.
1286 *
1287 * \param object the object to operate on.
1288 * \return appropriate error code.
1289 */
1291{
1292 /* ensure the source data is present if necessary */
1293 if ((object->source_data != NULL) ||
1294 (object->store_state != LLCACHE_STATE_DISC)) {
1295 /* source data does not require retrieving from
1296 * persistent store.
1297 */
1298 return NSERROR_OK;
1299 }
1300
1301 /* Source data for the object may be in the persistent store */
1302 return guit->llcache->fetch(object->url,
1304 &object->source_data,
1305 &object->source_len);
1306}
1307
1308/**
1309 * Generate a serialised version of an object's metadata
1310 *
1311 * The metadata includes object headers.
1312 *
1313 * \param object The cache object to serialise the metadata of.
1314 * \param data_out Where the serialised buffer will be placed.
1315 * \param datasize_out The size of the serialised data.
1316 * \return NSERROR_OK on success with \a data_out and \a datasize_out
1317 * updated, NSERROR_NOMEM on memory exhaustion or
1318 * NSERROR_INVALID if there was an error serialising the
1319 * stream.
1320 */
1321static nserror
1323 uint8_t **data_out,
1324 size_t *datasize_out)
1325{
1326 size_t allocsize;
1327 int datasize;
1328 uint8_t *data;
1329 char *op;
1330 unsigned int hloop;
1331 int use;
1332 size_t cert_chain_depth;
1333
1334 if (object->chain != NULL) {
1335 cert_chain_depth = object->chain->depth;
1336 } else {
1337 cert_chain_depth = 0;
1338 }
1339
1340 allocsize = 10 + 1; /* object length */
1341
1342 allocsize += 10 + 1; /* request time */
1343
1344 allocsize += 10 + 1; /* response time */
1345
1346 allocsize += 10 + 1; /* completion time */
1347
1348 allocsize += 10 + 1; /* space for number of header entries */
1349
1350 for (hloop = 0 ; hloop < object->num_headers ; hloop++) {
1351 allocsize += strlen(object->headers[hloop].name) + 1;
1352 allocsize += strlen(object->headers[hloop].value) + 1;
1353 }
1354
1355 allocsize += nsurl_length(object->url) + 1;
1356
1357 /* space for number of DER formatted certificates */
1358 allocsize += 10 + 1;
1359
1360 for (hloop = 0; hloop < cert_chain_depth; hloop++) {
1361 allocsize += 10 + 1; /* error status */
1362 allocsize += 4 * ((object->chain->certs[hloop].der_length + 2) / 3);
1363 }
1364
1365 data = malloc(allocsize);
1366 if (data == NULL) {
1367 return NSERROR_NOMEM;
1368 }
1369
1370 op = (char *)data;
1371 datasize = allocsize;
1372
1373 /* the url, used for checking for collisions */
1374 use = snprintf(op, datasize, "%s", nsurl_access(object->url));
1375 if (use < 0) {
1376 goto operror;
1377 }
1378 use++; /* does not count the null */
1379 if (use > datasize) {
1380 goto overflow;
1381 }
1382 op += use;
1383 datasize -= use;
1384
1385 /* object size */
1386 use = snprintf(op, datasize, "%" PRIsizet, object->source_len);
1387 if (use < 0) {
1388 goto operror;
1389 }
1390 use++; /* does not count the null */
1391 if (use > datasize)
1392 goto overflow;
1393 op += use;
1394 datasize -= use;
1395
1396 /* Time of request */
1397 use = nsc_sntimet(op, datasize, &object->cache.req_time);
1398 if (use == 0)
1399 goto overflow;
1400 use++; /* does not count the null */
1401 op += use;
1402 datasize -= use;
1403
1404 /* Time of response */
1405 use = nsc_sntimet(op, datasize, &object->cache.res_time);
1406 if (use == 0)
1407 goto overflow;
1408 use++; /* does not count the null */
1409 op += use;
1410 datasize -= use;
1411
1412 /* Time of completion */
1413 use = nsc_sntimet(op, datasize, &object->cache.fin_time);
1414 if (use == 0)
1415 goto overflow;
1416 use++; /* does not count the null */
1417 op += use;
1418 datasize -= use;
1419
1420 /* number of headers */
1421 use = snprintf(op, datasize, "%" PRIsizet, object->num_headers);
1422 if (use < 0) {
1423 goto operror;
1424 }
1425 use++; /* does not count the null */
1426 if (use > datasize)
1427 goto overflow;
1428 op += use;
1429 datasize -= use;
1430
1431 /* headers */
1432 for (hloop = 0 ; hloop < object->num_headers ; hloop++) {
1433 use = snprintf(op, datasize,
1434 "%s:%s",
1435 object->headers[hloop].name,
1436 object->headers[hloop].value);
1437 if (use < 0) {
1438 goto operror;
1439 }
1440 use++; /* does not count the null */
1441 if (use > datasize)
1442 goto overflow;
1443 op += use;
1444 datasize -= use;
1445 }
1446
1447 /* number of DER formatted ssl certificates */
1448 use = snprintf(op, datasize, "%" PRIsizet, cert_chain_depth);
1449 if (use < 0) {
1450 goto operror;
1451 }
1452 use++; /* does not count the null */
1453 if (use > datasize)
1454 goto overflow;
1455 op += use;
1456 datasize -= use;
1457
1458 /* SSL certificates */
1459 for (hloop = 0; hloop < cert_chain_depth; hloop++) {
1460 nsuerror res;
1461
1462 /* Certificate error code */
1463 use = snprintf(op, datasize, "%d",
1464 (int)(object->chain->certs[hloop].err));
1465 if (use < 0) {
1466 goto operror;
1467 }
1468 use++; /* does not count the null */
1469 if (use > datasize)
1470 goto overflow;
1471 op += use;
1472 datasize -= use;
1473
1474 /* DER certificate data in base64 encoding */
1475 if (object->chain->certs[hloop].der != NULL) {
1476 size_t output_length = datasize;
1477 res = nsu_base64_encode(
1478 object->chain->certs[hloop].der,
1479 object->chain->certs[hloop].der_length,
1480 (uint8_t *)op,
1481 &output_length);
1482 if (res != NSUERROR_OK) {
1483 goto operror;
1484 }
1485 use = output_length;
1486 } else {
1487 use = 0;
1488 }
1489 use++; /* allow for null */
1490 if (use > datasize)
1491 goto overflow;
1492 op += use;
1493 *(op - 1) = 0;
1494 datasize -= use;
1495 }
1496
1497 NSLOG(llcache, DEBUG, "Filled buffer with %d spare", datasize);
1498
1499 *data_out = data;
1500 *datasize_out = allocsize - datasize;
1501
1502 return NSERROR_OK;
1503
1504overflow:
1505 /* somehow we overflowed the buffer - hth? */
1506 NSLOG(llcache, INFO, "Overflowed metadata buffer");
1507 free(data);
1508 return NSERROR_INVALID;
1509
1510operror:
1511 /* output error */
1512 NSLOG(llcache, INFO, "Output error");
1513 free(data);
1514 return NSERROR_INVALID;
1515}
1516
1517/**
1518 * Deserialisation of an object's metadata.
1519 *
1520 * Attempt to retrieve and deserialise the metadata for an object from
1521 * the backing store.
1522 *
1523 * This must only update object if it is successful otherwise difficult
1524 * to debug crashes happen later by using bad leftover object state.
1525 *
1526 * \param object The object to retrieve the metadata for.
1527 * \return NSERROR_OK if the metatdata was retrieved and deserialised
1528 * or error code if URL is not in persistent storage or in
1529 * event of deserialisation error.
1530 */
1531static nserror
1533{
1534 nserror res;
1535 uint8_t *metadata = NULL;
1536 size_t metadatalen = 0;
1537 size_t remaining = 0;
1538 nsurl *metadataurl;
1539 unsigned int line;
1540 char *ln;
1541 int lnsize;
1542
1543 size_t source_length;
1544 time_t request_time;
1545 time_t response_time;
1546 time_t completion_time;
1547 size_t num_headers;
1548 size_t hloop;
1549 size_t ssl_cert_count = 0;
1550 struct cert_chain *chain = NULL;
1551
1552 NSLOG(llcache, INFO, "Retrieving metadata");
1553
1554 /* attempt to retrieve object metadata from the backing store */
1555 res = guit->llcache->fetch(object->url,
1557 &metadata,
1558 &metadatalen);
1559 if (res != NSERROR_OK) {
1560 return res;
1561 }
1562
1563 NSLOG(llcache, INFO, "Processing retrieved data");
1564
1565 /* metadata is stored as a sequence of NULL terminated strings
1566 * which we call 'line's here.
1567 */
1568
1569 /* We track remaining data because as we extend this data structure
1570 * we need to know if we should continue to parse
1571 */
1572 remaining = metadatalen;
1573
1574 /* metadata line 1 is the url the metadata referrs to */
1575 line = 1;
1576 ln = (char *)metadata;
1577 lnsize = strlen(ln);
1578 remaining -= lnsize + 1;
1579
1580 if (lnsize < 7) {
1581 res = NSERROR_INVALID;
1582 goto format_error;
1583 }
1584
1585 res = nsurl_create(ln, &metadataurl);
1586 if (res != NSERROR_OK)
1587 goto format_error;
1588
1589 if (nsurl_compare(object->url, metadataurl, NSURL_COMPLETE) != true) {
1590 /* backing store returned the wrong object for the
1591 * request. This may occur if the backing store had
1592 * a collision in its storage method. We cope with this
1593 * by simply skipping caching of this object.
1594 */
1595
1596 NSLOG(llcache, INFO, "Got metadata for %s instead of %s",
1597 nsurl_access(metadataurl), nsurl_access(object->url));
1598
1599 nsurl_unref(metadataurl);
1600
1602
1603 return NSERROR_BAD_URL;
1604 }
1605 nsurl_unref(metadataurl);
1606
1607
1608 /* metadata line 2 is the object's length */
1609 line = 2;
1610 ln += lnsize + 1;
1611 lnsize = strlen(ln);
1612 remaining -= lnsize + 1;
1613
1614 if ((lnsize < 1) || (sscanf(ln, "%" PRIsizet, &source_length) != 1)) {
1615 res = NSERROR_INVALID;
1616 goto format_error;
1617 }
1618
1619
1620 /* metadata line 3 is the time of request */
1621 line = 3;
1622 ln += lnsize + 1;
1623 lnsize = strlen(ln);
1624 remaining -= lnsize + 1;
1625
1626 res = nsc_snptimet(ln, lnsize, &request_time);
1627 if (res != NSERROR_OK)
1628 goto format_error;
1629
1630
1631 /* metadata line 4 is the time of response */
1632 line = 4;
1633 ln += lnsize + 1;
1634 lnsize = strlen(ln);
1635 remaining -= lnsize + 1;
1636
1637 res = nsc_snptimet(ln, lnsize, &response_time);
1638 if (res != NSERROR_OK)
1639 goto format_error;
1640
1641
1642 /* metadata line 5 is the time of request completion */
1643 line = 5;
1644 ln += lnsize + 1;
1645 lnsize = strlen(ln);
1646 remaining -= lnsize + 1;
1647
1648 res = nsc_snptimet(ln, lnsize, &completion_time);
1649 if (res != NSERROR_OK)
1650 goto format_error;
1651
1652
1653 /* metadata line 6 is the number of headers */
1654 line = 6;
1655 ln += lnsize + 1;
1656 lnsize = strlen(ln);
1657 remaining -= lnsize + 1;
1658
1659 if ((lnsize < 1) || (sscanf(ln, "%" PRIsizet, &num_headers) != 1)) {
1660 res = NSERROR_INVALID;
1661 goto format_error;
1662 }
1663
1664 /* read headers */
1665 for (hloop = 0 ; hloop < num_headers; hloop++) {
1666 line++;
1667 ln += lnsize + 1;
1668 lnsize = strlen(ln);
1669 remaining -= lnsize + 1;
1670
1671 res = llcache_fetch_process_header(object,
1672 (uint8_t *)ln,
1673 lnsize);
1674 if (res != NSERROR_OK)
1675 goto format_error;
1676 }
1677
1678 if (remaining == 0) {
1679 goto skip_ssl_certificates;
1680 }
1681
1682 /* Next line is the number of DER base64 encoded certificates */
1683 line++;
1684 ln += lnsize + 1;
1685 lnsize = strlen(ln);
1686 remaining -= lnsize + 1;
1687
1688 if ((lnsize < 1) || (sscanf(ln, "%" PRIsizet, &ssl_cert_count) != 1)) {
1689 res = NSERROR_INVALID;
1690 goto format_error;
1691 }
1692
1693 if (ssl_cert_count == 0) {
1694 goto skip_ssl_certificates;
1695 }
1696
1697 if (ssl_cert_count > MAX_CERT_DEPTH) {
1698 res = NSERROR_INVALID;
1699 goto format_error;
1700 }
1701
1702 res = cert_chain_alloc(ssl_cert_count, &chain);
1703 if (res != NSERROR_OK) {
1704 goto format_error;
1705 }
1706
1707 for (hloop = 0; hloop < ssl_cert_count; hloop++) {
1708 int errcode;
1709 nsuerror nsures;
1710
1711 /* Certificate error code */
1712 line++;
1713 ln += lnsize + 1;
1714 lnsize = strlen(ln);
1715 remaining -= lnsize + 1;
1716 if ((lnsize < 1) || (sscanf(ln, "%d", &errcode) != 1)) {
1717 res = NSERROR_INVALID;
1718 goto format_error;
1719 }
1720 if (errcode < SSL_CERT_ERR_OK ||
1721 errcode > SSL_CERT_ERR_MAX_KNOWN) {
1722 /* Error with the cert code, assume UNKNOWN */
1723 chain->certs[hloop].err = SSL_CERT_ERR_UNKNOWN;
1724 } else {
1725 chain->certs[hloop].err = (ssl_cert_err)errcode;
1726 }
1727
1728 /* base64 encoded DER certificate data */
1729 line++;
1730 ln += lnsize + 1;
1731 lnsize = strlen(ln);
1732 remaining -= lnsize + 1;
1733 if (lnsize > 0) {
1734 nsures = nsu_base64_decode_alloc((const uint8_t *)ln,
1735 lnsize,
1736 &chain->certs[hloop].der,
1737 &chain->certs[hloop].der_length);
1738 if (nsures != NSUERROR_OK) {
1739 res = NSERROR_NOMEM;
1740 goto format_error;
1741 }
1742 }
1743 }
1744
1745skip_ssl_certificates:
1747
1748 /* update object on successful parse of metadata */
1749 object->source_len = source_length;
1750
1751 /** \todo really not sure this is right, nothing is allocated here? */
1752 object->source_alloc = metadatalen;
1753
1754 object->cache.req_time = request_time;
1755 object->cache.res_time = response_time;
1756 object->cache.fin_time = completion_time;
1757
1758 object->chain = chain;
1759
1760 /* object stored in backing store */
1761 object->store_state = LLCACHE_STATE_DISC;
1762
1763 return NSERROR_OK;
1764
1765format_error:
1766 NSLOG(llcache, INFO,
1767 "metadata error on line %d error code %d\n",
1768 line, res);
1770
1771 cert_chain_free(chain);
1772
1773 return res;
1774}
1775
1776/**
1777 * Check whether a scheme is persistable.
1778 *
1779 * \param url URL to check.
1780 * \return true iff url has a persistable scheme.
1781 */
1782static inline bool llcache__scheme_is_persistable(const nsurl *url)
1783{
1784 lwc_string *scheme = nsurl_get_component(url, NSURL_SCHEME);
1785 bool persistable = false;
1786 bool match;
1787
1788 /* nsurl ensures lower case schemes, and corestrings are lower
1789 * case, so it's safe to use case-sensitive comparison. */
1790 if ((lwc_string_isequal(scheme, corestring_lwc_http,
1791 &match) == lwc_error_ok &&
1792 (match == true)) ||
1793 (lwc_string_isequal(scheme, corestring_lwc_https,
1794 &match) == lwc_error_ok &&
1795 (match == true))) {
1796 persistable = true;
1797 }
1798
1799 lwc_string_unref(scheme);
1800
1801 return persistable;
1802}
1803
1804/**
1805 * Check whether a scheme is cachable.
1806 *
1807 * \param url URL to check.
1808 * \return true iff url has a cachable scheme.
1809 */
1810static inline bool llcache__scheme_is_cachable(const nsurl *url)
1811{
1812 lwc_string *scheme = nsurl_get_component(url, NSURL_SCHEME);
1813 bool cachable = false;
1814 bool match;
1815
1816 /* nsurl ensures lower case schemes, and corestrings are lower
1817 * case, so it's safe to use case-sensitive comparison. */
1818 if ((lwc_string_isequal(scheme, corestring_lwc_http,
1819 &match) == lwc_error_ok &&
1820 (match == true)) ||
1821 (lwc_string_isequal(scheme, corestring_lwc_https,
1822 &match) == lwc_error_ok &&
1823 (match == true)) ||
1824 (lwc_string_isequal(scheme, corestring_lwc_data,
1825 &match) == lwc_error_ok &&
1826 (match == true)) ||
1827 (lwc_string_isequal(scheme, corestring_lwc_resource,
1828 &match) == lwc_error_ok &&
1829 (match == true)) ||
1830 (lwc_string_isequal(scheme, corestring_lwc_file,
1831 &match) == lwc_error_ok &&
1832 (match == true))) {
1833 cachable = true;
1834 }
1835
1836 lwc_string_unref(scheme);
1837
1838 return cachable;
1839}
1840
1841/**
1842 * Attempt to retrieve an object from persistent storage.
1843 *
1844 * \param object The object to populate from persistent store.
1845 * \param flags Fetch flags.
1846 * \param referer The referring url.
1847 * \param post Post data for fetch.
1848 * \param redirect_count how many times this fetch has been redirected.
1849 * \return NSERROR_OK if the object was successfully retrieved from the
1850 * cache else appropriate error code.
1851 */
1852static nserror
1854 uint32_t flags,
1855 nsurl *referer,
1856 const llcache_post_data *post,
1857 uint32_t redirect_count)
1858{
1859 nserror error;
1860 nsurl *referer_clone = NULL;
1861 llcache_post_data *post_clone = NULL;
1862
1863 if (!llcache__scheme_is_persistable(object->url)) {
1864 /* Don't bother looking up non-http(s) stuff; we don't
1865 * persist it. */
1866 return NSERROR_NOT_FOUND;
1867 }
1868
1869 object->cache.req_time = time(NULL);
1870 object->cache.fin_time = object->cache.req_time;
1871
1872 /* retrieve and process metadata */
1873 error = llcache_process_metadata(object);
1874 if (error != NSERROR_OK) {
1875 return error;
1876 }
1877
1878 /* entry came out of cache - need to setup object state */
1879 if (post != NULL) {
1880 error = llcache_post_data_clone(post, &post_clone);
1881 if (error != NSERROR_OK)
1882 return error;
1883 }
1884
1885 if (referer != NULL) {
1886 referer_clone = nsurl_ref(referer);
1887 }
1888
1889 object->fetch.flags = flags;
1890 object->fetch.referer = referer_clone;
1891 object->fetch.post = post_clone;
1892 object->fetch.redirect_count = redirect_count;
1893
1894 /* fetch is "finished" */
1895 object->fetch.state = LLCACHE_FETCH_COMPLETE;
1896 object->fetch.fetch = NULL;
1897
1898 return NSERROR_OK;
1899}
1900
1901/**
1902 * Retrieve a potentially cached object
1903 *
1904 * \param url URL of object to retrieve
1905 * \param flags Fetch flags
1906 * \param referer Referring URL, or NULL if none
1907 * \param post POST data, or NULL for a GET request
1908 * \param redirect_count Number of redirects followed so far
1909 * \param hsts_in_use Whether HSTS applies to this fetch
1910 * \param result Pointer to location to receive retrieved object
1911 * \return NSERROR_OK on success, appropriate error otherwise
1912 */
1913static nserror
1915 uint32_t flags,
1916 nsurl *referer,
1917 const llcache_post_data *post,
1918 uint32_t redirect_count,
1919 bool hsts_in_use,
1921{
1922 nserror error;
1923 llcache_object *obj, *newest = NULL;
1924
1925 NSLOG(llcache, DEBUG,
1926 "Searching cache for %s flags:%"PRIx32" referer:%s post:%p",
1927 nsurl_access(url), flags,
1928 referer==NULL?"":nsurl_access(referer),
1929 post);
1930
1931 /* Search for the most recently fetched matching object */
1932 for (obj = llcache->cached_objects; obj != NULL; obj = obj->next) {
1933
1934 if ((newest == NULL ||
1935 obj->cache.req_time > newest->cache.req_time) &&
1936 nsurl_compare(obj->url, url,
1937 NSURL_COMPLETE) == true) {
1938 newest = obj;
1939 }
1940 }
1941
1942 /* No viable object found in cache create one and attempt to
1943 * pull from persistent store.
1944 */
1945 if (newest == NULL) {
1946 NSLOG(llcache, DEBUG, "No viable object found in llcache");
1947
1948 error = llcache_object_new(url, &obj);
1949 if (error != NSERROR_OK)
1950 return error;
1951
1952 /* attempt to retrieve object from persistent store */
1953 error = llcache_object_fetch_persistent(obj, flags, referer, post, redirect_count);
1954 if (error == NSERROR_OK) {
1955 NSLOG(llcache, DEBUG, "retrieved object from persistent store");
1956
1957 /* set newest object from persistent store which
1958 * will cause the normal object handling to be used.
1959 */
1960 newest = obj;
1961
1962 /* Add new object to cached object list */
1964
1965 }
1966 /* else no object found and irretrievable from cache,
1967 * fall through with newest unset to start fetch
1968 */
1969 }
1970
1971 if ((newest != NULL) && (llcache_object_is_fresh(newest))) {
1972 /* Found a suitable object, and it's still fresh */
1973 NSLOG(llcache, DEBUG, "Found fresh %p", newest);
1974
1975 /* The client needs to catch up with the object's state.
1976 * This will occur the next time that llcache_poll is called.
1977 */
1978
1979 /* ensure the source data is present */
1980 error = llcache_retrieve_persisted_data(newest);
1981 if (error == NSERROR_OK) {
1982 /* source data was successfully retrieved from
1983 * persistent store
1984 */
1985 *result = newest;
1986
1987 return NSERROR_OK;
1988 }
1989
1990 /* retrieval of source data from persistent store
1991 * failed, destroy cache object and fall though to
1992 * cache miss to re-fetch
1993 */
1994 NSLOG(llcache, DEBUG, "Persistent retrieval failed for %p", newest);
1995
1997 llcache_object_destroy(newest);
1998
1999 error = llcache_object_new(url, &obj);
2000 if (error != NSERROR_OK) {
2001 return error;
2002 }
2003 } else if (newest != NULL) {
2004 /* Found a candidate object but it needs freshness validation */
2005
2006 /* ensure the source data is present */
2007 error = llcache_retrieve_persisted_data(newest);
2008 if (error == NSERROR_OK) {
2009
2010 /* Create a new object */
2011 error = llcache_object_new(url, &obj);
2012 if (error != NSERROR_OK)
2013 return error;
2014
2015 NSLOG(llcache, DEBUG, "Found candidate %p (%p)", obj, newest);
2016
2017 /* Clone candidate's cache data */
2018 error = llcache_object_clone_cache_data(newest, obj, true);
2019 if (error != NSERROR_OK) {
2021 return error;
2022 }
2023
2024 /* Record candidate, so we can fall back if it is still fresh */
2025 newest->candidate_count++;
2026 obj->candidate = newest;
2027
2028 /* Attempt to kick-off fetch */
2029 error = llcache_object_fetch(obj, flags, referer, post,
2030 redirect_count, hsts_in_use);
2031 if (error != NSERROR_OK) {
2032 newest->candidate_count--;
2034 return error;
2035 }
2036
2037 /* Add new object to cache */
2039
2040 *result = obj;
2041
2042 return NSERROR_OK;
2043 }
2044
2045 NSLOG(llcache, DEBUG, "Persistent retrieval failed for %p", newest);
2046
2047 /* retrieval of source data from persistent store
2048 * failed, destroy cache object and fall though to
2049 * cache miss to re-retch
2050 */
2053 llcache_object_destroy(newest);
2054
2055 error = llcache_object_new(url, &obj);
2056 if (error != NSERROR_OK) {
2057 return error;
2058 }
2059 }
2060
2061 /* Attempt to kick-off fetch */
2062 error = llcache_object_fetch(obj, flags, referer, post,
2063 redirect_count, hsts_in_use);
2064 if (error != NSERROR_OK) {
2066 return error;
2067 }
2068
2069 /* Add new object to cache */
2071
2072 *result = obj;
2073
2074 return NSERROR_OK;
2075}
2076
2077/**
2078 * Retrieve an object from the cache, fetching it if necessary.
2079 *
2080 * \param url URL of object to retrieve
2081 * \param flags Fetch flags
2082 * \param referer Referring URL, or NULL if none
2083 * \param post POST data, or NULL for a GET request
2084 * \param redirect_count Number of redirects followed so far
2085 * \param hsts_in_use Whether HSTS applies to this fetch
2086 * \param result Pointer to location to receive retrieved object
2087 * \return NSERROR_OK on success, appropriate error otherwise
2088 */
2089static nserror
2091 uint32_t flags,
2092 nsurl *referer,
2093 const llcache_post_data *post,
2094 uint32_t redirect_count,
2095 bool hsts_in_use,
2097{
2098 nserror error;
2099 llcache_object *obj;
2100 nsurl *defragmented_url;
2101 bool uncachable = false;
2102
2103 NSLOG(llcache, DEBUG, "Retrieve %s (%"PRIx32", %s, %p)", nsurl_access(url), flags,
2104 referer==NULL?"":nsurl_access(referer), post);
2105
2106
2107 /* Get rid of any url fragment */
2108 error = nsurl_defragment(url, &defragmented_url);
2109 if (error != NSERROR_OK)
2110 return error;
2111
2112 /* determine if content is cachable */
2113 if ((flags & LLCACHE_RETRIEVE_FORCE_FETCH) != 0) {
2114 /* Forced fetches are never cached */
2115 uncachable = true;
2116 } else if (post != NULL) {
2117 /* POST requests are never cached */
2118 uncachable = true;
2119 } else {
2120 uncachable = !llcache__scheme_is_cachable(defragmented_url);
2121 }
2122
2123 if (uncachable) {
2124 /* Create new object */
2125 error = llcache_object_new(defragmented_url, &obj);
2126 if (error != NSERROR_OK) {
2127 nsurl_unref(defragmented_url);
2128 return error;
2129 }
2130
2131 /* Attempt to kick-off fetch */
2132 error = llcache_object_fetch(obj, flags, referer, post,
2133 redirect_count, hsts_in_use);
2134 if (error != NSERROR_OK) {
2136 nsurl_unref(defragmented_url);
2137 return error;
2138 }
2139
2140 /* Add new object to uncached list */
2142 } else {
2143 error = llcache_object_retrieve_from_cache(defragmented_url,
2144 flags, referer, post, redirect_count,
2145 hsts_in_use, &obj);
2146 if (error != NSERROR_OK) {
2147 nsurl_unref(defragmented_url);
2148 return error;
2149 }
2150
2151 /* Returned object is already in the cached list */
2152 }
2153
2154 NSLOG(llcache, DEBUG, "Retrieved %p", obj);
2155
2156 *result = obj;
2157
2158 nsurl_unref(defragmented_url);
2159
2160 return NSERROR_OK;
2161}
2162
2163
2164/**
2165 * Add a user to a low-level cache object
2166 *
2167 * \param object Object to add user to
2168 * \param user User to add
2169 * \return NSERROR_OK.
2170 */
2172 llcache_object_user *user)
2173{
2174 assert(user->next == NULL);
2175 assert(user->prev == NULL);
2176 assert(user->handle != NULL);
2177
2178 user->handle->object = object;
2179
2180 user->prev = NULL;
2181 user->next = object->users;
2182
2183 if (object->users != NULL)
2184 object->users->prev = user;
2185 object->users = user;
2186
2187 NSLOG(llcache, DEBUG, "Adding user %p to %p", user, object);
2188
2189 return NSERROR_OK;
2190}
2191
2192/**
2193 * Transform a request-URI based on HSTS policy
2194 *
2195 * \param url URL to transform
2196 * \param result Pointer to location to receive transformed URL
2197 * \param hsts_in_use Pointer to location to receive HSTS in-use flag
2198 * \return NSERROR_OK on success, appropriate error otherwise
2199 */
2201 bool *hsts_in_use)
2202{
2203 lwc_string *scheme = NULL;
2204 bool match;
2205 nserror error = NSERROR_OK;
2206
2207 scheme = nsurl_get_component(url, NSURL_SCHEME);
2208 if (lwc_string_caseless_isequal(scheme, corestring_lwc_http,
2209 &match) != lwc_error_ok || match == false) {
2210 /* Non-HTTP fetch: no transform required */
2211 if (lwc_string_caseless_isequal(scheme, corestring_lwc_https,
2212 &match) == lwc_error_ok && match) {
2213 /* HTTPS: ask urldb if HSTS is enabled */
2214 *hsts_in_use = urldb_get_hsts_enabled(url);
2215 } else {
2216 /* Anything else: no HSTS */
2217 *hsts_in_use = false;
2218 }
2219 lwc_string_unref(scheme);
2220 *result = nsurl_ref(url);
2221 return error;
2222 }
2223 lwc_string_unref(scheme);
2224
2225 if (urldb_get_hsts_enabled(url)) {
2226 /* Only need to force HTTPS. If original port was explicitly
2227 * specified as 80, nsurl_create/join will remove it (as
2228 * it's redundant) */
2229 error = nsurl_replace_scheme(url, corestring_lwc_https,
2230 result);
2231 *hsts_in_use = (error == NSERROR_OK);
2232 } else {
2233 *result = nsurl_ref(url);
2234 *hsts_in_use = false;
2235 }
2236
2237 return error;
2238}
2239
2240/**
2241 * Update HSTS policy for target domain.
2242 *
2243 * \param object Newly-fetched cache object
2244 * \return NSERROR_OK on success, appropriate error otherwise
2245 */
2247{
2248 size_t i;
2249 lwc_string *scheme = NULL;
2250 bool match = false;
2251
2252 scheme = nsurl_get_component(object->url, NSURL_SCHEME);
2253 if (lwc_string_caseless_isequal(scheme, corestring_lwc_https,
2254 &match) != lwc_error_ok || match == false) {
2255 /* Non-HTTPS fetch: ignore */
2256 lwc_string_unref(scheme);
2257 return NSERROR_OK;
2258 }
2259 lwc_string_unref(scheme);
2260
2261 if (object->fetch.tainted_tls) {
2262 /* Transport is tainted: ignore */
2263 return NSERROR_OK;
2264 }
2265
2266 for (i = 0; i < object->num_headers; i++) {
2267 if (strcasecmp("Strict-Transport-Security",
2268 object->headers[i].name) == 0) {
2269 urldb_set_hsts_policy(object->url,
2270 object->headers[i].value);
2271 /* Only process the first one we find */
2272 break;
2273 }
2274 }
2275
2276 return NSERROR_OK;
2277}
2278
2279/**
2280 * Handle FETCH_REDIRECT event
2281 *
2282 * \param object Object being redirected
2283 * \param target Target of redirect (may be relative)
2284 * \param replacement Pointer to location to receive replacement object
2285 * \return NSERROR_OK on success, appropriate error otherwise
2286 */
2288 const char *target, llcache_object **replacement)
2289{
2290 nserror error;
2291 llcache_object *dest;
2292 llcache_object_user *user, *next;
2293 const llcache_post_data *post = object->fetch.post;
2294 nsurl *url, *hsts_url;
2295 lwc_string *scheme;
2296 lwc_string *object_scheme;
2297 bool match, hsts_in_use;
2298 /* Extract HTTP response code from the fetch object */
2299 long http_code = fetch_http_code(object->fetch.fetch);
2300 llcache_event event;
2301
2302 /* Abort fetch for this object */
2303 fetch_abort(object->fetch.fetch);
2304 object->fetch.fetch = NULL;
2305
2306 /* Invalidate the cache control data */
2308
2309 /* And mark it complete */
2310 object->fetch.state = LLCACHE_FETCH_COMPLETE;
2311
2312 (void) llcache_hsts_update_policy(object);
2313
2314 /* Forcibly stop redirecting if we've followed too many redirects */
2315#define REDIRECT_LIMIT 10
2316 if (object->fetch.redirect_count > REDIRECT_LIMIT) {
2317 NSLOG(llcache, INFO, "Too many nested redirects");
2318
2319 event.type = LLCACHE_EVENT_ERROR;
2320 event.data.error.code = NSERROR_BAD_REDIRECT;
2321 event.data.error.msg = messages_get("BadRedirect");
2322
2323 return llcache_send_event_to_users(object, &event);
2324 }
2325#undef REDIRECT_LIMIT
2326
2327 /* Make target absolute */
2328 error = nsurl_join(object->url, target, &url);
2329 if (error != NSERROR_OK)
2330 return error;
2331
2332 /* Perform HSTS transform */
2333 error = llcache_hsts_transform_url(url, &hsts_url, &hsts_in_use);
2334 if (error != NSERROR_OK) {
2335 nsurl_unref(url);
2336 return error;
2337 }
2338 nsurl_unref(url);
2339
2340 /* Inform users of redirect */
2341 event.type = LLCACHE_EVENT_REDIRECT;
2342 event.data.redirect.from = object->url;
2343 event.data.redirect.to = hsts_url;
2344
2345 error = llcache_send_event_to_users(object, &event);
2346
2347 if (error != NSERROR_OK) {
2348 nsurl_unref(hsts_url);
2349 return error;
2350 }
2351
2352 /* Reject attempts to redirect from unvalidated to validated schemes
2353 * A "validated" scheme is one over which we have some guarantee that
2354 * the source is trustworthy. */
2355 object_scheme = nsurl_get_component(object->url, NSURL_SCHEME);
2356 scheme = nsurl_get_component(hsts_url, NSURL_SCHEME);
2357
2358 /* resource: and about: are allowed to redirect anywhere */
2359 if ((lwc_string_isequal(object_scheme, corestring_lwc_resource,
2360 &match) == lwc_error_ok && match == false) &&
2361 (lwc_string_isequal(object_scheme, corestring_lwc_about,
2362 &match) == lwc_error_ok && match == false)) {
2363 /* file, about and resource are not valid redirect targets */
2364 if ((lwc_string_isequal(object_scheme, corestring_lwc_file,
2365 &match) == lwc_error_ok && match == true) ||
2366 (lwc_string_isequal(object_scheme, corestring_lwc_about,
2367 &match) == lwc_error_ok && match == true) ||
2368 (lwc_string_isequal(object_scheme, corestring_lwc_resource,
2369 &match) == lwc_error_ok && match == true)) {
2370 lwc_string_unref(object_scheme);
2371 lwc_string_unref(scheme);
2372 nsurl_unref(hsts_url);
2373 return NSERROR_OK;
2374 }
2375 }
2376
2377 lwc_string_unref(scheme);
2378 lwc_string_unref(object_scheme);
2379
2380 /* Bail out if we've no way of handling this URL */
2381 if (fetch_can_fetch(hsts_url) == false) {
2382 nsurl_unref(hsts_url);
2383 return NSERROR_OK;
2384 }
2385
2386 if (http_code == 301 || http_code == 302 || http_code == 303) {
2387 /* 301, 302, 303 redirects are all unconditional GET requests */
2388 post = NULL;
2389 } else if (http_code != 307 || post != NULL) {
2390 /** \todo 300, 305, 307 with POST */
2391 nsurl_unref(hsts_url);
2392 return NSERROR_OK;
2393 }
2394
2395 /* Attempt to fetch target URL */
2396 error = llcache_object_retrieve(hsts_url, object->fetch.flags,
2397 object->fetch.referer, post,
2398 object->fetch.redirect_count + 1,
2399 hsts_in_use, &dest);
2400
2401 /* No longer require url */
2402 nsurl_unref(hsts_url);
2403
2404 if (error != NSERROR_OK)
2405 return error;
2406
2407 /* Move user(s) to replacement object */
2408 for (user = object->users; user != NULL; user = next) {
2409 next = user->next;
2410
2411 llcache_object_remove_user(object, user);
2412 llcache_object_add_user(dest, user);
2413 }
2414
2415 /* Dest is now our object */
2416 *replacement = dest;
2417
2418 return NSERROR_OK;
2419}
2420
2421/**
2422 * Update an object's cache state
2423 *
2424 * \param object Object to update cache for
2425 * \return NSERROR_OK.
2426 */
2428{
2429 if (object->cache.date == 0)
2430 object->cache.date = time(NULL);
2431
2432 return NSERROR_OK;
2433}
2434
2435/**
2436 * Handle FETCH_NOTMODIFIED event
2437 *
2438 * \param object Object to process
2439 * \param replacement Pointer to location to receive replacement object
2440 * \return NSERROR_OK.
2441 */
2443 llcache_object **replacement)
2444{
2445 /* There may be no candidate if the server erroneously responded
2446 * to an unconditional request with a 304 Not Modified response.
2447 * In this case, we simply retain the initial object, having
2448 * invalidated it and marked it as complete.
2449 */
2450 if (object->candidate != NULL) {
2451 llcache_object_user *user, *next;
2452
2453 /* Move user(s) to candidate content */
2454 for (user = object->users; user != NULL; user = next) {
2455 next = user->next;
2456
2457 llcache_object_remove_user(object, user);
2458 llcache_object_add_user(object->candidate, user);
2459 }
2460
2461 /* Candidate is no longer a candidate for us */
2462 object->candidate->candidate_count--;
2463
2464 /* Clone our cache control data into the candidate */
2466 false);
2467 /* Bring candidate's cache data up to date */
2469 /* Revert no-cache to normal, if required */
2470 if (object->candidate->cache.no_cache ==
2472 object->candidate->cache.no_cache =
2474 }
2475
2476 /* Candidate is now our object */
2477 *replacement = object->candidate;
2478 object->candidate = NULL;
2479 } else {
2480 /* There was no candidate: retain object */
2481 *replacement = object;
2482 }
2483
2484 /* Ensure fetch has stopped */
2485 fetch_abort(object->fetch.fetch);
2486 object->fetch.fetch = NULL;
2487
2488 /* Invalidate our cache-control data */
2490
2491 /* Mark it complete */
2492 object->fetch.state = LLCACHE_FETCH_COMPLETE;
2493
2494 (void) llcache_hsts_update_policy(object);
2495
2496 /* Old object will be flushed from the cache on the next poll */
2497
2498 return NSERROR_OK;
2499}
2500
2501/**
2502 * Process a chunk of fetched data
2503 *
2504 * \param object Object being fetched
2505 * \param data Data to process
2506 * \param len Byte length of data
2507 * \return NSERROR_OK on success, appropriate error otherwise.
2508 */
2509static nserror
2511 const uint8_t *data,
2512 size_t len)
2513{
2514 if (object->fetch.state != LLCACHE_FETCH_DATA) {
2515 /**
2516 * \note
2517 * On entry into this state, check if we need to
2518 * invalidate the cache control data. We are guaranteed
2519 * to have received all response headers.
2520 *
2521 * There are two cases in which we want to suppress
2522 * cacheing of an object:
2523 *
2524 * 1) The HTTP response code is not 200 or 203
2525 * 2) The request URI had a query string and the
2526 * response headers did not provide an explicit
2527 * object expiration time.
2528 */
2529 long http_code = fetch_http_code(object->fetch.fetch);
2530
2531 if ((http_code != 200 && http_code != 203) ||
2532 (nsurl_has_component(object->url, NSURL_QUERY) &&
2533 (object->cache.max_age == INVALID_AGE &&
2534 object->cache.expires == 0))) {
2535 /* Invalidate cache control data */
2537 }
2538
2539 /* Release candidate, if any */
2540 if (object->candidate != NULL) {
2541 object->candidate->candidate_count--;
2542 object->candidate = NULL;
2543 }
2544
2545 object->fetch.state = LLCACHE_FETCH_DATA;
2546 }
2547
2548 /* Resize source buffer if it's too small */
2549 if (object->source_len + len >= object->source_alloc) {
2550 const size_t new_len = object->source_len + len + 64 * 1024;
2551 uint8_t *temp = realloc(object->source_data, new_len);
2552 if (temp == NULL)
2553 return NSERROR_NOMEM;
2554
2555 object->source_data = temp;
2556 object->source_alloc = new_len;
2557 }
2558
2559 /* Append this data chunk to source buffer */
2560 memcpy(object->source_data + object->source_len, data, len);
2561 object->source_len += len;
2562
2563 return NSERROR_OK;
2564}
2565
2566
2567/**
2568 * Handle an authentication request
2569 *
2570 * \param object Object being fetched
2571 * \param realm Authentication realm
2572 * \return NSERROR_OK on success, appropriate error otherwise.
2573 */
2574static nserror llcache_fetch_auth(llcache_object *object, const char *realm)
2575{
2576 const char *auth;
2577 nserror error = NSERROR_OK;
2578
2579 /* Abort fetch for this object */
2580 fetch_abort(object->fetch.fetch);
2581 object->fetch.fetch = NULL;
2582
2583 /* Invalidate cache-control data */
2585
2586 /* Destroy headers */
2588
2589 /* If there was no realm, then default to the URL */
2590 /** \todo If there was no WWW-Authenticate header, use response body */
2591 if (realm == NULL)
2592 realm = nsurl_access(object->url);
2593
2594 auth = urldb_get_auth_details(object->url, realm);
2595
2596 if (auth == NULL || object->fetch.tried_with_auth == true) {
2597 llcache_event event;
2598 /* No authentication details, or tried what we had, so ask */
2599 object->fetch.tried_with_auth = false;
2600
2601 /* Mark object complete */
2602 object->fetch.state = LLCACHE_FETCH_COMPLETE;
2603
2604 /* Inform client(s) that object fetch failed */
2605 event.type = LLCACHE_EVENT_ERROR;
2606 /** \todo More appropriate error message */
2607 event.data.error.code = NSERROR_BAD_AUTH;
2608 event.data.error.msg = realm;
2609
2610 error = llcache_send_event_to_users(object, &event);
2611 } else {
2612 /* Flag that we've tried to refetch with credentials, so
2613 * that if the fetch fails again, we ask the user again */
2614 object->fetch.tried_with_auth = true;
2615 error = llcache_object_refetch(object);
2616 }
2617
2618 return error;
2619}
2620
2621/**
2622 * Handle a TLS certificate verification failure
2623 *
2624 * \param object Object being fetched
2625 * \return NSERROR_OK on success, appropriate error otherwise
2626 */
2628{
2629 nserror error = NSERROR_OK;
2630
2631 /* Fetch has been stopped, and destroyed. Invalidate object's pointer */
2632 object->fetch.fetch = NULL;
2633
2634 /* Invalidate cache-control data */
2636
2637 /* Consider the TLS transport tainted */
2638 object->fetch.tainted_tls = true;
2639
2640 /* Only give the user a chance if HSTS isn't in use for this fetch */
2641 if (object->fetch.hsts_in_use == false) {
2642 llcache_event event;
2643
2644 /* Mark object complete */
2645 object->fetch.state = LLCACHE_FETCH_COMPLETE;
2646
2647 /* Inform client(s) that object fetch failed */
2648 event.type = LLCACHE_EVENT_ERROR;
2649 /** \todo More appropriate error message */
2650 event.data.error.code = NSERROR_BAD_CERTS;
2651 event.data.error.msg = messages_get("FetchFailed");
2652
2653 error = llcache_send_event_to_users(object, &event);
2654 } else {
2655 llcache_event event;
2656
2657 /* Mark object complete */
2658 object->fetch.state = LLCACHE_FETCH_COMPLETE;
2659
2660 /* Inform client(s) that object fetch failed */
2661 event.type = LLCACHE_EVENT_ERROR;
2662 /** \todo More appropriate error message */
2663 event.data.error.code = NSERROR_UNKNOWN;
2664 event.data.error.msg = messages_get("FetchFailed");
2665
2666 error = llcache_send_event_to_users(object, &event);
2667 }
2668
2669 return error;
2670}
2671
2672
2673/**
2674 * Handle a TLS connection setup failure
2675 *
2676 * \param object Object being fetched
2677 * \return NSERROR_OK on success, appropriate error otherwise
2678 */
2680{
2681 nserror error = NSERROR_OK;
2682
2683 /* Fetch has been stopped, and destroyed. Invalidate object's pointer */
2684 object->fetch.fetch = NULL;
2685
2686 /* Invalidate cache-control data */
2688
2689 /* Consider the TLS transport tainted */
2690 object->fetch.tainted_tls = true;
2691
2692 /* Make no attempt to downgrade if HSTS is in use
2693 * (i.e. assume server does TLS properly) */
2694 if (object->fetch.hsts_in_use ||
2696 /* Have already tried to downgrade, so give up */
2697 llcache_event event;
2698
2699 /* Mark object complete */
2700 object->fetch.state = LLCACHE_FETCH_COMPLETE;
2701
2702 /* Inform client(s) that object fetch failed */
2703 event.type = LLCACHE_EVENT_ERROR;
2704 /** \todo More appropriate error message */
2705 event.data.error.code = NSERROR_UNKNOWN;
2706 event.data.error.msg = messages_get("FetchFailed");
2707
2708 error = llcache_send_event_to_users(object, &event);
2709 } else {
2710 /* Flag that we've tried to downgrade, so that if the
2711 * fetch fails again, we give up */
2712 object->fetch.tried_with_tls_downgrade = true;
2713 error = llcache_object_refetch(object);
2714 }
2715
2716 return error;
2717}
2718
2719
2720/**
2721 * handle time out while trying to fetch.
2722 *
2723 * \param object Object being fetched
2724 * \return NSERROR_OK on success otherwise error code
2725 */
2727{
2728 llcache_event event;
2729
2730 /* The fetch has already been cleaned up by the fetcher but
2731 * we would like to retry if we can.
2732 */
2733 if (object->fetch.retries_remaining > 1) {
2734 object->fetch.retries_remaining--;
2735 return llcache_object_refetch(object);
2736 }
2737
2738 /* The fetch has has already been cleaned up by the fetcher */
2739 object->fetch.state = LLCACHE_FETCH_COMPLETE;
2740 object->fetch.fetch = NULL;
2741
2742 /* Release candidate, if any */
2743 if (object->candidate != NULL) {
2744 object->candidate->candidate_count--;
2745 object->candidate = NULL;
2746 }
2747
2748 /* Invalidate cache control data */
2750
2751 event.type = LLCACHE_EVENT_ERROR;
2752 event.data.error.code = NSERROR_TIMEOUT;
2753 event.data.error.msg = NULL;
2754
2755 return llcache_send_event_to_users(object, &event);
2756}
2757
2758
2759/**
2760 * Construct a sorted list of objects available for writeout operation.
2761 *
2762 * The list contains fresh cacheable objects held in RAM with no
2763 * pending fetches. Any objects with a remaining lifetime less than
2764 * the configured minimum lifetime are simply not considered, they will
2765 * become stale before pushing to backing store is worth the cost.
2766 *
2767 * \todo calculate useful cost metrics to improve sorting.
2768 *
2769 * \param[out] lst_out list of candidate objects.
2770 * \param[out] lst_len_out Number of candidate objects in result.
2771 * \return NSERROR_OK with \a lst_out and \a lst_len_out updated or
2772 * error code.
2773 */
2774static nserror
2775build_candidate_list(struct llcache_object ***lst_out, int *lst_len_out)
2776{
2777 llcache_object *object, *next;
2778 struct llcache_object **lst;
2779 int lst_len = 0;
2780 int remaining_lifetime;
2781
2782#define MAX_PERSIST_PER_RUN 128
2783
2784 lst = calloc(MAX_PERSIST_PER_RUN, sizeof(struct llcache_object *));
2785 if (lst == NULL) {
2786 return NSERROR_NOMEM;
2787 }
2788
2789 for (object = llcache->cached_objects; object != NULL; object = next) {
2790 next = object->next;
2791
2792 /* Only consider http(s) for the disc cache. */
2793 if (!llcache__scheme_is_persistable(object->url)) {
2794 continue;
2795 }
2796
2797 remaining_lifetime = llcache_object_rfc2616_remaining_lifetime(
2798 &object->cache);
2799
2800 /* cacehable objects with no pending fetches, not
2801 * already on disc and with sufficient lifetime to
2802 * make disc cache worthwhile
2803 */
2804 if ((object->candidate_count == 0) &&
2805 (object->fetch.fetch == NULL) &&
2806 (object->store_state == LLCACHE_STATE_RAM) &&
2807 (remaining_lifetime > llcache->minimum_lifetime)) {
2808 lst[lst_len] = object;
2809 lst_len++;
2810 if (lst_len == MAX_PERSIST_PER_RUN)
2811 break;
2812 }
2813 }
2814
2815 if (lst_len == 0) {
2816 free(lst);
2817 return NSERROR_NOT_FOUND;
2818 }
2819
2820 /** \todo sort list here */
2821
2822 *lst_len_out = lst_len;
2823 *lst_out = lst;
2824
2825#undef MAX_PERSIST_PER_RUN
2826
2827 return NSERROR_OK;
2828}
2829
2830/**
2831 * Write an object to the backing store.
2832 *
2833 * \param object The object to put in the backing store.
2834 * \param written_out The amount of data written out.
2835 * \param elapsed The time in ms it took to complete the write to backing store.
2836 * \return NSERROR_OK on success or appropriate error code.
2837 */
2838static nserror
2840 size_t *written_out,
2841 unsigned long *elapsed)
2842{
2843 nserror ret;
2844 uint8_t *metadata;
2845 size_t metadatasize;
2846 uint64_t startms = 0;
2847 uint64_t endms = 1000;
2848
2849 nsu_getmonotonic_ms(&startms);
2850
2851 /* put object data in backing store */
2852 ret = guit->llcache->store(object->url,
2854 object->source_data,
2855 object->source_len);
2856 if (ret != NSERROR_OK) {
2857 /* unable to put source data in backing store */
2858 return ret;
2859 }
2860
2861 ret = llcache_serialise_metadata(object, &metadata, &metadatasize);
2862 if (ret != NSERROR_OK) {
2863 /* There has been a metadata serialisation error. Ensure the
2864 * already written data object is invalidated.
2865 */
2866 guit->llcache->invalidate(object->url);
2867 return ret;
2868 }
2869
2870 ret = guit->llcache->store(object->url,
2872 metadata,
2873 metadatasize);
2875 if (ret != NSERROR_OK) {
2876 /* There has been an error putting the metadata in the
2877 * backing store. Ensure the data object is invalidated.
2878 */
2879 guit->llcache->invalidate(object->url);
2880 return ret;
2881 }
2882 nsu_getmonotonic_ms(&endms);
2883
2884 object->store_state = LLCACHE_STATE_DISC;
2885
2886 *written_out = object->source_len + metadatasize;
2887
2888 /* by ignoring the overflow this assumes the writeout took
2889 * less than 5 weeks.
2890 */
2891 *elapsed = endms - startms;
2892
2893 /* ensure the writeout is reported to have taken at least the
2894 * minimal amount of time
2895 */
2896 if (*elapsed == 0) {
2897 *elapsed = 1;
2898 }
2899
2900 return NSERROR_OK;
2901}
2902
2903/**
2904 * Check for overall write performance.
2905 *
2906 * If the overall write bandwidth has fallen below a useful level for
2907 * the backing store to be effective disable it.
2908 *
2909 * It is important to ensure a useful amount of data has been written
2910 * before calculating bandwidths otherwise tiny files taking a
2911 * disproportionately long time to write might trigger this erroneously.
2912 *
2913 * \param p The context pointer passed to the callback.
2914 */
2915static void llcache_persist_slowcheck(void *p)
2916{
2917 uint64_t total_bandwidth; /* total bandwidth */
2918
2920
2921 total_bandwidth = (llcache->total_written * 1000) / llcache->total_elapsed;
2922
2923 if (total_bandwidth < llcache->minimum_bandwidth) {
2924 NSLOG(llcache, INFO,
2925 "Current bandwidth %"PRIu64" less than minimum %"PRIsizet,
2926 total_bandwidth,
2928 guit->llcache->finalise();
2929 }
2930 }
2931}
2932
2933/**
2934 * Possibly write objects data to backing store.
2935 *
2936 * \param p The context pointer passed to the callback.
2937 */
2938static void llcache_persist(void *p)
2939{
2940 nserror ret;
2941 struct llcache_object **lst; /* candidate object list */
2942 int lst_count; /* number of candidates in list */
2943 int idx; /* current candidate object index in list */
2944 int next = -1; /* when the next run should be scheduled for */
2945
2946 unsigned long write_limit; /* max number of bytes to write in this run*/
2947
2948 size_t written; /* all bytes written for a single object */
2949 unsigned long elapsed; /* how long writing an object took */
2950
2951 size_t total_written = 0; /* total bytes written in this run */
2952 unsigned long total_elapsed = 1; /* total ms used to write bytes */
2953 unsigned long total_bandwidth = 0; /* total bandwidth */
2954
2955 ret = build_candidate_list(&lst, &lst_count);
2956 if (ret != NSERROR_OK) {
2957 NSLOG(llcache, DEBUG, "Unable to construct candidate list for persistent writeout");
2958 return;
2959 }
2960
2961 write_limit = (llcache->maximum_bandwidth * llcache->time_quantum) / 1000;
2962
2963 /* obtained a candidate list, make each object persistent in turn */
2964 for (idx = 0; idx < lst_count; idx++) {
2965 ret = write_backing_store(lst[idx], &written, &elapsed);
2966 if (ret != NSERROR_OK) {
2967 continue;
2968 }
2969
2970 /* successfully wrote object to backing store */
2971 total_written += written;
2972 total_elapsed += elapsed;
2973 total_bandwidth = (total_written * 1000) / total_elapsed;
2974
2975 NSLOG(llcache, DEBUG,
2976 "Wrote %"PRIsizet" bytes in %lums bw:%lu %s",
2977 written, elapsed, (written * 1000) / elapsed,
2978 nsurl_access(lst[idx]->url) );
2979
2980 /* check to for the time quantum or the size
2981 * (bandwidth) for this run being exceeded.
2982 */
2983 if (total_elapsed > llcache->time_quantum) {
2984 NSLOG(llcache, INFO, "Overran timeslot");
2985 /* writeout has exhausted the available time.
2986 * Either the writeout is slow or the last
2987 * object was very large.
2988 */
2989 if (total_bandwidth < llcache->minimum_bandwidth) {
2990 /* Writeout was slow in this time quantum.
2991 * Schedule a check in the future to see if
2992 * overall performance is too slow to be useful.
2993 */
2994 guit->misc->schedule(
2995 llcache->time_quantum * 100,
2997 NULL);
2998 break;
2999 } else {
3000 if (total_bandwidth > llcache->maximum_bandwidth) {
3001 /* fast writeout of large file
3002 * so calculate delay as if
3003 * write happened only at max
3004 * limit
3005 */
3006 next = ((total_written * llcache->time_quantum) / write_limit) - total_elapsed;
3007 } else {
3009 }
3010 break;
3011 }
3012 } else if (total_written > write_limit) {
3013 /* The bandwidth limit has been reached. */
3014
3015 if (total_bandwidth > llcache->maximum_bandwidth) {
3016 /* fast writeout of large file so
3017 * calculate delay as if write
3018 * happened only at max limit
3019 */
3020 next = ((total_written * llcache->time_quantum) / write_limit) - total_elapsed;
3021 } else {
3022 next = llcache->time_quantum - total_elapsed;
3023 }
3024 break;
3025 }
3026
3027 }
3028 free(lst);
3029
3030 /* Completed list without running out of allowed bytes or time */
3031 if (idx == lst_count) {
3032 /* only reschedule if writing is making any progress at all */
3033 if (total_written > 0) {
3034 next = llcache->time_quantum - total_elapsed;
3035 } else {
3036 next = -1;
3037 }
3038 }
3039
3040 llcache->total_written += total_written;
3041 llcache->total_elapsed += total_elapsed;
3042
3043 NSLOG(llcache, DEBUG,
3044 "writeout size:%"PRIsizet" time:%lu bandwidth:%lubytes/s",
3045 total_written, total_elapsed, total_bandwidth);
3046
3047 NSLOG(llcache, DEBUG, "Rescheduling writeout in %dms", next);
3049}
3050
3051
3052/**
3053 * Handler for fetch events
3054 *
3055 * \param msg Fetch event
3056 * \param p Our private data
3057 */
3058static void llcache_fetch_callback(const fetch_msg *msg, void *p)
3059{
3060 nserror error = NSERROR_OK;
3061 llcache_object *object = p;
3062 llcache_event event;
3063
3064 if (llcache == NULL) {
3065 NSLOG(llcache, CRITICAL, "Callback happened after llcache finalisation");
3066 assert(false);
3067 /* In case assertions are off, return here */
3068 return;
3069 }
3070
3071 NSLOG(llcache, DEBUG, "Fetch event %d for %p", msg->type, object);
3072
3073 switch (msg->type) {
3074 case FETCH_HEADER:
3075 /* Received a fetch header */
3076 object->fetch.state = LLCACHE_FETCH_HEADERS;
3077
3078 error = llcache_fetch_process_header(object,
3079 msg->data.header_or_data.buf,
3080 msg->data.header_or_data.len);
3081 break;
3082
3083 /* 3xx responses */
3084 case FETCH_REDIRECT:
3085 /* Request resulted in a redirect */
3086
3087 /* Release candidate, if any */
3088 if (object->candidate != NULL) {
3089 object->candidate->candidate_count--;
3090 object->candidate = NULL;
3091 }
3092
3093 error = llcache_fetch_redirect(object,
3094 msg->data.redirect, &object);
3095 break;
3096
3097 case FETCH_NOTMODIFIED:
3098 /* Conditional request determined that cached object is fresh */
3099 error = llcache_fetch_notmodified(object, &object);
3100 break;
3101
3102 /* Normal 2xx state machine */
3103 case FETCH_DATA:
3104 /* Received some data */
3105 error = llcache_fetch_process_data(object,
3106 msg->data.header_or_data.buf,
3107 msg->data.header_or_data.len);
3108 break;
3109
3110 case FETCH_FINISHED:
3111 /* Finished fetching */
3112 {
3113 uint8_t *temp;
3114
3115 object->fetch.state = LLCACHE_FETCH_COMPLETE;
3116 object->fetch.fetch = NULL;
3117
3118 /* Shrink source buffer to required size */
3119 temp = realloc(object->source_data,
3120 object->source_len);
3121 /* If source_len is 0, then temp may be NULL */
3122 if (temp != NULL || object->source_len == 0) {
3123 object->source_data = temp;
3124 object->source_alloc = object->source_len;
3125 }
3126
3128
3129 /* record when the fetch finished */
3130 object->cache.fin_time = time(NULL);
3131
3132 (void) llcache_hsts_update_policy(object);
3133
3134 guit->misc->schedule(5000, llcache_persist, NULL);
3135 }
3136 break;
3137
3138 /* Out-of-band information */
3139 case FETCH_TIMEDOUT:
3140 /* Timed out while trying to fetch. */
3141 error = llcache_fetch_timeout(object);
3142 break;
3143
3144 case FETCH_ERROR:
3145 /* An error occurred while fetching */
3146 /* The fetch has has already been cleaned up by the fetcher */
3147 object->fetch.state = LLCACHE_FETCH_COMPLETE;
3148 object->fetch.fetch = NULL;
3149
3150 /* Release candidate, if any */
3151 if (object->candidate != NULL) {
3152 object->candidate->candidate_count--;
3153 object->candidate = NULL;
3154 }
3155
3156 /* Invalidate cache control data */
3158
3159 /** \todo Consider using errorcode for something */
3160
3161 event.type = LLCACHE_EVENT_ERROR;
3162 event.data.error.code = NSERROR_UNKNOWN;
3163 event.data.error.msg = msg->data.error;
3164
3165 error = llcache_send_event_to_users(object, &event);
3166
3167 break;
3168
3169 case FETCH_PROGRESS:
3170 /* Progress update */
3171 event.type = LLCACHE_EVENT_PROGRESS;
3172 event.data.progress_msg = msg->data.progress;
3173
3174 error = llcache_send_event_to_users(object, &event);
3175
3176 break;
3177
3178 case FETCH_CERTS:
3179 /* Certificate information from the fetch */
3180
3181 if (object->chain != NULL) {
3182 cert_chain_free(object->chain);
3183 object->chain = NULL;
3184 }
3185
3186 /* Persist the chain onto our object */
3187 error = cert_chain_dup(msg->data.chain, &object->chain);
3188 if (error != NSERROR_OK) {
3189 NSLOG(llcache, ERROR,
3190 "Unable to duplicate cert chain into cache: %s",
3191 messages_get_errorcode(error));
3192 }
3193
3194 /* Now pass on the event */
3195 event.type = LLCACHE_EVENT_GOT_CERTS;
3196 event.data.chain = msg->data.chain;
3197
3198 error = llcache_send_event_to_users(object, &event);
3199
3200 break;
3201
3202 /* Events requiring action */
3203 case FETCH_AUTH:
3204 /* Need Authentication */
3205
3206 /* Release candidate, if any */
3207 if (object->candidate != NULL) {
3208 object->candidate->candidate_count--;
3209 object->candidate = NULL;
3210 }
3211
3212 error = llcache_fetch_auth(object, msg->data.auth.realm);
3213 break;
3214
3215 case FETCH_CERT_ERR:
3216 /* Something went wrong when validating TLS certificates */
3217
3218 /* Release candidate, if any */
3219 if (object->candidate != NULL) {
3220 object->candidate->candidate_count--;
3221 object->candidate = NULL;
3222 }
3223
3224 error = llcache_fetch_cert_error(object);
3225 break;
3226
3227 case FETCH_SSL_ERR:
3228 /* TLS connection setup failed */
3229
3230 /* Release candidate, if any */
3231 if (object->candidate != NULL) {
3232 object->candidate->candidate_count--;
3233 object->candidate = NULL;
3234 }
3235
3236 error = llcache_fetch_ssl_error(object);
3237 break;
3238 }
3239
3240 /* Deal with any errors reported by event handlers */
3241 if (error != NSERROR_OK) {
3242 if (error == NSERROR_NOMEM) {
3243 /* attempt to purge the cache to free some
3244 * memory. will not help this fetch, but may
3245 * allow the UI to report errors etc.
3246 */
3247 llcache_clean(true);
3248 }
3249
3250 if (object->fetch.fetch != NULL) {
3251 fetch_abort(object->fetch.fetch);
3252 object->fetch.fetch = NULL;
3253
3254 /* Invalidate cache control data */
3256
3257 object->fetch.state = LLCACHE_FETCH_COMPLETE;
3258 }
3259 }
3260
3261 /* There may be users which are not caught up so schedule ourselves */
3263}
3264
3265/**
3266 * Find a user of a low-level cache object
3267 *
3268 * \param handle External cache handle to search for
3269 * \return Pointer to corresponding user, or NULL if not found
3270 */
3272 const llcache_handle *handle)
3273{
3274 llcache_object_user *user;
3275
3276 assert(handle->object != NULL);
3277
3278 for (user = handle->object->users; user != NULL; user = user->next) {
3279 if (user->handle == handle)
3280 break;
3281 }
3282
3283 return user;
3284}
3285
3286
3287/**
3288 * Determine if a low-level cache object resides in a given list
3289 *
3290 * \param object Object to search for
3291 * \param list List to search in
3292 * \return True if object resides in list, false otherwise
3293 */
3294static bool llcache_object_in_list(const llcache_object *object,
3295 const llcache_object *list)
3296{
3297 while (list != NULL) {
3298 if (list == object)
3299 break;
3300
3301 list = list->next;
3302 }
3303
3304 return list != NULL;
3305}
3306
3307/**
3308 * Notify users of an object's current state
3309 *
3310 * \param object Object to notify users about
3311 * \return NSERROR_OK on success, appropriate error otherwise
3312 */
3314{
3315 nserror error;
3316 llcache_object_user *user, *next_user;
3317 llcache_event event;
3318 bool emitted_notify = false;
3319
3320 /**
3321 * State transitions and event emission for users.
3322 * Rows: user state. Cols: object state.
3323 *
3324 * User - Obj INIT HEADERS DATA COMPLETE
3325 * INIT - T T* T*
3326 * HEADERS - - T T*
3327 * DATA - - M T
3328 * COMPLETE - - - -
3329 *
3330 * T => transition user to object state
3331 * M => no transition required, but may need to emit event
3332 *
3333 * The transitions marked with an asterisk can be removed by moving
3334 * the user context into the subsequent state and then reevaluating.
3335 *
3336 * Events are issued as follows:
3337 *
3338 * HAD_HEADERS: on transition from HEADERS -> DATA state
3339 * HAD_DATA : in DATA state, whenever there's new source data
3340 * DONE : on transition from DATA -> COMPLETE state
3341 */
3342
3343 for (user = object->users; user != NULL; user = next_user) {
3344 /* Emit necessary events to bring the user up-to-date */
3345 llcache_handle *handle = user->handle;
3346 const llcache_fetch_state objstate = object->fetch.state;
3347
3348 /* Flag that this user is the current iteration target
3349 * in case the client attempts to destroy it underneath us */
3350 user->iterator_target = true;
3351
3352 /* A note on the computation of next_user:
3353 *
3354 * Within this loop, we may make a number of calls to
3355 * client code. Our contract with clients is that they
3356 * can do whatever they like from within their callback
3357 * handlers. This is so that we limit the pain of
3358 * reentrancy to this module alone.
3359 *
3360 * One of the things a client can do from within its
3361 * callback handler is to remove users from this object's
3362 * user list. In the common case, the user they attempt
3363 * to remove is the current iteration target, and we
3364 * already protect against that causing problems here.
3365 * However, no such protection exists if the client
3366 * attempts to remove other users from this object's
3367 * user list.
3368 *
3369 * Therefore, we cannot compute next_user up-front
3370 * and expect it to remain valid across calls to
3371 * client code (as the identity of the next user
3372 * in the list may change underneath us). Instead,
3373 * we must compute next_user at the point where we
3374 * are about to cause another iteration of this loop
3375 * (i.e. at the very end, and also at the points where
3376 * continue is used)
3377 */
3378
3379 if (handle->state != objstate) {
3380 if (emitted_notify == false) {
3381 NSLOG(llcache, DEBUG,
3382 "Notifying users of %p",
3383 object);
3384 emitted_notify = true;
3385 }
3386
3387 NSLOG(llcache, DEBUG,
3388 "User %p state: %d Object state: %d",
3389 user,
3390 handle->state,
3391 objstate);
3392 }
3393
3394 /* User: INIT, Obj: HEADERS, DATA, COMPLETE => User->HEADERS */
3395 if (handle->state == LLCACHE_FETCH_INIT &&
3396 objstate > LLCACHE_FETCH_INIT) {
3397 handle->state = LLCACHE_FETCH_HEADERS;
3398
3399 /* Emit any certificate data we hold */
3400 if (object->chain != NULL) {
3401 event.type = LLCACHE_EVENT_GOT_CERTS;
3402 event.data.chain = object->chain;
3403 error = handle->cb(handle, &event, handle->pw);
3404 } else {
3405 error = NSERROR_OK;
3406 }
3407
3408 if (user->queued_for_delete) {
3409 next_user = user->next;
3410 llcache_object_remove_user(object, user);
3412
3413 if (error != NSERROR_OK)
3414 return error;
3415
3416 continue;
3417 } else if (error == NSERROR_NEED_DATA) {
3418 /* User requested replay */
3419 handle->state = LLCACHE_FETCH_HEADERS;
3420
3421 /* Continue with the next user -- we'll
3422 * reemit the event next time round */
3423 user->iterator_target = false;
3424 next_user = user->next;
3426 continue;
3427 } else if (error != NSERROR_OK) {
3428 user->iterator_target = false;
3429 return error;
3430 }
3431 }
3432
3433 /* User: HEADERS, Obj: DATA, COMPLETE => User->DATA */
3434 if (handle->state == LLCACHE_FETCH_HEADERS &&
3435 objstate > LLCACHE_FETCH_HEADERS) {
3436 handle->state = LLCACHE_FETCH_DATA;
3437
3438 /* Emit HAD_HEADERS event */
3439 event.type = LLCACHE_EVENT_HAD_HEADERS;
3440
3441 error = handle->cb(handle, &event, handle->pw);
3442
3443 if (user->queued_for_delete) {
3444 next_user = user->next;
3445 llcache_object_remove_user(object, user);
3447
3448 if (error != NSERROR_OK)
3449 return error;
3450
3451 continue;
3452 } else if (error == NSERROR_NEED_DATA) {
3453 /* User requested replay */
3454 handle->state = LLCACHE_FETCH_HEADERS;
3455
3456 /* Continue with the next user -- we'll
3457 * reemit the event next time round */
3458 user->iterator_target = false;
3459 next_user = user->next;
3461 continue;
3462 } else if (error != NSERROR_OK) {
3463 user->iterator_target = false;
3464 return error;
3465 }
3466 }
3467
3468 /* User: DATA, Obj: DATA, COMPLETE, more source available */
3469 if (handle->state == LLCACHE_FETCH_DATA &&
3470 objstate >= LLCACHE_FETCH_DATA &&
3471 object->source_len > handle->bytes) {
3472 size_t orig_handle_read;
3473
3474 /* Construct HAD_DATA event */
3475 event.type = LLCACHE_EVENT_HAD_DATA;
3476 event.data.data.buf =
3477 object->source_data + handle->bytes;
3478 event.data.data.len =
3479 object->source_len - handle->bytes;
3480
3481 /* Update record of last byte emitted */
3482 if (object->fetch.flags &
3484 /* Streaming, so reset to zero to
3485 * minimise amount of cached source data.
3486 * Additionally, we don't support replay
3487 * when streaming. */
3488 orig_handle_read = 0;
3489 handle->bytes = object->source_len = 0;
3490 } else {
3491 orig_handle_read = handle->bytes;
3492 handle->bytes = object->source_len;
3493 }
3494
3495 /* Emit event */
3496 error = handle->cb(handle, &event, handle->pw);
3497 if (user->queued_for_delete) {
3498 next_user = user->next;
3499 llcache_object_remove_user(object, user);
3501
3502 if (error != NSERROR_OK)
3503 return error;
3504
3505 continue;
3506 } else if (error == NSERROR_NEED_DATA) {
3507 /* User requested replay */
3508 handle->bytes = orig_handle_read;
3509
3510 /* Continue with the next user -- we'll
3511 * reemit the data next time round */
3512 user->iterator_target = false;
3513 next_user = user->next;
3515 continue;
3516 } else if (error != NSERROR_OK) {
3517 user->iterator_target = false;
3518 return error;
3519 }
3520 }
3521
3522 /* User: DATA, Obj: COMPLETE => User->COMPLETE */
3523 if (handle->state == LLCACHE_FETCH_DATA &&
3524 objstate > LLCACHE_FETCH_DATA) {
3525 handle->state = LLCACHE_FETCH_COMPLETE;
3526
3527 /* Emit DONE event */
3528 event.type = LLCACHE_EVENT_DONE;
3529
3530 error = handle->cb(handle, &event, handle->pw);
3531 if (user->queued_for_delete) {
3532 next_user = user->next;
3533 llcache_object_remove_user(object, user);
3535
3536 if (error != NSERROR_OK)
3537 return error;
3538
3539 continue;
3540 } else if (error == NSERROR_NEED_DATA) {
3541 /* User requested replay */
3542 handle->state = LLCACHE_FETCH_DATA;
3543
3544 /* Continue with the next user -- we'll
3545 * reemit the event next time round */
3546 user->iterator_target = false;
3547 next_user = user->next;
3549 continue;
3550 } else if (error != NSERROR_OK) {
3551 user->iterator_target = false;
3552 return error;
3553 }
3554 }
3555
3556 /* No longer the target of an iterator */
3557 user->iterator_target = false;
3558
3559 next_user = user->next;
3560 }
3561
3562 return NSERROR_OK;
3563}
3564
3565/**
3566 * Make a snapshot of the current state of an llcache_object.
3567 *
3568 * This has the side-effect of the new object being non-cacheable,
3569 * also not-fetching and not a candidate for any other object.
3570 *
3571 * Also note that this new object has no users and at least one
3572 * should be assigned to it before llcache_clean is entered or it
3573 * will be immediately cleaned up.
3574 *
3575 * \param object The object to take a snapshot of
3576 * \param snapshot Pointer to receive snapshot of \a object
3577 * \return NSERROR_OK on success, appropriate error otherwise
3578 */
3579static nserror
3581{
3582 llcache_object *newobj;
3583 nserror error;
3584
3585 error = llcache_object_new(object->url, &newobj);
3586
3587 if (error != NSERROR_OK)
3588 return error;
3589
3590 newobj->source_alloc = newobj->source_len = object->source_len;
3591
3592 if (object->source_len > 0) {
3593 newobj->source_data = malloc(newobj->source_alloc);
3594 if (newobj->source_data == NULL) {
3595 llcache_object_destroy(newobj);
3596 return NSERROR_NOMEM;
3597 }
3598 memcpy(newobj->source_data, object->source_data,
3599 newobj->source_len);
3600 }
3601
3602 if (object->num_headers > 0) {
3603 newobj->headers = calloc(object->num_headers,
3604 sizeof(llcache_header));
3605 if (newobj->headers == NULL) {
3606 llcache_object_destroy(newobj);
3607 return NSERROR_NOMEM;
3608 }
3609 while (newobj->num_headers < object->num_headers) {
3610 llcache_header *nh =
3611 &(newobj->headers[newobj->num_headers]);
3612 llcache_header *oh =
3613 &(object->headers[newobj->num_headers]);
3614 newobj->num_headers += 1;
3615 nh->name = strdup(oh->name);
3616 nh->value = strdup(oh->value);
3617 if (nh->name == NULL || nh->value == NULL) {
3618 llcache_object_destroy(newobj);
3619 return NSERROR_NOMEM;
3620 }
3621 }
3622 }
3623
3624 if (object->chain != NULL) {
3625 error = cert_chain_dup(object->chain, &newobj->chain);
3626 if (error != NSERROR_OK) {
3627 llcache_object_destroy(newobj);
3628 return error;
3629 }
3630 }
3631
3633
3634 *snapshot = newobj;
3635
3636 return NSERROR_OK;
3637}
3638
3639/**
3640 * total ram usage of object
3641 *
3642 * \param object The object to calculate the total RAM usage of.
3643 * \return The total RAM usage in bytes.
3644 */
3645static inline uint32_t
3647{
3648 uint32_t tot;
3649 size_t hdrc;
3650
3651 tot = sizeof(*object);
3652 tot += nsurl_length(object->url);
3653
3654 if (object->source_data != NULL) {
3655 tot += object->source_len;
3656 }
3657
3658 tot += sizeof(llcache_header) * object->num_headers;
3659
3660 for (hdrc = 0; hdrc < object->num_headers; hdrc++) {
3661 if (object->headers[hdrc].name != NULL) {
3662 tot += strlen(object->headers[hdrc].name);
3663 }
3664 if (object->headers[hdrc].value != NULL) {
3665 tot += strlen(object->headers[hdrc].value);
3666 }
3667 }
3668
3669 tot += cert_chain_size(object->chain);
3670
3671 return tot;
3672}
3673
3674/**
3675 * Catch up the cache users with state changes from fetchers.
3676 *
3677 * \param ignored We ignore this because all our state comes from llcache.
3678 */
3679static void llcache_catch_up_all_users(void *ignored)
3680{
3681 llcache_object *object;
3682
3683 /* Assume after this we'll be all caught up. If any user of a handle
3684 * defers then we'll invalidate all_caught_up and reschedule via
3685 * llcache_users_not_caught_up()
3686 */
3687 llcache->all_caught_up = true;
3688
3689 /* Catch new users up with state of objects */
3690 for (object = llcache->cached_objects; object != NULL;
3691 object = object->next) {
3693 }
3694
3695 for (object = llcache->uncached_objects; object != NULL;
3696 object = object->next) {
3698 }
3699}
3700
3701/**
3702 * Ask for ::llcache_catch_up_all_users to be scheduled ASAP to pump the
3703 * user state machines.
3704 */
3706{
3707 if (llcache->all_caught_up) {
3708 llcache->all_caught_up = false;
3710 }
3711}
3712
3713
3714/******************************************************************************
3715 * Public API *
3716 ******************************************************************************/
3717
3718/*
3719 * Attempt to clean the cache
3720 *
3721 * The memory cache cleaning discards objects in order of increasing value.
3722 *
3723 * Exported interface documented in llcache.h
3724 */
3725void llcache_clean(bool purge)
3726{
3727 llcache_object *object, *next;
3728 uint32_t llcache_size = 0;
3729 int remaining_lifetime;
3730 uint32_t limit;
3731
3732 NSLOG(llcache, DEBUG, "Attempting cache clean");
3733
3734 /* If the cache is being purged set the size limit to zero. */
3735 if (purge) {
3736 limit = 0;
3737 } else {
3738 limit = llcache->limit;
3739 }
3740
3741 /* Uncacheable objects with no users or fetches */
3742 for (object = llcache->uncached_objects;
3743 object != NULL;
3744 object = next) {
3745 next = object->next;
3746
3747 /* The candidate count of uncacheable objects is always 0 */
3748 if ((object->users == NULL) &&
3749 (object->candidate_count == 0) &&
3750 (object->fetch.fetch == NULL)) {
3751 NSLOG(llcache, DEBUG, "Discarding uncachable object with no users (%p) %s",
3752 object, nsurl_access(object->url));
3753
3756 llcache_object_destroy(object);
3757 } else {
3758 llcache_size += total_object_size(object);
3759 }
3760 }
3761
3762
3763 /* Stale cacheable objects with no users or pending fetches */
3764 for (object = llcache->cached_objects;
3765 object != NULL;
3766 object = next) {
3767 next = object->next;
3768
3769 remaining_lifetime = llcache_object_rfc2616_remaining_lifetime(
3770 &object->cache);
3771
3772 if ((object->users == NULL) &&
3773 (object->candidate_count == 0) &&
3774 (object->fetch.fetch == NULL) &&
3775 (remaining_lifetime <= 0)) {
3776 /* object is stale */
3777 NSLOG(llcache, DEBUG, "discarding stale cacheable object with no "
3778 "users or pending fetches (%p) %s",
3779 object, nsurl_access(object->url));
3780
3783
3784 if (object->store_state == LLCACHE_STATE_DISC) {
3785 guit->llcache->invalidate(object->url);
3786 }
3787
3788 llcache_object_destroy(object);
3789
3790 } else {
3791 /* object has users so account for the storage */
3792 llcache_size += total_object_size(object);
3793 }
3794 }
3795
3796 /* if the cache limit is exceeded try to make some objects
3797 * persistent so their RAM can be reclaimed in the next
3798 * step
3799 */
3800 if (limit < llcache_size) {
3801 llcache_persist(NULL);
3802 }
3803
3804 /* Source data of fresh cacheable objects with no users, no
3805 * pending fetches and pushed to persistent store while the
3806 * cache exceeds the configured size.
3807 */
3808 for (object = llcache->cached_objects;
3809 ((limit < llcache_size) && (object != NULL));
3810 object = next) {
3811 next = object->next;
3812 if ((object->users == NULL) &&
3813 (object->candidate_count == 0) &&
3814 (object->fetch.fetch == NULL) &&
3815 (object->store_state == LLCACHE_STATE_DISC)) {
3817
3818 object->source_data = NULL;
3819
3820 llcache_size -= object->source_len;
3821
3822 NSLOG(llcache, DEBUG,
3823 "Freeing source data for %p len:%"PRIsizet,
3824 object, object->source_len);
3825 }
3826 }
3827
3828 /* Fresh cacheable objects with no users, no pending fetches
3829 * and pushed to persistent store while the cache exceeds
3830 * the configured size. Effectively just the llcache object metadata.
3831 */
3832 for (object = llcache->cached_objects;
3833 ((limit < llcache_size) && (object != NULL));
3834 object = next) {
3835 next = object->next;
3836 if ((object->users == NULL) &&
3837 (object->candidate_count == 0) &&
3838 (object->fetch.fetch == NULL) &&
3839 (object->store_state == LLCACHE_STATE_DISC) &&
3840 (object->source_data == NULL)) {
3841 NSLOG(llcache, DEBUG,
3842 "discarding backed object len:%"PRIsizet" age:%ld (%p) %s",
3843 object->source_len,
3844 (long)(time(NULL) - object->last_used),
3845 object,
3846 nsurl_access(object->url));
3847
3848 llcache_size -= total_object_size(object);
3849
3852 llcache_object_destroy(object);
3853
3854 }
3855 }
3856
3857 /* Fresh cacheable objects with no users or pending fetches
3858 * while the cache exceeds the configured size. These are the
3859 * most valuable objects as replacing them is a full network
3860 * fetch
3861 */
3862 for (object = llcache->cached_objects;
3863 ((limit < llcache_size) && (object != NULL));
3864 object = next) {
3865 next = object->next;
3866
3867 if ((object->users == NULL) &&
3868 (object->candidate_count == 0) &&
3869 (object->fetch.fetch == NULL) &&
3870 (object->store_state == LLCACHE_STATE_RAM)) {
3871 NSLOG(llcache, DEBUG,
3872 "discarding fresh object len:%"PRIsizet" age:%ld (%p) %s",
3873 object->source_len,
3874 (long)(time(NULL) - object->last_used),
3875 object,
3876 nsurl_access(object->url));
3877
3878 llcache_size -= object->source_len + sizeof(*object);
3879
3882 llcache_object_destroy(object);
3883 }
3884 }
3885
3886 NSLOG(llcache, DEBUG, "Size: %"PRIu32" (limit: %"PRIu32")", llcache_size, limit);
3887}
3888
3889/* Exported interface documented in content/llcache.h */
3890nserror
3892{
3893 llcache = calloc(1, sizeof(struct llcache_s));
3894 if (llcache == NULL) {
3895 return NSERROR_NOMEM;
3896 }
3897
3898 llcache->limit = prm->limit;
3904 llcache->all_caught_up = true;
3905
3906 NSLOG(llcache, INFO,
3907 "llcache initialising with a limit of %"PRIu32" bytes",
3908 llcache->limit);
3909
3910 /* backing store initialisation */
3911 return guit->llcache->initialise(&prm->store);
3912}
3913
3914
3915/* Exported interface documented in content/llcache.h */
3917{
3918 llcache_object *object, *next;
3919 uint64_t total_bandwidth = 0; /* total bandwidth */
3920
3921 /* Attempt to persist anything we have left lying around */
3922 llcache_persist(NULL);
3923 /* Now clear the persistence callback */
3924 guit->misc->schedule(-1, llcache_persist, NULL);
3925
3926 /* Clean uncached objects */
3927 for (object = llcache->uncached_objects; object != NULL; object = next) {
3928 llcache_object_user *user, *next_user;
3929
3930 next = object->next;
3931
3932 for (user = object->users; user != NULL; user = next_user) {
3933 next_user = user->next;
3934
3935 if (user->handle != NULL)
3936 free(user->handle);
3937
3938 free(user);
3939 }
3940
3941 /* Fetch system has already been destroyed */
3942 object->fetch.fetch = NULL;
3943
3944 llcache_object_destroy(object);
3945 }
3946
3947 /* Clean cached objects */
3948 for (object = llcache->cached_objects; object != NULL; object = next) {
3949 llcache_object_user *user, *next_user;
3950
3951 next = object->next;
3952
3953 for (user = object->users; user != NULL; user = next_user) {
3954 next_user = user->next;
3955
3956 if (user->handle != NULL)
3957 free(user->handle);
3958
3959 free(user);
3960 }
3961
3962 /* Fetch system has already been destroyed */
3963 object->fetch.fetch = NULL;
3964
3965 llcache_object_destroy(object);
3966 }
3967
3968 /* backing store finalisation */
3969 guit->llcache->finalise();
3970
3971 if (llcache->total_elapsed > 0) {
3972 total_bandwidth = (llcache->total_written * 1000) /
3974 }
3975
3976 NSLOG(llcache, INFO,
3977 "Backing store wrote %"PRIu64" bytes in %"PRIu64" ms ""(average %"PRIu64" bytes/second)",
3980 total_bandwidth);
3981
3982 free(llcache);
3983 llcache = NULL;
3984}
3985
3986
3987
3988/* Exported interface documented in content/llcache.h */
3989nserror
3991 uint32_t flags,
3992 nsurl *referer,
3993 const llcache_post_data *post,
3994 llcache_handle_callback cb, void *pw,
3996{
3997 nserror error;
3998 llcache_object_user *user;
3999 llcache_object *object;
4000 nsurl *hsts_url;
4001 bool hsts_in_use;
4002
4003 /* Perform HSTS transform */
4004 error = llcache_hsts_transform_url(url, &hsts_url, &hsts_in_use);
4005 if (error != NSERROR_OK) {
4006 return error;
4007 }
4008
4009 /* Can we fetch this URL at all? */
4010 if (fetch_can_fetch(hsts_url) == false) {
4011 nsurl_unref(hsts_url);
4013 }
4014
4015 /* Create a new object user */
4016 error = llcache_object_user_new(cb, pw, &user);
4017 if (error != NSERROR_OK) {
4018 nsurl_unref(hsts_url);
4019 return error;
4020 }
4021
4022 /* Retrieve a suitable object from the cache,
4023 * creating a new one if needed. */
4024 error = llcache_object_retrieve(hsts_url, flags, referer, post, 0,
4025 hsts_in_use, &object);
4026 if (error != NSERROR_OK) {
4028 nsurl_unref(hsts_url);
4029 return error;
4030 }
4031
4032 /* Add user to object */
4033 llcache_object_add_user(object, user);
4034
4035 *result = user->handle;
4036
4037 /* Users exist which are now not caught up! */
4039
4040 nsurl_unref(hsts_url);
4041
4042 return NSERROR_OK;
4043}
4044
4045
4046/* Exported interface documented in content/llcache.h */
4048 llcache_handle_callback cb, void *pw)
4049{
4050 handle->cb = cb;
4051 handle->pw = pw;
4052
4053 return NSERROR_OK;
4054}
4055
4056
4057/* Exported interface documented in content/llcache.h */
4059{
4060 nserror error = NSERROR_OK;
4061 llcache_object *object = handle->object;
4063
4064 assert(user != NULL);
4065
4066 if (user->iterator_target) {
4067 /* Can't remove / delete user object if it's
4068 * the target of an iterator */
4069 user->queued_for_delete = true;
4070 } else {
4071 /* Remove the user from the object and destroy it */
4072 error = llcache_object_remove_user(object, user);
4073 if (error == NSERROR_OK) {
4074 error = llcache_object_user_destroy(user);
4075 }
4076 }
4077
4078 return error;
4079}
4080
4081/* Exported interface documented in content/llcache.h */
4083{
4084 nserror error;
4085 llcache_object_user *newuser;
4086
4087 error = llcache_object_user_new(handle->cb, handle->pw, &newuser);
4088 if (error == NSERROR_OK) {
4089 llcache_object_add_user(handle->object, newuser);
4090 newuser->handle->state = handle->state;
4091 *result = newuser->handle;
4092 }
4093
4094 return error;
4095}
4096
4097/* See llcache.h for documentation */
4099{
4101 llcache_object *object = handle->object, *newobject;
4102 nserror error = NSERROR_OK;
4103 bool all_alone = true;
4104
4105 /* Determine if we are the only user */
4106 if (user->prev != NULL)
4107 all_alone = false;
4108 if (user->next != NULL)
4109 all_alone = false;
4110
4111 if (all_alone == false) {
4112 /* We must snapshot this object */
4113 error = llcache_object_snapshot(object, &newobject);
4114 if (error != NSERROR_OK)
4115 return error;
4116
4117 /* Move across to the new object */
4118 if (user->iterator_target) {
4119 /* User is current iterator target, clone it */
4120 llcache_object_user *newuser =
4121 calloc(1, sizeof(llcache_object_user));
4122 if (newuser == NULL) {
4123 llcache_object_destroy(newobject);
4124 return NSERROR_NOMEM;
4125 }
4126
4127 /* Move handle across to clone */
4128 newuser->handle = user->handle;
4129 user->handle = NULL;
4130
4131 /* Mark user as needing deletion */
4132 user->queued_for_delete = true;
4133
4134 llcache_object_add_user(newobject, newuser);
4135 } else {
4136 llcache_object_remove_user(object, user);
4137 llcache_object_add_user(newobject, user);
4138 }
4139
4140 /* Add new object to uncached list */
4143 } else {
4144 /* We're the only user, so abort any fetch in progress */
4145 if (object->fetch.fetch != NULL) {
4146 fetch_abort(object->fetch.fetch);
4147 object->fetch.fetch = NULL;
4148 }
4149
4150 object->fetch.state = LLCACHE_FETCH_COMPLETE;
4151
4152 /* Invalidate cache control data */
4154 }
4155
4156 return error;
4157}
4158
4159/* See llcache.h for documentation */
4161{
4163 llcache_object *object = handle->object;
4164
4165 /* Cannot stream if there are multiple users */
4166 if (user->prev != NULL || user->next != NULL)
4167 return NSERROR_OK;
4168
4169 /* Forcibly uncache this object */
4174 }
4175
4176 object->fetch.flags |= LLCACHE_RETRIEVE_STREAM_DATA;
4177
4178 return NSERROR_OK;
4179}
4180
4181/* See llcache.h for documentation */
4183{
4184 if ((handle->object != NULL) &&
4185 (handle->object->fetch.fetch == NULL) &&
4187 /* mark the cached object as requiring validation */
4189 }
4190
4191 return NSERROR_OK;
4192}
4193
4194/* See llcache.h for documentation */
4196{
4197 return handle->object != NULL ? handle->object->url : NULL;
4198}
4199
4200/* See llcache.h for documentation */
4202 size_t *size)
4203{
4204 *size = handle->object != NULL ? handle->object->source_len : 0;
4205
4206 return handle->object != NULL ? handle->object->source_data : NULL;
4207}
4208
4209/* See llcache.h for documentation */
4211 const char *key)
4212{
4213 const llcache_object *object = handle->object;
4214 size_t i;
4215
4216 if (object == NULL)
4217 return NULL;
4218
4219 /* About as trivial as possible */
4220 for (i = 0; i < object->num_headers; i++) {
4221 if (strcasecmp(key, object->headers[i].name) == 0)
4222 return object->headers[i].value;
4223 }
4224
4225 return NULL;
4226}
4227
4228/* See llcache.h for documentation */
4230 const llcache_handle *b)
4231{
4232 return a->object == b->object;
4233}
STATIC char result[100]
Definition: arexx.c:77
static struct bitmap snapshot
Definition: plot.c:206
Low-level source data cache backing store interface.
@ BACKING_STORE_NONE
no special processing
Definition: backing_store.h:32
@ BACKING_STORE_META
data is metadata
Definition: backing_store.h:34
uint32_t http_cache_control_max_age(http_cache_control *cc)
Get the value of a cache control's max-age.
bool http_cache_control_has_max_age(http_cache_control *cc)
Determine if a valid max-age directive is present.
nserror http_parse_cache_control(const char *header_value, http_cache_control **result)
Parse an HTTP Cache-Control header value.
void http_cache_control_destroy(http_cache_control *victim)
Destroy a cache_control object.
bool http_cache_control_no_cache(http_cache_control *cc)
Get the value of a cache control's no-cache flag.
bool http_cache_control_no_store(http_cache_control *cc)
Get the value of a cache control's no-store flag.
char * strndup(const char *s, size_t n)
Duplicate up to n characters of a string.
Definition: utils.c:332
bool fetch_can_fetch(const nsurl *url)
Check if a URL's scheme can be fetched.
Definition: fetch.c:586
long fetch_http_code(struct fetch *fetch)
Get the HTTP response code.
Definition: fetch.c:612
nserror fetch_start(nsurl *url, nsurl *referer, fetch_callback callback, void *p, bool only_2xx, const char *post_urlenc, const struct fetch_multipart_data *post_multipart, bool verifiable, bool downgrade_tls, const char *headers[], struct fetch **fetch_out)
Start fetching data for the given URL.
Definition: fetch.c:449
void fetch_abort(struct fetch *f)
Abort a fetch.
Definition: fetch.c:537
struct fetch_multipart_data * fetch_multipart_data_clone(const struct fetch_multipart_data *list)
Clone a linked list of fetch_multipart_data.
Definition: fetch.c:620
void fetch_multipart_data_destroy(struct fetch_multipart_data *list)
Free a linked list of fetch_multipart_data.
Definition: fetch.c:701
Fetching of data from a URL (interface).
@ FETCH_REDIRECT
Definition: fetch.h:49
@ FETCH_DATA
Definition: fetch.h:44
@ FETCH_AUTH
Definition: fetch.h:51
@ FETCH_CERTS
Definition: fetch.h:42
@ FETCH_HEADER
Definition: fetch.h:43
@ FETCH_PROGRESS
Definition: fetch.h:41
@ FETCH_NOTMODIFIED
Definition: fetch.h:50
@ FETCH_FINISHED
Definition: fetch.h:46
@ FETCH_SSL_ERR
Definition: fetch.h:53
@ FETCH_ERROR
Definition: fetch.h:48
@ FETCH_TIMEDOUT
Definition: fetch.h:47
@ FETCH_CERT_ERR
Definition: fetch.h:52
Useful interned string pointers (interface).
nserror
Enumeration of error codes.
Definition: errors.h:29
@ NSERROR_TIMEOUT
Operation timed out.
Definition: errors.h:65
@ NSERROR_NOT_FOUND
Requested item not found.
Definition: errors.h:34
@ NSERROR_BAD_URL
Bad URL.
Definition: errors.h:55
@ NSERROR_BAD_AUTH
Fetch needs authentication data.
Definition: errors.h:63
@ NSERROR_BAD_CERTS
Fetch needs certificate chain check.
Definition: errors.h:64
@ NSERROR_NO_FETCH_HANDLER
No fetch handler for URL scheme.
Definition: errors.h:33
@ NSERROR_NEED_DATA
More data needed.
Definition: errors.h:46
@ NSERROR_BAD_REDIRECT
Fetch encountered a bad redirect.
Definition: errors.h:62
@ NSERROR_UNKNOWN
Unknown error - DO NOT USE.
Definition: errors.h:31
@ NSERROR_INVALID
Invalid data.
Definition: errors.h:49
@ NSERROR_NOMEM
Memory exhaustion.
Definition: errors.h:32
@ NSERROR_OK
No error.
Definition: errors.h:30
struct netsurf_table * guit
The global interface table.
Definition: gui_factory.c:50
Interface to core interface table.
HTTP header parsing functions.
Interface to platform-specific miscellaneous browser operation table.
Netsurf additional integer type formatting macros.
#define PRIsizet
c99 standard printf formatting for size_t type
Definition: inttypes.h:53
#define PRIu64
Definition: inttypes.h:38
static nserror build_candidate_list(struct llcache_object ***lst_out, int *lst_len_out)
Construct a sorted list of objects available for writeout operation.
Definition: llcache.c:2775
static nserror llcache_object_refetch(llcache_object *object)
(Re)fetch an object
Definition: llcache.c:903
nserror llcache_initialise(const struct llcache_parameters *prm)
Initialise the low-level cache.
Definition: llcache.c:3891
void llcache_clean(bool purge)
Cause the low-level cache to attempt to perform cleanup.
Definition: llcache.c:3725
static nserror llcache_post_data_clone(const llcache_post_data *orig, llcache_post_data **clone)
Clone a POST data object.
Definition: llcache.c:445
static nserror llcache_object_new(nsurl *url, llcache_object **result)
Create a new low-level cache object.
Definition: llcache.c:423
static void llcache_persist(void *p)
Possibly write objects data to backing store.
Definition: llcache.c:2938
static nserror llcache_fetch_parse_cache_control(llcache_object *object, char *value)
parse cache control header value
Definition: llcache.c:582
static void llcache_fetch_callback(const fetch_msg *msg, void *p)
Handler for fetch events.
Definition: llcache.c:3058
static nserror llcache_object_fetch(llcache_object *object, uint32_t flags, nsurl *referer, const llcache_post_data *post, uint32_t redirect_count, bool hsts_in_use)
Kick-off a fetch for an object.
Definition: llcache.c:1019
static bool llcache__scheme_is_cachable(const nsurl *url)
Check whether a scheme is cachable.
Definition: llcache.c:1810
nserror llcache_handle_clone(llcache_handle *handle, llcache_handle **result)
Clone a low-level cache handle, producing a new handle to the same fetch/content.
Definition: llcache.c:4082
static nserror llcache_object_retrieve_from_cache(nsurl *url, uint32_t flags, nsurl *referer, const llcache_post_data *post, uint32_t redirect_count, bool hsts_in_use, llcache_object **result)
Retrieve a potentially cached object.
Definition: llcache.c:1914
static nserror llcache_object_clone_cache_data(llcache_object *source, llcache_object *destination, bool deep)
Clone an object's cache data.
Definition: llcache.c:1211
static nserror llcache_object_user_new(llcache_handle_callback cb, void *pw, llcache_object_user **user)
Create a new object user.
Definition: llcache.c:288
static nserror llcache_object_fetch_persistent(llcache_object *object, uint32_t flags, nsurl *referer, const llcache_post_data *post, uint32_t redirect_count)
Attempt to retrieve an object from persistent storage.
Definition: llcache.c:1853
static nserror llcache_fetch_ssl_error(llcache_object *object)
Handle a TLS connection setup failure.
Definition: llcache.c:2679
static nserror llcache_fetch_cert_error(llcache_object *object)
Handle a TLS certificate verification failure.
Definition: llcache.c:2627
static nserror llcache_serialise_metadata(llcache_object *object, uint8_t **data_out, size_t *datasize_out)
Generate a serialised version of an object's metadata.
Definition: llcache.c:1322
static void llcache_destroy_headers(llcache_object *object)
Destroy headers.
Definition: llcache.c:700
static nserror llcache_object_user_destroy(llcache_object_user *user)
Destroy an object user.
Definition: llcache.c:326
#define REDIRECT_LIMIT
static nserror llcache_fetch_timeout(llcache_object *object)
handle time out while trying to fetch.
Definition: llcache.c:2726
static nserror llcache_fetch_process_data(llcache_object *object, const uint8_t *data, size_t len)
Process a chunk of fetched data.
Definition: llcache.c:2510
static nserror llcache_object_snapshot(llcache_object *object, llcache_object **snapshot)
Make a snapshot of the current state of an llcache_object.
Definition: llcache.c:3580
static void llcache_persist_slowcheck(void *p)
Check for overall write performance.
Definition: llcache.c:2915
static nserror get_referer_header(nsurl *url, nsurl *referer, char **header_out)
construct a Referer header appropriate for the request
Definition: llcache.c:819
static struct llcache_s * llcache
low level cache state
Definition: llcache.c:267
nserror llcache_handle_abort(llcache_handle *handle)
Abort a low-level fetch, informing all users of this action.
Definition: llcache.c:4098
static bool llcache__scheme_is_persistable(const nsurl *url)
Check whether a scheme is persistable.
Definition: llcache.c:1782
static nserror llcache_fetch_auth(llcache_object *object, const char *realm)
Handle an authentication request.
Definition: llcache.c:2574
static nserror llcache_retrieve_persisted_data(llcache_object *object)
Retrieve source data for an object from persistent store if necessary.
Definition: llcache.c:1290
static uint32_t total_object_size(llcache_object *object)
total ram usage of object
Definition: llcache.c:3646
#define MAX_PERSIST_PER_RUN
llcache_fetch_state
State of a low-level cache object fetch.
Definition: llcache.c:61
@ LLCACHE_FETCH_HEADERS
Fetching headers.
Definition: llcache.c:63
@ LLCACHE_FETCH_INIT
Initial state, before fetch.
Definition: llcache.c:62
@ LLCACHE_FETCH_DATA
Fetching object data.
Definition: llcache.c:64
@ LLCACHE_FETCH_COMPLETE
Fetch completed.
Definition: llcache.c:65
static llcache_object_user * llcache_object_find_user(const llcache_handle *handle)
Find a user of a low-level cache object.
Definition: llcache.c:3271
static nserror llcache_fetch_split_header(const uint8_t *data, size_t len, char **name, char **value)
Split a fetch header into name and value.
Definition: llcache.c:496
static nserror llcache_send_event_to_users(llcache_object *object, llcache_event *event)
Iterate the users of an object, calling their callbacks.
Definition: llcache.c:384
static nserror llcache_object_retrieve(nsurl *url, uint32_t flags, nsurl *referer, const llcache_post_data *post, uint32_t redirect_count, bool hsts_in_use, llcache_object **result)
Retrieve an object from the cache, fetching it if necessary.
Definition: llcache.c:2090
void llcache_finalise(void)
Finalise the low-level cache.
Definition: llcache.c:3916
static nserror llcache_fetch_redirect(llcache_object *object, const char *target, llcache_object **replacement)
Handle FETCH_REDIRECT event.
Definition: llcache.c:2287
static nserror llcache_object_add_to_list(llcache_object *object, llcache_object **list)
Add a low-level cache object to a cache list.
Definition: llcache.c:1116
bool llcache_handle_references_same_object(const llcache_handle *a, const llcache_handle *b)
Determine if the same underlying object is referenced by the given handles.
Definition: llcache.c:4229
nsurl * llcache_handle_get_url(const llcache_handle *handle)
Retrieve the post-redirect URL of a low-level cache object.
Definition: llcache.c:4195
llcache_validate
Validation control.
Definition: llcache.c:127
@ LLCACHE_VALIDATE_ALWAYS
Always revalidate.
Definition: llcache.c:129
@ LLCACHE_VALIDATE_FRESH
Only revalidate if not fresh.
Definition: llcache.c:128
@ LLCACHE_VALIDATE_ONCE
Revalidate once only.
Definition: llcache.c:130
static nserror llcache_hsts_transform_url(nsurl *url, nsurl **result, bool *hsts_in_use)
Transform a request-URI based on HSTS policy.
Definition: llcache.c:2200
const uint8_t * llcache_handle_get_source_data(const llcache_handle *handle, size_t *size)
Retrieve source data of a low-level cache object.
Definition: llcache.c:4201
static void llcache_users_not_caught_up(void)
Ask for llcache_catch_up_all_users to be scheduled ASAP to pump the user state machines.
Definition: llcache.c:3705
static nserror llcache_fetch_process_header(llcache_object *object, const uint8_t *data, size_t len)
Process a fetch header.
Definition: llcache.c:734
struct llcache_object_user llcache_object_user
Low-level cache object user record.
static nserror llcache_object_notify_users(llcache_object *object)
Notify users of an object's current state.
Definition: llcache.c:3313
const char * llcache_handle_get_header(const llcache_handle *handle, const char *key)
Retrieve a header value associated with a low-level cache object.
Definition: llcache.c:4210
static bool llcache_object_in_list(const llcache_object *object, const llcache_object *list)
Determine if a low-level cache object resides in a given list.
Definition: llcache.c:3294
static nserror llcache_process_metadata(llcache_object *object)
Deserialisation of an object's metadata.
Definition: llcache.c:1532
nserror llcache_handle_change_callback(llcache_handle *handle, llcache_handle_callback cb, void *pw)
Change the callback associated with a low-level cache handle.
Definition: llcache.c:4047
static nserror llcache_hsts_update_policy(llcache_object *object)
Update HSTS policy for target domain.
Definition: llcache.c:2246
static nserror llcache_object_remove_from_list(llcache_object *object, llcache_object **list)
Remove a low-level cache object from a cache list.
Definition: llcache.c:1267
nserror llcache_handle_retrieve(nsurl *url, uint32_t flags, nsurl *referer, const llcache_post_data *post, llcache_handle_callback cb, void *pw, llcache_handle **result)
Retrieve a handle for a low-level cache object.
Definition: llcache.c:3990
static int llcache_object_rfc2616_remaining_lifetime(const llcache_cache_control *cd)
Determine the remaining lifetime of a cache object using the.
Definition: llcache.c:1136
nserror llcache_handle_force_stream(llcache_handle *handle)
Force a low-level cache handle into streaming mode.
Definition: llcache.c:4160
static nserror llcache_object_destroy(llcache_object *object)
Destroy a low-level cache object.
Definition: llcache.c:1058
static nserror write_backing_store(struct llcache_object *object, size_t *written_out, unsigned long *elapsed)
Write an object to the backing store.
Definition: llcache.c:2839
static void llcache_invalidate_cache_control_data(llcache_object *object)
Invalidate cache control data.
Definition: llcache.c:717
static nserror llcache_fetch_notmodified(llcache_object *object, llcache_object **replacement)
Handle FETCH_NOTMODIFIED event.
Definition: llcache.c:2442
nserror llcache_handle_invalidate_cache_data(llcache_handle *handle)
Invalidate cache data for a low-level cache object.
Definition: llcache.c:4182
static nserror llcache_object_remove_user(llcache_object *object, llcache_object_user *user)
Remove a user from a low-level cache object.
Definition: llcache.c:348
static nserror llcache_object_cache_update(llcache_object *object)
Update an object's cache state.
Definition: llcache.c:2427
static nserror llcache_fetch_header_cache_control(llcache_object *object, char *name, char *value)
Update cache control from appropriate header.
Definition: llcache.c:620
static nserror llcache_object_add_user(llcache_object *object, llcache_object_user *user)
Add a user to a low-level cache object.
Definition: llcache.c:2171
static void llcache_catch_up_all_users(void *ignored)
Catch up the cache users with state changes from fetchers.
Definition: llcache.c:3679
static bool llcache_object_is_fresh(const llcache_object *object)
Determine if an object is still fresh.
Definition: llcache.c:1179
#define INVALID_AGE
Cache control value for invalid age.
Definition: llcache.c:136
llcache_store_state
Current status of an object's data.
Definition: llcache.c:159
@ LLCACHE_STATE_RAM
source data is stored in RAM only
Definition: llcache.c:160
@ LLCACHE_STATE_DISC
source data is stored on disc
Definition: llcache.c:161
nserror llcache_handle_release(llcache_handle *handle)
Release a low-level cache handle.
Definition: llcache.c:4058
nserror(* llcache_handle_callback)(llcache_handle *handle, const llcache_event *event, void *pw)
Client callback for low-level cache events.
Definition: llcache.h:113
@ LLCACHE_RETRIEVE_STREAM_DATA
Definition: llcache.h:63
@ LLCACHE_RETRIEVE_NO_ERROR_PAGES
Stream data (implies that object is not cacheable)
Definition: llcache.h:61
@ LLCACHE_RETRIEVE_VERIFIABLE
Requested URL was verified.
Definition: llcache.h:59
@ LLCACHE_RETRIEVE_FORCE_FETCH
Force a new fetch.
Definition: llcache.h:57
@ LLCACHE_EVENT_DONE
Finished fetching data.
Definition: llcache.h:71
@ LLCACHE_EVENT_ERROR
An error occurred during fetch.
Definition: llcache.h:73
@ LLCACHE_EVENT_GOT_CERTS
SSL certificates arrived.
Definition: llcache.h:68
@ LLCACHE_EVENT_REDIRECT
Fetch URL redirect occured.
Definition: llcache.h:76
@ LLCACHE_EVENT_PROGRESS
Fetch progress update.
Definition: llcache.h:74
@ LLCACHE_EVENT_HAD_HEADERS
Received all headers.
Definition: llcache.h:69
@ LLCACHE_EVENT_HAD_DATA
Received some data.
Definition: llcache.h:70
#define NSLOG(catname, level, logmsg, args...)
Definition: log.h:116
const char * messages_get_errorcode(nserror code)
lookup of a message by errorcode from the standard Messages hash.
Definition: messages.c:248
const char * messages_get(const char *key)
Fast lookup of a message by key from the standard Messages hash.
Definition: messages.c:241
Localised message support (interface).
NetSurf URL handling (interface).
bool nsurl_compare(const nsurl *url1, const nsurl *url2, nsurl_component parts)
Compare two URLs.
nserror nsurl_create(const char *const url_s, nsurl **url)
Create a NetSurf URL object from a URL string.
nserror nsurl_defragment(const nsurl *url, nsurl **no_frag)
Create a NetSurf URL object without a fragment from a NetSurf URL.
void nsurl_unref(nsurl *url)
Drop a reference to a NetSurf URL object.
const char * nsurl_access(const nsurl *url)
Access a NetSurf URL object as a string.
size_t nsurl_length(const nsurl *url)
Find the length of a NetSurf URL object's URL, as returned by nsurl_access.
bool nsurl_has_component(const nsurl *url, nsurl_component part)
Enquire about the existence of componenets in a given URL.
nserror nsurl_replace_scheme(const nsurl *url, lwc_string *scheme, nsurl **new_url)
Create a NetSurf URL object, with scheme replaced.
nsurl * nsurl_ref(nsurl *url)
Increment the reference count to a NetSurf URL object.
lwc_string * nsurl_get_component(const nsurl *url, nsurl_component part)
Get part of a URL as a lwc_string, from a NetSurf URL object.
@ NSURL_SCHEME
Definition: nsurl.h:45
@ NSURL_COMPLETE
Definition: nsurl.h:54
@ NSURL_QUERY
Definition: nsurl.h:53
nserror nsurl_join(const nsurl *base, const char *rel, nsurl **joined)
Join a base url to a relative link part, creating a new NetSurf URL object.
struct nsurl nsurl
NetSurf URL object.
Definition: nsurl.h:31
nserror cert_chain_alloc(size_t depth, struct cert_chain **chain_out)
create new certificate chain
Definition: ssl_certs.c:41
ssl_cert_err
ssl certificate error status
Definition: ssl_certs.h:36
@ SSL_CERT_ERR_OK
Nothing wrong with this certificate.
Definition: ssl_certs.h:37
@ SSL_CERT_ERR_UNKNOWN
Unknown error.
Definition: ssl_certs.h:38
#define SSL_CERT_ERR_MAX_KNOWN
Always the max known ssl certificate error type.
Definition: ssl_certs.h:51
nserror cert_chain_dup(const struct cert_chain *src, struct cert_chain **dst_out)
duplicate a certificate chain
Definition: ssl_certs.c:101
nserror cert_chain_free(struct cert_chain *chain)
free a certificate chain
Definition: ssl_certs.c:317
#define MAX_CERT_DEPTH
maximum number of X509 certificates in chain for TLS connection
Definition: ssl_certs.h:54
size_t cert_chain_size(const struct cert_chain *chain)
total number of data bytes in a chain
Definition: ssl_certs.c:340
Interface to utility string handling.
X509 certificate chain.
Definition: ssl_certs.h:59
struct cert_chain::@57 certs[MAX_CERT_DEPTH]
uint8_t * der
data in Distinguished Encoding Rules (DER) format
Definition: ssl_certs.h:73
size_t der_length
DER length.
Definition: ssl_certs.h:78
ssl_cert_err err
Whatever is wrong with this certificate.
Definition: ssl_certs.h:68
Fetcher message data.
Definition: fetch.h:72
const uint8_t * buf
Definition: fetch.h:79
fetch_msg_type type
Definition: fetch.h:73
struct fetch_msg::@118::@119 header_or_data
const char * realm
Definition: fetch.h:89
const struct cert_chain * chain
Definition: fetch.h:92
size_t len
Definition: fetch.h:80
struct fetch_msg::@118::@120 auth
union fetch_msg::@118 data
const char * progress
Definition: fetch.h:76
const char * redirect
Definition: fetch.h:86
const char * error
Definition: fetch.h:83
Fetch POST multipart data.
Definition: fetch.h:109
Information for a single fetch.
Definition: fetch.c:89
nserror(* initialise)(const struct llcache_store_parameters *parameters)
Initialise the backing store.
Definition: backing_store.h:51
nserror(* store)(struct nsurl *url, enum backing_store_flags flags, uint8_t *data, const size_t datalen)
Place an object in the backing store.
Definition: backing_store.h:80
nserror(* finalise)(void)
Finalise the backing store.
Definition: backing_store.h:58
nserror(* invalidate)(struct nsurl *url)
Invalidate a source object from the backing store.
nserror(* release)(struct nsurl *url, enum backing_store_flags flags)
release a previously fetched or stored memory object.
nserror(* fetch)(struct nsurl *url, enum backing_store_flags flags, uint8_t **data, size_t *datalen)
Retrieve an object from the backing store.
Definition: backing_store.h:99
nserror(* schedule)(int t, void(*callback)(void *p), void *p)
Schedule a callback.
Definition: misc.h:58
Representation of a Cache-Control.
Definition: cache-control.c:31
Cache control data.
Definition: llcache.c:139
time_t date
Date: response header.
Definition: llcache.c:143
int max_age
Max-Age Cache-control parameter.
Definition: llcache.c:146
time_t res_time
Time of response.
Definition: llcache.c:141
int age
Age: response header.
Definition: llcache.c:145
time_t fin_time
Time of request completion.
Definition: llcache.c:142
time_t expires
Expires: response header.
Definition: llcache.c:144
llcache_validate no_cache
No-Cache Cache-control parameter.
Definition: llcache.c:147
char * etag
Etag: response header.
Definition: llcache.c:148
time_t last_modified
Last-Modified: response header.
Definition: llcache.c:149
time_t req_time
Time of request.
Definition: llcache.c:140
Low-level cache events.
Definition: llcache.h:85
Low-level cache object fetch context.
Definition: llcache.c:102
bool tried_with_auth
Whether we've tried with auth.
Definition: llcache.c:117
bool hsts_in_use
Whether HSTS applies to this fetch.
Definition: llcache.c:115
bool tainted_tls
Whether the TLS transport is tainted.
Definition: llcache.c:121
struct fetch * fetch
Fetch handle for this object.
Definition: llcache.c:107
llcache_post_data * post
POST data, or NULL for GET.
Definition: llcache.c:105
llcache_fetch_state state
Current state of object fetch.
Definition: llcache.c:109
uint32_t flags
Fetch flags.
Definition: llcache.c:103
uint32_t retries_remaining
Number of times to retry on timeout.
Definition: llcache.c:113
bool tried_with_tls_downgrade
Whether we've tried TLS 1.2.
Definition: llcache.c:119
nsurl * referer
Referring URL, or NULL if none.
Definition: llcache.c:104
uint32_t redirect_count
Count of redirects followed.
Definition: llcache.c:111
Handle to low-level cache object.
Definition: llcache.c:76
llcache_handle_callback cb
Client callback.
Definition: llcache.c:79
size_t bytes
Last reported byte count.
Definition: llcache.c:83
llcache_object * object
Pointer to associated object.
Definition: llcache.c:77
llcache_fetch_state state
Last known state of object fetch.
Definition: llcache.c:82
void * pw
Client data.
Definition: llcache.c:80
Representation of a fetch header.
Definition: llcache.c:153
char * value
Header value.
Definition: llcache.c:155
char * name
Header name.
Definition: llcache.c:154
Low-level cache object user record.
Definition: llcache.c:89
bool iterator_target
This is the iterator target.
Definition: llcache.c:92
struct llcache_object_user * next
Next in list.
Definition: llcache.c:96
bool queued_for_delete
This user is queued for deletion.
Definition: llcache.c:93
llcache_handle * handle
Handle data for client.
Definition: llcache.c:90
struct llcache_object_user * prev
Previous in list.
Definition: llcache.c:95
Low-level cache object.
Definition: llcache.c:169
llcache_object * prev
Previous in list.
Definition: llcache.c:170
llcache_object * candidate
Object to use, if fetch determines that it is still fresh.
Definition: llcache.c:189
size_t source_len
Byte length of source data.
Definition: llcache.c:177
nsurl * url
Post-redirect URL for object.
Definition: llcache.c:173
llcache_cache_control cache
Cache control data for object.
Definition: llcache.c:188
time_t last_used
time the last user was removed from the object
Definition: llcache.c:204
llcache_object_user * users
List of users.
Definition: llcache.c:184
uint8_t * source_data
Source data for object.
Definition: llcache.c:176
uint32_t candidate_count
Count of objects this is a candidate for.
Definition: llcache.c:192
llcache_fetch_ctx fetch
Fetch context for object.
Definition: llcache.c:186
struct cert_chain * chain
Certificate chain from the fetch.
Definition: llcache.c:180
llcache_header * headers
Fetch headers.
Definition: llcache.c:196
llcache_store_state store_state
where the data for the object is stored
Definition: llcache.c:182
llcache_object * next
Next in list.
Definition: llcache.c:171
size_t num_headers
Number of fetch headers.
Definition: llcache.c:197
size_t source_alloc
Allocated size of source buffer.
Definition: llcache.c:178
Parameters to configure the low level cache.
Definition: llcache.h:129
struct llcache_store_parameters store
Definition: llcache.h:153
int minimum_lifetime
The minimum lifetime to consider sending objects to backing store.
Definition: llcache.h:134
size_t limit
The target upper bound for the RAM cache size.
Definition: llcache.h:130
size_t maximum_bandwidth
The maximum bandwidth to allow the backing store to use in bytes/second.
Definition: llcache.h:144
uint32_t fetch_attempts
The number of fetches to attempt when timing out.
Definition: llcache.h:151
size_t minimum_bandwidth
The minimum bandwidth to allow the backing store to use in bytes/second.
Definition: llcache.h:139
unsigned long time_quantum
The time quantum over which to calculate the bandwidth values.
Definition: llcache.h:148
POST data object for low-level cache requests.
Definition: llcache.h:40
struct fetch_multipart_data * multipart
Multipart data.
Definition: llcache.h:47
enum llcache_post_data::@122 type
Type of POST data.
union llcache_post_data::@123 data
POST data content.
char * urlenc
URL encoded data.
Definition: llcache.h:46
Core llcache control context.
Definition: llcache.c:210
llcache_object * cached_objects
Head of the low-level cached object list.
Definition: llcache.c:212
uint32_t fetch_attempts
The number of fetch attempts we make when timing out.
Definition: llcache.c:221
bool all_caught_up
Whether or not our users are caught up.
Definition: llcache.c:224
size_t maximum_bandwidth
The maximum bandwidth to allow the backing store to use in bytes/second.
Definition: llcache.c:252
llcache_object * uncached_objects
Head of the low-level uncached object list.
Definition: llcache.c:215
uint32_t limit
The target upper bound for the RAM cache size.
Definition: llcache.c:218
int minimum_lifetime
The minimum lifetime to consider sending objects to backing store.
Definition: llcache.c:234
size_t minimum_bandwidth
The minimum bandwidth to allow the backing store to use in bytes/second.
Definition: llcache.c:246
unsigned long time_quantum
The time over which to apply the bandwidth calculations in ms.
Definition: llcache.c:239
uint64_t total_elapsed
Total number of milliseconds taken to write to backing store.
Definition: llcache.c:262
uint64_t total_written
Total number of bytes written to backing store.
Definition: llcache.c:257
struct gui_misc_table * misc
Browser table.
Definition: gui_table.h:57
struct gui_llcache_table * llcache
Low level cache table.
Definition: gui_table.h:143
int nsc_sntimet(char *str, size_t size, time_t *timep)
Write the time in seconds since epoch to a buffer.
Definition: time.c:126
nserror nsc_strntimet(const char *str, size_t size, time_t *timep)
Converts a date string to a number of seconds since epoch.
Definition: time.c:980
nserror nsc_snptimet(const char *str, size_t size, time_t *timep)
Parse time in seconds since epoc.
Definition: time.c:147
const char * rfc1123_date(time_t t)
Create an RFC 1123 compliant date string from a Unix timestamp.
Definition: time.c:110
Interface to time operations.
bool urldb_get_hsts_enabled(struct nsurl *url)
Determine if HSTS policy is enabled for an URL.
Definition: urldb.c:3573
bool urldb_set_hsts_policy(struct nsurl *url, const char *header)
Set HSTS policy for an URL.
Definition: urldb.c:3502
const char * urldb_get_auth_details(nsurl *url, const char *realm)
Look up authentication details in database.
Definition: urldb.c:3405
Unified URL information database internal interface.
Option reading and saving interface.
#define nsoption_bool(OPTION)
Get the value of a boolean option.
Definition: nsoption.h:304
iconv_t cd
Iconv conversion descriptor.
Definition: utf8.c:145
Interface to a number of general purpose functionality.
#define max(x, y)
Definition: utils.h:50
#define SLEN(x)
Calculate length of constant C string.
Definition: utils.h:88
static nserror line(const struct redraw_context *ctx, const plot_style_t *style, const struct rect *line)
Plots a line.
Definition: plot.c:579