44#include <nsutils/unistd.h>
59#define CONTROL_VERSION 202
65#define CONTROL_MAINT_TIME 10000
68#define ENTRIES_FNAME "entries"
71#define BLOCKS_FNAME "blocks"
74#define BLOCK_ADDR_LEN 16
77#define BLOCK_ENTRY_COUNT 10
80#define BLOCK_FILE_COUNT (BLOCK_ADDR_LEN - BLOCK_ENTRY_COUNT)
83#define BLOCK_DATA_SIZE 13
86#define BLOCK_META_SIZE 13
89#define BLOCK_USE_MAP_SIZE (1 << (BLOCK_ENTRY_COUNT - 3))
326 const uint8_t *b32u_d[6];
329 const char *base_dir_table[] = {
330 "d",
"m",
"dblk",
"mblk"
334 const uint8_t encoding_table[64][3] = {
335 {
'A', 0, 0 }, {
'B', 0, 0 },
336 {
'C', 0, 0 }, {
'D', 0, 0 },
337 {
'E', 0, 0 }, {
'F', 0, 0 },
338 {
'G', 0, 0 }, {
'H', 0, 0 },
339 {
'I', 0, 0 }, {
'J', 0, 0 },
340 {
'K', 0, 0 }, {
'L', 0, 0 },
341 {
'M', 0, 0 }, {
'N', 0, 0 },
342 {
'O', 0, 0 }, {
'P', 0, 0 },
343 {
'Q', 0, 0 }, {
'R', 0, 0 },
344 {
'S', 0, 0 }, {
'T', 0, 0 },
345 {
'U', 0, 0 }, {
'V', 0, 0 },
346 {
'W', 0, 0 }, {
'X', 0, 0 },
347 {
'Y', 0, 0 }, {
'Z', 0, 0 },
348 {
'2', 0, 0 }, {
'3', 0, 0 },
349 {
'4', 0, 0 }, {
'5', 0, 0 },
350 {
'6', 0, 0 }, {
'7', 0, 0 },
351 {
'B',
'A', 0 }, {
'B',
'B', 0 },
352 {
'B',
'C', 0 }, {
'B',
'D', 0 },
353 {
'B',
'E', 0 }, {
'B',
'F', 0 },
354 {
'B',
'G', 0 }, {
'B',
'H', 0 },
355 {
'B',
'I', 0 }, {
'B',
'J', 0 },
356 {
'B',
'K', 0 }, {
'B',
'L', 0 },
357 {
'B',
'M', 0 }, {
'B',
'N', 0 },
358 {
'B',
'O', 0 }, {
'B',
'P', 0 },
359 {
'B',
'Q', 0 }, {
'B',
'R', 0 },
360 {
'B',
'S', 0 }, {
'B',
'T', 0 },
361 {
'B',
'U', 0 }, {
'B',
'V', 0 },
362 {
'B',
'W', 0 }, {
'B',
'X', 0 },
363 {
'B',
'Y', 0 }, {
'B',
'Z', 0 },
364 {
'B',
'2', 0 }, {
'B',
'3', 0 },
365 {
'B',
'4', 0 }, {
'B',
'5', 0 },
366 {
'B',
'6', 0 }, {
'B',
'7', 0 }
370 b32u_i[0] = encoding_table[(ident ) & 0x1f][0];
371 b32u_i[1] = encoding_table[(ident >> 5) & 0x1f][0];
372 b32u_i[2] = encoding_table[(ident >> 10) & 0x1f][0];
373 b32u_i[3] = encoding_table[(ident >> 15) & 0x1f][0];
374 b32u_i[4] = encoding_table[(ident >> 20) & 0x1f][0];
375 b32u_i[5] = encoding_table[(ident >> 25) & 0x1f][0];
376 b32u_i[6] = encoding_table[(ident >> 30) & 0x1f][0];
380 b32u_d[0] = (uint8_t*)base_dir_table[elem_idx];
381 b32u_d[1] = &encoding_table[(ident ) & 0x3f][0];
382 b32u_d[2] = &encoding_table[(ident >> 6) & 0x3f][0];
383 b32u_d[3] = &encoding_table[(ident >> 12) & 0x3f][0];
384 b32u_d[4] = &encoding_table[(ident >> 18) & 0x3f][0];
385 b32u_d[5] = &encoding_table[(ident >> 24) & 0x3f][0];
391 state->
path, b32u_d[0], b32u_d[1], b32u_d[2],
392 b32u_d[3], b32u_d[4], b32u_d[5], b32u_i);
398 state->
path, b32u_d[0], b32u_d[1]);
402 assert(
"bad element index" == NULL);
434 state->
blocks[elem_idx][bf].
use_map[bi >> 3] &= ~(1U << (bi & 7));
476 NSLOG(netsurf, DEBUG,
477 "invalidating entry with referenced allocation");
485 NSLOG(netsurf, ERROR,
"Error invalidating metadata element");
490 NSLOG(netsurf, ERROR,
"Error invalidating data element");
504static int compar(
const void *va,
const void *vb)
599 estate.
elist = malloc(
sizeof(
struct state_entry*) * old_count);
600 if (estate.
elist == NULL) {
605 NSLOG(netsurf, WARNING,
"Unexpected termination of eviction iterator");
611 NSLOG(netsurf, WARNING,
"Incorrect entry count after eviction iterator");
620 for (ent = 0; ent < estate.
ent_count; ent++) {
640 removed, ent, state->
total_alloc, old_count - ent);
657 if (write(fd, &len,
sizeof(len)) !=
sizeof(len))
661 if (write(fd, ent,
sizeof(*ent)) !=
sizeof(*ent))
701 memset(&weistate, 0,
sizeof(weistate));
713 weistate.
fd = open(tname, O_RDWR | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR);
714 if (weistate.
fd == -1) {
738 if (rename(tname, fname) != 0) {
780 fd = open(tname, O_RDWR | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR);
794 NSLOG(netsurf, DEBUG,
795 "writing block file %d use index on file number %d failed",
807 if (written != blocks_size) {
822 if (rename(tname, fname) != 0) {
853 NSLOG(netsurf, DEBUG,
"Starting");
856 if (state->
blocks[elem_idx][bfidx].
fd != -1) {
860 NSLOG(netsurf, ERROR,
861 "Truncate failed errno:%d",
867 NSLOG(netsurf, DEBUG,
"Complete");
943 if (*(map + idx) != 0xff) {
945 for (bit = 0; bit < 8;bit++) {
946 if (((*(map + idx)) & (1U << bit)) == 0) {
948 *(map + idx) |= 1U << bit;
979 const size_t datalen,
1005 elem = &se->
elem[elem_idx];
1012 NSLOG(netsurf, ERROR,
1013 "attempt to overwrite entry with in use data");
1028 elem->
size = datalen;
1070 if (fname == NULL) {
1071 NSLOG(netsurf, ERROR,
"filename error");
1076 if (openflags & O_CREAT) {
1079 NSLOG(netsurf, WARNING,
1080 "file path \"%s\" could not be created", fname);
1086 NSLOG(netsurf, DEBUG,
"opening %s", fname);
1087 fd = open(fname, openflags, S_IRUSR | S_IWUSR);
1146 fd = open(fname, O_RDWR);
1149 while (read(fd, &urllen,
sizeof(urllen)) ==
sizeof(urllen)) {
1150 url = calloc(1, urllen+1);
1156 if (read(fd,
url, urllen) != (ssize_t)urllen) {
1179 if (read(fd, ent,
sizeof(*ent)) !=
sizeof(*ent)) {
1230 NSLOG(netsurf, INFO,
"Initialising block use map from %s", fname);
1232 fd = open(fname, O_RDWR);
1242 NSLOG(netsurf, ERROR,
1243 "reading block file %d use index on file number %d failed",
1254 NSLOG(netsurf, INFO,
"Initialising block use map to defaults");
1287 fcachetag = fopen(fname,
"wb");
1291 if (fcachetag == NULL) {
1296 "Signature: 8a477f597d28d172789f06886806bc55\n"
1297 "# This file is a cache directory tag created by NetSurf.\n"
1298 "# For information about cache directory tags, see:\n"
1299 "# http://www.brynosaurus.com/cachedir/\n");
1324 NSLOG(netsurf, INFO,
"writing control file \"%s\"", fname);
1332 fcontrol = fopen(fname,
"wb");
1336 if (fcontrol == NULL) {
1359 unsigned int ctrlversion;
1367 NSLOG(netsurf, INFO,
"opening control file \"%s\"", fname);
1369 fcontrol = fopen(fname,
"rb");
1373 if (fcontrol == NULL) {
1375 if (errno == ENOENT) {
1385 if (fscanf(fcontrol,
"%u", &ctrlversion) != 1) {
1393 if (fgetc(fcontrol) != 0) {
1431 if (parameters->
limit == 0) {
1436 if (parameters->
path == NULL) {
1442 if (newstate == NULL) {
1446 newstate->
path = strdup(parameters->
path);
1454 NSLOG(netsurf, INFO,
"cache control file not found, making fresh");
1456 NSLOG(netsurf, ERROR,
"read control failed %s",
1460 NSLOG(netsurf, WARNING,
"Error `%s` while removing `%s`",
1462 NSLOG(netsurf, WARNING,
"Unable to clean up partial cache state.");
1463 NSLOG(netsurf, WARNING,
"Funky behaviour may ensue.");
1465 NSLOG(netsurf, INFO,
"Successfully removed old cache from `%s`",
1477 free(newstate->
path);
1486 free(newstate->
path);
1496 free(newstate->
path);
1503 NSLOG(netsurf, INFO,
"FS backing store init successful");
1505 NSLOG(netsurf, INFO,
1529 unsigned int op_count;
1550 NSLOG(netsurf, INFO,
1589 if (state->
blocks[elem_idx][bf].
fd == -1) {
1592 if (state->
blocks[elem_idx][bf].
fd == -1) {
1593 NSLOG(netsurf, ERROR,
"Open failed errno %d", errno);
1603 wr = nsu_pwrite(state->
blocks[elem_idx][bf].
fd,
1607 if (wr != (ssize_t)bse->
elem[elem_idx].
size) {
1608 NSLOG(netsurf, ERROR,
1609 "Write failed %"PRIssizet" of %"PRId32
" bytes from %p at %"PRIsizet" block %"PRIu16
" errno %d",
1619 NSLOG(netsurf, INFO,
1621 bse->
elem[elem_idx].
data, (
size_t)offst,
1646 NSLOG(netsurf, ERROR,
"Open failed %d errno %d", fd, errno);
1654 if (wr != (ssize_t)bse->
elem[elem_idx].
size) {
1655 NSLOG(netsurf, ERROR,
1656 "Write failed %"PRIssizet" of %"PRId32
" bytes from %p errno %d",
1687 const size_t datalen)
1708 NSLOG(netsurf, ERROR,
"store entry setting failed");
1759 if (state->
blocks[elem_idx][bf].
fd == -1) {
1762 if (state->
blocks[elem_idx][bf].
fd == -1) {
1763 NSLOG(netsurf, ERROR,
"Open failed errno %d", errno);
1773 rd = nsu_pread(state->
blocks[elem_idx][bf].
fd,
1777 if (rd != (ssize_t)bse->
elem[elem_idx].
size) {
1778 NSLOG(netsurf, ERROR,
1779 "Failed reading %"PRIssizet" of %"PRId32
" bytes into %p from %"PRIsizet" block %"PRIu16
" errno %d",
1789 NSLOG(netsurf, DEEPDEBUG,
1791 bse->
elem[elem_idx].
data, (
size_t)offst,
1817 NSLOG(netsurf, ERROR,
"Open failed %d errno %d", fd, errno);
1822 while (tot < bse->
elem[elem_idx].size) {
1827 NSLOG(netsurf, ERROR,
1828 "read error returned %"PRIssizet" errno %d",
1839 NSLOG(netsurf, DEEPDEBUG,
"Read %"PRIsizet" bytes into %p", tot,
1858 size_t *datalen_out)
1879 NSLOG(netsurf, DEBUG,
"retrieving cache data for url:%s",
1888 elem = &bse->
elem[elem_idx];
1895 NSLOG(netsurf, DEEPDEBUG,
1896 "Using existing entry (%p) allocation %p refs:%d", bse,
1902 if (elem->
data == NULL) {
1903 NSLOG(netsurf, ERROR,
1904 "Failed to create new heap allocation");
1907 NSLOG(netsurf, DEEPDEBUG,
"Created new heap allocation %p",
1915 if (elem->
block != 0) {
1929 *data_out = elem->
data;
1930 *datalen_out = elem->
size;
1957 NSLOG(netsurf, WARNING,
"entry not found");
Low-level source data cache backing store interface.
backing_store_flags
storage control flags
@ BACKING_STORE_META
data is metadata
nserror
Enumeration of error codes.
@ NSERROR_PERMISSION
Permission error.
@ NSERROR_SAVE_FAILED
Failed to save data.
@ NSERROR_NOT_FOUND
Requested item not found.
@ NSERROR_INIT_FAILED
Initialisation failed.
@ NSERROR_UNKNOWN
Unknown error - DO NOT USE.
@ NSERROR_NOMEM
Memory exhaustion.
Utility routines to obtain paths to file resources.
static void * entries_hashmap_value_alloc(void *key)
@ ENTRY_FLAGS_NONE
entry is normal
@ ENTRY_FLAGS_INVALID
entry has been invalidated but something still holding a reference
static nserror initialise(const struct llcache_store_parameters *parameters)
Initialise the backing store.
#define BLOCK_META_SIZE
log2 size of metadata blocks (8k)
#define CONTROL_VERSION
Backing store file format version.
static nserror store_read_block(struct store_state *state, struct store_entry *bse, int elem_idx)
Read an element of an entry from a small block file in the backing storage.
#define BLOCK_USE_MAP_SIZE
length in bytes of a block files use map
uint32_t entry_ident_t
The type used as a binary identifier for each entry derived from the URL.
store_entry_elem_flags
flags that indicate what additional information is contained within an entry element.
@ ENTRY_ELEM_FLAG_MMAP
entry data allocation is mmaped
@ ENTRY_ELEM_FLAG_NONE
store not managing any allocation on entry
@ ENTRY_ELEM_FLAG_SMALL
entry data allocation is in small object pool
@ ENTRY_ELEM_FLAG_HEAP
entry data allocation is on heap
static nserror unlink_entries(struct store_state *state)
Unlink entries file.
static nserror set_store_entry(struct store_state *state, nsurl *url, int elem_idx, uint8_t *data, const size_t datalen, struct store_entry **bse)
Set a backing store entry in the entry table from a url.
static nserror invalidate_entry(struct store_state *state, struct store_entry *bse)
Remove the entry and files associated with an identifier.
uint16_t block_index_t
The type used to store block file index values.
static nserror release(nsurl *url, enum backing_store_flags bsflags)
release a previously fetched or stored memory object.
static nserror store(nsurl *url, enum backing_store_flags bsflags, uint8_t *data, const size_t datalen)
Place an object in the backing store.
#define ENTRIES_FNAME
Filename of serialised entries.
static nserror write_entry(struct store_entry *ent, int fd)
Write a single store entry to disk.
static nserror store_read_file(struct store_state *state, struct store_entry *bse, int elem_idx)
Read an element of an entry from an individual file in the backing storage.
#define BLOCK_DATA_SIZE
log2 size of data blocks (8k)
static nserror read_blocks(struct store_state *state)
Read block file usage bitmaps.
static nserror store_write_block(struct store_state *state, struct store_entry *bse, int elem_idx)
Write an element of an entry to backing storage in a small block file.
static bool entry_eviction_iterator_cb(void *key, void *value, void *ctx)
Iterator for gathering entries to compute eviction order.
static nserror get_store_entry(struct store_state *state, nsurl *url, struct store_entry **bse)
Lookup a backing store entry in the entry table from a url.
static block_index_t alloc_block(struct store_state *state, int elem_idx)
Find next available small block.
static nserror read_entries(struct store_state *state)
Read description entries into memory.
static const unsigned int log2_block_size[ENTRY_ELEM_COUNT]
log2 of block size.
static int compar(const void *va, const void *vb)
Quick sort comparison.
static void control_maintenance(void *s)
maintenance of control structures.
static nserror read_control(struct store_state *state)
Read and parse the control file.
static nserror write_cache_tag(struct store_state *state)
Write the cache tag file.
#define BLOCK_ENTRY_COUNT
log2 number of entries per block file(1024)
#define BLOCK_FILE_COUNT
log2 number of data block files
static bool entries_hashmap_key_eq(void *key1, void *key2)
struct store_state * storestate
Global storage state.
static bool write_entry_iterator(void *key, void *value, void *ctx)
Callback for iterating the entries hashmap.
static nserror entry_release_alloc(struct store_entry_element *elem)
release any allocation for an entry
static char * store_fname(struct store_state *state, entry_ident_t ident, int elem_idx)
Generate a filename for an object.
#define BLOCKS_FNAME
Filename of block file index.
store_entry_elem_idx
Entry element index values.
@ ENTRY_ELEM_META
entry element is metadata
@ ENTRY_ELEM_COUNT
count of elements on an entry
@ ENTRY_ELEM_DATA
entry element is data
static nserror finalise(void)
Finalise the backing store.
static hashmap_parameters_t entries_hashmap_parameters
static nserror store_evict(struct store_state *state)
Evict entries from backing store as per configuration.
#define CONTROL_MAINT_TIME
Number of milliseconds after a update before control data maintenance is performed.
static struct gui_llcache_table llcache_table
static nserror store_write_file(struct store_state *state, struct store_entry *bse, int elem_idx)
Write an element of an entry to backing storage as an individual file.
static nserror write_blocks(struct store_state *state)
Write block file use map to file.
static nserror set_block_extents(struct store_state *state)
Ensures block files are of the correct extent.
static nserror fetch(nsurl *url, enum backing_store_flags bsflags, uint8_t **data_out, size_t *datalen_out)
Retrieve an object from the backing store.
static nserror write_control(struct store_state *state)
Write the control file for the current state.
static void entries_hashmap_value_destroy(void *value)
struct gui_llcache_table * filesystem_llcache_table
static nserror write_entries(struct store_state *state)
Write filesystem entries to file.
static nserror invalidate_element(struct store_state *state, struct store_entry *bse, int elem_idx)
invalidate an element of an entry
static nserror invalidate(nsurl *url)
Invalidate a source object from the backing store.
static int store_open(struct store_state *state, entry_ident_t ident, int elem_idx, int openflags)
Open a file using a store ident.
struct netsurf_table * guit
The global interface table.
Interface to core interface table.
bool hashmap_remove(hashmap_t *hashmap, void *key)
Remove an entry from the hashmap.
hashmap_t * hashmap_create(hashmap_parameters_t *params)
Create a hashmap.
size_t hashmap_count(hashmap_t *hashmap)
Get the number of entries in this map.
void * hashmap_lookup(hashmap_t *hashmap, void *key)
Look up a key in a hashmap.
void * hashmap_insert(hashmap_t *hashmap, void *key)
Create an entry in a hashmap.
bool hashmap_iterate(hashmap_t *hashmap, hashmap_iteration_cb_t cb, void *ctx)
Iterate the hashmap.
void hashmap_destroy(hashmap_t *hashmap)
Destroy a hashmap.
void(* hashmap_key_destroy_t)(void *)
Key destructor function type.
void *(* hashmap_key_clone_t)(void *)
Key cloning function type.
uint32_t(* hashmap_key_hash_t)(void *)
Key hashing function type.
Interface to platform-specific miscellaneous browser operation table.
Netsurf additional integer type formatting macros.
#define PRIssizet
c99 standard printf formatting for ssize_t type
#define PRIsizet
c99 standard printf formatting for size_t type
#define NSLOG(catname, level, logmsg, args...)
const char * messages_get_errorcode(nserror code)
lookup of a message by errorcode from the standard Messages hash.
Localised message support (interface).
NetSurf URL handling (interface).
bool nsurl_compare(const nsurl *url1, const nsurl *url2, nsurl_component parts)
Compare two URLs.
nserror nsurl_create(const char *const url_s, nsurl **url)
Create a NetSurf URL object from a URL string.
void nsurl_unref(nsurl *url)
Drop a reference to a NetSurf URL object.
uint32_t nsurl_hash(const nsurl *url)
Get a URL's hash value.
const char * nsurl_access(const nsurl *url)
Access a NetSurf URL object as a string.
nsurl * nsurl_ref(nsurl *url)
Increment the reference count to a NetSurf URL object.
struct nsurl nsurl
NetSurf URL object.
Interface to utility string handling.
uint8_t use_map[BLOCK_USE_MAP_SIZE]
map of used and unused entries within the block file
int fd
file descriptor of the block file
struct store_entry ** elist
low level cache backing store operation table
nserror(* initialise)(const struct llcache_store_parameters *parameters)
Initialise the backing store.
nserror(* schedule)(int t, void(*callback)(void *p), void *p)
Schedule a callback.
hashmap_key_clone_t key_clone
A function which when called will clone a key and give ownership of the returned object to the hashma...
The content of a hashmap.
Parameters to configure the low level cache backing store.
size_t hysteresis
The hysteresis around the target size.
size_t limit
The backing store upper bound target size.
const char * path
The path to the backing store.
struct gui_misc_table * misc
Browser table.
Backing store entry element.
block_index_t block
small object data block
uint8_t * data
data allocated
uint8_t ref
element data reference count
uint32_t size
size of entry element on disc
Backing store object index entry.
uint16_t use_count
number of times this entry has been accessed
struct store_entry_element elem[ENTRY_ELEM_COUNT]
Entry element (data or meta) specific information.
nsurl * url
The URL for this entry.
int64_t last_used
UNIX time the entry was last used.
Parameters controlling the backing store.
bool blocks_opened
flag indicating if a block file has been opened for update since maintenance was previously done.
uint64_t total_alloc
total size of all allocated storage.
bool blocks_dirty
flag indicating if the block file use maps have been made persistent since they were last changed.
uint64_t hit_size
size of storage served
hashmap_t * entries
The cache object hash.
char * path
The path to the backing store.
struct block_file blocks[ENTRY_ELEM_COUNT][BLOCK_FILE_COUNT]
small block indexes
size_t hysteresis
The hysteresis around the target size.
size_t limit
The backing store upper bound target size.
bool entries_dirty
flag indicating if the entries have been made persistent since they were last changed.
size_t hit_count
number of cache hits
size_t miss_count
number of cache misses
Interface to time operations.
nserror netsurf_mkdir_all(const char *fname)
Ensure that all directory elements needed to store a filename exist.
nserror netsurf_mkpath(char **str, size_t *size, size_t nelm,...)
Generate a path from one or more component elemnts.
nserror netsurf_recursive_rm(const char *path)
Recursively remove a directory.
Default operations table for files.