fs_backing_store.c File Reference

Low-level resource cache persistent storage implementation. More...

#include <unistd.h>
#include <string.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <fcntl.h>
#include <errno.h>
#include <time.h>
#include <stdlib.h>
#include <nsutils/unistd.h>
#include "netsurf/inttypes.h"
#include "utils/filepath.h"
#include "utils/file.h"
#include "utils/nsurl.h"
#include "utils/log.h"
#include "utils/messages.h"
#include "utils/hashmap.h"
#include "desktop/gui_internal.h"
#include "netsurf/misc.h"
#include "content/backing_store.h"

Include dependency graph for fs_backing_store.c:

Go to the source code of this file.

Data Structures
struct	store_entry_element
	Backing store entry element. More...
struct	store_entry
	Backing store object index entry. More...
struct	block_file
	Small block file. More...
struct	store_state
	Parameters controlling the backing store. More...
struct	eviction_state_t
struct	write_entry_iteration_state

Macros
#define	CONTROL_VERSION   202
	Backing store file format version. More...
#define	CONTROL_MAINT_TIME   10000
	Number of milliseconds after a update before control data maintenance is performed. More...
#define	ENTRIES_FNAME   "entries"
	Filename of serialised entries. More...
#define	BLOCKS_FNAME   "blocks"
	Filename of block file index. More...
#define	BLOCK_ADDR_LEN   16
	log2 block data address length (64k) More...
#define	BLOCK_ENTRY_COUNT   10
	log2 number of entries per block file(1024) More...
#define	BLOCK_FILE_COUNT   (BLOCK_ADDR_LEN - BLOCK_ENTRY_COUNT)
	log2 number of data block files More...
#define	BLOCK_DATA_SIZE   13
	log2 size of data blocks (8k) More...
#define	BLOCK_META_SIZE   13
	log2 size of metadata blocks (8k) More...
#define	BLOCK_USE_MAP_SIZE   (1 << (BLOCK_ENTRY_COUNT - 3))
	length in bytes of a block files use map More...

Typedefs
typedef uint32_t	entry_ident_t
	The type used as a binary identifier for each entry derived from the URL. More...
typedef uint16_t	block_index_t
	The type used to store block file index values. More...

Enumerations
enum	store_entry_elem_idx { ENTRY_ELEM_DATA = 0 , ENTRY_ELEM_META = 1 , ENTRY_ELEM_COUNT = 2 }
	Entry element index values. More...
enum	store_entry_elem_flags { ENTRY_ELEM_FLAG_NONE = 0 , ENTRY_ELEM_FLAG_HEAP = 0x1 , ENTRY_ELEM_FLAG_MMAP = 0x2 , ENTRY_ELEM_FLAG_SMALL = 0x4 }
	flags that indicate what additional information is contained within an entry element. More...
enum	store_entry_flags { ENTRY_FLAGS_NONE = 0 , ENTRY_FLAGS_INVALID = 1 }

Functions
static bool	entries_hashmap_key_eq (void *key1, void *key2)
static void *	entries_hashmap_value_alloc (void *key)
static void	entries_hashmap_value_destroy (void *value)
static char *	store_fname (struct store_state *state, entry_ident_t ident, int elem_idx)
	Generate a filename for an object. More...
static nserror	invalidate_element (struct store_state *state, struct store_entry *bse, int elem_idx)
	invalidate an element of an entry More...
static nserror	invalidate_entry (struct store_state *state, struct store_entry *bse)
	Remove the entry and files associated with an identifier. More...
static int	compar (const void *va, const void *vb)
	Quick sort comparison. More...
static bool	entry_eviction_iterator_cb (void *key, void *value, void *ctx)
	Iterator for gathering entries to compute eviction order. More...
static nserror	store_evict (struct store_state *state)
	Evict entries from backing store as per configuration. More...
static nserror	write_entry (struct store_entry *ent, int fd)
	Write a single store entry to disk. More...
static bool	write_entry_iterator (void *key, void *value, void *ctx)
	Callback for iterating the entries hashmap. More...
static nserror	write_entries (struct store_state *state)
	Write filesystem entries to file. More...
static nserror	write_blocks (struct store_state *state)
	Write block file use map to file. More...
static nserror	set_block_extents (struct store_state *state)
	Ensures block files are of the correct extent. More...
static void	control_maintenance (void *s)
	maintenance of control structures. More...
static nserror	get_store_entry (struct store_state *state, nsurl *url, struct store_entry **bse)
	Lookup a backing store entry in the entry table from a url. More...
static block_index_t	alloc_block (struct store_state *state, int elem_idx)
	Find next available small block. More...
static nserror	set_store_entry (struct store_state *state, nsurl *url, int elem_idx, uint8_t *data, const size_t datalen, struct store_entry **bse)
	Set a backing store entry in the entry table from a url. More...
static int	store_open (struct store_state *state, entry_ident_t ident, int elem_idx, int openflags)
	Open a file using a store ident. More...
static nserror	unlink_entries (struct store_state *state)
	Unlink entries file. More...
static nserror	read_entries (struct store_state *state)
	Read description entries into memory. More...
static nserror	read_blocks (struct store_state *state)
	Read block file usage bitmaps. More...
static nserror	write_cache_tag (struct store_state *state)
	Write the cache tag file. More...
static nserror	write_control (struct store_state *state)
	Write the control file for the current state. More...
static nserror	read_control (struct store_state *state)
	Read and parse the control file. More...
static nserror	initialise (const struct llcache_store_parameters *parameters)
	Initialise the backing store. More...
static nserror	finalise (void)
	Finalise the backing store. More...
static nserror	store_write_block (struct store_state *state, struct store_entry *bse, int elem_idx)
	Write an element of an entry to backing storage in a small block file. More...
static nserror	store_write_file (struct store_state *state, struct store_entry *bse, int elem_idx)
	Write an element of an entry to backing storage as an individual file. More...
static nserror	store (nsurl *url, enum backing_store_flags bsflags, uint8_t *data, const size_t datalen)
	Place an object in the backing store. More...
static nserror	entry_release_alloc (struct store_entry_element *elem)
	release any allocation for an entry More...
static nserror	store_read_block (struct store_state *state, struct store_entry *bse, int elem_idx)
	Read an element of an entry from a small block file in the backing storage. More...
static nserror	store_read_file (struct store_state *state, struct store_entry *bse, int elem_idx)
	Read an element of an entry from an individual file in the backing storage. More...
static nserror	fetch (nsurl *url, enum backing_store_flags bsflags, uint8_t **data_out, size_t *datalen_out)
	Retrieve an object from the backing store. More...
static nserror	release (nsurl *url, enum backing_store_flags bsflags)
	release a previously fetched or stored memory object. More...
static nserror	invalidate (nsurl *url)
	Invalidate a source object from the backing store. More...

Variables
static const unsigned int	log2_block_size [ENTRY_ELEM_COUNT]
	log2 of block size. More...
struct store_state *	storestate
	Global storage state. More...
static hashmap_parameters_t	entries_hashmap_parameters
static struct gui_llcache_table	llcache_table
struct gui_llcache_table *	filesystem_llcache_table = &llcache_table

Detailed Description

Low-level resource cache persistent storage implementation.

file based backing store.

Todo:

Consider improving eviction sorting to include objects size and remaining lifetime and other cost metrics.

Todo:

Implement mmap retrieval where supported.

Todo:

Implement static retrieval for metadata objects as their heap lifetime is typically very short, though this may be obsoleted by a small object storage strategy.

Definition in file fs_backing_store.c.

Macro Definition Documentation

BLOCK_ADDR_LEN

#define BLOCK_ADDR_LEN   16

log2 block data address length (64k)

Definition at line 74 of file fs_backing_store.c.

BLOCK_DATA_SIZE

#define BLOCK_DATA_SIZE   13

log2 size of data blocks (8k)

Definition at line 83 of file fs_backing_store.c.

BLOCK_ENTRY_COUNT

#define BLOCK_ENTRY_COUNT   10

log2 number of entries per block file(1024)

Definition at line 77 of file fs_backing_store.c.

BLOCK_FILE_COUNT

#define BLOCK_FILE_COUNT   (BLOCK_ADDR_LEN - BLOCK_ENTRY_COUNT)

log2 number of data block files

Definition at line 80 of file fs_backing_store.c.

BLOCK_META_SIZE

#define BLOCK_META_SIZE   13

log2 size of metadata blocks (8k)

Definition at line 86 of file fs_backing_store.c.

BLOCK_USE_MAP_SIZE

#define BLOCK_USE_MAP_SIZE   (1 << (BLOCK_ENTRY_COUNT - 3))

length in bytes of a block files use map

Definition at line 89 of file fs_backing_store.c.

BLOCKS_FNAME

#define BLOCKS_FNAME   "blocks"

Filename of block file index.

Definition at line 71 of file fs_backing_store.c.

CONTROL_MAINT_TIME

#define CONTROL_MAINT_TIME   10000

Number of milliseconds after a update before control data maintenance is performed.

Definition at line 65 of file fs_backing_store.c.

CONTROL_VERSION

#define CONTROL_VERSION   202

Backing store file format version.

Definition at line 59 of file fs_backing_store.c.

ENTRIES_FNAME

#define ENTRIES_FNAME   "entries"

Filename of serialised entries.

Definition at line 68 of file fs_backing_store.c.

Typedef Documentation

block_index_t

typedef uint16_t block_index_t

The type used to store block file index values.

If this is changed it will affect the entry storage/alignment and BLOCK_ADDR_LEN must also be updated.

Definition at line 103 of file fs_backing_store.c.

entry_ident_t

typedef uint32_t entry_ident_t

The type used as a binary identifier for each entry derived from the URL.

A larger identifier will have fewer collisions but requires proportionately more storage.

Definition at line 96 of file fs_backing_store.c.

Enumeration Type Documentation

store_entry_elem_flags

enum store_entry_elem_flags

flags that indicate what additional information is contained within an entry element.

Enumerator
ENTRY_ELEM_FLAG_NONE	store not managing any allocation on entry
ENTRY_ELEM_FLAG_HEAP	entry data allocation is on heap
ENTRY_ELEM_FLAG_MMAP	entry data allocation is mmaped
ENTRY_ELEM_FLAG_SMALL	entry data allocation is in small object pool

Definition at line 119 of file fs_backing_store.c.

store_entry_elem_idx

enum store_entry_elem_idx

Entry element index values.

Enumerator
ENTRY_ELEM_DATA	entry element is data
ENTRY_ELEM_META	entry element is metadata
ENTRY_ELEM_COUNT	count of elements on an entry

Definition at line 109 of file fs_backing_store.c.

store_entry_flags

enum store_entry_flags

Enumerator
ENTRY_FLAGS_NONE	entry is normal
ENTRY_FLAGS_INVALID	entry has been invalidated but something still holding a reference

Definition at line 131 of file fs_backing_store.c.

Function Documentation

alloc_block()

static block_index_t alloc_block ( struct store_state *  state, int  elem_idx  )

static

Find next available small block.

Definition at line 932 of file fs_backing_store.c.

References BLOCK_FILE_COUNT, BLOCK_USE_MAP_SIZE, store_state::blocks, store_state::blocks_dirty, and block_file::use_map.

Referenced by set_store_entry().

Here is the caller graph for this function:

compar()

static int compar ( const void *  va, const void *  vb  )

static

Quick sort comparison.

Definition at line 504 of file fs_backing_store.c.

References store_entry::elem, ENTRY_ELEM_DATA, ENTRY_ELEM_FLAG_NONE, ENTRY_ELEM_META, store_entry_element::flags, store_entry::last_used, and store_entry::use_count.

Referenced by scandir(), and store_evict().

Here is the caller graph for this function:

control_maintenance()

static void control_maintenance ( void *  s )

static

maintenance of control structures.

callback scheduled when control data has been update. Currently this is for when the entries table is dirty and requires serialising.

Parameters

s	store state to maintain.

Definition at line 883 of file fs_backing_store.c.

References set_block_extents(), write_blocks(), and write_entries().

Referenced by finalise(), get_store_entry(), and set_store_entry().

Here is the call graph for this function:

Here is the caller graph for this function:

entries_hashmap_key_eq()

static bool entries_hashmap_key_eq ( void *  key1, void *  key2  )

static

Definition at line 250 of file fs_backing_store.c.

References nsurl_compare(), and NSURL_COMPLETE.

Here is the call graph for this function:

entries_hashmap_value_alloc()

static void * entries_hashmap_value_alloc ( void *  key )

static

Definition at line 256 of file fs_backing_store.c.

References nsurl_ref(), and store_entry::url.

Here is the call graph for this function:

entries_hashmap_value_destroy()

static void entries_hashmap_value_destroy ( void *  value )

static

Todo:

Do we need to do any disk cleanup here? if so, meep!

Definition at line 266 of file fs_backing_store.c.

References nsurl_unref(), and store_entry::url.

Here is the call graph for this function:

entry_eviction_iterator_cb()

static bool entry_eviction_iterator_cb ( void *  key, void *  value, void *  ctx  )

static

Iterator for gathering entries to compute eviction order.

Definition at line 555 of file fs_backing_store.c.

References eviction_state_t::elist, and eviction_state_t::ent_count.

Referenced by store_evict().

Here is the caller graph for this function:

entry_release_alloc()

static nserror entry_release_alloc ( struct store_entry_element *  elem )

static

release any allocation for an entry

Definition at line 1726 of file fs_backing_store.c.

References store_entry_element::data, store_entry::elem, ENTRY_ELEM_FLAG_HEAP, store_entry_element::flags, NSERROR_OK, NSLOG, and store_entry_element::ref.

Referenced by fetch(), and release().

Here is the caller graph for this function:

fetch()

static nserror fetch ( nsurl *  url, enum backing_store_flags  bsflags, uint8_t **  data_out, size_t *  datalen_out  )

static

Retrieve an object from the backing store.

Parameters

[in]	url	The url is used as the unique primary key for the data.
[in]	bsflags	The flags to control how the object is retrieved.
[out]	data_out	The objects data.
[out]	datalen_out	The length of the data retrieved.

Returns

NSERROR_OK on success or error code on failure.

Definition at line 1855 of file fs_backing_store.c.

References BACKING_STORE_META, store_entry_element::block, store_entry_element::data, store_entry::elem, ENTRY_ELEM_DATA, ENTRY_ELEM_FLAG_HEAP, ENTRY_ELEM_META, entry_release_alloc(), store_entry_element::flags, get_store_entry(), store_state::hit_count, store_state::hit_size, store_state::miss_count, NSERROR_INIT_FAILED, NSERROR_NOMEM, NSERROR_OK, NSLOG, nsurl_access(), store_entry_element::ref, store_entry_element::size, store_read_block(), store_read_file(), and storestate.

Referenced by fetch_alloc(), fetch_curl_setup(), and fetch_start().

Here is the call graph for this function:

Here is the caller graph for this function:

finalise()

static nserror finalise ( void  )

static

Finalise the backing store.

Todo:

This will cause the backing store to leak any outstanding memory allocations. This will probably best be done by a global use count.

Returns

NSERROR_OK on success.

Definition at line 1526 of file fs_backing_store.c.

References BLOCK_FILE_COUNT, store_state::blocks, control_maintenance(), store_state::entries, ENTRY_ELEM_DATA, ENTRY_ELEM_META, block_file::fd, guit, hashmap_destroy(), store_state::hit_count, netsurf_table::misc, store_state::miss_count, NSERROR_OK, NSLOG, store_state::path, PRIsizet, gui_misc_table::schedule, storestate, write_blocks(), and write_entries().

Referenced by ro_gui_configure_register().

Here is the call graph for this function:

Here is the caller graph for this function:

get_store_entry()

static nserror get_store_entry ( struct store_state *  state, nsurl *  url, struct store_entry **  bse  )

static

Lookup a backing store entry in the entry table from a url.

This finds the store entry associated with the given key. Additionally if an entry is found it updates the usage data about the entry.

Parameters

state	The store state to use.
url	The value used as the unique key to search entries for.
bse	Pointer used to return value.

Returns

NSERROR_OK and bse updated on success or NSERROR_NOT_FOUND if no entry corresponds to the url.

Definition at line 907 of file fs_backing_store.c.

References CONTROL_MAINT_TIME, control_maintenance(), store_state::entries, store_state::entries_dirty, guit, hashmap_lookup(), store_entry::last_used, netsurf_table::misc, NSERROR_NOT_FOUND, NSERROR_OK, gui_misc_table::schedule, store_entry::url, and store_entry::use_count.

Referenced by fetch(), invalidate(), and release().

Here is the call graph for this function:

Here is the caller graph for this function:

initialise()

static nserror initialise ( const struct llcache_store_parameters *  parameters )

static

Initialise the backing store.

Parameters

parameters

to configure backing store.

Returns

NSERROR_OK on success or error code on failure.

Definition at line 1420 of file fs_backing_store.c.

References store_state::entries, hashmap_destroy(), store_state::hysteresis, llcache_store_parameters::hysteresis, store_state::limit, llcache_store_parameters::limit, messages_get_errorcode(), netsurf_recursive_rm(), NSERROR_INIT_FAILED, NSERROR_NOMEM, NSERROR_NOT_FOUND, NSERROR_OK, NSLOG, store_state::path, llcache_store_parameters::path, PRIsizet, PRIu64, read_blocks(), read_control(), read_entries(), storestate, store_state::total_alloc, unlink_entries(), write_cache_tag(), and write_control().

Referenced by ro_gui_configure_register().

Here is the call graph for this function:

Here is the caller graph for this function:

invalidate()

static nserror invalidate ( nsurl *  url )

static

Invalidate a source object from the backing store.

The entry (if present in the backing store) must no longer be returned as a result to the fetch or meta operations.

Parameters

url	The url is used as the unique primary key to invalidate.

Returns

NSERROR_OK on success or error code on failure.

Definition at line 1993 of file fs_backing_store.c.

References get_store_entry(), invalidate_entry(), NSERROR_INIT_FAILED, NSERROR_OK, storestate, and store_entry::url.

Here is the call graph for this function:

invalidate_element()

static nserror invalidate_element ( struct store_state *  state, struct store_entry *  bse, int  elem_idx  )

static

invalidate an element of an entry

Parameters

state	The store state to use.
bse	The entry to invalidate.
elem_idx	The element index to invalidate.

Returns

NSERROR_OK on success or error code on failure.

Definition at line 418 of file fs_backing_store.c.

References store_entry_element::block, BLOCK_ENTRY_COUNT, BLOCK_FILE_COUNT, store_state::blocks, store_entry::elem, NSERROR_NOMEM, NSERROR_OK, nsurl_hash(), store_entry_element::size, store_fname(), store_state::total_alloc, store_entry::url, and block_file::use_map.

Referenced by invalidate_entry().

Here is the call graph for this function:

Here is the caller graph for this function:

invalidate_entry()

static nserror invalidate_entry ( struct store_state *  state, struct store_entry *  bse  )

static

Remove the entry and files associated with an identifier.

Parameters

state	The store state to use.
bse	The entry to invalidate.

Returns

NSERROR_OK on success or error code on failure.

Definition at line 460 of file fs_backing_store.c.

References store_entry::elem, store_state::entries, ENTRY_ELEM_DATA, ENTRY_ELEM_FLAG_HEAP, ENTRY_ELEM_FLAG_MMAP, ENTRY_ELEM_META, ENTRY_FLAGS_INVALID, store_entry_element::flags, store_entry::flags, hashmap_remove(), invalidate_element(), NSERROR_OK, NSLOG, nsurl_access(), and store_entry::url.

Referenced by invalidate(), release(), and store_evict().

Here is the call graph for this function:

Here is the caller graph for this function:

read_blocks()

static nserror read_blocks ( struct store_state *  state )

static

Read block file usage bitmaps.

Parameters

state

The backing store state to put the loaded entries in.

Returns

NSERROR_OK on success or error code on failure.

Definition at line 1216 of file fs_backing_store.c.

References BLOCK_FILE_COUNT, BLOCK_USE_MAP_SIZE, store_state::blocks, BLOCKS_FNAME, ENTRY_ELEM_COUNT, ENTRY_ELEM_DATA, ENTRY_ELEM_META, block_file::fd, netsurf_mkpath(), NSERROR_OK, NSLOG, store_state::path, and block_file::use_map.

Referenced by initialise().

Here is the call graph for this function:

Here is the caller graph for this function:

read_control()

static nserror read_control ( struct store_state *  state )

static

Read and parse the control file.

Parameters

state

The state to read from the control file.

Returns

NSERROR_OK on success or error code on failure.

Definition at line 1355 of file fs_backing_store.c.

References CONTROL_VERSION, netsurf_mkpath(), NSERROR_INIT_FAILED, NSERROR_NOT_FOUND, NSERROR_OK, NSLOG, and store_state::path.

Referenced by initialise().

Here is the call graph for this function:

Here is the caller graph for this function:

read_entries()

static nserror read_entries ( struct store_state *  state )

static

Read description entries into memory.

Parameters

state

The backing store state to put the loaded entries in.

Returns

NSERROR_OK on success or error code on faliure.

Definition at line 1125 of file fs_backing_store.c.

References store_entry::elem, store_state::entries, ENTRIES_FNAME, entries_hashmap_parameters, ENTRY_ELEM_DATA, ENTRY_ELEM_FLAG_HEAP, ENTRY_ELEM_FLAG_MMAP, ENTRY_ELEM_META, store_entry_element::flags, hashmap_create(), hashmap_insert(), netsurf_mkpath(), NSERROR_INIT_FAILED, NSERROR_NOMEM, NSERROR_OK, NSLOG, nsurl_access(), nsurl_create(), nsurl_unref(), store_state::path, PRIsizet, read_entries(), store_entry_element::size, store_state::total_alloc, and store_entry::url.

Referenced by initialise(), and read_entries().

Here is the call graph for this function:

Here is the caller graph for this function:

release()

static nserror release ( nsurl *  url, enum backing_store_flags  bsflags  )

static

release a previously fetched or stored memory object.

Parameters

[in]	url	The url is used as the unique primary key to invalidate.
[in]	bsflags	The flags to control how the object data is released.

Returns

NSERROR_OK on success or error code on failure.

Definition at line 1944 of file fs_backing_store.c.

References BACKING_STORE_META, store_entry::elem, ENTRY_ELEM_DATA, ENTRY_ELEM_META, ENTRY_FLAGS_INVALID, entry_release_alloc(), store_entry::flags, get_store_entry(), invalidate_entry(), NSERROR_INIT_FAILED, NSERROR_OK, NSLOG, and storestate.

Here is the call graph for this function:

set_block_extents()

static nserror set_block_extents ( struct store_state *  state )

static

Ensures block files are of the correct extent.

block files have their extent set to their maximum size to ensure subsequent reads and writes do not need to extend the file and are therefore faster.

Parameters

state

The backing store state to set block extent for.

Returns

NSERROR_OK on success or error code on failure.

Definition at line 842 of file fs_backing_store.c.

References BLOCK_ENTRY_COUNT, BLOCK_FILE_COUNT, store_state::blocks, store_state::blocks_opened, ENTRY_ELEM_COUNT, block_file::fd, log2_block_size, NSERROR_OK, and NSLOG.

Referenced by control_maintenance().

Here is the caller graph for this function:

set_store_entry()

static nserror set_store_entry ( struct store_state *  state, nsurl *  url, int  elem_idx, uint8_t *  data, const size_t  datalen, struct store_entry **  bse  )

static

Set a backing store entry in the entry table from a url.

This creates a backing store entry in the entry table for a url.

Parameters

state	The store state to use.
url	The value used as the unique key to search entries for.
elem_idx	The index of the entry element to use.
data	The data to store
datalen	The length of data in data
bse	Pointer used to return value.

Returns

NSERROR_OK and bse updated on success or NSERROR_NOT_FOUND if no entry corresponds to the url.

Definition at line 975 of file fs_backing_store.c.

References alloc_block(), store_entry_element::block, CONTROL_MAINT_TIME, control_maintenance(), store_entry_element::data, store_entry::elem, store_state::entries, store_state::entries_dirty, ENTRY_ELEM_FLAG_HEAP, ENTRY_ELEM_FLAG_MMAP, store_entry_element::flags, guit, hashmap_insert(), hashmap_lookup(), store_entry::last_used, log2_block_size, netsurf_table::misc, NSERROR_NOMEM, NSERROR_OK, NSERROR_PERMISSION, NSLOG, nsurl_access(), store_entry_element::ref, gui_misc_table::schedule, store_entry_element::size, store_evict(), store_state::total_alloc, and store_entry::use_count.

Referenced by store().

Here is the call graph for this function:

Here is the caller graph for this function:

store()

static nserror store ( nsurl *  url, enum backing_store_flags  bsflags, uint8_t *  data, const size_t  datalen  )

static

Place an object in the backing store.

takes ownership of the heap block passed in.

Parameters

url	The url is used as the unique primary key for the data.
bsflags	The flags to control how the object is stored.
data	The objects source data.
datalen	The length of the data.

Returns

NSERROR_OK on success or error code on failure.

Definition at line 1684 of file fs_backing_store.c.

References BACKING_STORE_META, store_entry_element::block, store_entry::elem, ENTRY_ELEM_DATA, ENTRY_ELEM_META, NSERROR_INIT_FAILED, NSERROR_OK, NSLOG, set_store_entry(), store_write_block(), store_write_file(), storestate, and store_entry::url.

Referenced by main().

Here is the call graph for this function:

Here is the caller graph for this function:

store_evict()

static nserror store_evict ( struct store_state *  state )

static

Evict entries from backing store as per configuration.

Entries are evicted to ensure the cache remains within the configured limits on size and number of entries.

The approach is to check if the cache limits have been exceeded and if so build and sort list of entries to evict. The list is sorted by use count and then by age, so oldest object with least number of uses get evicted first.

Parameters

state

The store state to use.

Returns

NSERROR_OK on success or error code on failure.

Definition at line 577 of file fs_backing_store.c.

References compar(), store_entry::elem, eviction_state_t::elist, eviction_state_t::ent_count, store_state::entries, ENTRY_ELEM_DATA, ENTRY_ELEM_META, entry_eviction_iterator_cb(), hashmap_count(), hashmap_iterate(), store_state::hysteresis, invalidate_entry(), store_state::limit, NSERROR_NOMEM, NSERROR_OK, NSERROR_UNKNOWN, NSLOG, PRIsizet, PRIu64, store_entry_element::size, and store_state::total_alloc.

Referenced by set_store_entry().

Here is the call graph for this function:

Here is the caller graph for this function:

store_fname()

static char * store_fname ( struct store_state *  state, entry_ident_t  ident, int  elem_idx  )

static

Generate a filename for an object.

this generates the filename for an object on disc. It is necessary for this to generate a filename which conforms to the limitations of all the filesystems the cache can be placed upon.

From http://en.wikipedia.org/wiki/Comparison_of_file_systems#Limits the relevant subset is:

• path elements no longer than 8 characters
• acceptable characters are A-Z, 0-9
• short total path lengths (255 or less)
• no more than 77 entries per directory (6bits worth)

The short total path lengths mean the encoding must represent as much data as possible in the least number of characters.

To achieve all these goals we use RFC4648 base32 encoding which packs 5bits into each character of the filename. To represent a 32 bit ident this requires a total path length of between 17 and 22 bytes (including directory separators) BA/BB/BC/BD/BE/ABCDEFG

Note

Version 1.00 of the cache implementation used base64 to encode this, however that did not meet the requirement for only using uppercase characters.

Versions prior to 1.30 only packed 5 bits per directory level A/B/C/D/E/F/ABCDEFG which only required 19 characters to represent but resulted in requiring an extra level of directory which is less desirable than the three extra characters using six bits.

Parameters

state	The store state to use.
ident	The identifier to use.
elem_idx	The element index.

Returns

The filename string or NULL on allocation error.

Definition at line 320 of file fs_backing_store.c.

References ENTRY_ELEM_COUNT, ENTRY_ELEM_DATA, ENTRY_ELEM_META, netsurf_mkpath(), and store_state::path.

Referenced by invalidate_element(), and store_open().

Here is the call graph for this function:

Here is the caller graph for this function:

store_open()

static int store_open ( struct store_state *  state, entry_ident_t  ident, int  elem_idx, int  openflags  )

static

Open a file using a store ident.

Parameters

state	The store state to use.
ident	The identifier to open file for.
elem_idx	The element within the store entry to open. The value should be be one of the values in the store_entry_elem_idx enum. Additionally it may have ENTRY_ELEM_COUNT added to it to indicate block file names.
openflags	The flags used with the open call.

Returns

An fd from the open call or -1 on error.

Definition at line 1060 of file fs_backing_store.c.

References netsurf_mkdir_all(), NSERROR_OK, NSLOG, and store_fname().

Referenced by store_read_block(), store_read_file(), store_write_block(), and store_write_file().

Here is the call graph for this function:

Here is the caller graph for this function:

store_read_block()

static nserror store_read_block ( struct store_state *  state, struct store_entry *  bse, int  elem_idx  )

static

Read an element of an entry from a small block file in the backing storage.

Parameters

state	The backing store state to use.
bse	The entry to read.
elem_idx	The element index within the entry.

Returns

NSERROR_OK on success or error code.

Definition at line 1748 of file fs_backing_store.c.

References store_entry_element::block, BLOCK_ENTRY_COUNT, BLOCK_FILE_COUNT, store_state::blocks, store_state::blocks_opened, store_entry_element::data, store_entry::elem, ENTRY_ELEM_COUNT, block_file::fd, log2_block_size, NSERROR_OK, NSERROR_SAVE_FAILED, NSLOG, PRIsizet, PRIssizet, store_entry_element::size, and store_open().

Referenced by fetch().

Here is the call graph for this function:

Here is the caller graph for this function:

store_read_file()

static nserror store_read_file ( struct store_state *  state, struct store_entry *  bse, int  elem_idx  )

static

Read an element of an entry from an individual file in the backing storage.

Parameters

state	The backing store state to use.
bse	The entry to read.
elem_idx	The element index within the entry.

Returns

NSERROR_OK on success or error code.

Todo:

should this invalidate the entry?

Definition at line 1805 of file fs_backing_store.c.

References store_entry_element::data, store_entry::elem, NSERROR_NOT_FOUND, NSERROR_OK, NSLOG, nsurl_hash(), PRIsizet, PRIssizet, store_entry_element::size, store_open(), storestate, and store_entry::url.

Referenced by fetch().

Here is the call graph for this function:

Here is the caller graph for this function:

store_write_block()

static nserror store_write_block ( struct store_state *  state, struct store_entry *  bse, int  elem_idx  )

static

Write an element of an entry to backing storage in a small block file.

Parameters

state	The backing store state to use.
bse	The entry to store
elem_idx	The element index within the entry.

Returns

NSERROR_OK on success or error code.

Definition at line 1578 of file fs_backing_store.c.

References store_entry_element::block, BLOCK_ENTRY_COUNT, BLOCK_FILE_COUNT, store_state::blocks, store_state::blocks_opened, store_entry_element::data, store_entry::elem, ENTRY_ELEM_COUNT, block_file::fd, log2_block_size, NSERROR_OK, NSERROR_SAVE_FAILED, NSLOG, PRIsizet, PRIssizet, store_entry_element::size, and store_open().

Referenced by store().

Here is the call graph for this function:

Here is the caller graph for this function:

store_write_file()

static nserror store_write_file ( struct store_state *  state, struct store_entry *  bse, int  elem_idx  )

static

Write an element of an entry to backing storage as an individual file.

Parameters

state	The backing store state to use.
bse	The entry to store
elem_idx	The element index within the entry.

Returns

NSERROR_OK on success or error code.

Todo:

Delete the file?

Definition at line 1635 of file fs_backing_store.c.

References store_entry_element::data, store_entry::elem, NSERROR_OK, NSERROR_SAVE_FAILED, NSLOG, nsurl_hash(), PRIssizet, store_entry_element::size, store_open(), and store_entry::url.

Referenced by store().

Here is the call graph for this function:

Here is the caller graph for this function:

unlink_entries()

static nserror unlink_entries ( struct store_state *  state )

static

Unlink entries file.

Parameters

state

The backing store state.

Returns

NSERROR_OK on success or error code on failure.

Definition at line 1102 of file fs_backing_store.c.

References ENTRIES_FNAME, netsurf_mkpath(), NSERROR_OK, and store_state::path.

Referenced by initialise().

Here is the call graph for this function:

Here is the caller graph for this function:

write_blocks()

static nserror write_blocks ( struct store_state *  state )

static

Write block file use map to file.

Serialise block file use map out to storage.

Parameters

state

The backing store state to serialise.

Returns

NSERROR_OK on success or error code on failure.

Definition at line 758 of file fs_backing_store.c.

References BLOCK_FILE_COUNT, BLOCK_USE_MAP_SIZE, store_state::blocks, store_state::blocks_dirty, BLOCKS_FNAME, ENTRY_ELEM_COUNT, netsurf_mkpath(), NSERROR_OK, NSERROR_SAVE_FAILED, NSLOG, store_state::path, and block_file::use_map.

Referenced by control_maintenance(), and finalise().

Here is the call graph for this function:

Here is the caller graph for this function:

write_cache_tag()

static nserror write_cache_tag ( struct store_state *  state )

static

Write the cache tag file.

Parameters

state

The cache state.

Returns

NSERROR_OK on success or error code on failure.

Definition at line 1276 of file fs_backing_store.c.

References netsurf_mkpath(), NSERROR_NOT_FOUND, NSERROR_OK, and store_state::path.

Referenced by initialise().

Here is the call graph for this function:

Here is the caller graph for this function:

write_control()

static nserror write_control ( struct store_state *  state )

static

Write the control file for the current state.

Parameters

state

The state to write to the control file.

Returns

NSERROR_OK on success or error code on failure.

Definition at line 1313 of file fs_backing_store.c.

References CONTROL_VERSION, netsurf_mkdir_all(), netsurf_mkpath(), NSERROR_NOT_FOUND, NSERROR_OK, NSLOG, and store_state::path.

Referenced by initialise().

Here is the call graph for this function:

Here is the caller graph for this function:

write_entries()

static nserror write_entries ( struct store_state *  state )

static

Write filesystem entries to file.

Serialise entry index out to storage.

Parameters

state

The backing store state to serialise.

Returns

NSERROR_OK on success or error code on failure.

Definition at line 694 of file fs_backing_store.c.

References store_state::entries, store_state::entries_dirty, ENTRIES_FNAME, write_entry_iteration_state::fd, hashmap_iterate(), netsurf_mkpath(), NSERROR_OK, NSERROR_SAVE_FAILED, NSLOG, store_state::path, PRIsizet, write_entry_iterator(), and write_entry_iteration_state::written.

Referenced by control_maintenance(), and finalise().

Here is the call graph for this function:

Here is the caller graph for this function:

write_entry()

static nserror write_entry ( struct store_entry *  ent, int  fd  )

static

Write a single store entry to disk.

To serialise a single store entry for now we write out a 32bit int which is the length of the url, then that many bytes of the url. Then we write out the full store entry struct as-is, which includes a useless nsurl pointer.

Definition at line 654 of file fs_backing_store.c.

References NSERROR_OK, NSERROR_SAVE_FAILED, nsurl_access(), and store_entry::url.

Referenced by write_entry_iterator().

Here is the call graph for this function:

Here is the caller graph for this function:

write_entry_iterator()

static bool write_entry_iterator ( void *  key, void *  value, void *  ctx  )

static

Callback for iterating the entries hashmap.

Definition at line 676 of file fs_backing_store.c.

References write_entry_iteration_state::fd, NSERROR_OK, write_entry(), and write_entry_iteration_state::written.

Referenced by write_entries().

Here is the call graph for this function:

Here is the caller graph for this function:

Variable Documentation

entries_hashmap_parameters

hashmap_parameters_t entries_hashmap_parameters

static

Initial value:

= {
        .key_clone = (hashmap_key_clone_t)nsurl_ref,
        .key_destroy = (hashmap_key_destroy_t)nsurl_unref,
        .key_hash = (hashmap_key_hash_t)nsurl_hash,
        .key_eq = entries_hashmap_key_eq,
        .value_alloc = entries_hashmap_value_alloc,
        .value_destroy = entries_hashmap_value_destroy,
}

Definition at line 274 of file fs_backing_store.c.

Referenced by read_entries().

filesystem_llcache_table

struct gui_llcache_table* filesystem_llcache_table = &llcache_table

Definition at line 2021 of file fs_backing_store.c.

Referenced by main().

llcache_table

struct gui_llcache_table llcache_table

static

Initial value:

= {
        .initialise = initialise,
        .finalise = finalise,
        .store = store,
        .fetch = fetch,
        .invalidate = invalidate,
        .release = release,
}

Definition at line 2012 of file fs_backing_store.c.

log2_block_size

const unsigned int log2_block_size[ENTRY_ELEM_COUNT]

static

Initial value:

= {
        BLOCK_DATA_SIZE, 
        BLOCK_META_SIZE  
}

log2 of block size.

Definition at line 189 of file fs_backing_store.c.

Referenced by set_block_extents(), set_store_entry(), store_read_block(), and store_write_block().

storestate

struct store_state* storestate

Global storage state.

Todo:

Investigate if there is a way to have a context rather than use a global.

Definition at line 242 of file fs_backing_store.c.

Referenced by fetch(), finalise(), initialise(), invalidate(), release(), store(), and store_read_file().