Unified URL information database implementation. More...

#include <assert.h>
#include <stdbool.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <strings.h>
#include <time.h>
#include "utils/inet.h"
#include "utils/nsoption.h"
#include "utils/log.h"
#include "utils/corestrings.h"
#include "utils/url.h"
#include "utils/utils.h"
#include "utils/bloom.h"
#include "utils/nsurl.h"
#include "utils/ascii.h"
#include "utils/http.h"
#include "netsurf/bitmap.h"
#include "desktop/cookie_manager.h"
#include "content/content.h"
#include "content/urldb.h"

Include dependency graph for urldb.c:

Data Structures
struct	cookie_internal_data
	cookie entry. More...

struct	prot_space_data
	A protection space. More...

struct	url_internal_data
	meta data about a url More...

struct	path_data
	data entry for url More...

struct	hsts_data

struct	host_part

struct	search_node
	search index node More...

Macros
#define	NUM_SEARCH_TREES 28
	Search trees - one per letter + 1 for IPs + 1 for Everything Else. More...

#define	ST_IP 0

#define	ST_EE 1

#define	ST_DN 2

#define	MIN_COOKIE_FILE_VERSION 100
	Minimum cookie database file version. More...

#define	COOKIE_FILE_VERSION 102
	Current cookie database file version. More...

#define	MIN_URL_FILE_VERSION 106
	Minimum URL database file version. More...

#define	URL_FILE_VERSION 107
	Current URL database file version. More...

#define	BLOOM_SIZE (1024 * 32)
	Size of url filter. More...

#define	MAXIMUM_URL_LENGTH 4096

#define	GROW_MATCHED_COOKIES

#define	FIND_T

#define	SKIP_T

Functions
static nserror	urldb_write_timet (FILE *fp, time_t val)
	write a time_t to a file portably More...

static void	urldb_write_paths (const struct path_data parent, const char host, FILE fp, char path, int path_alloc, int *path_used, time_t expiry)
	Write paths associated with a host. More...

static void	urldb_count_urls (const struct path_data root, time_t expiry, unsigned int count)
	Count number of URLs associated with a host. More...

static void	urldb_save_search_tree (struct search_node parent, FILE fp)
	Save a search (sub)tree. More...

static bool	urldb_iterate_entries_path (const struct path_data parent, bool(url_callback)(nsurl url, const struct url_data data), bool(cookie_callback)(const struct cookie_data data))
	Path data iterator (internal) More...

static bool	urldb__host_is_ip_address (const char *host)
	Check whether a host string is an IP address. More...

static int	urldb_search_match_prefix (const struct host_part a, const char b)
	Compare host_part with prefix. More...

static bool	urldb_iterate_partial_host (struct search_node root, const char prefix, bool(callback)(nsurl url, const struct url_data *data))
	Partial host iterator (internal) More...

static bool	urldb_iterate_partial_path (const struct path_data parent, const char prefix, bool(callback)(nsurl url, const struct url_data *data))
	Partial path iterator (internal) More...

static bool	urldb_iterate_entries_host (struct search_node parent, bool(url_callback)(nsurl url, const struct url_data data), bool(cookie_callback)(const struct cookie_data data))
	Host data iterator (internal) More...

static struct host_part *	urldb_add_host_node (const char part, struct host_part parent)
	Add a host node to the tree. More...

static int	urldb_add_path_fragment_cmp (const void a, const void b)
	Fragment comparator callback for qsort. More...

static struct path_data *	urldb_add_path_fragment (struct path_data segment, lwc_string fragment)
	Add a fragment to a path segment. More...

static struct path_data *	urldb_add_path_node (lwc_string scheme, unsigned int port, const char segment, lwc_string fragment, struct path_data parent)
	Add a path node to the tree. More...

static struct search_node **	urldb_get_search_tree_direct (const char *host)
	Get the search tree for a particular host. More...

static struct search_node *	urldb_get_search_tree (const char *host)
	Get the search tree for a particular host. More...

static int	urldb_search_match_string (const struct host_part a, const char b)
	Compare host part with a string. More...

static const struct host_part *	urldb_search_find (struct search_node root, const char host)
	Find a node in a search tree. More...

static struct path_data *	urldb_match_path (const struct path_data parent, const char path, lwc_string *scheme, unsigned short port)
	Match a path string. More...

static struct path_data *	urldb_find_url (nsurl *url)
	Find an URL in the database. More...

static void	urldb_dump_paths (struct path_data *parent)
	Dump URL database paths to stderr. More...

static void	urldb_dump_hosts (struct host_part *parent)
	Dump URL database hosts to stderr. More...

static void	urldb_dump_search (struct search_node *parent, int depth)
	Dump search tree. More...

static int	urldb_search_match_host (const struct host_part a, const struct host_part b)
	Compare a pair of host parts. More...

static struct search_node *	urldb_search_skew (struct search_node *root)
	Rotate a subtree right. More...

static struct search_node *	urldb_search_split (struct search_node *root)
	Rotate a node left, increasing the parent's level. More...

static struct search_node *	urldb_search_insert_internal (struct search_node root, struct search_node n)
	Insert node into search tree. More...

static struct search_node *	urldb_search_insert (struct search_node root, const struct host_part data)
	Insert a node into the search tree. More...

static bool	urldb_parse_avpair (struct cookie_internal_data c, char n, char *v, bool was_quoted)
	Parse a cookie avpair. More...

static void	urldb_free_cookie (struct cookie_internal_data *c)
	Free a cookie. More...

static struct cookie_internal_data *	urldb_parse_cookie (nsurl url, const char *cookie)
	Parse a cookie. More...

static struct path_data *	urldb_add_path (lwc_string scheme, unsigned int port, const struct host_part host, char path_query, lwc_string fragment, nsurl *url)
	Add a path to the database, creating any intermediate entries. More...

static struct host_part *	urldb_add_host (const char *host)
	Add a host to the database, creating any intermediate entries. More...

static bool	urldb_insert_cookie (struct cookie_internal_data c, lwc_string scheme, nsurl *url)
	Insert a cookie into the database. More...

static bool	urldb_concat_cookie (struct cookie_internal_data c, int version, int used, int alloc, char *buf)
	Concatenate a cookie into the provided buffer. More...

static void	urldb_delete_cookie_paths (const char domain, const char path, const char name, struct path_data parent)
	deletes paths from a cookie. More...

static void	urldb_delete_cookie_hosts (const char domain, const char path, const char name, struct host_part parent)
	Deletes cookie hosts and their assoicated paths. More...

static void	urldb_save_cookie_paths (FILE fp, struct path_data parent)
	Save a path subtree's cookies. More...

static void	urldb_save_cookie_hosts (FILE fp, struct host_part parent)
	Save a host subtree's cookies. More...

static void	urldb_destroy_cookie (struct cookie_internal_data *c)
	Destroy a cookie node. More...

static void	urldb_destroy_path_node_content (struct path_data *node)
	Destroy the contents of a path node. More...

static void	urldb_destroy_prot_space (struct prot_space_data *space)
	Destroy protection space data. More...

static void	urldb_destroy_path_tree (struct path_data *root)
	Destroy a path tree. More...

static void	urldb_destroy_host_tree (struct host_part *root)
	Destroy a host tree. More...

static void	urldb_destroy_search_tree (struct search_node *root)
	Destroy a search tree. More...

void	urldb_destroy (void)
	Destroy urldb. More...

nserror	urldb_load (const char *filename)
	Import an URL database from file, replacing any existing database. More...

nserror	urldb_save (const char *filename)
	Export the current database to file. More...

nserror	urldb_set_url_persistence (nsurl *url, bool persist)
	Set the cross-session persistence of the entry for an URL. More...

bool	urldb_add_url (nsurl *url)
	Insert an URL into the database. More...

nserror	urldb_set_url_title (nsurl url, const char title)
	Set an URL's title string, replacing any existing one. More...

nserror	urldb_set_url_content_type (nsurl *url, content_type type)
	Set an URL's content type. More...

nserror	urldb_update_url_visit_data (nsurl *url)
	Update an URL's visit data. More...

void	urldb_reset_url_visit_data (nsurl *url)
	Reset an URL's visit statistics. More...

const struct url_data *	urldb_get_url_data (nsurl *url)
	Find data for an URL. More...

nsurl *	urldb_get_url (nsurl *url)
	Extract an URL from the db. More...

void	urldb_set_auth_details (nsurl url, const char realm, const char *auth)
	Set authentication data for an URL. More...

const char *	urldb_get_auth_details (nsurl url, const char realm)
	Look up authentication details in database. More...

void	urldb_set_cert_permissions (nsurl *url, bool permit)
	Set certificate verification permissions. More...

bool	urldb_get_cert_permissions (nsurl *url)
	Retrieve certificate verification permissions from database. More...

bool	urldb_set_hsts_policy (struct nsurl url, const char header)
	Set HSTS policy for an URL. More...

bool	urldb_get_hsts_enabled (struct nsurl *url)
	Determine if HSTS policy is enabled for an URL. More...

void	urldb_iterate_partial (const char prefix, bool(callback)(nsurl url, const struct url_data data))

void	urldb_iterate_entries (bool(callback)(nsurl url, const struct url_data *data))

void	urldb_iterate_cookies (bool(callback)(const struct cookie_data data))

bool	urldb_set_cookie (const char header, nsurl url, nsurl *referer)
	Parse Set-Cookie header and insert cookie(s) into database. More...

char *	urldb_get_cookie (nsurl *url, bool include_http_only)
	Retrieve cookies for an URL. More...

void	urldb_delete_cookie (const char domain, const char path, const char *name)
	Delete a cookie. More...

void	urldb_load_cookies (const char *filename)
	Load a cookie file into the database. More...

void	urldb_save_cookies (const char *filename)
	Save persistent cookies to file. More...

void	urldb_dump (void)
	Dump URL database to stderr. More...

Variables
static struct host_part	db_root
	Root database handle. More...

static struct search_node	empty = { 0, 0, &empty, &empty }

static struct search_node *	search_trees [NUM_SEARCH_TREES]

static int	loaded_cookie_file_version
	loaded cookie file version More...

static struct bloom_filter *	url_bloom
	filter for url presence in database More...

Detailed Description

Unified URL information database implementation.

URLs are stored in a tree-based structure as follows:

The host component is extracted from each URL and, if a FQDN, split on every '.'.The tree is constructed by inserting each FQDN segment in reverse order. Duplicate nodes are merged.

If the host part of an URL is an IP address, then this is added to the tree verbatim (as if it were a TLD).

This provides something looking like:

                  root (a sentinel)
                    |
 -------------------------------------------------
 |  |       |       |       |       |       |
com     edu     gov  127.0.0.1  net     org     uk  TLDs
 |  |       |               |       |       |

google ... ... ... ... co 2LDs | | www bbc Hosts/Subdomains | www ...

Each of the nodes in this tree is a struct host_part. This stores the FQDN segment (or IP address) with which the node is concerned. Each node may contain further information about paths on a host (struct path_data) or SSL certificate processing on a host-wide basis (host_part::permit_invalid_certs).

Path data is concerned with storing various metadata about the path in question. This includes global history data, HTTP authentication details and any associated HTTP cookies. This is stored as a tree of path segments hanging off the relevant host_part node.

Therefore, to find the last visited time of the URL http://www.example.com/path/to/resource.html, the FQDN tree would be traversed in the order root -> "com" -> "example" -> "www". The "www" node would have attached to it a tree of struct path_data:

                (sentinel)
                    |
                   path
                    |
                   to
                    |
               resource.html

This represents the absolute path "/path/to/resource.html". The leaf node "resource.html" contains the last visited time of the resource.

The mechanism described above is, however, not particularly conducive to fast searching of the database for a given URL (or URLs beginning with a given prefix). Therefore, an anciliary data structure is used to enable fast searching. This structure simply reflects the contents of the database, with entries being added/removed at the same time as for the core database. In order to ensure that degenerate cases are kept to a minimum, we use an AAtree. This is an approximation of a Red-Black tree with similar performance characteristics, but with a significantly simpler implementation. Entries in this tree comprise pointers to the leaf nodes of the host tree described above.

REALLY IMPORTANT NOTE: urldb expects all URLs to be normalised. Use of non-normalised URLs with urldb will result in undefined behaviour and potential crashes.

Definition in file urldb.c.

Macro Definition Documentation

◆ BLOOM_SIZE

#define BLOOM_SIZE (1024 * 32)

Size of url filter.

Definition at line 317 of file urldb.c.

◆ COOKIE_FILE_VERSION

#define COOKIE_FILE_VERSION 102

Current cookie database file version.

Definition at line 293 of file urldb.c.

◆ FIND_T

#define FIND_T

Value:

                {                               \
                for (; *p && *p != '\t'; p++)   \
                        ; /* do nothing */      \
                if (p >= end) {                 \
                        NSLOG(netsurf, INFO, "Overran input");  \
                        continue;               \
                }                               \
                *p++ = '\0';                    \
        }

◆ GROW_MATCHED_COOKIES

#define GROW_MATCHED_COOKIES

Value:

        do {                                                            \
                if (count == matched_cookies_size) {                    \
                        struct cookie_internal_data **temp;             \
                        temp = realloc(matched_cookies,                 \
                                       (matched_cookies_size + 20) *    \
                                       sizeof(struct cookie_internal_data *)); \
                                                                        \
                        if (temp == NULL) {                             \
                                free(ret);                              \
                                free(matched_cookies);                  \
                                return NULL;                            \
                        }                                               \
                                                                        \
                        matched_cookies = temp;                         \
                        matched_cookies_size += 20;                     \
                }                                                       \
        } while(0)

◆ MAXIMUM_URL_LENGTH

#define MAXIMUM_URL_LENGTH 4096

◆ MIN_COOKIE_FILE_VERSION

#define MIN_COOKIE_FILE_VERSION 100

Minimum cookie database file version.

Definition at line 291 of file urldb.c.

◆ MIN_URL_FILE_VERSION

#define MIN_URL_FILE_VERSION 106

Minimum URL database file version.

Definition at line 298 of file urldb.c.

◆ NUM_SEARCH_TREES

#define NUM_SEARCH_TREES 28

Search trees - one per letter + 1 for IPs + 1 for Everything Else.

Definition at line 278 of file urldb.c.

◆ SKIP_T

#define SKIP_T

Value:

                {                               \
                for (; *p && *p == '\t'; p++)   \
                        ; /* do nothing */      \
                if (p >= end) {                 \
                        NSLOG(netsurf, INFO, "Overran input");  \
                        continue;               \
                }                               \
        }

◆ ST_DN

#define ST_DN 2

Definition at line 281 of file urldb.c.

◆ ST_EE

#define ST_EE 1

Definition at line 280 of file urldb.c.

◆ ST_IP

#define ST_IP 0

Definition at line 279 of file urldb.c.

◆ URL_FILE_VERSION

#define URL_FILE_VERSION 107

Current URL database file version.

Definition at line 300 of file urldb.c.

Function Documentation

◆ urldb__host_is_ip_address()

static bool urldb__host_is_ip_address ( const char * host )

static

Check whether a host string is an IP address.

This call detects IPv4 addresses (all of dotted-quad or subsets, decimal or hexadecimal notations) and IPv6 addresses (including those containing embedded IPv4 addresses.)

Parameters

host	a hostname terminated by '\0'

Returns: true if the hostname is an IP address, false otherwise

Todo:: FIXME Some parts of urldb.c make confusions between hosts and "prefixes", we can sometimes be erroneously passed more than just a host. Sometimes we may be passed trailing slashes, or even whole path segments. A specific criminal in this class is urldb_iterate_partial, which takes a prefix to search for, but passes that prefix to functions that expect only hosts.

For the time being, we will accept such calls; we check if there is a / in the host parameter, and if there is, we take a copy and replace the / with a \0. This is not a permanent solution; we should search through NetSurf and find all the callers that are in error and fix them. When doing this task, it might be wise to replace the hideousness below with code that doesn't have to do this, and add assert(strchr(host, '/') == NULL); somewhere. – rjek - 2010-11-04

Definition at line 647 of file urldb.c.

References inet_aton(), inet_pton(), and NSLOG.

Referenced by urldb_add_host(), urldb_get_hsts_enabled(), urldb_get_search_tree_direct(), urldb_search_match_prefix(), urldb_search_match_string(), urldb_set_cookie(), and urldb_set_hsts_policy().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ urldb_add_host()

static struct host_part * urldb_add_host ( const char * host )

static

Add a host to the database, creating any intermediate entries.

Parameters

host	Hostname to add

Returns: Pointer to leaf node, or NULL on memory exhaustion

Definition at line 2164 of file urldb.c.

References host_part::children, db_root, host_part::next, search_trees, ST_IP, urldb__host_is_ip_address(), urldb_add_host_node(), urldb_get_search_tree_direct(), and urldb_search_insert().

Referenced by urldb_add_url(), urldb_insert_cookie(), and urldb_load().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ urldb_add_host_node()

static struct host_part * urldb_add_host_node	(	const char *	part,
		struct host_part *	parent
	)

static

Add a host node to the tree.

Parameters

part	Host segment to add (or whole IP address) (copied)
parent	Parent node to add to

Returns: Pointer to added node, or NULL on memory exhaustion

Definition at line 1006 of file urldb.c.

References host_part::next, host_part::parent, parent, and host_part::part.

Referenced by urldb_add_host().

Here is the caller graph for this function:

◆ urldb_add_path()

static struct path_data * urldb_add_path	(	lwc_string *	scheme,
		unsigned int	port,
		const struct host_part *	host,
		char *	path_query,
		lwc_string *	fragment,
		nsurl *	url
	)

static

Add a path to the database, creating any intermediate entries.

Parameters

scheme	URL scheme associated with path
port	Port number on host associated with path
host	Host tree node to attach to
path_query	Absolute path plus query to add (freed)
fragment	URL fragment, or NULL
url	URL (fragment ignored)

Returns: Pointer to leaf node, or NULL on memory exhaustion

Definition at line 2086 of file urldb.c.

References path_data::children, path_data::fragment, path_data::next, NSERROR_OK, nsurl_defragment(), host_part::paths, path_data::port, path_data::scheme, path_data::segment, path_data::url, urldb_add_path_fragment(), and urldb_add_path_node().

Referenced by urldb_add_url(), urldb_insert_cookie(), and urldb_load().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ urldb_add_path_fragment()

static struct path_data * urldb_add_path_fragment	(	struct path_data *	segment,
		lwc_string *	fragment
	)

static

Add a fragment to a path segment.

Parameters

segment	Path segment to add to
fragment	Fragment to add (copied), or NULL

Returns: segment or NULL on memory exhaustion

Definition at line 1055 of file urldb.c.

References path_data::fragment, path_data::segment, and urldb_add_path_fragment_cmp().

Referenced by urldb_add_path(), and urldb_add_path_node().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ urldb_add_path_fragment_cmp()

static int urldb_add_path_fragment_cmp	(	const void *	a,
		const void *	b
	)

static

Fragment comparator callback for qsort.

Parameters

a	first value
b	second value

Returns: 0 for equal else positive or negative value on comparison

Definition at line 1041 of file urldb.c.

Referenced by urldb_add_path_fragment().

Here is the caller graph for this function:

◆ urldb_add_path_node()

static struct path_data * urldb_add_path_node	(	lwc_string *	scheme,
		unsigned int	port,
		const char *	segment,
		lwc_string *	fragment,
		struct path_data *	parent
	)

static

Add a path node to the tree.

Parameters

scheme	URL scheme associated with path (copied)
port	Port number on host associated with path
segment	Path segment to add (copied)
fragment	URL fragment (copied), or NULL
parent	Parent node to add to

Returns: Pointer to added node, or NULL on memory exhaustion

Definition at line 1104 of file urldb.c.

References path_data::fragment, path_data::next, path_data::parent, parent, path_data::port, path_data::prev, path_data::scheme, path_data::segment, and urldb_add_path_fragment().

Referenced by urldb_add_path().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ urldb_add_url()

bool urldb_add_url ( struct nsurl * url )

Insert an URL into the database.

Parameters

url	Absolute URL to insert

Returns: true on success, false otherwise

Definition at line 3140 of file urldb.c.

References bloom_create(), bloom_insert_hash(), BLOOM_SIZE, path_data::fragment, NSERROR_OK, NSURL_FRAGMENT, nsurl_get(), nsurl_get_component(), nsurl_hash(), NSURL_HOST, NSURL_PATH, NSURL_PORT, NSURL_QUERY, NSURL_SCHEME, path_data::port, path_data::scheme, path_data::url, url_bloom, urldb_add_host(), and urldb_add_path().

Referenced by browser_window_callback(), browser_window_content_ready(), hotlist_create_entry(), hotlist_update_url_walk_cb(), urldb_get_auth_details(), urldb_get_cookie(), urldb_get_hsts_enabled(), urldb_set_auth_details(), urldb_set_cert_permissions(), and urldb_set_hsts_policy().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ urldb_concat_cookie()

static bool urldb_concat_cookie	(	struct cookie_internal_data *	c,
		int	version,
		int *	used,
		int *	alloc,
		char **	buf
	)

static

Concatenate a cookie into the provided buffer.

Parameters

c	Cookie to concatenate
version	The version of the cookie string to output
used	Pointer to amount of buffer used (updated)
alloc	Pointer to allocated size of buffer (updated)
buf	Pointer to Pointer to buffer (updated)

Returns: true on success, false on memory exhaustion

Todo:: should we XX-encode [;HT,SP] ?

Todo:: Should we strip escaping backslashes?

Definition at line 2376 of file urldb.c.

References COOKIE_NETSCAPE, cookie_internal_data::domain, cookie_internal_data::domain_from_set, cookie_internal_data::name, cookie_internal_data::path, cookie_internal_data::path_from_set, cookie_internal_data::value, cookie_internal_data::value_was_quoted, and version.

Referenced by urldb_get_cookie().

Here is the caller graph for this function:

◆ urldb_count_urls()

static void urldb_count_urls	(	const struct path_data *	root,
		time_t	expiry,
		unsigned int *	count
	)

static

Count number of URLs associated with a host.

Parameters

root	Root of path data tree
expiry	Expiry time for URLs
count	Pointer to count

Definition at line 474 of file urldb.c.

References path_data::children, url_internal_data::last_visit, path_data::next, path_data::parent, path_data::persistent, root, path_data::urld, and url_internal_data::visits.

Referenced by urldb_save_search_tree().

Here is the caller graph for this function:

◆ urldb_delete_cookie()

void urldb_delete_cookie	(	const char *	domain,
		const char *	path,
		const char *	name
	)

Delete a cookie.

Parameters

domain	The cookie's domain
path	The cookie's path
name	The cookie's name

Definition at line 4273 of file urldb.c.

References db_root, cookie_internal_data::domain, cookie_internal_data::name, path(), and urldb_delete_cookie_hosts().

Referenced by CookieWindow::_DeleteCookies(), and cookie_manager_delete_entry().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ urldb_delete_cookie_hosts()

static void urldb_delete_cookie_hosts	(	const char *	domain,
		const char *	path,
		const char *	name,
		struct host_part *	parent
	)

static

Deletes cookie hosts and their assoicated paths.

Parameters

domain	the cookie domain
path	the cookie path
name	The cookie name
parent	The url data of the cookie

Definition at line 2596 of file urldb.c.

References host_part::next, parent, path(), urldb_delete_cookie_hosts(), and urldb_delete_cookie_paths().

Referenced by urldb_delete_cookie(), and urldb_delete_cookie_hosts().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ urldb_delete_cookie_paths()

static void urldb_delete_cookie_paths	(	const char *	domain,
		const char *	path,
		const char *	name,
		struct path_data *	parent
	)

static

deletes paths from a cookie.

Parameters

domain	the cookie domain
path	the cookie path
name	The cookie name
parent	The url data of the cookie

Definition at line 2538 of file urldb.c.

References path_data::children, path_data::cookies, path_data::cookies_end, cookie_internal_data::domain, cookie_internal_data::name, cookie_internal_data::next, path_data::next, path_data::parent, parent, cookie_internal_data::path, path(), cookie_internal_data::prev, and urldb_free_cookie().

Referenced by urldb_delete_cookie_hosts().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ urldb_destroy()

void urldb_destroy ( void )

Destroy urldb.

Definition at line 2847 of file urldb.c.

References bloom_destroy(), host_part::children, db_root, empty, host_part::next, NUM_SEARCH_TREES, search_trees, url_bloom, urldb_destroy_host_tree(), and urldb_destroy_search_tree().

Referenced by netsurf_exit().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ urldb_destroy_cookie()

static void urldb_destroy_cookie ( struct cookie_internal_data * c )

static

Destroy a cookie node.

Parameters

c	Cookie to destroy

Definition at line 2692 of file urldb.c.

References cookie_internal_data::comment, cookie_internal_data::domain, cookie_internal_data::name, cookie_internal_data::path, and cookie_internal_data::value.

Referenced by urldb_destroy_path_node_content().

Here is the caller graph for this function:

◆ urldb_destroy_host_tree()

static void urldb_destroy_host_tree ( struct host_part * root )

static

Destroy a host tree.

Parameters

root	Root node of tree to destroy

Definition at line 2792 of file urldb.c.

References prot_space_data::next, path_data::next, host_part::next, root, urldb_destroy_host_tree(), urldb_destroy_path_node_content(), urldb_destroy_path_tree(), and urldb_destroy_prot_space().

Referenced by urldb_destroy(), and urldb_destroy_host_tree().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ urldb_destroy_path_node_content()

static void urldb_destroy_path_node_content ( struct path_data * node )

static

Destroy the contents of a path node.

Parameters

node	Node to destroy contents of (does not destroy node)

Definition at line 2709 of file urldb.c.

References path_data::cookies, path_data::frag_cnt, path_data::fragment, cookie_internal_data::next, nsurl_unref(), path_data::scheme, path_data::segment, url_internal_data::title, path_data::url, path_data::urld, and urldb_destroy_cookie().

Referenced by urldb_destroy_host_tree(), and urldb_destroy_path_tree().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ urldb_destroy_path_tree()

static void urldb_destroy_path_tree ( struct path_data * root )

static

Destroy a path tree.

Parameters

root	Root node of tree to destroy

Definition at line 2756 of file urldb.c.

References path_data::children, path_data::next, path_data::parent, root, and urldb_destroy_path_node_content().

Referenced by urldb_destroy_host_tree().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ urldb_destroy_prot_space()

static void urldb_destroy_prot_space ( struct prot_space_data * space )

static

Destroy protection space data.

Parameters

space Protection space to destroy

Definition at line 2741 of file urldb.c.

References prot_space_data::auth, prot_space_data::realm, and prot_space_data::scheme.

Referenced by urldb_destroy_host_tree().

Here is the caller graph for this function:

◆ urldb_destroy_search_tree()

static void urldb_destroy_search_tree ( struct search_node * root )

static

Destroy a search tree.

Parameters

root	Root node of tree to destroy

Definition at line 2830 of file urldb.c.

References empty, root, and urldb_destroy_search_tree().

Referenced by urldb_destroy(), and urldb_destroy_search_tree().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ urldb_dump()

void urldb_dump ( void )

Dump URL database to stderr.

Definition at line 4481 of file urldb.c.

References db_root, NUM_SEARCH_TREES, search_trees, urldb_dump_hosts(), and urldb_dump_search().

Referenced by ro_gui_window_handle_local_keypress().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ urldb_dump_hosts()

static void urldb_dump_hosts ( struct host_part * parent )

static

Dump URL database hosts to stderr.

Parameters

parent Parent node of tree to dump

Definition at line 1477 of file urldb.c.

References host_part::next, NSLOG, parent, urldb_dump_hosts(), and urldb_dump_paths().

Referenced by urldb_dump(), and urldb_dump_hosts().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ urldb_dump_paths()

static void urldb_dump_paths ( struct path_data * parent )

static

Dump URL database paths to stderr.

Parameters

parent Parent node of tree to dump

Definition at line 1438 of file urldb.c.

References path_data::children, path_data::frag_cnt, path_data::fragment, path_data::next, NSLOG, path_data::parent, parent, path_data::port, path_data::scheme, and path_data::segment.

Referenced by urldb_dump_hosts().

Here is the caller graph for this function:

◆ urldb_dump_search()

static void urldb_dump_search	(	struct search_node *	parent,
		int	depth
	)

static

Dump search tree.

Parameters

parent	Parent node of tree to dump
depth	Tree depth

Definition at line 1504 of file urldb.c.

References empty, NSLOG, host_part::parent, parent, host_part::part, and urldb_dump_search().

Referenced by urldb_dump(), and urldb_dump_search().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ urldb_find_url()

static struct path_data * urldb_find_url ( nsurl * url )

static

Find an URL in the database.

Parameters

url	Absolute URL to find

Returns: Pointer to path data, or NULL if not found

Definition at line 1358 of file urldb.c.

References bloom_search_hash(), NSERROR_OK, nsurl_get(), nsurl_get_component(), nsurl_hash(), NSURL_HOST, NSURL_PATH, NSURL_PORT, NSURL_QUERY, NSURL_SCHEME, host_part::paths, url_bloom, urldb_get_search_tree(), urldb_match_path(), and urldb_search_find().

Referenced by urldb_get_auth_details(), urldb_get_cert_permissions(), urldb_get_cookie(), urldb_get_hsts_enabled(), urldb_get_url(), urldb_get_url_data(), urldb_reset_url_visit_data(), urldb_set_auth_details(), urldb_set_cert_permissions(), urldb_set_hsts_policy(), urldb_set_url_content_type(), urldb_set_url_persistence(), urldb_set_url_title(), and urldb_update_url_visit_data().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ urldb_free_cookie()

static void urldb_free_cookie ( struct cookie_internal_data * c )

static

Free a cookie.

Parameters

c	The cookie to free

Definition at line 1817 of file urldb.c.

References cookie_internal_data::comment, cookie_internal_data::domain, cookie_internal_data::name, cookie_internal_data::path, and cookie_internal_data::value.

Referenced by urldb_delete_cookie_paths(), urldb_insert_cookie(), urldb_load_cookies(), urldb_parse_cookie(), and urldb_set_cookie().

Here is the caller graph for this function:

◆ urldb_get_auth_details()

const char * urldb_get_auth_details	(	struct nsurl *	url,
		const char *	realm
	)

Look up authentication details in database.

Parameters

url	Absolute URL to search for
realm	When non-NULL, it is realm which can be used to determine the protection space when that's not been done before for given URL.

Returns: Pointer to authentication details, or NULL if not found

Definition at line 3405 of file urldb.c.

References prot_space_data::auth, prot_space_data::next, path_data::parent, prot_space_data::port, path_data::port, path_data::prot_space, host_part::prot_space, prot_space_data::realm, prot_space_data::scheme, path_data::scheme, path_data::url, urldb_add_url(), and urldb_find_url().

Referenced by browser_window__handle_login(), fetch_curl_set_options(), and llcache_fetch_auth().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ urldb_get_cert_permissions()

bool urldb_get_cert_permissions ( struct nsurl * url )

Retrieve certificate verification permissions from database.

Parameters

url	Absolute URL to search for

Returns: true to permit connections to hosts with invalid certificates, false otherwise.

Definition at line 3480 of file urldb.c.

References path_data::parent, host_part::permit_invalid_certs, and urldb_find_url().

Referenced by browser_window_get_page_info_state(), content_saw_insecure_objects(), and fetch_curl_set_options().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ urldb_get_cookie()

char * urldb_get_cookie	(	struct nsurl *	url,
		bool	include_http_only
	)

Retrieve cookies for an URL.

Parameters

url	URL being fetched
include_http_only	Whether to include HTTP(S) only cookies.

Returns: Cookies string for libcurl (on heap), or NULL on error/no cookies

Definition at line 3997 of file urldb.c.

References path_data::children, cookie_manager_add(), COOKIE_NETSCAPE, COOKIE_RFC2965, path_data::cookies, count(), db_root, cookie_internal_data::expires, GROW_MATCHED_COOKIES, cookie_internal_data::http_only, cookie_internal_data::last_used, cookie_internal_data::next, path_data::next, nsurl_get_component(), NSURL_PATH, path_data::parent, host_part::parent, cookie_internal_data::path, path(), host_part::paths, path_data::scheme, cookie_internal_data::secure, path_data::segment, urldb_add_url(), urldb_concat_cookie(), urldb_find_url(), cookie_internal_data::version, and version.

Referenced by browser_window_get_cookie_count(), and fetch_curl_set_options().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ urldb_get_hsts_enabled()

bool urldb_get_hsts_enabled ( struct nsurl * url )

Determine if HSTS policy is enabled for an URL.

Parameters

url	URL being fetched

Returns: true if HSTS policy is enabled, false otherwise

Definition at line 3573 of file urldb.c.

References db_root, hsts_data::expires, host_part::hsts, hsts_data::include_sub_domains, nsurl_get_component(), NSURL_HOST, path_data::parent, host_part::parent, urldb__host_is_ip_address(), urldb_add_url(), and urldb_find_url().

Referenced by llcache_hsts_transform_url().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ urldb_get_search_tree()

static struct search_node * urldb_get_search_tree ( const char * host )

static

Get the search tree for a particular host.

Parameters

host	the host to lookup

Returns: the corresponding search tree

Definition at line 1191 of file urldb.c.

References urldb_get_search_tree_direct().

Referenced by urldb_find_url(), urldb_insert_cookie(), and urldb_iterate_partial().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ urldb_get_search_tree_direct()

static struct search_node ** urldb_get_search_tree_direct ( const char * host )

static

Get the search tree for a particular host.

Parameters

host	the host to lookup

Returns: the corresponding search tree

Definition at line 1172 of file urldb.c.

References ascii_is_alpha(), ascii_to_lower(), search_trees, ST_DN, ST_EE, ST_IP, and urldb__host_is_ip_address().

Referenced by urldb_add_host(), and urldb_get_search_tree().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ urldb_get_url()

nsurl * urldb_get_url ( struct nsurl * url )

Extract an URL from the db.

Parameters

url	URL to extract

Returns: Pointer to database's copy of URL or NULL if not found

Definition at line 3327 of file urldb.c.

References path_data::url, and urldb_find_url().

Referenced by browser_window_content_ready().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ urldb_get_url_data()

const struct url_data * urldb_get_url_data ( struct nsurl * url )

Find data for an URL.

Parameters

url	Absolute URL to look for

Returns: Pointer to result struct, or NULL

Definition at line 3309 of file urldb.c.

References path_data::urld, and urldb_find_url().

Referenced by global_history_add(), hotlist_create_entry(), hotlist_update_url_walk_cb(), node_is_visited(), ro_gui_hotlist_add_page(), and ro_gui_url_complete_redraw().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ urldb_insert_cookie()

static bool urldb_insert_cookie	(	struct cookie_internal_data *	c,
		lwc_string *	scheme,
		nsurl *	url
	)

static

Insert a cookie into the database.

Parameters

c	The cookie to insert
scheme	URL scheme associated with cookie path
url	URL (sans fragment) associated with cookie

Returns: true on success, false on memory exhaustion (c will be freed)

Definition at line 2257 of file urldb.c.

References cookie_manager_add(), cookie_manager_remove(), path_data::cookies, path_data::cookies_end, cookie_internal_data::domain, cookie_internal_data::expires, cookie_internal_data::name, cookie_internal_data::next, cookie_internal_data::path, host_part::paths, cookie_internal_data::prev, path_data::scheme, path_data::url, urldb_add_host(), urldb_add_path(), urldb_free_cookie(), urldb_get_search_tree(), and urldb_search_find().

Referenced by urldb_load_cookies(), and urldb_set_cookie().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ urldb_iterate_cookies()

void urldb_iterate_cookies ( bool(*)(const struct cookie_data *data) callback )

Definition at line 3719 of file urldb.c.

References NUM_SEARCH_TREES, search_trees, and urldb_iterate_entries_host().

Here is the call graph for this function:

◆ urldb_iterate_entries()

void urldb_iterate_entries ( bool(*)(nsurl *url, const struct url_data *data) callback )

Definition at line 3702 of file urldb.c.

References NUM_SEARCH_TREES, search_trees, and urldb_iterate_entries_host().

Here is the call graph for this function:

◆ urldb_iterate_entries_host()

static bool urldb_iterate_entries_host	(	struct search_node *	parent,
		bool()(nsurl url, const struct url_data *data)	url_callback,
		bool()(const struct cookie_data data)	cookie_callback
	)

static

Host data iterator (internal)

Parameters

parent	Root of subtree to iterate over
url_callback	Callback function
cookie_callback	Callback function

Returns: true to continue, false otherwise

Definition at line 963 of file urldb.c.

References empty, parent, urldb_iterate_entries_host(), and urldb_iterate_entries_path().

Referenced by urldb_iterate_cookies(), urldb_iterate_entries(), and urldb_iterate_entries_host().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ urldb_iterate_entries_path()

static bool urldb_iterate_entries_path	(	const struct path_data *	parent,
		bool()(nsurl url, const struct url_data *data)	url_callback,
		bool()(const struct cookie_data data)	cookie_callback
	)

static

Path data iterator (internal)

Parameters

parent	Root of subtree to iterate over
url_callback	Callback function
cookie_callback	Callback function

Returns: true to continue, false otherwise

Todo:: handle fragments?

Definition at line 584 of file urldb.c.

References path_data::children, path_data::cookies, path_data::next, cookie_data::next, path_data::parent, parent, path_data::url, and path_data::urld.

Referenced by urldb_iterate_entries_host(), urldb_iterate_partial_host(), and urldb_iterate_partial_path().

Here is the caller graph for this function:

◆ urldb_iterate_partial()

void urldb_iterate_partial	(	const char *	prefix,
		bool()(nsurl url, const struct url_data *data)	callback
	)

Definition at line 3635 of file urldb.c.

References path_data::children, host_part::paths, search_trees, ST_DN, urldb_get_search_tree(), urldb_iterate_partial_host(), urldb_iterate_partial_path(), and urldb_search_find().

Here is the call graph for this function:

◆ urldb_iterate_partial_host()

static bool urldb_iterate_partial_host	(	struct search_node *	root,
		const char *	prefix,
		bool()(nsurl url, const struct url_data *data)	callback
	)

static

Partial host iterator (internal)

Parameters

root	Root of (sub)tree to traverse
prefix	Prefix to match
callback	Callback function

Returns: true to continue, false otherwise

Definition at line 816 of file urldb.c.

References empty, root, urldb_iterate_entries_path(), urldb_iterate_partial_host(), and urldb_search_match_prefix().

Referenced by urldb_iterate_partial(), and urldb_iterate_partial_host().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ urldb_iterate_partial_path()

static bool urldb_iterate_partial_path	(	const struct path_data *	parent,
		const char *	prefix,
		bool()(nsurl url, const struct url_data *data)	callback
	)

static

Partial path iterator (internal)

Given: http://www.example.org/a/b/c/d//e and assuming a path tree: ^ / \ a1 b1 / \ a2 b2 /|\ a b c 3 3 | d | e / \ f g

Prefix will be: p will be:

a/b/c/d//e a1 b/c/d//e a2 b/c/d//e b3 c/d//e a3 c/d//e b3 c/d//e c d//e d /e e (skip /) e e

I.E. perform a breadth-first search of the tree.

Parameters

parent	Root of (sub)tree to traverse
prefix	Prefix to match
callback	Callback function

Returns: true to continue, false otherwise

Definition at line 906 of file urldb.c.

References path_data::children, path_data::next, parent, path_data::segment, and urldb_iterate_entries_path().

Referenced by urldb_iterate_partial().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ urldb_load()

nserror urldb_load ( const char * filename )

Import an URL database from file, replacing any existing database.

Parameters

filename Name of file containing data

Definition at line 2876 of file urldb.c.

References bloom_create(), bloom_insert_hash(), BLOOM_SIZE, hsts_data::expires, host_part::hsts, hsts_data::include_sub_domains, url_internal_data::last_visit, MAXIMUM_URL_LENGTH, MIN_URL_FILE_VERSION, nsc_snptimet(), NSERROR_INVALID, NSERROR_NEED_DATA, NSERROR_NOMEM, NSERROR_NOT_FOUND, NSERROR_OK, NSLOG, nsurl_create(), NSURL_FRAGMENT, nsurl_get(), nsurl_get_component(), nsurl_hash(), NSURL_PATH, NSURL_QUERY, NSURL_SCHEME, nsurl_unref(), path_data::port, path_data::scheme, url_internal_data::title, url_internal_data::type, path_data::url, url_bloom, URL_FILE_VERSION, path_data::urld, urldb_add_host(), urldb_add_path(), version, and url_internal_data::visits.

Referenced by gui_init(), main(), nsgtk_setup(), and WinMain().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ urldb_load_cookies()

void urldb_load_cookies ( const char * filename )

Load a cookie file into the database.

Parameters

filename File to load

Definition at line 4281 of file urldb.c.

Referenced by gui_init(), main(), nsgtk_setup(), and WinMain().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ urldb_match_path()

static struct path_data * urldb_match_path	(	const struct path_data *	parent,
		const char *	path,
		lwc_string *	scheme,
		unsigned short	port
	)

static

Match a path string.

Parameters

parent	Path (sub)tree to look in
path	The path to search for
scheme	The URL scheme associated with the path
port	The port associated with the path

Returns: Pointer to path data or NULL if not found.

Definition at line 1302 of file urldb.c.

References path_data::children, path_data::next, NSLOG, parent, path(), path_data::port, path_data::scheme, and path_data::segment.

Referenced by urldb_find_url().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ urldb_parse_avpair()

static bool urldb_parse_avpair	(	struct cookie_internal_data *	c,
		char *	n,
		char *	v,
		bool	was_quoted
	)

static

Parse a cookie avpair.

Parameters

c	Cookie struct to populate
n	Name component
v	Value component
was_quoted	Whether v was quoted in the input

Returns: true on success, false on memory exhaustion

Definition at line 1709 of file urldb.c.

References ascii_is_digit(), cookie_internal_data::comment, cookie_internal_data::domain, cookie_internal_data::domain_from_set, cookie_internal_data::expires, cookie_internal_data::http_only, cookie_internal_data::name, nsc_strntimet(), NSERROR_OK, cookie_internal_data::path, cookie_internal_data::path_from_set, cookie_internal_data::secure, cookie_internal_data::value, cookie_internal_data::value_was_quoted, and cookie_internal_data::version.

Referenced by urldb_parse_cookie().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ urldb_parse_cookie()

static struct cookie_internal_data * urldb_parse_cookie	(	nsurl *	url,
		const char **	cookie
	)

static

Parse a cookie.

Parameters

url	URL being fetched
cookie	Pointer to cookie string (updated on exit)

Returns: Pointer to cookie structure (on heap, caller frees) or NULL

Todo:: is silently truncating overlong names/values wise?

Definition at line 1838 of file urldb.c.

References cookie_internal_data::domain, cookie_internal_data::expires, cookie_internal_data::name, nsurl_get_component(), NSURL_HOST, NSURL_PATH, cookie_internal_data::path, path(), strndup(), urldb_free_cookie(), urldb_parse_avpair(), and cookie_internal_data::value.

Referenced by urldb_set_cookie().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ urldb_reset_url_visit_data()

void urldb_reset_url_visit_data ( struct nsurl * url )

Reset an URL's visit statistics.

Parameters

url	The URL to reset

Definition at line 3293 of file urldb.c.

References url_internal_data::last_visit, path_data::url, path_data::urld, urldb_find_url(), and url_internal_data::visits.

Referenced by global_history_delete_entry_internal().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ urldb_save()

nserror urldb_save ( const char * filename )

Export the current database to file.

Parameters

filename Name of file to export to

Definition at line 3094 of file urldb.c.

References NSERROR_OK, NSERROR_SAVE_FAILED, NSLOG, NUM_SEARCH_TREES, search_trees, URL_FILE_VERSION, and urldb_save_search_tree().

Referenced by gui_quit(), monkey_quit(), nsgtk_finalise(), and WinMain().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ urldb_save_cookie_hosts()

static void urldb_save_cookie_hosts	(	FILE *	fp,
		struct host_part *	parent
	)

static

Save a host subtree's cookies.

Parameters

fp	File pointer to write to
parent	Parent host

Definition at line 2675 of file urldb.c.

References host_part::next, parent, urldb_save_cookie_hosts(), and urldb_save_cookie_paths().

Referenced by urldb_save_cookie_hosts(), and urldb_save_cookies().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ urldb_save_cookie_paths()

static void urldb_save_cookie_paths	(	FILE *	fp,
		struct path_data *	parent
	)

static

Save a path subtree's cookies.

Parameters

fp	File pointer to write to
parent	Parent path

Definition at line 2618 of file urldb.c.

Referenced by urldb_save_cookie_hosts().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ urldb_save_cookies()

void urldb_save_cookies ( const char * filename )

Save persistent cookies to file.

Parameters

filename Path to save to

Definition at line 4448 of file urldb.c.

References COOKIE_FILE_VERSION, db_root, loaded_cookie_file_version, max, and urldb_save_cookie_hosts().

Referenced by gui_quit(), monkey_quit(), nsgtk_finalise(), and WinMain().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ urldb_save_search_tree()

static void urldb_save_search_tree	(	struct search_node *	parent,
		FILE *	fp
	)

static

Save a search (sub)tree.

Parameters

parent	root node of search tree to save.
fp	File to write to

Definition at line 514 of file urldb.c.

References db_root, empty, hsts_data::expires, host_part::hsts, hsts_data::include_sub_domains, nsoption_int, host_part::parent, parent, host_part::part, path(), urldb_count_urls(), urldb_save_search_tree(), urldb_write_paths(), and urldb_write_timet().

Referenced by urldb_save(), and urldb_save_search_tree().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ urldb_search_find()

static const struct host_part * urldb_search_find	(	struct search_node *	root,
		const char *	host
	)

static

Find a node in a search tree.

Parameters

root	Tree to look in
host	Host to find

Returns: Pointer to host tree node, or NULL if not found

Definition at line 1270 of file urldb.c.

References empty, root, urldb_search_find(), and urldb_search_match_string().

Referenced by urldb_find_url(), urldb_insert_cookie(), urldb_iterate_partial(), and urldb_search_find().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ urldb_search_insert()

static struct search_node * urldb_search_insert	(	struct search_node *	root,
		const struct host_part *	data
	)

static

Insert a node into the search tree.

Parameters

root	Root of tree to insert into
data	User data to insert

Returns: Pointer to updated root, or NULL if failed

Definition at line 1679 of file urldb.c.

References search_node::data, empty, search_node::left, search_node::level, search_node::right, root, and urldb_search_insert_internal().

Referenced by urldb_add_host().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ urldb_search_insert_internal()

static struct search_node * urldb_search_insert_internal	(	struct search_node *	root,
		struct search_node *	n
	)

static

Insert node into search tree.

Parameters

root	Root of (sub)tree to insert into
n	Node to insert

Returns: Pointer to updated root

Definition at line 1642 of file urldb.c.

References search_node::data, empty, root, urldb_search_insert_internal(), urldb_search_match_host(), urldb_search_skew(), and urldb_search_split().

Referenced by urldb_search_insert(), and urldb_search_insert_internal().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ urldb_search_match_host()

static int urldb_search_match_host	(	const struct host_part *	a,
		const struct host_part *	b
	)

static

Compare a pair of host parts.

Parameters

a	first host part
b	second host part

Returns: 0 if match, non-zero, otherwise

Definition at line 1554 of file urldb.c.

References db_root, host_part::parent, and host_part::part.

Referenced by urldb_search_insert_internal().

Here is the caller graph for this function:

◆ urldb_search_match_prefix()

static int urldb_search_match_prefix	(	const struct host_part *	a,
		const char *	b
	)

static

Compare host_part with prefix.

Parameters

a	host part
b	prefix

Returns: 0 if match, non-zero, otherwise

Definition at line 746 of file urldb.c.

References db_root, host_part::parent, host_part::part, and urldb__host_is_ip_address().

Referenced by urldb_iterate_partial_host().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ urldb_search_match_string()

static int urldb_search_match_string	(	const struct host_part *	a,
		const char *	b
	)

static

Compare host part with a string.

Parameters

a	host part
b	string to compare

Returns: 0 if match, non-zero, otherwise

Definition at line 1204 of file urldb.c.

References db_root, host_part::parent, host_part::part, and urldb__host_is_ip_address().

Referenced by urldb_search_find().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ urldb_search_skew()

static struct search_node * urldb_search_skew ( struct search_node * root )

static

Rotate a subtree right.

Parameters

root	Root of subtree to rotate

Returns: new root of subtree

Definition at line 1592 of file urldb.c.

References search_node::right, and root.

Referenced by urldb_search_insert_internal().

Here is the caller graph for this function:

◆ urldb_search_split()

static struct search_node * urldb_search_split ( struct search_node * root )

static

Rotate a node left, increasing the parent's level.

Parameters

root	Root of subtree to rotate

Returns: New root of subtree

Definition at line 1615 of file urldb.c.

References search_node::left, and root.

Referenced by urldb_search_insert_internal().

Here is the caller graph for this function:

◆ urldb_set_auth_details()

void urldb_set_auth_details	(	struct nsurl *	url,
		const char *	realm,
		const char *	auth
	)

Set authentication data for an URL.

Parameters

url	The URL to consider
realm	The authentication realm
auth	The authentication details (in form username:password)

Definition at line 3342 of file urldb.c.

References prot_space_data::auth, prot_space_data::next, path_data::parent, prot_space_data::port, path_data::port, path_data::prot_space, host_part::prot_space, prot_space_data::realm, prot_space_data::scheme, path_data::scheme, urldb_add_url(), and urldb_find_url().

Referenced by browser_window__handle_userpass_response(), and navigate_internal_query_auth().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ urldb_set_cert_permissions()

void urldb_set_cert_permissions	(	struct nsurl *	url,
		bool	permit
	)

Set certificate verification permissions.

Parameters

url	URL to consider
permit	Set to true to allow invalid certificates

Definition at line 3455 of file urldb.c.

References path_data::parent, host_part::permit_invalid_certs, urldb_add_url(), and urldb_find_url().

Referenced by browser_window__handle_ssl_query_response().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ urldb_set_cookie()

bool urldb_set_cookie	(	const char *	header,
		struct nsurl *	url,
		struct nsurl *	referrer
	)

Parse Set-Cookie header and insert cookie(s) into database.

Parameters

header	Header to parse, with Set-Cookie: stripped
url	URL being fetched
referrer	Referring resource, or 0 for verifiable transaction

Returns: true on success, false otherwise

Todo:: In future, we should consult a TLD service instead of just looking for embedded dots.

Definition at line 3734 of file urldb.c.

References cookie_internal_data::domain, cookie_internal_data::name, NSERROR_OK, NSLOG, nsurl_defragment(), nsurl_get_component(), NSURL_HOST, NSURL_PATH, NSURL_SCHEME, nsurl_unref(), cookie_internal_data::path, path(), urldb__host_is_ip_address(), urldb_free_cookie(), urldb_insert_cookie(), urldb_parse_cookie(), and cookie_internal_data::value.

Referenced by fetch_set_cookie().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ urldb_set_hsts_policy()

bool urldb_set_hsts_policy	(	struct nsurl *	url,
		const char *	header
	)

Set HSTS policy for an URL.

Parameters

url	URL being fetched
header	Strict-Transport-Security header value

Returns: true on success, false otherwise

Definition at line 3502 of file urldb.c.

References hsts_data::expires, host_part::hsts, http_parse_strict_transport_security(), http_strict_transport_security_destroy(), http_strict_transport_security_include_subdomains(), http_strict_transport_security_max_age(), hsts_data::include_sub_domains, NSERROR_OK, nsurl_get_component(), NSURL_HOST, path_data::parent, host_part::permit_invalid_certs, urldb__host_is_ip_address(), urldb_add_url(), and urldb_find_url().

Referenced by llcache_hsts_update_policy().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ urldb_set_url_content_type()

nserror urldb_set_url_content_type	(	struct nsurl *	url,
		content_type	type
	)

Set an URL's content type.

Parameters

url	The URL to look for
type	The type to set

Returns: NSERROR_OK on success or NSERROR_NOT_FOUND if url not in database

Definition at line 3256 of file urldb.c.

References NSERROR_NOT_FOUND, NSERROR_OK, url_internal_data::type, type, path_data::url, path_data::urld, and urldb_find_url().

Referenced by browser_window_content_ready().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ urldb_set_url_persistence()

nserror urldb_set_url_persistence	(	struct nsurl *	url,
		bool	persist
	)

Set the cross-session persistence of the entry for an URL.

Parameters

url	Absolute URL to persist
persist	True to persist, false otherwise

Returns: NSERROR_OK on success or NSERROR_NOT_FOUND if url not in database

Definition at line 3122 of file urldb.c.

References NSERROR_NOT_FOUND, NSERROR_OK, path_data::persistent, path_data::url, and urldb_find_url().

Referenced by hotlist_add_entry_internal().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ urldb_set_url_title()

nserror urldb_set_url_title	(	struct nsurl *	url,
		const char *	title
	)

Set an URL's title string, replacing any existing one.

Parameters

url	The URL to look for
title	The title string to use (copied)

Returns: NSERROR_OK on success otherwise appropriate error code

Definition at line 3226 of file urldb.c.

References NSERROR_NOMEM, NSERROR_NOT_FOUND, NSERROR_OK, url_internal_data::title, path_data::url, path_data::urld, and urldb_find_url().

Referenced by browser_window_content_ready().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ urldb_update_url_visit_data()

nserror urldb_update_url_visit_data ( struct nsurl * url )

Update an URL's visit data.

Parameters

url	The URL to update

Returns: NSERROR_OK on success or NSERROR_NOT_FOUND if url not in database

Definition at line 3274 of file urldb.c.

References url_internal_data::last_visit, NSERROR_NOT_FOUND, NSERROR_OK, path_data::url, path_data::urld, urldb_find_url(), and url_internal_data::visits.

Referenced by browser_window_callback(), and browser_window_content_ready().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ urldb_write_paths()

static void urldb_write_paths	(	const struct path_data *	parent,
		const char *	host,
		FILE *	fp,
		char **	path,
		int *	path_alloc,
		int *	path_used,
		time_t	expiry
	)

static

Write paths associated with a host.

Parameters

parent	Root of (sub)tree to write
host	Current host name
fp	File to write to
path	Current path string
path_alloc	Allocated size of path
path_used	Used size of path
expiry	Expiry time of URLs

Todo:: handle fragments?

Definition at line 353 of file urldb.c.

References path_data::children, url_internal_data::last_visit, path_data::next, path_data::parent, parent, path(), path_data::persistent, path_data::port, path_data::scheme, path_data::segment, url_internal_data::title, url_internal_data::type, path_data::urld, urldb_write_timet(), and url_internal_data::visits.

Referenced by urldb_save_search_tree().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ urldb_write_timet()

static nserror urldb_write_timet	(	FILE *	fp,
		time_t	val
	)

static

write a time_t to a file portably

Parameters

fp	File to write to
val	the unix time value to output

Returns: NSERROR_OK on success

Definition at line 327 of file urldb.c.

References nsc_sntimet(), and NSERROR_OK.

Referenced by urldb_save_search_tree(), and urldb_write_paths().

Here is the call graph for this function:

Here is the caller graph for this function:

Variable Documentation

◆ db_root

struct host_part db_root

static

Root database handle.

Definition at line 275 of file urldb.c.

Referenced by urldb_add_host(), urldb_delete_cookie(), urldb_destroy(), urldb_dump(), urldb_get_cookie(), urldb_get_hsts_enabled(), urldb_save_cookies(), urldb_save_search_tree(), urldb_search_match_host(), urldb_search_match_prefix(), and urldb_search_match_string().

◆ empty

struct search_node empty = { 0, 0, &empty, &empty }

static

Definition at line 282 of file urldb.c.

Referenced by urldb_destroy(), urldb_destroy_search_tree(), urldb_dump_search(), urldb_iterate_entries_host(), urldb_iterate_partial_host(), urldb_save_search_tree(), urldb_search_find(), urldb_search_insert(), and urldb_search_insert_internal().

◆ loaded_cookie_file_version

int loaded_cookie_file_version

static

loaded cookie file version

Definition at line 295 of file urldb.c.

Referenced by urldb_load_cookies(), and urldb_save_cookies().

◆ search_trees

struct search_node* search_trees[NUM_SEARCH_TREES]

static

Initial value:

= {
        &empty, &empty, &empty, &empty, &empty, &empty, &empty, &empty,
        &empty, &empty, &empty, &empty, &empty, &empty, &empty, &empty,
        &empty, &empty, &empty, &empty, &empty, &empty, &empty, &empty,
        &empty, &empty, &empty, &empty
}

Definition at line 283 of file urldb.c.

Referenced by urldb_add_host(), urldb_destroy(), urldb_dump(), urldb_get_search_tree_direct(), urldb_iterate_cookies(), urldb_iterate_entries(), urldb_iterate_partial(), and urldb_save().

◆ url_bloom

struct bloom_filter* url_bloom

static

filter for url presence in database

Bloom filter used for short-circuting the false case of "is this URL in the database?". BLOOM_SIZE controls how large the filter is in bytes. Primitive experimentation shows that for a filter of X bytes filled with X items, searching for X items not in the filter has a 5% false-positive rate. We set it to 32kB, which should be enough for all but the largest databases, while not being shockingly wasteful on memory.

Definition at line 313 of file urldb.c.

Referenced by urldb_add_url(), urldb_destroy(), urldb_find_url(), and urldb_load().

Data Structures

Macros

Functions

Variables

Detailed Description

Macro Definition Documentation

◆ BLOOM_SIZE

◆ COOKIE_FILE_VERSION

◆ FIND_T

◆ GROW_MATCHED_COOKIES

◆ MAXIMUM_URL_LENGTH

◆ MIN_COOKIE_FILE_VERSION

◆ MIN_URL_FILE_VERSION

◆ NUM_SEARCH_TREES

◆ SKIP_T

◆ ST_DN

◆ ST_EE

◆ ST_IP

◆ URL_FILE_VERSION

Function Documentation

◆ urldb__host_is_ip_address()

◆ urldb_add_host()

◆ urldb_add_host_node()

◆ urldb_add_path()

◆ urldb_add_path_fragment()

◆ urldb_add_path_fragment_cmp()

◆ urldb_add_path_node()

◆ urldb_add_url()

◆ urldb_concat_cookie()

◆ urldb_count_urls()

◆ urldb_delete_cookie()

◆ urldb_delete_cookie_hosts()

◆ urldb_delete_cookie_paths()

◆ urldb_destroy()

◆ urldb_destroy_cookie()

◆ urldb_destroy_host_tree()

◆ urldb_destroy_path_node_content()

◆ urldb_destroy_path_tree()

◆ urldb_destroy_prot_space()

◆ urldb_destroy_search_tree()

◆ urldb_dump()

◆ urldb_dump_hosts()

◆ urldb_dump_paths()

◆ urldb_dump_search()

◆ urldb_find_url()

◆ urldb_free_cookie()

◆ urldb_get_auth_details()

◆ urldb_get_cert_permissions()

◆ urldb_get_cookie()

◆ urldb_get_hsts_enabled()

◆ urldb_get_search_tree()

◆ urldb_get_search_tree_direct()

◆ urldb_get_url()

◆ urldb_get_url_data()

◆ urldb_insert_cookie()

◆ urldb_iterate_cookies()

◆ urldb_iterate_entries()

◆ urldb_iterate_entries_host()

◆ urldb_iterate_entries_path()

◆ urldb_iterate_partial()

◆ urldb_iterate_partial_host()

◆ urldb_iterate_partial_path()

◆ urldb_load()

◆ urldb_load_cookies()

◆ urldb_match_path()

◆ urldb_parse_avpair()

◆ urldb_parse_cookie()

◆ urldb_reset_url_visit_data()

◆ urldb_save()

◆ urldb_save_cookie_hosts()

◆ urldb_save_cookie_paths()

◆ urldb_save_cookies()

◆ urldb_save_search_tree()

◆ urldb_search_find()

◆ urldb_search_insert()

◆ urldb_search_insert_internal()

◆ urldb_search_match_host()

◆ urldb_search_match_prefix()

◆ urldb_search_match_string()

◆ urldb_search_skew()