NetSurf
|
Unified URL information database implementation. More...
#include <assert.h>
#include <stdbool.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <strings.h>
#include <time.h>
#include "utils/inet.h"
#include "utils/nsoption.h"
#include "utils/log.h"
#include "utils/corestrings.h"
#include "utils/url.h"
#include "utils/utils.h"
#include "utils/bloom.h"
#include "utils/nsurl.h"
#include "utils/ascii.h"
#include "utils/http.h"
#include "netsurf/bitmap.h"
#include "desktop/cookie_manager.h"
#include "content/content.h"
#include "content/urldb.h"
Go to the source code of this file.
Data Structures | |
struct | cookie_internal_data |
cookie entry. More... | |
struct | prot_space_data |
A protection space. More... | |
struct | url_internal_data |
meta data about a url More... | |
struct | path_data |
data entry for url More... | |
struct | hsts_data |
struct | host_part |
struct | search_node |
search index node More... | |
Macros | |
#define | NUM_SEARCH_TREES 28 |
Search trees - one per letter + 1 for IPs + 1 for Everything Else. More... | |
#define | ST_IP 0 |
#define | ST_EE 1 |
#define | ST_DN 2 |
#define | MIN_COOKIE_FILE_VERSION 100 |
Minimum cookie database file version. More... | |
#define | COOKIE_FILE_VERSION 102 |
Current cookie database file version. More... | |
#define | MIN_URL_FILE_VERSION 106 |
Minimum URL database file version. More... | |
#define | URL_FILE_VERSION 107 |
Current URL database file version. More... | |
#define | BLOOM_SIZE (1024 * 32) |
Size of url filter. More... | |
#define | MAXIMUM_URL_LENGTH 4096 |
#define | GROW_MATCHED_COOKIES |
#define | FIND_T |
#define | SKIP_T |
Functions | |
static nserror | urldb_write_timet (FILE *fp, time_t val) |
write a time_t to a file portably More... | |
static void | urldb_write_paths (const struct path_data *parent, const char *host, FILE *fp, char **path, int *path_alloc, int *path_used, time_t expiry) |
Write paths associated with a host. More... | |
static void | urldb_count_urls (const struct path_data *root, time_t expiry, unsigned int *count) |
Count number of URLs associated with a host. More... | |
static void | urldb_save_search_tree (struct search_node *parent, FILE *fp) |
Save a search (sub)tree. More... | |
static bool | urldb_iterate_entries_path (const struct path_data *parent, bool(*url_callback)(nsurl *url, const struct url_data *data), bool(*cookie_callback)(const struct cookie_data *data)) |
Path data iterator (internal) More... | |
static bool | urldb__host_is_ip_address (const char *host) |
Check whether a host string is an IP address. More... | |
static int | urldb_search_match_prefix (const struct host_part *a, const char *b) |
Compare host_part with prefix. More... | |
static bool | urldb_iterate_partial_host (struct search_node *root, const char *prefix, bool(*callback)(nsurl *url, const struct url_data *data)) |
Partial host iterator (internal) More... | |
static bool | urldb_iterate_partial_path (const struct path_data *parent, const char *prefix, bool(*callback)(nsurl *url, const struct url_data *data)) |
Partial path iterator (internal) More... | |
static bool | urldb_iterate_entries_host (struct search_node *parent, bool(*url_callback)(nsurl *url, const struct url_data *data), bool(*cookie_callback)(const struct cookie_data *data)) |
Host data iterator (internal) More... | |
static struct host_part * | urldb_add_host_node (const char *part, struct host_part *parent) |
Add a host node to the tree. More... | |
static int | urldb_add_path_fragment_cmp (const void *a, const void *b) |
Fragment comparator callback for qsort. More... | |
static struct path_data * | urldb_add_path_fragment (struct path_data *segment, lwc_string *fragment) |
Add a fragment to a path segment. More... | |
static struct path_data * | urldb_add_path_node (lwc_string *scheme, unsigned int port, const char *segment, lwc_string *fragment, struct path_data *parent) |
Add a path node to the tree. More... | |
static struct search_node ** | urldb_get_search_tree_direct (const char *host) |
Get the search tree for a particular host. More... | |
static struct search_node * | urldb_get_search_tree (const char *host) |
Get the search tree for a particular host. More... | |
static int | urldb_search_match_string (const struct host_part *a, const char *b) |
Compare host part with a string. More... | |
static const struct host_part * | urldb_search_find (struct search_node *root, const char *host) |
Find a node in a search tree. More... | |
static struct path_data * | urldb_match_path (const struct path_data *parent, const char *path, lwc_string *scheme, unsigned short port) |
Match a path string. More... | |
static struct path_data * | urldb_find_url (nsurl *url) |
Find an URL in the database. More... | |
static void | urldb_dump_paths (struct path_data *parent) |
Dump URL database paths to stderr. More... | |
static void | urldb_dump_hosts (struct host_part *parent) |
Dump URL database hosts to stderr. More... | |
static void | urldb_dump_search (struct search_node *parent, int depth) |
Dump search tree. More... | |
static int | urldb_search_match_host (const struct host_part *a, const struct host_part *b) |
Compare a pair of host parts. More... | |
static struct search_node * | urldb_search_skew (struct search_node *root) |
Rotate a subtree right. More... | |
static struct search_node * | urldb_search_split (struct search_node *root) |
Rotate a node left, increasing the parent's level. More... | |
static struct search_node * | urldb_search_insert_internal (struct search_node *root, struct search_node *n) |
Insert node into search tree. More... | |
static struct search_node * | urldb_search_insert (struct search_node *root, const struct host_part *data) |
Insert a node into the search tree. More... | |
static bool | urldb_parse_avpair (struct cookie_internal_data *c, char *n, char *v, bool was_quoted) |
Parse a cookie avpair. More... | |
static void | urldb_free_cookie (struct cookie_internal_data *c) |
Free a cookie. More... | |
static struct cookie_internal_data * | urldb_parse_cookie (nsurl *url, const char **cookie) |
Parse a cookie. More... | |
static struct path_data * | urldb_add_path (lwc_string *scheme, unsigned int port, const struct host_part *host, char *path_query, lwc_string *fragment, nsurl *url) |
Add a path to the database, creating any intermediate entries. More... | |
static struct host_part * | urldb_add_host (const char *host) |
Add a host to the database, creating any intermediate entries. More... | |
static bool | urldb_insert_cookie (struct cookie_internal_data *c, lwc_string *scheme, nsurl *url) |
Insert a cookie into the database. More... | |
static bool | urldb_concat_cookie (struct cookie_internal_data *c, int version, int *used, int *alloc, char **buf) |
Concatenate a cookie into the provided buffer. More... | |
static void | urldb_delete_cookie_paths (const char *domain, const char *path, const char *name, struct path_data *parent) |
deletes paths from a cookie. More... | |
static void | urldb_delete_cookie_hosts (const char *domain, const char *path, const char *name, struct host_part *parent) |
Deletes cookie hosts and their assoicated paths. More... | |
static void | urldb_save_cookie_paths (FILE *fp, struct path_data *parent) |
Save a path subtree's cookies. More... | |
static void | urldb_save_cookie_hosts (FILE *fp, struct host_part *parent) |
Save a host subtree's cookies. More... | |
static void | urldb_destroy_cookie (struct cookie_internal_data *c) |
Destroy a cookie node. More... | |
static void | urldb_destroy_path_node_content (struct path_data *node) |
Destroy the contents of a path node. More... | |
static void | urldb_destroy_prot_space (struct prot_space_data *space) |
Destroy protection space data. More... | |
static void | urldb_destroy_path_tree (struct path_data *root) |
Destroy a path tree. More... | |
static void | urldb_destroy_host_tree (struct host_part *root) |
Destroy a host tree. More... | |
static void | urldb_destroy_search_tree (struct search_node *root) |
Destroy a search tree. More... | |
void | urldb_destroy (void) |
Destroy urldb. More... | |
nserror | urldb_load (const char *filename) |
Import an URL database from file, replacing any existing database. More... | |
nserror | urldb_save (const char *filename) |
Export the current database to file. More... | |
nserror | urldb_set_url_persistence (nsurl *url, bool persist) |
Set the cross-session persistence of the entry for an URL. More... | |
bool | urldb_add_url (nsurl *url) |
Insert an URL into the database. More... | |
nserror | urldb_set_url_title (nsurl *url, const char *title) |
Set an URL's title string, replacing any existing one. More... | |
nserror | urldb_set_url_content_type (nsurl *url, content_type type) |
Set an URL's content type. More... | |
nserror | urldb_update_url_visit_data (nsurl *url) |
Update an URL's visit data. More... | |
void | urldb_reset_url_visit_data (nsurl *url) |
Reset an URL's visit statistics. More... | |
const struct url_data * | urldb_get_url_data (nsurl *url) |
Find data for an URL. More... | |
nsurl * | urldb_get_url (nsurl *url) |
Extract an URL from the db. More... | |
void | urldb_set_auth_details (nsurl *url, const char *realm, const char *auth) |
Set authentication data for an URL. More... | |
const char * | urldb_get_auth_details (nsurl *url, const char *realm) |
Look up authentication details in database. More... | |
void | urldb_set_cert_permissions (nsurl *url, bool permit) |
Set certificate verification permissions. More... | |
bool | urldb_get_cert_permissions (nsurl *url) |
Retrieve certificate verification permissions from database. More... | |
bool | urldb_set_hsts_policy (struct nsurl *url, const char *header) |
Set HSTS policy for an URL. More... | |
bool | urldb_get_hsts_enabled (struct nsurl *url) |
Determine if HSTS policy is enabled for an URL. More... | |
void | urldb_iterate_partial (const char *prefix, bool(*callback)(nsurl *url, const struct url_data *data)) |
void | urldb_iterate_entries (bool(*callback)(nsurl *url, const struct url_data *data)) |
void | urldb_iterate_cookies (bool(*callback)(const struct cookie_data *data)) |
bool | urldb_set_cookie (const char *header, nsurl *url, nsurl *referer) |
Parse Set-Cookie header and insert cookie(s) into database. More... | |
char * | urldb_get_cookie (nsurl *url, bool include_http_only) |
Retrieve cookies for an URL. More... | |
void | urldb_delete_cookie (const char *domain, const char *path, const char *name) |
Delete a cookie. More... | |
void | urldb_load_cookies (const char *filename) |
Load a cookie file into the database. More... | |
void | urldb_save_cookies (const char *filename) |
Save persistent cookies to file. More... | |
void | urldb_dump (void) |
Dump URL database to stderr. More... | |
Variables | |
static struct host_part | db_root |
Root database handle. More... | |
static struct search_node | empty = { 0, 0, &empty, &empty } |
static struct search_node * | search_trees [NUM_SEARCH_TREES] |
static int | loaded_cookie_file_version |
loaded cookie file version More... | |
static struct bloom_filter * | url_bloom |
filter for url presence in database More... | |
Unified URL information database implementation.
URLs are stored in a tree-based structure as follows:
The host component is extracted from each URL and, if a FQDN, split on every '.'.The tree is constructed by inserting each FQDN segment in reverse order. Duplicate nodes are merged.
If the host part of an URL is an IP address, then this is added to the tree verbatim (as if it were a TLD).
This provides something looking like:
root (a sentinel) | ------------------------------------------------- | | | | | | | com edu gov 127.0.0.1 net org uk TLDs | | | | | |
google ... ... ... ... co 2LDs | | www bbc Hosts/Subdomains | www ...
Each of the nodes in this tree is a struct host_part. This stores the FQDN segment (or IP address) with which the node is concerned. Each node may contain further information about paths on a host (struct path_data) or SSL certificate processing on a host-wide basis (host_part::permit_invalid_certs).
Path data is concerned with storing various metadata about the path in question. This includes global history data, HTTP authentication details and any associated HTTP cookies. This is stored as a tree of path segments hanging off the relevant host_part node.
Therefore, to find the last visited time of the URL http://www.example.com/path/to/resource.html, the FQDN tree would be traversed in the order root -> "com" -> "example" -> "www". The "www" node would have attached to it a tree of struct path_data:
(sentinel) | path | to | resource.html
This represents the absolute path "/path/to/resource.html". The leaf node "resource.html" contains the last visited time of the resource.
The mechanism described above is, however, not particularly conducive to fast searching of the database for a given URL (or URLs beginning with a given prefix). Therefore, an anciliary data structure is used to enable fast searching. This structure simply reflects the contents of the database, with entries being added/removed at the same time as for the core database. In order to ensure that degenerate cases are kept to a minimum, we use an AAtree. This is an approximation of a Red-Black tree with similar performance characteristics, but with a significantly simpler implementation. Entries in this tree comprise pointers to the leaf nodes of the host tree described above.
REALLY IMPORTANT NOTE: urldb expects all URLs to be normalised. Use of non-normalised URLs with urldb will result in undefined behaviour and potential crashes.
Definition in file urldb.c.
#define COOKIE_FILE_VERSION 102 |
#define FIND_T |
#define GROW_MATCHED_COOKIES |
#define MAXIMUM_URL_LENGTH 4096 |
#define MIN_COOKIE_FILE_VERSION 100 |
#define MIN_URL_FILE_VERSION 106 |
#define NUM_SEARCH_TREES 28 |
#define SKIP_T |
#define URL_FILE_VERSION 107 |
|
static |
Check whether a host string is an IP address.
This call detects IPv4 addresses (all of dotted-quad or subsets, decimal or hexadecimal notations) and IPv6 addresses (including those containing embedded IPv4 addresses.)
host | a hostname terminated by '\0' |
For the time being, we will accept such calls; we check if there is a / in the host parameter, and if there is, we take a copy and replace the / with a \0. This is not a permanent solution; we should search through NetSurf and find all the callers that are in error and fix them. When doing this task, it might be wise to replace the hideousness below with code that doesn't have to do this, and add assert(strchr(host, '/') == NULL); somewhere. – rjek - 2010-11-04
Definition at line 647 of file urldb.c.
References inet_aton(), inet_pton(), and NSLOG.
Referenced by urldb_add_host(), urldb_get_hsts_enabled(), urldb_get_search_tree_direct(), urldb_search_match_prefix(), urldb_search_match_string(), urldb_set_cookie(), and urldb_set_hsts_policy().
|
static |
Add a host to the database, creating any intermediate entries.
host | Hostname to add |
Definition at line 2164 of file urldb.c.
References host_part::children, db_root, host_part::next, search_trees, ST_IP, urldb__host_is_ip_address(), urldb_add_host_node(), urldb_get_search_tree_direct(), and urldb_search_insert().
Referenced by urldb_add_url(), urldb_insert_cookie(), and urldb_load().
|
static |
Add a host node to the tree.
part | Host segment to add (or whole IP address) (copied) |
parent | Parent node to add to |
Definition at line 1006 of file urldb.c.
References host_part::next, host_part::parent, parent, and host_part::part.
Referenced by urldb_add_host().
|
static |
Add a path to the database, creating any intermediate entries.
scheme | URL scheme associated with path |
port | Port number on host associated with path |
host | Host tree node to attach to |
path_query | Absolute path plus query to add (freed) |
fragment | URL fragment, or NULL |
url | URL (fragment ignored) |
Definition at line 2086 of file urldb.c.
References path_data::children, path_data::fragment, path_data::next, NSERROR_OK, nsurl_defragment(), host_part::paths, path_data::port, path_data::scheme, path_data::segment, path_data::url, urldb_add_path_fragment(), and urldb_add_path_node().
Referenced by urldb_add_url(), urldb_insert_cookie(), and urldb_load().
|
static |
Add a fragment to a path segment.
segment | Path segment to add to |
fragment | Fragment to add (copied), or NULL |
Definition at line 1055 of file urldb.c.
References path_data::fragment, path_data::segment, and urldb_add_path_fragment_cmp().
Referenced by urldb_add_path(), and urldb_add_path_node().
|
static |
Fragment comparator callback for qsort.
a | first value |
b | second value |
Definition at line 1041 of file urldb.c.
Referenced by urldb_add_path_fragment().
|
static |
Add a path node to the tree.
scheme | URL scheme associated with path (copied) |
port | Port number on host associated with path |
segment | Path segment to add (copied) |
fragment | URL fragment (copied), or NULL |
parent | Parent node to add to |
Definition at line 1104 of file urldb.c.
References path_data::fragment, path_data::next, path_data::parent, parent, path_data::port, path_data::prev, path_data::scheme, path_data::segment, and urldb_add_path_fragment().
Referenced by urldb_add_path().
bool urldb_add_url | ( | struct nsurl * | url | ) |
Insert an URL into the database.
url | Absolute URL to insert |
Definition at line 3140 of file urldb.c.
References bloom_create(), bloom_insert_hash(), BLOOM_SIZE, path_data::fragment, NSERROR_OK, NSURL_FRAGMENT, nsurl_get(), nsurl_get_component(), nsurl_hash(), NSURL_HOST, NSURL_PATH, NSURL_PORT, NSURL_QUERY, NSURL_SCHEME, path_data::port, path_data::scheme, path_data::url, url_bloom, urldb_add_host(), and urldb_add_path().
Referenced by browser_window_callback(), browser_window_content_ready(), hotlist_create_entry(), hotlist_update_url_walk_cb(), urldb_get_auth_details(), urldb_get_cookie(), urldb_get_hsts_enabled(), urldb_set_auth_details(), urldb_set_cert_permissions(), and urldb_set_hsts_policy().
|
static |
Concatenate a cookie into the provided buffer.
c | Cookie to concatenate |
version | The version of the cookie string to output |
used | Pointer to amount of buffer used (updated) |
alloc | Pointer to allocated size of buffer (updated) |
buf | Pointer to Pointer to buffer (updated) |
Definition at line 2376 of file urldb.c.
References COOKIE_NETSCAPE, cookie_internal_data::domain, cookie_internal_data::domain_from_set, cookie_internal_data::name, cookie_internal_data::path, cookie_internal_data::path_from_set, cookie_internal_data::value, cookie_internal_data::value_was_quoted, and version.
Referenced by urldb_get_cookie().
|
static |
Count number of URLs associated with a host.
root | Root of path data tree |
expiry | Expiry time for URLs |
count | Pointer to count |
Definition at line 474 of file urldb.c.
References path_data::children, url_internal_data::last_visit, path_data::next, path_data::parent, path_data::persistent, root, path_data::urld, and url_internal_data::visits.
Referenced by urldb_save_search_tree().
void urldb_delete_cookie | ( | const char * | domain, |
const char * | path, | ||
const char * | name | ||
) |
Delete a cookie.
domain | The cookie's domain |
path | The cookie's path |
name | The cookie's name |
Definition at line 4273 of file urldb.c.
References db_root, cookie_internal_data::domain, cookie_internal_data::name, path(), and urldb_delete_cookie_hosts().
Referenced by CookieWindow::_DeleteCookies(), and cookie_manager_delete_entry().
|
static |
Deletes cookie hosts and their assoicated paths.
domain | the cookie domain |
path | the cookie path |
name | The cookie name |
parent | The url data of the cookie |
Definition at line 2596 of file urldb.c.
References host_part::next, parent, path(), urldb_delete_cookie_hosts(), and urldb_delete_cookie_paths().
Referenced by urldb_delete_cookie(), and urldb_delete_cookie_hosts().
|
static |
deletes paths from a cookie.
domain | the cookie domain |
path | the cookie path |
name | The cookie name |
parent | The url data of the cookie |
Definition at line 2538 of file urldb.c.
References path_data::children, path_data::cookies, path_data::cookies_end, cookie_internal_data::domain, cookie_internal_data::name, cookie_internal_data::next, path_data::next, path_data::parent, parent, cookie_internal_data::path, path(), cookie_internal_data::prev, and urldb_free_cookie().
Referenced by urldb_delete_cookie_hosts().
void urldb_destroy | ( | void | ) |
Destroy urldb.
Definition at line 2847 of file urldb.c.
References bloom_destroy(), host_part::children, db_root, empty, host_part::next, NUM_SEARCH_TREES, search_trees, url_bloom, urldb_destroy_host_tree(), and urldb_destroy_search_tree().
Referenced by netsurf_exit().
|
static |
Destroy a cookie node.
c | Cookie to destroy |
Definition at line 2692 of file urldb.c.
References cookie_internal_data::comment, cookie_internal_data::domain, cookie_internal_data::name, cookie_internal_data::path, and cookie_internal_data::value.
Referenced by urldb_destroy_path_node_content().
|
static |
Destroy a host tree.
root | Root node of tree to destroy |
Definition at line 2792 of file urldb.c.
References prot_space_data::next, path_data::next, host_part::next, root, urldb_destroy_host_tree(), urldb_destroy_path_node_content(), urldb_destroy_path_tree(), and urldb_destroy_prot_space().
Referenced by urldb_destroy(), and urldb_destroy_host_tree().
|
static |
Destroy the contents of a path node.
node | Node to destroy contents of (does not destroy node) |
Definition at line 2709 of file urldb.c.
References path_data::cookies, path_data::frag_cnt, path_data::fragment, cookie_internal_data::next, nsurl_unref(), path_data::scheme, path_data::segment, url_internal_data::title, path_data::url, path_data::urld, and urldb_destroy_cookie().
Referenced by urldb_destroy_host_tree(), and urldb_destroy_path_tree().
|
static |
Destroy a path tree.
root | Root node of tree to destroy |
Definition at line 2756 of file urldb.c.
References path_data::children, path_data::next, path_data::parent, root, and urldb_destroy_path_node_content().
Referenced by urldb_destroy_host_tree().
|
static |
Destroy protection space data.
space | Protection space to destroy |
Definition at line 2741 of file urldb.c.
References prot_space_data::auth, prot_space_data::realm, and prot_space_data::scheme.
Referenced by urldb_destroy_host_tree().
|
static |
Destroy a search tree.
root | Root node of tree to destroy |
Definition at line 2830 of file urldb.c.
References empty, root, and urldb_destroy_search_tree().
Referenced by urldb_destroy(), and urldb_destroy_search_tree().
void urldb_dump | ( | void | ) |
Dump URL database to stderr.
Definition at line 4481 of file urldb.c.
References db_root, NUM_SEARCH_TREES, search_trees, urldb_dump_hosts(), and urldb_dump_search().
Referenced by ro_gui_window_handle_local_keypress().
|
static |
Dump URL database hosts to stderr.
parent | Parent node of tree to dump |
Definition at line 1477 of file urldb.c.
References host_part::next, NSLOG, parent, urldb_dump_hosts(), and urldb_dump_paths().
Referenced by urldb_dump(), and urldb_dump_hosts().
|
static |
Dump URL database paths to stderr.
parent | Parent node of tree to dump |
Definition at line 1438 of file urldb.c.
References path_data::children, path_data::frag_cnt, path_data::fragment, path_data::next, NSLOG, path_data::parent, parent, path_data::port, path_data::scheme, and path_data::segment.
Referenced by urldb_dump_hosts().
|
static |
Dump search tree.
parent | Parent node of tree to dump |
depth | Tree depth |
Definition at line 1504 of file urldb.c.
References empty, NSLOG, host_part::parent, parent, host_part::part, and urldb_dump_search().
Referenced by urldb_dump(), and urldb_dump_search().
Find an URL in the database.
url | Absolute URL to find |
Definition at line 1358 of file urldb.c.
References bloom_search_hash(), NSERROR_OK, nsurl_get(), nsurl_get_component(), nsurl_hash(), NSURL_HOST, NSURL_PATH, NSURL_PORT, NSURL_QUERY, NSURL_SCHEME, host_part::paths, url_bloom, urldb_get_search_tree(), urldb_match_path(), and urldb_search_find().
Referenced by urldb_get_auth_details(), urldb_get_cert_permissions(), urldb_get_cookie(), urldb_get_hsts_enabled(), urldb_get_url(), urldb_get_url_data(), urldb_reset_url_visit_data(), urldb_set_auth_details(), urldb_set_cert_permissions(), urldb_set_hsts_policy(), urldb_set_url_content_type(), urldb_set_url_persistence(), urldb_set_url_title(), and urldb_update_url_visit_data().
|
static |
Free a cookie.
c | The cookie to free |
Definition at line 1817 of file urldb.c.
References cookie_internal_data::comment, cookie_internal_data::domain, cookie_internal_data::name, cookie_internal_data::path, and cookie_internal_data::value.
Referenced by urldb_delete_cookie_paths(), urldb_insert_cookie(), urldb_load_cookies(), urldb_parse_cookie(), and urldb_set_cookie().
const char * urldb_get_auth_details | ( | struct nsurl * | url, |
const char * | realm | ||
) |
Look up authentication details in database.
url | Absolute URL to search for |
realm | When non-NULL, it is realm which can be used to determine the protection space when that's not been done before for given URL. |
Definition at line 3405 of file urldb.c.
References prot_space_data::auth, prot_space_data::next, path_data::parent, prot_space_data::port, path_data::port, path_data::prot_space, host_part::prot_space, prot_space_data::realm, prot_space_data::scheme, path_data::scheme, path_data::url, urldb_add_url(), and urldb_find_url().
Referenced by browser_window__handle_login(), fetch_curl_set_options(), and llcache_fetch_auth().
bool urldb_get_cert_permissions | ( | struct nsurl * | url | ) |
Retrieve certificate verification permissions from database.
url | Absolute URL to search for |
Definition at line 3480 of file urldb.c.
References path_data::parent, host_part::permit_invalid_certs, and urldb_find_url().
Referenced by browser_window_get_page_info_state(), content_saw_insecure_objects(), and fetch_curl_set_options().
char * urldb_get_cookie | ( | struct nsurl * | url, |
bool | include_http_only | ||
) |
Retrieve cookies for an URL.
url | URL being fetched |
include_http_only | Whether to include HTTP(S) only cookies. |
Definition at line 3997 of file urldb.c.
References path_data::children, cookie_manager_add(), COOKIE_NETSCAPE, COOKIE_RFC2965, path_data::cookies, count(), db_root, cookie_internal_data::expires, GROW_MATCHED_COOKIES, cookie_internal_data::http_only, cookie_internal_data::last_used, cookie_internal_data::next, path_data::next, nsurl_get_component(), NSURL_PATH, path_data::parent, host_part::parent, cookie_internal_data::path, path(), host_part::paths, path_data::scheme, cookie_internal_data::secure, path_data::segment, urldb_add_url(), urldb_concat_cookie(), urldb_find_url(), cookie_internal_data::version, and version.
Referenced by browser_window_get_cookie_count(), and fetch_curl_set_options().
bool urldb_get_hsts_enabled | ( | struct nsurl * | url | ) |
Determine if HSTS policy is enabled for an URL.
url | URL being fetched |
Definition at line 3573 of file urldb.c.
References db_root, hsts_data::expires, host_part::hsts, hsts_data::include_sub_domains, nsurl_get_component(), NSURL_HOST, path_data::parent, host_part::parent, urldb__host_is_ip_address(), urldb_add_url(), and urldb_find_url().
Referenced by llcache_hsts_transform_url().
|
static |
Get the search tree for a particular host.
host | the host to lookup |
Definition at line 1191 of file urldb.c.
References urldb_get_search_tree_direct().
Referenced by urldb_find_url(), urldb_insert_cookie(), and urldb_iterate_partial().
|
static |
Get the search tree for a particular host.
host | the host to lookup |
Definition at line 1172 of file urldb.c.
References ascii_is_alpha(), ascii_to_lower(), search_trees, ST_DN, ST_EE, ST_IP, and urldb__host_is_ip_address().
Referenced by urldb_add_host(), and urldb_get_search_tree().
Extract an URL from the db.
url | URL to extract |
Definition at line 3327 of file urldb.c.
References path_data::url, and urldb_find_url().
Referenced by browser_window_content_ready().
Find data for an URL.
url | Absolute URL to look for |
Definition at line 3309 of file urldb.c.
References path_data::urld, and urldb_find_url().
Referenced by global_history_add(), hotlist_create_entry(), hotlist_update_url_walk_cb(), node_is_visited(), ro_gui_hotlist_add_page(), and ro_gui_url_complete_redraw().
|
static |
Insert a cookie into the database.
c | The cookie to insert |
scheme | URL scheme associated with cookie path |
url | URL (sans fragment) associated with cookie |
Definition at line 2257 of file urldb.c.
References cookie_manager_add(), cookie_manager_remove(), path_data::cookies, path_data::cookies_end, cookie_internal_data::domain, cookie_internal_data::expires, cookie_internal_data::name, cookie_internal_data::next, cookie_internal_data::path, host_part::paths, cookie_internal_data::prev, path_data::scheme, path_data::url, urldb_add_host(), urldb_add_path(), urldb_free_cookie(), urldb_get_search_tree(), and urldb_search_find().
Referenced by urldb_load_cookies(), and urldb_set_cookie().
void urldb_iterate_cookies | ( | bool(*)(const struct cookie_data *data) | callback | ) |
Definition at line 3719 of file urldb.c.
References NUM_SEARCH_TREES, search_trees, and urldb_iterate_entries_host().
Definition at line 3702 of file urldb.c.
References NUM_SEARCH_TREES, search_trees, and urldb_iterate_entries_host().
|
static |
Host data iterator (internal)
parent | Root of subtree to iterate over |
url_callback | Callback function |
cookie_callback | Callback function |
Definition at line 963 of file urldb.c.
References empty, parent, urldb_iterate_entries_host(), and urldb_iterate_entries_path().
Referenced by urldb_iterate_cookies(), urldb_iterate_entries(), and urldb_iterate_entries_host().
|
static |
Path data iterator (internal)
parent | Root of subtree to iterate over |
url_callback | Callback function |
cookie_callback | Callback function |
Definition at line 584 of file urldb.c.
References path_data::children, path_data::cookies, path_data::next, cookie_data::next, path_data::parent, parent, path_data::url, and path_data::urld.
Referenced by urldb_iterate_entries_host(), urldb_iterate_partial_host(), and urldb_iterate_partial_path().
void urldb_iterate_partial | ( | const char * | prefix, |
bool(*)(nsurl *url, const struct url_data *data) | callback | ||
) |
Definition at line 3635 of file urldb.c.
References path_data::children, host_part::paths, search_trees, ST_DN, urldb_get_search_tree(), urldb_iterate_partial_host(), urldb_iterate_partial_path(), and urldb_search_find().
|
static |
Partial host iterator (internal)
root | Root of (sub)tree to traverse |
prefix | Prefix to match |
callback | Callback function |
Definition at line 816 of file urldb.c.
References empty, root, urldb_iterate_entries_path(), urldb_iterate_partial_host(), and urldb_search_match_prefix().
Referenced by urldb_iterate_partial(), and urldb_iterate_partial_host().
|
static |
Partial path iterator (internal)
Given: http://www.example.org/a/b/c/d//e and assuming a path tree: ^ / \ a1 b1 / \ a2 b2 /|\ a b c 3 3 | d | e / \ f g
Prefix will be: p will be:
a/b/c/d//e a1 b/c/d//e a2 b/c/d//e b3 c/d//e a3 c/d//e b3 c/d//e c d//e d /e e (skip /) e e
I.E. perform a breadth-first search of the tree.
parent | Root of (sub)tree to traverse |
prefix | Prefix to match |
callback | Callback function |
Definition at line 906 of file urldb.c.
References path_data::children, path_data::next, parent, path_data::segment, and urldb_iterate_entries_path().
Referenced by urldb_iterate_partial().
nserror urldb_load | ( | const char * | filename | ) |
Import an URL database from file, replacing any existing database.
filename | Name of file containing data |
Definition at line 2876 of file urldb.c.
References bloom_create(), bloom_insert_hash(), BLOOM_SIZE, hsts_data::expires, host_part::hsts, hsts_data::include_sub_domains, url_internal_data::last_visit, MAXIMUM_URL_LENGTH, MIN_URL_FILE_VERSION, nsc_snptimet(), NSERROR_INVALID, NSERROR_NEED_DATA, NSERROR_NOMEM, NSERROR_NOT_FOUND, NSERROR_OK, NSLOG, nsurl_create(), NSURL_FRAGMENT, nsurl_get(), nsurl_get_component(), nsurl_hash(), NSURL_PATH, NSURL_QUERY, NSURL_SCHEME, nsurl_unref(), path_data::port, path_data::scheme, url_internal_data::title, url_internal_data::type, path_data::url, url_bloom, URL_FILE_VERSION, path_data::urld, urldb_add_host(), urldb_add_path(), version, and url_internal_data::visits.
Referenced by gui_init(), main(), nsgtk_setup(), and WinMain().
void urldb_load_cookies | ( | const char * | filename | ) |
Load a cookie file into the database.
filename | File to load |
Definition at line 4281 of file urldb.c.
References cookie_internal_data::comment, cookie_internal_data::domain, cookie_internal_data::domain_from_set, cookie_internal_data::expires, FIND_T, cookie_internal_data::http_only, cookie_internal_data::last_used, loaded_cookie_file_version, MIN_COOKIE_FILE_VERSION, cookie_internal_data::name, cookie_internal_data::no_destroy, NSERROR_OK, NSLOG, nsurl_create(), nsurl_get_component(), NSURL_SCHEME, nsurl_unref(), cookie_internal_data::path, path(), cookie_internal_data::path_from_set, cookie_internal_data::secure, SKIP_T, urldb_free_cookie(), urldb_insert_cookie(), cookie_internal_data::value, cookie_internal_data::value_was_quoted, cookie_internal_data::version, and version.
Referenced by gui_init(), main(), nsgtk_setup(), and WinMain().
|
static |
Match a path string.
parent | Path (sub)tree to look in |
path | The path to search for |
scheme | The URL scheme associated with the path |
port | The port associated with the path |
Definition at line 1302 of file urldb.c.
References path_data::children, path_data::next, NSLOG, parent, path(), path_data::port, path_data::scheme, and path_data::segment.
Referenced by urldb_find_url().
|
static |
Parse a cookie avpair.
c | Cookie struct to populate |
n | Name component |
v | Value component |
was_quoted | Whether v was quoted in the input |
Definition at line 1709 of file urldb.c.
References ascii_is_digit(), cookie_internal_data::comment, cookie_internal_data::domain, cookie_internal_data::domain_from_set, cookie_internal_data::expires, cookie_internal_data::http_only, cookie_internal_data::name, nsc_strntimet(), NSERROR_OK, cookie_internal_data::path, cookie_internal_data::path_from_set, cookie_internal_data::secure, cookie_internal_data::value, cookie_internal_data::value_was_quoted, and cookie_internal_data::version.
Referenced by urldb_parse_cookie().
|
static |
Parse a cookie.
url | URL being fetched |
cookie | Pointer to cookie string (updated on exit) |
Definition at line 1838 of file urldb.c.
References cookie_internal_data::domain, cookie_internal_data::expires, cookie_internal_data::name, nsurl_get_component(), NSURL_HOST, NSURL_PATH, cookie_internal_data::path, path(), strndup(), urldb_free_cookie(), urldb_parse_avpair(), and cookie_internal_data::value.
Referenced by urldb_set_cookie().
void urldb_reset_url_visit_data | ( | struct nsurl * | url | ) |
Reset an URL's visit statistics.
url | The URL to reset |
Definition at line 3293 of file urldb.c.
References url_internal_data::last_visit, path_data::url, path_data::urld, urldb_find_url(), and url_internal_data::visits.
Referenced by global_history_delete_entry_internal().
nserror urldb_save | ( | const char * | filename | ) |
Export the current database to file.
filename | Name of file to export to |
Definition at line 3094 of file urldb.c.
References NSERROR_OK, NSERROR_SAVE_FAILED, NSLOG, NUM_SEARCH_TREES, search_trees, URL_FILE_VERSION, and urldb_save_search_tree().
Referenced by gui_quit(), monkey_quit(), nsgtk_finalise(), and WinMain().
|
static |
Save a host subtree's cookies.
fp | File pointer to write to |
parent | Parent host |
Definition at line 2675 of file urldb.c.
References host_part::next, parent, urldb_save_cookie_hosts(), and urldb_save_cookie_paths().
Referenced by urldb_save_cookie_hosts(), and urldb_save_cookies().
|
static |
Save a path subtree's cookies.
fp | File pointer to write to |
parent | Parent path |
Definition at line 2618 of file urldb.c.
References path_data::children, cookie_internal_data::comment, path_data::cookies, cookie_internal_data::domain, cookie_internal_data::domain_from_set, cookie_internal_data::expires, cookie_internal_data::http_only, cookie_internal_data::last_used, cookie_internal_data::name, cookie_internal_data::next, path_data::next, cookie_internal_data::no_destroy, nsurl_access(), path_data::parent, parent, cookie_internal_data::path, cookie_internal_data::path_from_set, path_data::scheme, cookie_internal_data::secure, path_data::url, cookie_internal_data::value, cookie_internal_data::value_was_quoted, and cookie_internal_data::version.
Referenced by urldb_save_cookie_hosts().
void urldb_save_cookies | ( | const char * | filename | ) |
Save persistent cookies to file.
filename | Path to save to |
Definition at line 4448 of file urldb.c.
References COOKIE_FILE_VERSION, db_root, loaded_cookie_file_version, max, and urldb_save_cookie_hosts().
Referenced by gui_quit(), monkey_quit(), nsgtk_finalise(), and WinMain().
|
static |
Save a search (sub)tree.
parent | root node of search tree to save. |
fp | File to write to |
Definition at line 514 of file urldb.c.
References db_root, empty, hsts_data::expires, host_part::hsts, hsts_data::include_sub_domains, nsoption_int, host_part::parent, parent, host_part::part, path(), urldb_count_urls(), urldb_save_search_tree(), urldb_write_paths(), and urldb_write_timet().
Referenced by urldb_save(), and urldb_save_search_tree().
|
static |
Find a node in a search tree.
root | Tree to look in |
host | Host to find |
Definition at line 1270 of file urldb.c.
References empty, root, urldb_search_find(), and urldb_search_match_string().
Referenced by urldb_find_url(), urldb_insert_cookie(), urldb_iterate_partial(), and urldb_search_find().
|
static |
Insert a node into the search tree.
root | Root of tree to insert into |
data | User data to insert |
Definition at line 1679 of file urldb.c.
References search_node::data, empty, search_node::left, search_node::level, search_node::right, root, and urldb_search_insert_internal().
Referenced by urldb_add_host().
|
static |
Insert node into search tree.
root | Root of (sub)tree to insert into |
n | Node to insert |
Definition at line 1642 of file urldb.c.
References search_node::data, empty, root, urldb_search_insert_internal(), urldb_search_match_host(), urldb_search_skew(), and urldb_search_split().
Referenced by urldb_search_insert(), and urldb_search_insert_internal().
|
static |
Compare a pair of host parts.
a | first host part |
b | second host part |
Definition at line 1554 of file urldb.c.
References db_root, host_part::parent, and host_part::part.
Referenced by urldb_search_insert_internal().
|
static |
Compare host_part with prefix.
a | host part |
b | prefix |
Definition at line 746 of file urldb.c.
References db_root, host_part::parent, host_part::part, and urldb__host_is_ip_address().
Referenced by urldb_iterate_partial_host().
|
static |
Compare host part with a string.
a | host part |
b | string to compare |
Definition at line 1204 of file urldb.c.
References db_root, host_part::parent, host_part::part, and urldb__host_is_ip_address().
Referenced by urldb_search_find().
|
static |
Rotate a subtree right.
root | Root of subtree to rotate |
Definition at line 1592 of file urldb.c.
References search_node::right, and root.
Referenced by urldb_search_insert_internal().
|
static |
Rotate a node left, increasing the parent's level.
root | Root of subtree to rotate |
Definition at line 1615 of file urldb.c.
References search_node::left, and root.
Referenced by urldb_search_insert_internal().
void urldb_set_auth_details | ( | struct nsurl * | url, |
const char * | realm, | ||
const char * | auth | ||
) |
Set authentication data for an URL.
url | The URL to consider |
realm | The authentication realm |
auth | The authentication details (in form username:password) |
Definition at line 3342 of file urldb.c.
References prot_space_data::auth, prot_space_data::next, path_data::parent, prot_space_data::port, path_data::port, path_data::prot_space, host_part::prot_space, prot_space_data::realm, prot_space_data::scheme, path_data::scheme, urldb_add_url(), and urldb_find_url().
Referenced by browser_window__handle_userpass_response(), and navigate_internal_query_auth().
void urldb_set_cert_permissions | ( | struct nsurl * | url, |
bool | permit | ||
) |
Set certificate verification permissions.
url | URL to consider |
permit | Set to true to allow invalid certificates |
Definition at line 3455 of file urldb.c.
References path_data::parent, host_part::permit_invalid_certs, urldb_add_url(), and urldb_find_url().
Referenced by browser_window__handle_ssl_query_response().
Parse Set-Cookie header and insert cookie(s) into database.
header | Header to parse, with Set-Cookie: stripped |
url | URL being fetched |
referrer | Referring resource, or 0 for verifiable transaction |
Definition at line 3734 of file urldb.c.
References cookie_internal_data::domain, cookie_internal_data::name, NSERROR_OK, NSLOG, nsurl_defragment(), nsurl_get_component(), NSURL_HOST, NSURL_PATH, NSURL_SCHEME, nsurl_unref(), cookie_internal_data::path, path(), urldb__host_is_ip_address(), urldb_free_cookie(), urldb_insert_cookie(), urldb_parse_cookie(), and cookie_internal_data::value.
Referenced by fetch_set_cookie().
bool urldb_set_hsts_policy | ( | struct nsurl * | url, |
const char * | header | ||
) |
Set HSTS policy for an URL.
url | URL being fetched |
header | Strict-Transport-Security header value |
Definition at line 3502 of file urldb.c.
References hsts_data::expires, host_part::hsts, http_parse_strict_transport_security(), http_strict_transport_security_destroy(), http_strict_transport_security_include_subdomains(), http_strict_transport_security_max_age(), hsts_data::include_sub_domains, NSERROR_OK, nsurl_get_component(), NSURL_HOST, path_data::parent, host_part::permit_invalid_certs, urldb__host_is_ip_address(), urldb_add_url(), and urldb_find_url().
Referenced by llcache_hsts_update_policy().
nserror urldb_set_url_content_type | ( | struct nsurl * | url, |
content_type | type | ||
) |
Set an URL's content type.
url | The URL to look for |
type | The type to set |
Definition at line 3256 of file urldb.c.
References NSERROR_NOT_FOUND, NSERROR_OK, url_internal_data::type, type, path_data::url, path_data::urld, and urldb_find_url().
Referenced by browser_window_content_ready().
Set the cross-session persistence of the entry for an URL.
url | Absolute URL to persist |
persist | True to persist, false otherwise |
Definition at line 3122 of file urldb.c.
References NSERROR_NOT_FOUND, NSERROR_OK, path_data::persistent, path_data::url, and urldb_find_url().
Referenced by hotlist_add_entry_internal().
Set an URL's title string, replacing any existing one.
url | The URL to look for |
title | The title string to use (copied) |
Definition at line 3226 of file urldb.c.
References NSERROR_NOMEM, NSERROR_NOT_FOUND, NSERROR_OK, url_internal_data::title, path_data::url, path_data::urld, and urldb_find_url().
Referenced by browser_window_content_ready().
Update an URL's visit data.
url | The URL to update |
Definition at line 3274 of file urldb.c.
References url_internal_data::last_visit, NSERROR_NOT_FOUND, NSERROR_OK, path_data::url, path_data::urld, urldb_find_url(), and url_internal_data::visits.
Referenced by browser_window_callback(), and browser_window_content_ready().
|
static |
Write paths associated with a host.
parent | Root of (sub)tree to write |
host | Current host name |
fp | File to write to |
path | Current path string |
path_alloc | Allocated size of path |
path_used | Used size of path |
expiry | Expiry time of URLs |
Definition at line 353 of file urldb.c.
References path_data::children, url_internal_data::last_visit, path_data::next, path_data::parent, parent, path(), path_data::persistent, path_data::port, path_data::scheme, path_data::segment, url_internal_data::title, url_internal_data::type, path_data::urld, urldb_write_timet(), and url_internal_data::visits.
Referenced by urldb_save_search_tree().
|
static |
write a time_t to a file portably
fp | File to write to |
val | the unix time value to output |
Definition at line 327 of file urldb.c.
References nsc_sntimet(), and NSERROR_OK.
Referenced by urldb_save_search_tree(), and urldb_write_paths().
|
static |
Root database handle.
Definition at line 275 of file urldb.c.
Referenced by urldb_add_host(), urldb_delete_cookie(), urldb_destroy(), urldb_dump(), urldb_get_cookie(), urldb_get_hsts_enabled(), urldb_save_cookies(), urldb_save_search_tree(), urldb_search_match_host(), urldb_search_match_prefix(), and urldb_search_match_string().
|
static |
Definition at line 282 of file urldb.c.
Referenced by urldb_destroy(), urldb_destroy_search_tree(), urldb_dump_search(), urldb_iterate_entries_host(), urldb_iterate_partial_host(), urldb_save_search_tree(), urldb_search_find(), urldb_search_insert(), and urldb_search_insert_internal().
|
static |
loaded cookie file version
Definition at line 295 of file urldb.c.
Referenced by urldb_load_cookies(), and urldb_save_cookies().
|
static |
Definition at line 283 of file urldb.c.
Referenced by urldb_add_host(), urldb_destroy(), urldb_dump(), urldb_get_search_tree_direct(), urldb_iterate_cookies(), urldb_iterate_entries(), urldb_iterate_partial(), and urldb_save().
|
static |
filter for url presence in database
Bloom filter used for short-circuting the false case of "is this URL in the database?". BLOOM_SIZE controls how large the filter is in bytes. Primitive experimentation shows that for a filter of X bytes filled with X items, searching for X items not in the filter has a 5% false-positive rate. We set it to 32kB, which should be enough for all but the largest databases, while not being shockingly wasteful on memory.
Definition at line 313 of file urldb.c.
Referenced by urldb_add_url(), urldb_destroy(), urldb_find_url(), and urldb_load().