From: Florent Bruneau Date: Mon, 10 Nov 2008 20:38:25 +0000 (+0100) Subject: Reload strlist and iplist resource-files only when needed. X-Git-Url: http://git.madism.org/?p=apps%2Fpfixtools.git;a=commitdiff_plain;h=56eeb7d73ed0c82f2a8165b6aba525af73c58f73 Reload strlist and iplist resource-files only when needed. Signed-off-by: Florent Bruneau --- diff --git a/common/file.c b/common/file.c index dd6fc91..a23f0e6 100644 --- a/common/file.c +++ b/common/file.c @@ -58,7 +58,6 @@ void file_map_delete(file_map_t **map) bool file_map_open(file_map_t *map, const char *file, bool memlock) { - struct stat st; int fd; fd = open(file, O_RDONLY, 0000); @@ -67,13 +66,13 @@ bool file_map_open(file_map_t *map, const char *file, bool memlock) return false; } - if (fstat(fd, &st) < 0) { + if (fstat(fd, &map->st) < 0) { UNIXERR("fstat"); close(fd); return false; } - map->map = mmap(NULL, st.st_size, PROT_READ, MAP_PRIVATE, fd, 0); + map->map = mmap(NULL, map->st.st_size, PROT_READ, MAP_PRIVATE, fd, 0); if (map->map == MAP_FAILED) { UNIXERR("mmap"); close(fd); @@ -82,8 +81,8 @@ bool file_map_open(file_map_t *map, const char *file, bool memlock) } close(fd); - map->end = map->map + st.st_size; - map->locked = memlock && mlock(map->map, st.st_size) == 0; + map->end = map->map + map->st.st_size; + map->locked = memlock && mlock(map->map, map->st.st_size) == 0; return true; } diff --git a/common/file.h b/common/file.h index ead058c..47ff5fc 100644 --- a/common/file.h +++ b/common/file.h @@ -42,6 +42,8 @@ */ typedef struct file_map_t { + struct stat st; + const char *map; const char *end; diff --git a/postlicyd/Makefile b/postlicyd/Makefile index 386d2af..cdbca81 100644 --- a/postlicyd/Makefile +++ b/postlicyd/Makefile @@ -42,16 +42,16 @@ UB_LIBS = -lunbound FILTERS = iplist.c greylist.c strlist.c match.c counters.c -postlicyd_SOURCES = main-postlicyd.c ../common/lib.a filter.c config.c query.c $(FILTERS) $(GENERATED) +postlicyd_SOURCES = main-postlicyd.c ../common/lib.a filter.c config.c query.c resources.c $(FILTERS) $(GENERATED) postlicyd_LIBADD = $(UB_LIBS) $(TC_LIBS) -lev -tst-rbl_SOURCES = tst-rbl.c ../common/lib.a filter.c config.c query.c iplist.c $(GENERATED) +tst-rbl_SOURCES = tst-rbl.c ../common/lib.a filter.c config.c query.c iplist.c resources.c $(GENERATED) tst-rbl_LIBADD = $(UB_LIBS) -lev -tst-filters_SOURCES = tst-filters.c ../common/lib.a config.c filter.c query.c $(FILTERS) $(GENERATED) +tst-filters_SOURCES = tst-filters.c ../common/lib.a config.c filter.c query.c resources.c $(FILTERS) $(GENERATED) tst-filters_LIBADD = $(UB_LIBS) $(TC_LIBS) -lev -tst-greylist_SOURCES = tst-greylist.c ../common/lib.a +tst-greylist_SOURCES = tst-greylist.c resources.c ../common/lib.a tst-greylist_LIBADD = $(TC_LIBS) hook_tokens.h hook_tokens.c: $(FILTERS) diff --git a/postlicyd/config.c b/postlicyd/config.c index 0111b7a..9c28895 100644 --- a/postlicyd/config.c +++ b/postlicyd/config.c @@ -36,6 +36,7 @@ #include "file.h" #include "config.h" #include "str.h" +#include "resources.h" #define config_param_register(Param) @@ -132,7 +133,6 @@ static bool config_parse(config_t *config) return false; } - config_close(config); filter_init(&filter); linep = p = map.map; @@ -419,6 +419,8 @@ static bool config_build_filters(config_t *config) } static bool config_load(config_t *config) { + config_close(config); + if (!config_parse(config)) { err("Invalid configuration: cannot parse configuration file \"%s\"", config->filename); return false; @@ -431,6 +433,8 @@ static bool config_load(config_t *config) { err("Invalid configuration: invalid filter"); return false; } + + resource_garbage_collect(); return true; } diff --git a/postlicyd/iplist.c b/postlicyd/iplist.c index cddc70a..b37d64b 100644 --- a/postlicyd/iplist.c +++ b/postlicyd/iplist.c @@ -43,6 +43,7 @@ #include "str.h" #include "file.h" #include "array.h" +#include "resources.h" #include "rbl.h" #define IPv4_BITS 5 @@ -62,10 +63,25 @@ enum { }; struct rbldb_t { - A(uint16_t) ips[1 << 16]; + char *filename; + A(uint16_t) *ips; }; ARRAY(rbldb_t) +typedef struct rbldb_resource_t { + time_t mtime; + off_t size; + A(uint16_t) ips[1 << 16]; +} rbldb_resource_t; + +static void rbldb_resource_wipe(rbldb_resource_t *res) +{ + for (int i = 0 ; i < 1 << 16 ; ++i) { + array_wipe(res->ips[i]); + } + p_delete(&res); +} + static int get_o(const char *s, const char **out) { int res = 0; @@ -129,6 +145,25 @@ rbldb_t *rbldb_create(const char *file, bool lock) return NULL; } + rbldb_resource_t *res = resource_get("iplist", file); + if (res == NULL) { + debug("No resource found"); + res = p_new(rbldb_resource_t, 1); + resource_set("iplist", file, res, (resource_destructor_t)rbldb_resource_wipe); + } + + db = p_new(rbldb_t, 1); + db->filename = m_strdup(file); + db->ips = res->ips; + if (map.st.st_size == res->size && map.st.st_mtime == res->mtime) { + info("rbl %s up to date", file); + file_map_close(&map); + return db; + } + debug("mtime %d/%d, size %d/%d", (int)map.st.st_mtime, (int)res->mtime, (int)map.st.st_size, (int)res->size); + res->size = map.st.st_size; + res->mtime = map.st.st_mtime; + p = map.map; end = map.end; while (end > p && end[-1] != '\n') { @@ -139,7 +174,6 @@ rbldb_t *rbldb_create(const char *file, bool lock) file); } - db = p_new(rbldb_t, 1); while (p < end) { uint32_t ip; @@ -149,7 +183,7 @@ rbldb_t *rbldb_create(const char *file, bool lock) if (parse_ipv4(p, &p, &ip) < 0) { p = (char *)memchr(p, '\n', end - p) + 1; } else { - array_add(db->ips[ip >> 16], ip & 0xffff); + array_add(res->ips[ip >> 16], ip & 0xffff); ++ips; } } @@ -158,14 +192,14 @@ rbldb_t *rbldb_create(const char *file, bool lock) /* Lookup may perform serveral I/O, so avoid swap. */ for (int i = 0 ; i < 1 << 16 ; ++i) { - array_adjust(db->ips[i]); - if (lock && !array_lock(db->ips[i])) { + array_adjust(res->ips[i]); + if (lock && !array_lock(res->ips[i])) { UNIXERR("mlock"); } - if (db->ips[i].len) { + if (res->ips[i].len) { # define QSORT_TYPE uint16_t -# define QSORT_BASE db->ips[i].data -# define QSORT_NELT db->ips[i].len +# define QSORT_BASE res->ips[i].data +# define QSORT_NELT res->ips[i].len # define QSORT_LT(a,b) *a < *b # include "qsort.c" } @@ -177,9 +211,9 @@ rbldb_t *rbldb_create(const char *file, bool lock) static void rbldb_wipe(rbldb_t *db) { - for (int i = 0 ; i < 1 << 16 ; ++i) { - array_wipe(db->ips[i]); - } + resource_release("iplist", db->filename); + p_delete(&db->filename); + db->ips = NULL; } void rbldb_delete(rbldb_t **db) diff --git a/postlicyd/resources.c b/postlicyd/resources.c index 759c717..f875891 100644 --- a/postlicyd/resources.c +++ b/postlicyd/resources.c @@ -123,14 +123,19 @@ void resource_garbage_collect(void) uint32_t used = 0; foreach (resource_t *res, resources) { if (res->key != NULL && res->refcount == 0) { + debug("resource gc: %s not referenced anymore", res->key); resource_wipe(res); } else if (res->key != NULL) { + debug("resource gc: keeping %s, still %d references", + res->key, res->refcount); if (used < __Ai) { array_elt(resources, used) = *res; } ++used; } }} + debug("resource gc: before %d resources, after %d", + array_len(resources), used); array_len(resources) = used; } diff --git a/postlicyd/strlist.c b/postlicyd/strlist.c index 07b358e..29fe79e 100644 --- a/postlicyd/strlist.c +++ b/postlicyd/strlist.c @@ -39,12 +39,26 @@ #include "str.h" #include "rbl.h" #include "policy_tokens.h" +#include "resources.h" + +typedef struct strlist_local_t { + char *filename; + trie_t **db; + int weight; + unsigned reverse :1; + unsigned partial :1; +} strlist_local_t; +ARRAY(strlist_local_t) + +typedef struct strlist_resource_t { + off_t size; + time_t mtime; + trie_t *trie1; + trie_t *trie2; +} strlist_resource_t; typedef struct strlist_config_t { - PA(trie_t) tries; - A(int) weights; - A(bool) reverses; - A(bool) partiales; + A(strlist_local_t) locals; A(char) hosts; A(int) host_offsets; @@ -74,6 +88,21 @@ typedef struct strlist_async_data_t { static filter_type_t filter_type = FTK_UNKNOWN; +static void strlist_local_wipe(strlist_local_t *entry) +{ + if (entry->filename != NULL) { + resource_release("strlist", entry->filename); + p_delete(&entry->filename); + } +} + +static void strlist_resource_wipe(strlist_resource_t *res) +{ + trie_delete(&res->trie1); + trie_delete(&res->trie2); + p_delete(&res); +} + static strlist_config_t *strlist_config_new(void) { return p_new(strlist_config_t, 1); @@ -82,10 +111,7 @@ static strlist_config_t *strlist_config_new(void) static void strlist_config_delete(strlist_config_t **config) { if (*config) { - array_deep_wipe((*config)->tries, trie_delete); - array_wipe((*config)->weights); - array_wipe((*config)->reverses); - array_wipe((*config)->partiales); + array_deep_wipe((*config)->locals, strlist_local_wipe); array_wipe((*config)->hosts); array_wipe((*config)->host_offsets); array_wipe((*config)->host_weights); @@ -113,16 +139,17 @@ static inline void strlist_copy(char *dest, const char *str, ssize_t str_len, } -static trie_t *strlist_create(const char *file, bool reverse, bool lock) +static bool strlist_create(strlist_local_t *local, + const char *file, int weight, + bool reverse, bool partial, bool lock) { - trie_t *db; file_map_t map; const char *p, *end; char line[BUFSIZ]; uint32_t count = 0; if (!file_map_open(&map, file, false)) { - return NULL; + return false; } p = map.map; end = map.end; @@ -134,7 +161,34 @@ static trie_t *strlist_create(const char *file, bool reverse, bool lock) file); } - db = trie_new(); + strlist_resource_t *res = resource_get("strlist", file); + if (res == NULL) { + res = p_new(strlist_resource_t, 1); + resource_set("strlist", file, res, (resource_destructor_t)strlist_resource_wipe); + } else if (res->trie2 != NULL) { + err("A file (%s) cannot be used as a rbldns zone file and a strlist file at the same time", + file); + resource_release("strlist", file); + file_map_close(&map); + return false; + } + + p_clear(local, 1); + local->filename = m_strdup(file); + local->db = &res->trie1; + local->weight = weight; + local->reverse = reverse; + local->partial = partial; + if (res->size == map.st.st_size && res->mtime == map.st.st_mtime) { + info("strlist %s up to date", file); + file_map_close(&map); + return true; + } + trie_delete(&res->trie1); + res->trie1 = trie_new(); + res->size = map.st.st_size; + res->mtime = map.st.st_mtime; + while (p < end && p != NULL) { const char *eol = (char *)memchr(p, '\n', end - p); if (eol == NULL) { @@ -143,8 +197,9 @@ static trie_t *strlist_create(const char *file, bool reverse, bool lock) if (eol - p >= BUFSIZ) { err("unreasonnable long line"); file_map_close(&map); - trie_delete(&db); - return NULL; + trie_delete(&res->trie1); + strlist_local_wipe(local); + return false; } if (*p != '#') { const char *eos = eol; @@ -156,22 +211,21 @@ static trie_t *strlist_create(const char *file, bool reverse, bool lock) } if (p < eos) { strlist_copy(line, p, eos - p, reverse); - trie_insert(db, line); + trie_insert(res->trie1, line); ++count; } } p = eol + 1; } file_map_close(&map); - trie_compile(db, lock); + trie_compile(res->trie1, lock); info("%s loaded, %u entries", file, count); - return db; + return true; } -static bool strlist_create_from_rhbl(const char *file, bool lock, - trie_t **phosts, trie_t **pdomains) +static bool strlist_create_from_rhbl(strlist_local_t *hosts, strlist_local_t *domains, + const char *file, int weight, bool lock) { - trie_t *hosts, *domains; uint32_t host_count, domain_count; file_map_t map; const char *p, *end; @@ -190,10 +244,47 @@ static bool strlist_create_from_rhbl(const char *file, bool lock, file); } - hosts = trie_new(); + + strlist_resource_t *res = resource_get("strlist", file); + if (res == NULL) { + res = p_new(strlist_resource_t, 1); + resource_set("strlist", file, res, (resource_destructor_t)strlist_resource_wipe); + } else if (res->trie2 == NULL) { + err("A file (%s) cannot be used as a rbldns zone file and a strlist file at the same time", + file); + resource_release("strlist", file); + file_map_close(&map); + return false; + } + + p_clear(hosts, 1); + hosts->filename = m_strdup(file); + hosts->db = &res->trie1; + hosts->weight = weight; + hosts->reverse = true; host_count = 0; - domains = trie_new(); + + p_clear(domains, 1); + /* don't set filename */ + domains->db = &res->trie2; + domains->weight = weight; + domains->reverse = true; + domains->partial = true; domain_count = 0; + + if (map.st.st_size == res->size && map.st.st_mtime == res->mtime) { + info("rbldns %s up to date", file); + file_map_close(&map); + return true; + } + + trie_delete(&res->trie1); + trie_delete(&res->trie2); + res->trie1 = trie_new(); + res->trie2 = trie_new(); + res->size = map.st.st_size; + res->mtime = map.st.st_mtime; + while (p < end && p != NULL) { const char *eol = (char *)memchr(p, '\n', end - p); if (eol == NULL) { @@ -202,8 +293,9 @@ static bool strlist_create_from_rhbl(const char *file, bool lock, if (eol - p >= BUFSIZ) { err("unreasonnable long line"); file_map_close(&map); - trie_delete(&hosts); - trie_delete(&domains); + trie_delete(&res->trie1); + trie_delete(&res->trie2); + strlist_local_wipe(hosts); return false; } if (*p != '#') { @@ -217,12 +309,12 @@ static bool strlist_create_from_rhbl(const char *file, bool lock, if (p < eos) { if (isalnum(*p)) { strlist_copy(line, p, eos - p, true); - trie_insert(hosts, line); + trie_insert(res->trie1, line); ++host_count; } else if (*p == '*') { ++p; strlist_copy(line, p, eos - p, true); - trie_insert(domains, line); + trie_insert(res->trie2, line); ++domain_count; } } @@ -231,22 +323,21 @@ static bool strlist_create_from_rhbl(const char *file, bool lock, } file_map_close(&map); if (host_count > 0) { - trie_compile(hosts, lock); - *phosts = hosts; + trie_compile(res->trie1, lock); } else { - trie_delete(&hosts); - *phosts = NULL; + trie_delete(&res->trie1); } if (domain_count > 0) { - trie_compile(domains, lock); - *pdomains = domains; + trie_compile(res->trie2, lock); } else { - trie_delete(&domains); - *pdomains = NULL; + trie_delete(&res->trie2); } info("rhbl %s loaded, %u hosts, %u domains", file, host_count, domain_count); - return hosts != NULL || domains != NULL; - + if (res->trie1 == NULL && res->trie2 == NULL) { + strlist_local_wipe(hosts); + return false; + } + return true; } @@ -282,7 +373,6 @@ static bool strlist_filter_constructor(filter_t *filter) int weight = 0; bool reverse = false; bool partial = false; - trie_t *trie = NULL; const char *current = param->value; const char *p = m_strchrnul(param->value, ':'); char *next = NULL; @@ -325,15 +415,13 @@ static bool strlist_filter_constructor(filter_t *filter) (int)(p - current), current); break; - case 3: - trie = strlist_create(current, reverse, lock); - PARSE_CHECK(trie != NULL, + case 3: { + strlist_local_t entry; + PARSE_CHECK(strlist_create(&entry, current, weight, + reverse, partial, lock), "cannot load string list from %s", current); - array_add(config->tries, trie); - array_add(config->weights, weight); - array_add(config->reverses, reverse); - array_add(config->partiales, partial); - break; + array_add(config->locals, entry); + } break; } if (i != 3) { current = p + 1; @@ -354,8 +442,6 @@ static bool strlist_filter_constructor(filter_t *filter) case ATK_RBLDNS: { bool lock = false; int weight = 0; - trie_t *trie_hosts = NULL; - trie_t *trie_domains = NULL; const char *current = param->value; const char *p = m_strchrnul(param->value, ':'); char *next = NULL; @@ -382,24 +468,19 @@ static bool strlist_filter_constructor(filter_t *filter) (int)(p - current), current); break; - case 2: - PARSE_CHECK(strlist_create_from_rhbl(current, lock, - &trie_hosts, &trie_domains), + case 2: { + strlist_local_t trie_hosts, trie_domains; + PARSE_CHECK(strlist_create_from_rhbl(&trie_hosts, &trie_domains, + current, weight, lock), "cannot load string list from rhbl %s", current); - if (trie_hosts != NULL) { - array_add(config->tries, trie_hosts); - array_add(config->weights, weight); - array_add(config->reverses, true); - array_add(config->partiales, false); + if (trie_hosts.db != NULL) { + array_add(config->locals, trie_hosts); } - if (trie_domains != NULL) { - array_add(config->tries, trie_domains); - array_add(config->weights, weight); - array_add(config->reverses, true); - array_add(config->partiales, true); + if (trie_domains.db != NULL) { + array_add(config->locals, trie_domains); } config->is_hostname = true; - break; + } break; } if (i != 2) { current = p + 1; @@ -500,7 +581,7 @@ static bool strlist_filter_constructor(filter_t *filter) PARSE_CHECK(config->is_email != config->is_hostname, "matched field MUST be emails XOR hostnames"); - PARSE_CHECK(config->tries.len || config->host_offsets.len, + PARSE_CHECK(config->locals.len || config->host_offsets.len, "no file parameter in the filter %s", filter->name); filter->data = config; return true; @@ -603,20 +684,18 @@ static filter_result_t strlist_filter(const filter_t *filter, const query_t *que const int len = m_strlen(query->Field); \ strlist_copy(normal, query->Field, len, false); \ strlist_copy(reverse, query->Field, len, true); \ - for (uint32_t i = 0 ; i < config->tries.len ; ++i) { \ - const int weight = array_elt(config->weights, i); \ - const trie_t *trie = array_elt(config->tries, i); \ - const bool rev = array_elt(config->reverses, i); \ - const bool part = array_elt(config->partiales, i); \ - if ((!part && trie_lookup(trie, rev ? reverse : normal)) \ - || (part && trie_prefix(trie, rev ? reverse : normal))) { \ - async->sum += weight; \ + foreach (strlist_local_t *entry, config->locals) { \ + if ((!entry->partial && trie_lookup(*(entry->db), \ + entry->reverse ? reverse : normal)) \ + || (entry->partial && trie_prefix(*(entry->db), \ + entry->reverse ? reverse : normal))) { \ + async->sum += entry->weight; \ if (async->sum >= (uint32_t)config->hard_threshold) { \ return HTK_HARD_MATCH; \ } \ } \ async->error = false; \ - } \ + }} \ } #define DNS(Flag, Field) \ if (config->match_ ## Flag) { \