[dovecot-cvs] dovecot/src/plugins/fts-squat fts-backend-squat.c, 1.3, 1.4 squat-trie.c, 1.3, 1.4 squat-uidlist.c, 1.3, 1.4 squat-uidlist.h, 1.3, 1.4
tss at dovecot.org
tss at dovecot.org
Sat Dec 9 21:08:58 UTC 2006
Update of /var/lib/cvs/dovecot/src/plugins/fts-squat
In directory talvi:/tmp/cvs-serv17594
Modified Files:
fts-backend-squat.c squat-trie.c squat-uidlist.c
squat-uidlist.h
Log Message:
Added support for mmap_disable=yes and several other fixes.
Index: fts-backend-squat.c
===================================================================
RCS file: /var/lib/cvs/dovecot/src/plugins/fts-squat/fts-backend-squat.c,v
retrieving revision 1.3
retrieving revision 1.4
diff -u -d -r1.3 -r1.4
--- fts-backend-squat.c 6 Dec 2006 23:43:15 -0000 1.3
+++ fts-backend-squat.c 9 Dec 2006 21:08:56 -0000 1.4
@@ -38,7 +38,9 @@
if (mailbox_get_status(box, STATUS_UIDVALIDITY, &status) < 0)
return NULL;
- mmap_disable = (storage->flags & MAIL_STORAGE_FLAG_MMAP_DISABLE) != 0;
+ mmap_disable = (storage->flags &
+ (MAIL_STORAGE_FLAG_MMAP_DISABLE |
+ MAIL_STORAGE_FLAG_MMAP_NO_WRITE)) != 0;
backend = i_new(struct squat_fts_backend, 1);
backend->backend = fts_backend_squat;
Index: squat-trie.c
===================================================================
RCS file: /var/lib/cvs/dovecot/src/plugins/fts-squat/squat-trie.c,v
retrieving revision 1.3
retrieving revision 1.4
diff -u -d -r1.3 -r1.4
--- squat-trie.c 6 Dec 2006 23:43:15 -0000 1.3
+++ squat-trie.c 9 Dec 2006 21:08:56 -0000 1.4
@@ -3,9 +3,11 @@
#include "lib.h"
#include "array.h"
#include "bsearch-insert-pos.h"
+#include "file-cache.h"
#include "file-lock.h"
#include "istream.h"
#include "ostream.h"
+#include "read-full.h"
#include "write-full.h"
#include "mmap-util.h"
#include "squat-uidlist.h"
@@ -43,9 +45,12 @@
struct file_lock *file_lock;
int lock_count;
int lock_type; /* F_RDLCK / F_WRLCK */
- bool mmap_disable;
- void *mmap_base;
+ struct file_cache *file_cache;
+ uint32_t file_cache_modify_counter;
+
+ void *mmap_base; /* NULL with mmap_disable=yes */
+ const uint8_t *const_mmap_base;
size_t mmap_size;
const struct squat_trie_header *hdr;
@@ -57,6 +62,7 @@
buffer_t *buf;
unsigned int corrupted:1;
+ unsigned int mmap_disable:1;
};
struct squat_trie_build_context {
@@ -71,6 +77,7 @@
unsigned int node_count;
unsigned int deleted_space;
+ unsigned int modified:1;
unsigned int failed:1;
unsigned int locked:1;
};
@@ -94,6 +101,7 @@
uint32_t used_file_size;
uint32_t deleted_space;
uint32_t node_count;
+ uint32_t modify_counter;
uint32_t root_offset;
};
@@ -273,6 +281,24 @@
#endif
}
+static int trie_map_area(struct squat_trie *trie, uoff_t offset, size_t len)
+{
+ ssize_t ret;
+
+ if (trie->file_cache == NULL)
+ return 0;
+
+ ret = file_cache_read(trie->file_cache, offset, len);
+ if (ret < 0) {
+ squat_trie_set_syscall_error(trie, "file_cache_read()");
+ return -1;
+ }
+ trie->const_mmap_base =
+ file_cache_get_map(trie->file_cache, &trie->mmap_size);
+ trie->hdr = (const void *)trie->const_mmap_base;
+ return 0;
+}
+
static int
trie_map_node(struct squat_trie *trie, uint32_t offset, unsigned int level,
struct trie_node **node_r)
@@ -280,52 +306,74 @@
struct trie_node *node;
const uint8_t *p, *end, *chars8_src, *chars16_src;
uint32_t num, chars8_count, chars16_count;
- unsigned int chars8_size, chars16_size;
+ unsigned int chars8_offset, chars8_size, chars8_memsize;
+ unsigned int chars16_offset, chars16_size, chars16_memsize;
i_assert(trie->fd != -1);
+ if (trie_map_area(trie, offset, 2+256) < 0)
+ return -1;
+
if (offset >= trie->mmap_size) {
squat_trie_set_corrupted(trie, "trie offset too large");
return -1;
}
- p = CONST_PTR_OFFSET(trie->mmap_base, offset);
- end = CONST_PTR_OFFSET(trie->mmap_base, trie->mmap_size);
+ p = trie->const_mmap_base + offset;
+ end = trie->const_mmap_base + trie->mmap_size;
/* get 8bit char count and check that it's valid */
num = _squat_trie_unpack_num(&p, end);
chars8_count = num >> 1;
- if (chars8_count > 256 || p + chars8_count >= end) {
+ chars8_offset = p - trie->const_mmap_base;
+ chars8_size = chars8_count * (sizeof(uint8_t) + sizeof(uint32_t));
+
+ if (chars8_count > 256 ||
+ chars8_offset + chars8_size > trie->mmap_size) {
squat_trie_set_corrupted(trie, "trie offset broken");
return -1;
}
- chars8_src = p;
- chars8_size = ALIGN(chars8_count * sizeof(uint8_t)) +
+ chars8_memsize = ALIGN(chars8_count * sizeof(uint8_t)) +
chars8_count * sizeof(struct trie_node *);
+
+ if (trie_map_area(trie, chars8_offset, chars8_size + 8) < 0)
+ return -1;
+
if ((num & 1) == 0) {
/* no 16bit chars */
chars16_count = 0;
- chars16_size = 0;
- chars16_src = NULL;
+ chars16_memsize = 0;
+ chars16_offset = 0;
} else {
- /* get the 16bit char count and check that it's valid */
- p = CONST_PTR_OFFSET(p, chars8_count *
- (sizeof(uint8_t) + sizeof(uint32_t)));
+ /* get the 16bit char count */
+ p = trie->const_mmap_base + chars8_offset + chars8_size;
+ end = trie->const_mmap_base + trie->mmap_size;
+
chars16_count = _squat_trie_unpack_num(&p, end);
- if (chars16_count > 65536 ||
- p + chars16_count*sizeof(uint16_t) >= end) {
+ if (chars16_count > 65536) {
squat_trie_set_corrupted(trie, "trie offset broken");
return -1;
}
+ chars16_offset = p - trie->const_mmap_base;
- chars16_src = p;
- chars16_size = ALIGN(chars16_count * sizeof(uint16_t)) +
+ /* map the required area size and make sure it exists */
+ chars16_size = chars16_count *
+ (sizeof(uint16_t) + sizeof(uint32_t));
+ if (trie_map_area(trie, chars16_offset, chars16_size) < 0)
+ return -1;
+
+ if (chars16_offset + chars16_size > trie->mmap_size) {
+ squat_trie_set_corrupted(trie, "trie offset broken");
+ return -1;
+ }
+
+ chars16_memsize = ALIGN(chars16_count * sizeof(uint16_t)) +
chars16_count * sizeof(struct trie_node *);
}
- node = i_malloc(sizeof(*node) + chars8_size + chars16_size);
+ node = i_malloc(sizeof(*node) + chars8_memsize + chars16_memsize);
node->chars_8bit_count = chars8_count;
node->chars_16bit_count = chars16_count;
node->file_offset = offset;
@@ -338,6 +386,9 @@
const uint32_t *src_idx;
const void *end_offset;
+ chars8_src = trie->const_mmap_base + chars8_offset;
+ chars16_src = trie->const_mmap_base + chars16_offset;
+
memcpy(chars8, chars8_src, sizeof(uint8_t) * chars8_count);
memcpy(chars16, chars16_src, sizeof(uint16_t) * chars16_count);
@@ -355,8 +406,8 @@
end_offset = &src_idx[chars16_count];
}
- node->orig_size = ((const char *)end_offset -
- (const char *)trie->mmap_base) - offset;
+ node->orig_size = ((const uint8_t *)end_offset -
+ trie->const_mmap_base) - offset;
}
*node_r = node;
@@ -365,6 +416,9 @@
static void squat_trie_unmap(struct squat_trie *trie)
{
+ if (trie->file_cache != NULL)
+ file_cache_invalidate(trie->file_cache, 0, (uoff_t)-1);
+
if (trie->mmap_base != NULL) {
if (munmap(trie->mmap_base, trie->mmap_size) < 0)
squat_trie_set_syscall_error(trie, "munmap()");
@@ -373,10 +427,13 @@
trie->mmap_size = 0;
trie->hdr = NULL;
+ trie->const_mmap_base = NULL;
}
static void trie_file_close(struct squat_trie *trie)
{
+ if (trie->file_cache != NULL)
+ file_cache_free(&trie->file_cache);
if (trie->file_lock != NULL)
file_lock_free(&trie->file_lock);
@@ -391,18 +448,19 @@
trie->corrupted = FALSE;
}
-static int trie_map_check_header(struct squat_trie *trie,
- const struct squat_trie_header *hdr)
+static int
+trie_map_check_header(struct squat_trie *trie,
+ const struct squat_trie_header *hdr, uoff_t file_size)
{
if (hdr->version != SQUAT_TRIE_VERSION)
return -1;
- if (hdr->used_file_size > trie->mmap_size) {
+ if (hdr->used_file_size > file_size) {
squat_trie_set_corrupted(trie, "used_file_size too large");
return -1;
}
if (hdr->root_offset != 0 &&
- (hdr->root_offset > trie->mmap_size ||
+ (hdr->root_offset > file_size ||
hdr->root_offset < sizeof(*hdr))) {
squat_trie_set_corrupted(trie, "invalid root_offset");
return -1;
@@ -415,14 +473,42 @@
return 0;
}
+static int squat_trie_file_was_modified(struct squat_trie *trie)
+{
+ struct squat_trie_header hdr;
+ int ret;
+
+ ret = pread_full(trie->fd, &hdr.modify_counter,
+ sizeof(hdr.modify_counter),
+ offsetof(struct squat_trie_header, modify_counter));
+ if (ret < 0) {
+ squat_trie_set_syscall_error(trie, "pread_full()");
+ return -1;
+ }
+ if (ret == 0) {
+ /* broken file, treat as modified */
+ return 1;
+ }
+ return hdr.modify_counter == trie->file_cache_modify_counter ? 0 : 1;
+}
+
static int squat_trie_map(struct squat_trie *trie)
{
+ const struct squat_trie_header *hdr;
struct stat st;
+ ssize_t ret;
- if (trie->hdr != NULL &&
- trie->hdr->used_file_size <= trie->mmap_size) {
- /* everything is already mapped */
- return 1;
+ if (trie->hdr != NULL) {
+ if (!trie->mmap_disable) {
+ if (trie->hdr->used_file_size <= trie->mmap_size) {
+ /* everything is already mapped */
+ return 1;
+ }
+ } else {
+ ret = squat_trie_file_was_modified(trie);
+ if (ret <= 0)
+ return ret < 0 ? -1 : 1;
+ }
}
if (fstat(trie->fd, &st) < 0) {
@@ -434,58 +520,71 @@
squat_trie_unmap(trie);
- trie->mmap_size = st.st_size;
- trie->mmap_base = mmap(NULL, trie->mmap_size, PROT_READ | PROT_WRITE,
- MAP_SHARED, trie->fd, 0);
- if (trie->mmap_base == MAP_FAILED) {
- trie->mmap_size = 0;
- trie->mmap_base = NULL;
- squat_trie_set_syscall_error(trie, "mmap()");
- return -1;
+ if (!trie->mmap_disable) {
+ trie->mmap_size = st.st_size;
+ trie->mmap_base = mmap(NULL, trie->mmap_size,
+ PROT_READ | PROT_WRITE,
+ MAP_SHARED, trie->fd, 0);
+ if (trie->mmap_base == MAP_FAILED) {
+ trie->mmap_size = 0;
+ trie->mmap_base = NULL;
+ squat_trie_set_syscall_error(trie, "mmap()");
+ return -1;
+ }
+ trie->const_mmap_base = trie->mmap_base;
+ } else {
+ ret = file_cache_read(trie->file_cache, 0, sizeof(*trie->hdr));
+ if (ret < 0) {
+ squat_trie_set_syscall_error(trie, "file_cache_read()");
+ return -1;
+ }
+ if ((size_t)ret < sizeof(*trie->hdr)) {
+ squat_trie_set_corrupted(trie, "file too small");
+ return -1;
+ }
+ trie->const_mmap_base =
+ file_cache_get_map(trie->file_cache, &trie->mmap_size);
}
- if (trie_map_check_header(trie, trie->mmap_base) < 0)
+ hdr = (const void *)trie->const_mmap_base;
+ if (trie_map_check_header(trie, hdr, st.st_size) < 0)
return -1;
- trie->hdr = trie->mmap_base;
+ trie->hdr = hdr;
+ trie->file_cache_modify_counter = trie->hdr->modify_counter;
- if (trie_map_node(trie, trie->hdr->root_offset, 1, &trie->root) < 0) {
- trie_file_close(trie);
+ if (trie_map_node(trie, trie->hdr->root_offset, 1, &trie->root) < 0)
return 0;
- }
return 1;
}
-static int trie_file_open(struct squat_trie *trie)
+static int trie_file_open(struct squat_trie *trie, bool create)
{
- trie->fd = open(trie->filepath, O_RDWR);
- if (trie->fd == -1) {
- if (errno == ENOENT)
- return 0;
-
- squat_trie_set_syscall_error(trie, "open()");
- return -1;
- }
+ struct stat st;
+ int fd;
- return 1;
-}
+ i_assert(trie->fd == -1);
-static int trie_file_create(struct squat_trie *trie)
-{
- struct stat st;
+ fd = open(trie->filepath, O_RDWR | (create ? O_CREAT : 0), 0660);
+ if (fd == -1) {
+ if (errno == ENOENT)
+ return 0;
- trie->fd = open(trie->filepath, O_RDWR | O_CREAT, 0660);
- if (trie->fd == -1) {
squat_trie_set_syscall_error(trie, "open()");
return -1;
}
-
- if (fstat(trie->fd, &st) < 0) {
+ if (fstat(fd, &st) < 0) {
squat_trie_set_syscall_error(trie, "fstat()");
+ (void)close(fd);
return -1;
}
+
+ trie->fd = fd;
trie->dev = st.st_dev;
trie->ino = st.st_ino;
- return 0;
+
+ if (trie->mmap_disable)
+ trie->file_cache = file_cache_new(trie->fd);
+ return 1;
}
static int trie_file_create_finish(struct squat_trie *trie)
@@ -513,15 +612,6 @@
return 0;
}
-static int squat_trie_reopen(struct squat_trie *trie)
-{
- trie_file_close(trie);
- if (trie_file_open(trie) < 0)
- return -1;
-
- return 0;
-}
-
struct squat_trie *
squat_trie_open(const char *path, uint32_t uidvalidity,
enum file_lock_method lock_method, bool mmap_disable)
@@ -538,7 +628,8 @@
trie->uidlist_filepath = i_strconcat(path, ".uids", NULL);
trie->uidlist =
- squat_uidlist_init(trie, trie->uidlist_filepath, uidvalidity);
+ squat_uidlist_init(trie, trie->uidlist_filepath,
+ uidvalidity, mmap_disable);
return trie;
}
@@ -553,7 +644,23 @@
int squat_trie_get_last_uid(struct squat_trie *trie, uint32_t *uid_r)
{
- return squat_uidlist_get_last_uid(trie->uidlist, uid_r);
+ int ret;
+
+ if (trie->fd == -1) {
+ if ((ret = trie_file_open(trie, FALSE)) < 0)
+ return ret;
+ if (ret == 0) {
+ *uid_r = 0;
+ return 0;
+ }
+ }
+
+ if (squat_trie_lock(trie, F_RDLCK) <= 0)
+ return -1;
+
+ ret = squat_uidlist_get_last_uid(trie->uidlist, uid_r);
+ squat_trie_unlock(trie);
+ return ret;
}
static int squat_trie_is_file_stale(struct squat_trie *trie)
@@ -561,6 +668,9 @@
struct stat st;
if (stat(trie->filepath, &st) < 0) {
+ if (errno == ENOENT)
+ return 1;
+
squat_trie_set_syscall_error(trie, "stat()");
return -1;
}
@@ -587,15 +697,15 @@
if (trie->fd == -1 || trie->corrupted) {
trie_file_close(trie);
if (lock_type == F_WRLCK) {
- if ((ret = trie_file_open(trie)) < 0)
+ if ((ret = trie_file_open(trie, FALSE)) < 0)
return -1;
if (ret == 0) {
- if (trie_file_create(trie) < 0)
+ if (trie_file_open(trie, TRUE) < 0)
return -1;
created = TRUE;
}
} else {
- if (trie_file_open(trie) <= 0)
+ if (trie_file_open(trie, FALSE) <= 0)
return -1;
}
}
@@ -605,8 +715,13 @@
ret = file_wait_lock(trie->fd, trie->filepath, lock_type,
trie->lock_method, SQUAT_TRIE_LOCK_TIMEOUT,
&trie->file_lock);
- if (ret <= 0)
+ if (ret <= 0) {
+ if (ret == 0) {
+ squat_trie_set_syscall_error(trie,
+ "file_wait_lock()");
+ }
return ret;
+ }
/* if the trie has been compressed, we need to reopen the
file and try to lock again */
@@ -618,7 +733,8 @@
if (ret < 0)
return -1;
- if (squat_trie_reopen(trie) < 0)
+ trie_file_close(trie);
+ if (trie_file_open(trie, FALSE) <= 0)
return -1;
}
@@ -636,6 +752,10 @@
file_unlock(&trie->file_lock);
return -1;
}
+ if (squat_uidlist_refresh(trie->uidlist) < 0) {
+ file_unlock(&trie->file_lock);
+ return -1;
+ }
trie->lock_count++;
trie->lock_type = lock_type;
@@ -681,6 +801,7 @@
*chrp = chr;
}
+ node->modified = TRUE;
node->resized = TRUE;
return node;
}
@@ -1141,12 +1262,22 @@
struct trie_node **children8 = NODE_CHILDREN8(node);
struct trie_node **children16 = NODE_CHILDREN16(node);
- trie_write_node_children(ctx, level + 1,
- children8, node->chars_8bit_count);
- trie_write_node_children(ctx, level + 1,
- children16, node->chars_16bit_count);
+ if (trie_write_node_children(ctx, level + 1,
+ children8,
+ node->chars_8bit_count) < 0)
+ return -1;
+ if (trie_write_node_children(ctx, level + 1,
+ children16,
+ node->chars_16bit_count) < 0)
+ return -1;
+ }
+
+ if (!node->modified)
+ return 0;
+
+ if (level < BLOCK_SIZE)
node_pack(trie->buf, node);
- } else {
+ else {
if (node_leaf_finish(trie, node) < 0)
return -1;
@@ -1165,7 +1296,7 @@
o_stream_send(ctx->output, trie->buf->data, trie->buf->used);
ctx->deleted_space += node->orig_size;
- } else if (node->modified) {
+ } else {
/* overwrite node's contents */
i_assert(node->file_offset != 0);
i_assert(trie->buf->used <= node->orig_size);
@@ -1177,6 +1308,8 @@
ctx->deleted_space += trie->buf->used - node->orig_size;
}
+
+ ctx->modified = TRUE;
return 0;
}
@@ -1193,20 +1326,25 @@
}
ctx->output = o_stream_create_file(trie->fd, default_pool, 0, FALSE);
- if (hdr.used_file_size == 0)
+ if (hdr.used_file_size == 0) {
o_stream_send(ctx->output, &hdr, sizeof(hdr));
+ ctx->modified = TRUE;
+ }
ctx->deleted_space = 0;
if (trie_write_node(ctx, 1, trie->root) < 0)
return -1;
- /* update the header */
- hdr.root_offset = trie->root->file_offset;
- hdr.used_file_size = ctx->output->offset;
- hdr.deleted_space += ctx->deleted_space;
- hdr.node_count = ctx->node_count;
- o_stream_seek(ctx->output, 0);
- o_stream_send(ctx->output, &hdr, sizeof(hdr));
+ if (ctx->modified) {
+ /* update the header */
+ hdr.root_offset = trie->root->file_offset;
+ hdr.used_file_size = ctx->output->offset;
+ hdr.deleted_space += ctx->deleted_space;
+ hdr.node_count = ctx->node_count;
+ hdr.modify_counter++;
+ o_stream_seek(ctx->output, 0);
+ o_stream_send(ctx->output, &hdr, sizeof(hdr));
+ }
o_stream_destroy(&ctx->output);
*uidvalidity_r = hdr.uidvalidity;
@@ -1250,6 +1388,11 @@
return -1;
if (squat_trie_need_compress(trie, (unsigned int)-1)) {
+ if (ctx->locked) {
+ squat_trie_unlock(ctx->trie);
+ ctx->locked = FALSE;
+ }
+
if (squat_trie_compress(trie, NULL) < 0)
return -1;
}
@@ -1433,6 +1576,8 @@
struct squat_trie_header hdr;
int fd;
+ memset(ctx, 0, sizeof(*ctx));
+
ctx->tmp_path = t_strconcat(trie->filepath, ".tmp", NULL);
fd = open(ctx->tmp_path, O_RDWR | O_CREAT | O_TRUNC, 0600);
if (fd == -1) {
@@ -1440,7 +1585,6 @@
return -1;
}
- memset(ctx, 0, sizeof(*ctx));
ctx->trie = trie;
ctx->output = o_stream_create_file(fd, default_pool, 0, TRUE);
ctx->node_count = trie->hdr->node_count;
@@ -1475,6 +1619,9 @@
struct trie_node *node;
int ret;
+ /* reopening the file loses locks, so we can't be locked initially */
+ i_assert(trie->lock_count == 0);
+
if (squat_trie_lock(trie, F_WRLCK) <= 0)
return -1;
@@ -1511,7 +1658,8 @@
if (ret < 0)
(void)unlink(ctx.tmp_path);
else {
- if (squat_trie_reopen(trie) < 0)
+ trie_file_close(trie);
+ if (trie_file_open(trie, FALSE) < 0)
return -1;
}
return ret;
@@ -1524,7 +1672,11 @@
bool compress;
int ret;
+ if ((ret = squat_trie_lock(trie, F_RDLCK)) <= 0)
+ return ret;
compress = squat_trie_need_compress(trie, current_message_count);
+ squat_trie_unlock(trie);
+
ret = squat_uidlist_mark_having_expunges(trie->uidlist, compress);
if (compress)
Index: squat-uidlist.c
===================================================================
RCS file: /var/lib/cvs/dovecot/src/plugins/fts-squat/squat-uidlist.c,v
retrieving revision 1.3
retrieving revision 1.4
diff -u -d -r1.3 -r1.4
--- squat-uidlist.c 6 Dec 2006 23:43:15 -0000 1.3
+++ squat-uidlist.c 9 Dec 2006 21:08:56 -0000 1.4
@@ -3,7 +3,9 @@
#include "lib.h"
#include "array.h"
#include "ostream.h"
+#include "file-cache.h"
#include "mmap-util.h"
+#include "read-full.h"
#include "write-full.h"
#include "squat-trie.h"
#include "squat-uidlist.h"
@@ -53,8 +55,14 @@
int fd;
struct ostream *output;
+ dev_t dev;
+ ino_t ino;
+
void *mmap_base;
+ const uint8_t *const_mmap_base;
size_t mmap_size;
+
+ struct file_cache *file_cache;
struct squat_uidlist_header hdr;
ARRAY_DEFINE(lists, struct uid_node);
@@ -65,6 +73,7 @@
unsigned int check_expunges:1;
unsigned int write_failed:1;
+ unsigned int mmap_disable:1;
};
struct squat_uidlist_compress_ctx {
@@ -84,6 +93,8 @@
unsigned int failed:1;
};
+static void squat_uidlist_close(struct squat_uidlist *uidlist);
+
static void
squat_uidlist_set_syscall_error(struct squat_uidlist *uidlist,
const char *function)
@@ -93,14 +104,15 @@
}
static int squat_uidlist_check_header(struct squat_uidlist *uidlist,
- const struct squat_uidlist_header *hdr)
+ const struct squat_uidlist_header *hdr,
+ uoff_t file_size)
{
if (hdr->uidvalidity != uidlist->uidvalidity) {
squat_trie_set_corrupted(uidlist->trie,
"uidlist: uidvalidity changed");
return -1;
}
- if (hdr->used_file_size > uidlist->mmap_size) {
+ if (hdr->used_file_size > file_size) {
squat_trie_set_corrupted(uidlist->trie,
"uidlist: used_file_size too large");
return -1;
@@ -109,68 +121,135 @@
return 0;
}
+static int squat_uidlist_read_header(struct squat_uidlist *uidlist)
+{
+ int ret;
+
+ ret = pread_full(uidlist->fd, &uidlist->hdr, sizeof(uidlist->hdr), 0);
+ if (ret < 0)
+ squat_uidlist_set_syscall_error(uidlist, "pread_full()");
+ return ret;
+}
+
static int squat_uidlist_map(struct squat_uidlist *uidlist)
{
struct stat st;
+ int ret;
+
+ if (!uidlist->mmap_disable) {
+ const struct squat_uidlist_header *hdr = uidlist->mmap_base;
+
+ if (hdr != NULL && hdr->used_file_size <= uidlist->mmap_size) {
+ /* everything is already mapped */
+ uidlist->hdr = *hdr;
+ return 1;
+ }
+ } else {
+ if ((ret = squat_uidlist_read_header(uidlist)) < 0)
+ return -1;
+ if (ret == 0)
+ return 0;
+ }
if (fstat(uidlist->fd, &st) < 0) {
squat_uidlist_set_syscall_error(uidlist, "fstat()");
return -1;
}
+ uidlist->dev = st.st_dev;
+ uidlist->ino = st.st_ino;
- if (st.st_size <= sizeof(uidlist->hdr)) {
- memset(&uidlist->hdr, 0, sizeof(uidlist->hdr));
- uidlist->hdr.used_file_size = sizeof(uidlist->hdr);
+ if (st.st_size <= sizeof(uidlist->hdr))
return 0;
- }
if (uidlist->mmap_base != NULL) {
if (munmap(uidlist->mmap_base, uidlist->mmap_size) < 0)
squat_uidlist_set_syscall_error(uidlist, "munmap()");
}
- uidlist->mmap_size = st.st_size;
+ uidlist->const_mmap_base = NULL;
- uidlist->mmap_base =
- mmap(NULL, uidlist->mmap_size, PROT_READ | PROT_WRITE,
- MAP_SHARED, uidlist->fd, 0);
- if (uidlist->mmap_base == MAP_FAILED) {
- uidlist->mmap_size = 0;
- uidlist->mmap_base = NULL;
- squat_uidlist_set_syscall_error(uidlist, "mmap()");
- return -1;
+ if (!uidlist->mmap_disable) {
+ uidlist->mmap_size = st.st_size;
+ uidlist->mmap_base =
+ mmap(NULL, uidlist->mmap_size, PROT_READ | PROT_WRITE,
+ MAP_SHARED, uidlist->fd, 0);
+ if (uidlist->mmap_base == MAP_FAILED) {
+ uidlist->mmap_size = 0;
+ uidlist->mmap_base = NULL;
+ squat_uidlist_set_syscall_error(uidlist, "mmap()");
+ return -1;
+ }
+ uidlist->const_mmap_base = uidlist->mmap_base;
+ memcpy(&uidlist->hdr, uidlist->mmap_base, sizeof(uidlist->hdr));
+ } else {
+ /* the header is always read separately. everything between it
+ and the used_file_size doesn't change */
+ file_cache_invalidate(uidlist->file_cache,
+ uidlist->hdr.used_file_size, (uoff_t)-1);
}
- memcpy(&uidlist->hdr, uidlist->mmap_base, sizeof(uidlist->hdr));
- if (squat_uidlist_check_header(uidlist, &uidlist->hdr) < 0)
- return -1;
+ if (squat_uidlist_check_header(uidlist, &uidlist->hdr, st.st_size) < 0)
+ return 0;
if (uidlist->hdr.uids_expunged)
uidlist->check_expunges = TRUE;
- uidlist->first_new_list_idx = uidlist->mmap_size;
+ uidlist->first_new_list_idx = uidlist->hdr.used_file_size;
return 1;
}
static int squat_uidlist_open(struct squat_uidlist *uidlist)
{
+ int ret;
+
i_assert(uidlist->fd == -1);
- uidlist->fd = open(uidlist->filepath, O_RDWR | O_CREAT, 0600);
+ uidlist->fd = open(uidlist->filepath, O_RDWR, 0600);
if (uidlist->fd == -1) {
+ if (errno == ENOENT)
+ return 0;
+
squat_uidlist_set_syscall_error(uidlist, "open()");
return -1;
}
- return squat_uidlist_map(uidlist);
+ if (uidlist->mmap_disable)
+ uidlist->file_cache = file_cache_new(uidlist->fd);
+
+ if ((ret = squat_uidlist_map(uidlist)) == 0) {
+ /* broken */
+ if (unlink(uidlist->filepath) < 0)
+ squat_uidlist_set_syscall_error(uidlist, "unlink()");
+ squat_uidlist_close(uidlist);
+ }
+ return ret;
+}
+
+static int squat_uidlist_create(struct squat_uidlist *uidlist)
+{
+ i_assert(uidlist->fd == -1);
+
+ /* we should get here only if normal file opening failed */
+ uidlist->fd = open(uidlist->filepath, O_RDWR | O_CREAT | O_TRUNC, 0600);
+ if (uidlist->fd == -1) {
+ squat_uidlist_set_syscall_error(uidlist, "open()");
+ return -1;
+ }
+
+ if (uidlist->mmap_disable)
+ uidlist->file_cache = file_cache_new(uidlist->fd);
+ return 0;
}
static void squat_uidlist_close(struct squat_uidlist *uidlist)
{
+ if (uidlist->file_cache != NULL)
+ file_cache_free(&uidlist->file_cache);
if (uidlist->mmap_base != NULL) {
if (munmap(uidlist->mmap_base, uidlist->mmap_size) < 0)
squat_uidlist_set_syscall_error(uidlist, "munmap()");
uidlist->mmap_base = NULL;
}
+ uidlist->const_mmap_base = NULL;
uidlist->mmap_size = 0;
if (uidlist->fd != -1) {
@@ -182,7 +261,7 @@
struct squat_uidlist *
squat_uidlist_init(struct squat_trie *trie, const char *path,
- uint32_t uidvalidity)
+ uint32_t uidvalidity, bool mmap_disable)
{
struct squat_uidlist *uidlist;
@@ -192,12 +271,12 @@
uidlist->uidvalidity = uidvalidity;
uidlist->fd = -1;
uidlist->first_new_list_idx = 1;
+ uidlist->mmap_disable = mmap_disable;
i_array_init(&uidlist->lists, 65536);
uidlist->node_pool =
pool_alloconly_create("squat uidlist node pool", 65536);
uidlist->tmp_buf = buffer_create_dynamic(default_pool, 16);
uidlist->list_buf = buffer_create_dynamic(default_pool, 256);
- (void)squat_uidlist_open(uidlist);
return uidlist;
}
@@ -212,6 +291,34 @@
i_free(uidlist);
}
+int squat_uidlist_refresh(struct squat_uidlist *uidlist)
+{
+ struct stat st;
+ int ret;
+
+ if (uidlist->fd != -1) {
+ if (stat(uidlist->filepath, &st) < 0) {
+ if (errno == ENOENT)
+ return 0;
+
+ squat_uidlist_set_syscall_error(uidlist, "stat()");
+ return -1;
+ }
+ if (st.st_ino == uidlist->ino &&
+ CMP_DEV_T(st.st_dev, uidlist->dev)) {
+ /* no need to reopen, just remap */
+ if ((ret = squat_uidlist_map(uidlist)) != 0)
+ return ret < 0 ? -1 : 0;
+ /* broken file */
+ }
+ squat_uidlist_close(uidlist);
+ }
+
+ if (squat_uidlist_open(uidlist) < 0)
+ return -1;
+ return 0;
+}
+
int squat_uidlist_get_last_uid(struct squat_uidlist *uidlist, uint32_t *uid_r)
{
*uid_r = uidlist->hdr.uid_max;
@@ -289,20 +396,61 @@
}
static int
-squat_uidlist_copy_existing(struct squat_uidlist *uidlist, size_t offset,
- uint32_t *prev_uid_r, uint32_t *written_uid_r)
+squat_uidlist_map_area(struct squat_uidlist *uidlist,
+ size_t offset, size_t size)
{
- const uint8_t *data, *data_start, *end, *p = NULL;
- uint32_t size, num, prev_uid, next_uid;
+ ssize_t ret;
+ if (uidlist->file_cache == NULL)
+ return 0;
+
+ ret = file_cache_read(uidlist->file_cache, offset, size);
+ if (ret < 0) {
+ squat_uidlist_set_syscall_error(uidlist, "file_cache_read()");
+ return -1;
+ }
+ uidlist->const_mmap_base =
+ file_cache_get_map(uidlist->file_cache, &uidlist->mmap_size);
+ return 0;
+}
+
+static int
+squat_uidlist_map_list(struct squat_uidlist *uidlist, size_t offset,
+ const uint8_t **data_r, uint32_t *size_r)
+{
+ const uint8_t *data, *end;
+ size_t data_offset;
+ uint32_t size;
+
+ if (squat_uidlist_map_area(uidlist, offset, 128) < 0)
+ return -1;
if (offset >= uidlist->mmap_size)
return -1;
- data = CONST_PTR_OFFSET(uidlist->mmap_base, offset);
- end = CONST_PTR_OFFSET(uidlist->mmap_base, uidlist->mmap_size);
+ data = uidlist->const_mmap_base + offset;
+ end = uidlist->const_mmap_base + uidlist->mmap_size;
size = _squat_trie_unpack_num(&data, end);
- if (data + size > end)
+ data_offset = data - uidlist->const_mmap_base;
+
+ if (squat_uidlist_map_area(uidlist, data_offset, size) < 0)
+ return -1;
+ if (data_offset + size > uidlist->mmap_size)
+ return -1;
+
+ *data_r = uidlist->const_mmap_base + data_offset;
+ *size_r = size;
+ return 0;
+}
+
+static int
+squat_uidlist_copy_existing(struct squat_uidlist *uidlist, size_t offset,
+ uint32_t *prev_uid_r, uint32_t *written_uid_r)
+{
+ const uint8_t *data, *data_start, *end, *p = NULL;
+ uint32_t size, num, prev_uid, next_uid;
+
+ if (squat_uidlist_map_list(uidlist, offset, &data, &size) < 0)
return -1;
data_start = data;
@@ -337,7 +485,7 @@
*prev_uid_r = next_uid;
uidlist->hdr.deleted_space +=
- (end - (const uint8_t *)uidlist->mmap_base) - offset;
+ (end - (const uint8_t *)uidlist->const_mmap_base) - offset;
buffer_append(uidlist->list_buf, data_start, p - data_start);
return 0;
@@ -410,10 +558,15 @@
return 0;
}
-static void squat_uidlist_write_init(struct squat_uidlist *uidlist)
+static int squat_uidlist_write_init(struct squat_uidlist *uidlist)
{
i_assert(uidlist->output == NULL);
+ if (uidlist->fd == -1) {
+ if (squat_uidlist_create(uidlist) < 0)
+ return -1;
+ }
+
uidlist->output = o_stream_create_file(uidlist->fd, default_pool,
0, FALSE);
if (uidlist->hdr.used_file_size < sizeof(uidlist->hdr)) {
@@ -425,6 +578,7 @@
o_stream_seek(uidlist->output,
uidlist->hdr.used_file_size);
}
+ return 0;
}
static int squat_uidlist_write_listbuf(struct squat_uidlist *uidlist,
@@ -476,8 +630,12 @@
return -1;
}
- if (uidlist->output == NULL)
- squat_uidlist_write_init(uidlist);
+ if (uidlist->output == NULL) {
+ if (squat_uidlist_write_init(uidlist) < 0) {
+ uidlist->write_failed = TRUE;
+ return -1;
+ }
+ }
/* new uidlist index is the offset in uidlist file */
*_uid_list_idx = uidlist->output->offset;
@@ -511,8 +669,6 @@
p_clear(uidlist->node_pool);
uidlist->write_failed = FALSE;
-
- (void)squat_uidlist_map(uidlist);
return ret;
}
@@ -588,6 +744,9 @@
ctx->output = o_stream_create_file(fd, default_pool, 0, TRUE);
o_stream_send(ctx->output, &ctx->hdr, sizeof(ctx->hdr));
}
+
+ if (squat_uidlist_refresh(uidlist) < 0)
+ ctx->failed = TRUE;
return ctx;
}
@@ -683,8 +842,8 @@
uint32_t *uid_list_idx)
{
struct squat_uidlist *uidlist = ctx->uidlist;
- const uint8_t *data, *p, *end;
- uint32_t size;
+ const uint8_t *data, *data_start;
+ uint32_t size, old_offset;
int ret;
if ((*uid_list_idx & UID_LIST_IDX_FLAG_SINGLE) != 0) {
@@ -700,22 +859,13 @@
if (ctx->output == NULL)
return -1;
- if (*uid_list_idx >= uidlist->mmap_size) {
- squat_trie_set_corrupted(uidlist->trie,
- "uidlist index points outside file (compressing)");
- return -1;
- }
-
- data = p = CONST_PTR_OFFSET(uidlist->mmap_base, *uid_list_idx);
- end = CONST_PTR_OFFSET(uidlist->mmap_base, uidlist->mmap_size);
-
- size = _squat_trie_unpack_num(&p, end);
- if (data + size > end) {
+ if (squat_uidlist_map_list(uidlist, *uid_list_idx, &data, &size) < 0) {
squat_trie_set_corrupted(uidlist->trie,
"corrupted uidlist index (compressing)");
return -1;
}
+ old_offset = *uid_list_idx;
*uid_list_idx = ctx->output->offset;
if (!ctx->uidlist->check_expunges)
@@ -723,7 +873,7 @@
else {
bool all_expunged;
- ret = squat_uidlist_remove_expunged(ctx, p, size,
+ ret = squat_uidlist_remove_expunged(ctx, data, size,
&all_expunged);
if (ret < 0) {
ctx->failed = TRUE;
@@ -734,7 +884,11 @@
}
if (ret == 0) {
- if (o_stream_send(ctx->output, data, p - data + size) < 0) {
+ data_start = data = uidlist->const_mmap_base + old_offset;
+ (void)_squat_trie_unpack_num(&data, NULL);
+
+ if (o_stream_send(ctx->output, data_start,
+ data - data_start + size) < 0) {
ctx->failed = TRUE;
return -1;
}
@@ -826,17 +980,8 @@
const uint8_t *data, *end;
uint32_t size, num, prev_uid, next_uid;
- if (offset >= ctx->uidlist->mmap_size)
- return -1;
-
- data = CONST_PTR_OFFSET(ctx->uidlist->mmap_base, offset);
- end = CONST_PTR_OFFSET(ctx->uidlist->mmap_base,
- ctx->uidlist->mmap_size);
-
- size = _squat_trie_unpack_num(&data, end);
- if (data + size > end)
+ if (squat_uidlist_map_list(ctx->uidlist, offset, &data, &size) < 0)
return -1;
-
end = data + size;
prev_uid = _squat_trie_unpack_num(&data, end);
Index: squat-uidlist.h
===================================================================
RCS file: /var/lib/cvs/dovecot/src/plugins/fts-squat/squat-uidlist.h,v
retrieving revision 1.3
retrieving revision 1.4
diff -u -d -r1.3 -r1.4
--- squat-uidlist.h 6 Dec 2006 23:43:15 -0000 1.3
+++ squat-uidlist.h 9 Dec 2006 21:08:56 -0000 1.4
@@ -8,9 +8,13 @@
struct squat_uidlist *
squat_uidlist_init(struct squat_trie *trie, const char *path,
- uint32_t uidvalidity);
+ uint32_t uidvalidity, bool mmap_disable);
void squat_uidlist_deinit(struct squat_uidlist *uidlist);
+/* Make sure that we've the latest uidlist file fully mapped. */
+int squat_uidlist_refresh(struct squat_uidlist *uidlist);
+
+/* Get the last UID added to the file. */
int squat_uidlist_get_last_uid(struct squat_uidlist *uidlist, uint32_t *uid_r);
/* Add new UID to given UID list. The uid_list_idx is updated to contain the
More information about the dovecot-cvs
mailing list