[dovecot-cvs] dovecot/src/lib-index mail-cache-compress.c, NONE,
1.1 mail-cache-lookup.c, NONE, 1.1 mail-cache-old.c, NONE,
1.1 mail-cache-private.h, NONE, 1.1 mail-cache-transaction.c,
NONE, 1.1 mail-cache.c, 1.25, 1.26 mail-cache.h, 1.6,
1.7 mail-index-fsck.c, 1.15, 1.16 mail-index-lock.c, NONE,
1.1 mail-index-private.h, NONE, 1.1 mail-index-reset.c, NONE,
1.1 mail-index-sync-private.h, NONE,
1.1 mail-index-sync-update.c, NONE, 1.1 mail-index-sync.c,
NONE, 1.1 mail-index-transaction-private.h, NONE,
1.1 mail-index-transaction.c, NONE,
1.1 mail-index-view-private.h, NONE,
1.1 mail-index-view-sync.c, NONE, 1.1 mail-index-view.c, NONE,
1.1 mail-index.c, 1.105, 1.106 mail-index.h, 1.97,
1.98 mail-transaction-log-private.h, NONE,
1.1 mail-transaction-log-view.c, NONE,
1.1 mail-transaction-log.c, NONE, 1.1 mail-transaction-log.h,
NONE, 1.1 mail-transaction-util.c, NONE,
1.1 mail-transaction-util.h, NONE, 1.1 Makefile.am, 1.15, 1.16
cras at procontrol.fi
cras at procontrol.fi
Tue Apr 27 23:25:55 EEST 2004
Update of /home/cvs/dovecot/src/lib-index
In directory talvi:/tmp/cvs-serv29236/src/lib-index
Modified Files:
Makefile.am
Added Files:
mail-cache-compress.c mail-cache-lookup.c mail-cache-old.c
mail-cache-private.h mail-cache-transaction.c mail-cache.c
mail-cache.h mail-index-fsck.c mail-index-lock.c
mail-index-private.h mail-index-reset.c
mail-index-sync-private.h mail-index-sync-update.c
mail-index-sync.c mail-index-transaction-private.h
mail-index-transaction.c mail-index-view-private.h
mail-index-view-sync.c mail-index-view.c mail-index.c
mail-index.h mail-transaction-log-private.h
mail-transaction-log-view.c mail-transaction-log.c
mail-transaction-log.h mail-transaction-util.c
mail-transaction-util.h
Log Message:
importing new index code. mbox still broken.
--- NEW FILE: mail-cache-compress.c ---
static const struct mail_cache_record *
mail_cache_compress_record(struct mail_cache *cache,
struct mail_index_record *rec, int header_idx,
uint32_t *size_r)
{
enum mail_cache_field orig_cached_fields, cached_fields, field;
struct mail_cache_record cache_rec;
buffer_t *buffer;
const void *data;
size_t size, pos;
uint32_t nb_size;
int i;
memset(&cache_rec, 0, sizeof(cache_rec));
buffer = buffer_create_dynamic(pool_datastack_create(),
4096, (size_t)-1);
orig_cached_fields = mail_cache_get_fields(cache, rec);
cached_fields = orig_cached_fields & ~MAIL_CACHE_HEADERS_MASK;
buffer_append(buffer, &cache_rec, sizeof(cache_rec));
for (i = 0, field = 1; i < 31; i++, field <<= 1) {
if ((cached_fields & field) == 0)
continue;
if (!mail_cache_lookup_field(cache, rec, field, &data, &size)) {
cached_fields &= ~field;
continue;
}
nb_size = uint32_to_nbo((uint32_t)size);
if ((field & MAIL_CACHE_FIXED_MASK) == 0)
buffer_append(buffer, &nb_size, sizeof(nb_size));
buffer_append(buffer, data, size);
if ((size & 3) != 0)
buffer_append(buffer, null4, 4 - (size & 3));
}
/* now merge all the headers if we have them all */
if ((orig_cached_fields & mail_cache_header_fields[header_idx]) != 0) {
nb_size = 0;
pos = buffer_get_used_size(buffer);
buffer_append(buffer, &nb_size, sizeof(nb_size));
for (i = 0; i <= header_idx; i++) {
field = mail_cache_header_fields[i];
if (mail_cache_lookup_field(cache, rec, field,
&data, &size) && size > 1) {
size--; /* terminating \0 */
buffer_append(buffer, data, size);
nb_size += size;
}
}
buffer_append(buffer, "", 1);
nb_size++;
if ((nb_size & 3) != 0)
buffer_append(buffer, null4, 4 - (nb_size & 3));
nb_size = uint32_to_nbo(nb_size);
buffer_write(buffer, pos, &nb_size, sizeof(nb_size));
cached_fields |= MAIL_CACHE_HEADERS1;
}
cache_rec.fields = cached_fields;
cache_rec.size = uint32_to_nbo(buffer_get_used_size(buffer));
buffer_write(buffer, 0, &cache_rec, sizeof(cache_rec));
data = buffer_get_data(buffer, &size);
*size_r = size;
return data;
}
static int mail_cache_copy(struct mail_cache *cache, int fd)
{
#if 0
struct mail_cache_header *hdr;
const struct mail_cache_record *cache_rec;
struct mail_index_record *rec;
enum mail_cache_field used_fields;
unsigned char *mmap_base;
const char *str;
uint32_t new_file_size, offset, size, nb_size;
int i, header_idx;
/* pick some reasonably good file size */
new_file_size = cache->used_file_size -
nbo_to_uint32(cache->hdr->deleted_space);
new_file_size = (new_file_size + 1023) & ~1023;
if (new_file_size < MAIL_CACHE_INITIAL_SIZE)
new_file_size = MAIL_CACHE_INITIAL_SIZE;
if (file_set_size(fd, new_file_size) < 0)
return mail_cache_set_syscall_error(cache, "file_set_size()");
mmap_base = mmap(NULL, new_file_size, PROT_READ | PROT_WRITE,
MAP_SHARED, fd, 0);
if (mmap_base == MAP_FAILED)
return mail_cache_set_syscall_error(cache, "mmap()");
/* skip file's header */
hdr = (struct mail_cache_header *) mmap_base;
offset = sizeof(*hdr);
/* merge all the header pieces into one. if some message doesn't have
all the required pieces, we'll just have to drop them all. */
for (i = MAIL_CACHE_HEADERS_COUNT-1; i >= 0; i--) {
str = mail_cache_get_header_fields_str(cache, i);
if (str != NULL)
break;
}
if (str == NULL)
header_idx = -1;
else {
hdr->header_offsets[0] = uint32_to_offset(offset);
header_idx = i;
size = strlen(str) + 1;
nb_size = uint32_to_nbo(size);
memcpy(mmap_base + offset, &nb_size, sizeof(nb_size));
offset += sizeof(nb_size);
memcpy(mmap_base + offset, str, size);
offset += (size + 3) & ~3;
}
// FIXME: recreate index file with new cache_offsets
used_fields = 0;
rec = cache->index->lookup(cache->index, 1);
while (rec != NULL) {
cache_rec = mail_cache_lookup(cache, rec, 0);
if (cache_rec == NULL)
rec->cache_offset = 0;
else if (offset_to_uint32(cache_rec->next_offset) == 0) {
/* just one unmodified block, copy it */
size = nbo_to_uint32(cache_rec->size);
i_assert(offset + size <= new_file_size);
memcpy(mmap_base + offset, cache_rec, size);
rec->cache_offset = uint32_to_offset(offset);
size = (size + 3) & ~3;
offset += size;
} else {
/* multiple blocks, sort them into buffer */
t_push();
cache_rec = mail_cache_compress_record(cache, rec,
header_idx,
&size);
i_assert(offset + size <= new_file_size);
memcpy(mmap_base + offset, cache_rec, size);
used_fields |= cache_rec->fields;
t_pop();
rec->cache_offset = uint32_to_offset(offset);
offset += size;
}
rec = cache->index->next(cache->index, rec);
}
/* update header */
hdr->indexid = cache->index->indexid;
hdr->file_seq = cache->index->hdr->cache_sync_id+1;
hdr->used_file_size = uint32_to_nbo(offset);
hdr->used_fields = used_fields;
hdr->field_usage_start = uint32_to_nbo(ioloop_time);
/* write everything to disk */
if (msync(mmap_base, offset, MS_SYNC) < 0)
return mail_cache_set_syscall_error(cache, "msync()");
if (munmap(mmap_base, new_file_size) < 0)
return mail_cache_set_syscall_error(cache, "munmap()");
if (fdatasync(fd) < 0)
return mail_cache_set_syscall_error(cache, "fdatasync()");
return TRUE;
#endif
}
int mail_cache_compress(struct mail_cache *cache)
{
int fd, ret = TRUE;
i_assert(cache->trans_ctx == NULL);
if (cache->anon_mmap)
return TRUE;
if (!cache->index->set_lock(cache->index, MAIL_LOCK_EXCLUSIVE))
return FALSE;
if (mail_cache_lock(cache, TRUE) <= 0)
return FALSE;
#ifdef DEBUG
i_warning("Compressing cache file %s", cache->filepath);
#endif
fd = file_dotlock_open(cache->filepath, NULL, MAIL_CACHE_LOCK_TIMEOUT,
MAIL_CACHE_LOCK_CHANGE_TIMEOUT,
MAIL_CACHE_LOCK_IMMEDIATE_TIMEOUT, NULL, NULL);
if (fd == -1) {
mail_cache_set_syscall_error(cache, "file_dotlock_open()");
return FALSE;
}
/* now we'll begin the actual moving. keep rebuild-flag on
while doing it. */
cache->index->hdr->flags |= MAIL_INDEX_HDR_FLAG_REBUILD;
if (!mail_index_fmdatasync(cache->index, cache->index->hdr_size))
return FALSE;
if (!mail_cache_copy(cache, fd)) {
(void)file_dotlock_delete(cache->filepath, fd);
ret = FALSE;
} else {
mail_cache_file_close(cache);
cache->fd = dup(fd);
if (file_dotlock_replace(cache->filepath, fd, FALSE) < 0) {
mail_cache_set_syscall_error(cache,
"file_dotlock_replace()");
ret = FALSE;
}
if (!mmap_update(cache, 0, 0))
ret = FALSE;
}
/* headers could have changed, reread them */
memset(cache->split_offsets, 0, sizeof(cache->split_offsets));
memset(cache->split_headers, 0, sizeof(cache->split_headers));
if (ret) {
cache->index->hdr->flags &=
~(MAIL_INDEX_HDR_FLAG_REBUILD |
MAIL_INDEX_HDR_FLAG_COMPRESS_CACHE);
}
if (!mail_cache_unlock(cache))
ret = FALSE;
return ret;
}
--- NEW FILE: mail-cache-lookup.c ---
/* Copyright (C) 2003-2004 Timo Sirainen */
#include "lib.h"
#include "buffer.h"
#include "byteorder.h"
#include "mail-cache-private.h"
#if 0
const char *
mail_cache_get_header_fields_str(struct mail_cache *cache, unsigned int idx)
{
uint32_t offset, data_size;
unsigned char *buf;
offset = mail_cache_offset_to_uint32(cache->hdr->header_offsets[idx]);
if (offset == 0)
return NULL;
if (!mmap_update(cache, offset, 1024))
return NULL;
if (offset + sizeof(data_size) > cache->mmap_length) {
mail_cache_set_corrupted(cache, "Header %u points outside file",
idx);
return NULL;
}
buf = cache->mmap_base;
memcpy(&data_size, buf + offset, sizeof(data_size));
data_size = nbo_to_uint32(data_size);
offset += sizeof(data_size);
if (data_size == 0) {
mail_cache_set_corrupted(cache,
"Header %u points to empty string", idx);
return NULL;
}
if (!mmap_update(cache, offset, data_size))
return NULL;
if (offset + data_size > cache->mmap_length) {
mail_cache_set_corrupted(cache, "Header %u points outside file",
idx);
return NULL;
}
buf = cache->mmap_base;
if (buf[offset + data_size - 1] != '\0') {
mail_cache_set_corrupted(cache,
"Header %u points to invalid string", idx);
return NULL;
}
return buf + offset;
}
const char *const *
mail_cache_split_header(struct mail_cache *cache, const char *header)
{
const char *const *arr, *const *tmp;
const char *null = NULL;
char *str;
buffer_t *buf;
if (header == NULL)
return NULL;
arr = t_strsplit(header, "\n");
buf = buffer_create_dynamic(cache->split_header_pool, 32, (size_t)-1);
for (tmp = arr; *tmp != NULL; tmp++) {
str = p_strdup(cache->split_header_pool, *tmp);
buffer_append(buf, &str, sizeof(str));
}
buffer_append(buf, &null, sizeof(null));
return buffer_get_data(buf, NULL);
}
const char *const *mail_cache_get_header_fields(struct mail_cache_view *view,
unsigned int idx)
{
struct mail_cache *cache = view->cache;
const char *str;
int i;
i_assert(idx < MAIL_CACHE_HEADERS_COUNT);
/* t_strsplit() is a bit slow, so we cache it */
if (cache->hdr->header_offsets[idx] != cache->split_offsets[idx]) {
p_clear(cache->split_header_pool);
t_push();
for (i = 0; i < MAIL_CACHE_HEADERS_COUNT; i++) {
cache->split_offsets[i] =
cache->hdr->header_offsets[i];
str = mail_cache_get_header_fields_str(cache, i);
cache->split_headers[i] =
mail_cache_split_header(cache, str);
}
t_pop();
}
return cache->split_headers[idx];
}
struct mail_cache_record *
mail_cache_get_record(struct mail_cache *cache, uint32_t offset)
{
#define CACHE_PREFETCH 1024
struct mail_cache_record *cache_rec;
size_t size;
offset = mail_cache_offset_to_uint32(offset);
if (offset == 0)
return NULL;
if (!mmap_update(cache, offset, sizeof(*cache_rec) + CACHE_PREFETCH))
return NULL;
if (offset + sizeof(*cache_rec) > cache->mmap_length) {
mail_cache_set_corrupted(cache, "record points outside file");
return NULL;
}
cache_rec = CACHE_RECORD(cache, offset);
size = nbo_to_uint32(cache_rec->size);
if (size < sizeof(*cache_rec)) {
mail_cache_set_corrupted(cache, "invalid record size");
return NULL;
}
if (size > CACHE_PREFETCH) {
if (!mmap_update(cache, offset, size))
return NULL;
}
if (offset + size > cache->mmap_length) {
mail_cache_set_corrupted(cache, "record points outside file");
return NULL;
}
return cache_rec;
}
struct mail_cache_record *
mail_cache_get_next_record(struct mail_cache *cache,
struct mail_cache_record *rec)
{
struct mail_cache_record *next;
next = mail_cache_get_record(cache, rec->next_offset);
if (next != NULL && next <= rec) {
mail_cache_set_corrupted(cache, "next_offset points backwards");
return NULL;
}
return next;
}
struct mail_cache_record *
mail_cache_lookup(struct mail_cache_view *view, uint32_t seq,
enum mail_cache_field fields)
{
const struct mail_index_record *rec;
if (mail_cache_transaction_autocommit(view, seq, fields) < 0)
return NULL;
// FIXME: check cache_offset in transaction
if (mail_index_lookup_latest(view->view, seq, &rec) < 0)
return NULL;
return mail_cache_get_record(view->cache, rec->cache_offset);
}
enum mail_cache_field
mail_cache_get_fields(struct mail_cache_view *view, uint32_t seq)
{
struct mail_cache_record *cache_rec;
enum mail_cache_field fields = 0;
cache_rec = mail_cache_lookup(view, seq, 0);
while (cache_rec != NULL) {
fields |= cache_rec->fields;
cache_rec = mail_cache_get_next_record(view->cache, cache_rec);
}
return fields;
}
static int cache_get_field(struct mail_cache *cache,
struct mail_cache_record *cache_rec,
enum mail_cache_field field,
void **data_r, size_t *size_r)
{
unsigned char *buf;
unsigned int mask;
uint32_t rec_size, data_size;
size_t offset, next_offset;
int i;
rec_size = nbo_to_uint32(cache_rec->size);
buf = (unsigned char *) cache_rec;
offset = sizeof(*cache_rec);
for (i = 0, mask = 1; i < 31; i++, mask <<= 1) {
if ((cache_rec->fields & mask) == 0)
continue;
/* all records are at least 32bit. we have to check this
before getting data_size. */
if (offset + sizeof(uint32_t) > rec_size) {
mail_cache_set_corrupted(cache,
"Record continues outside it's allocated size");
return FALSE;
}
if ((mask & MAIL_CACHE_FIXED_MASK) != 0)
data_size = mail_cache_field_sizes[i];
else {
memcpy(&data_size, buf + offset, sizeof(data_size));
data_size = nbo_to_uint32(data_size);
offset += sizeof(data_size);
}
next_offset = offset + ((data_size + 3) & ~3);
if (next_offset > rec_size) {
mail_cache_set_corrupted(cache,
"Record continues outside it's allocated size");
return FALSE;
}
if (field == mask) {
if (data_size == 0) {
mail_cache_set_corrupted(cache,
"Field size is 0");
return FALSE;
}
*data_r = buf + offset;
*size_r = data_size;
return TRUE;
}
offset = next_offset;
}
i_unreached();
return FALSE;
}
static int cache_lookup_field(struct mail_cache_view *view, uint32_t seq,
enum mail_cache_field field,
void **data_r, size_t *size_r)
{
struct mail_cache_record *cache_rec;
cache_rec = mail_cache_lookup(view, seq, field);
while (cache_rec != NULL) {
if ((cache_rec->fields & field) != 0) {
return cache_get_field(view->cache, cache_rec, field,
data_r, size_r);
}
cache_rec = mail_cache_get_next_record(view->cache, cache_rec);
}
return FALSE;
}
int mail_cache_lookup_field(struct mail_cache_view *view, uint32_t seq,
enum mail_cache_field field,
const void **data_r, size_t *size_r)
{
void *data;
if (!cache_lookup_field(view, seq, field, &data, size_r))
return FALSE;
*data_r = data;
return TRUE;
}
const char *
mail_cache_lookup_string_field(struct mail_cache_view *view, uint32_t seq,
enum mail_cache_field field)
{
const void *data;
size_t size;
i_assert((field & MAIL_CACHE_STRING_MASK) != 0);
if (!mail_cache_lookup_field(view, seq, field, &data, &size))
return NULL;
if (((const char *) data)[size-1] != '\0') {
mail_cache_set_corrupted(view->cache,
"String field %x doesn't end with NUL", field);
return NULL;
}
return data;
}
int mail_cache_copy_fixed_field(struct mail_cache_view *view, uint32_t seq,
enum mail_cache_field field,
void *buffer, size_t buffer_size)
{
const void *data;
size_t size;
i_assert((field & MAIL_CACHE_FIXED_MASK) != 0);
if (!mail_cache_lookup_field(view, seq, field, &data, &size))
return FALSE;
if (buffer_size != size) {
i_panic("cache: fixed field %x wrong size "
"(%"PRIuSIZE_T" vs %"PRIuSIZE_T")",
field, size, buffer_size);
}
memcpy(buffer, data, buffer_size);
return TRUE;
}
#else
#endif
--- NEW FILE: mail-cache-old.c ---
static const char *
mail_cache_get_header_fields_str(struct mail_cache *cache, unsigned int idx);
static int mail_cache_write(struct mail_cache_transaction_ctx *ctx);
static struct mail_cache_record *
mail_cache_lookup(struct mail_cache *cache,
const struct mail_index_record *rec,
enum mail_cache_field fields);
static void mail_cache_file_close(struct mail_cache *cache)
{
if (cache->mmap_base != NULL) {
if (munmap(cache->mmap_base, cache->mmap_length) < 0)
mail_cache_set_syscall_error(cache, "munmap()");
}
cache->mmap_base = NULL;
cache->hdr = NULL;
cache->mmap_length = 0;
if (cache->fd != -1) {
if (close(cache->fd) < 0)
mail_cache_set_syscall_error(cache, "close()");
cache->fd = -1;
}
}
static int mail_cache_file_reopen(struct mail_cache *cache)
{
int fd;
fd = open(cache->filepath, O_RDWR);
if (fd == -1) {
mail_cache_set_syscall_error(cache, "open()");
return -1;
}
mail_cache_file_close(cache);
cache->fd = fd;
return 0;
}
static int mmap_verify_header(struct mail_cache *cache)
{
struct mail_cache_header *hdr;
/* check that the header is still ok */
if (cache->mmap_length < sizeof(struct mail_cache_header)) {
mail_cache_set_corrupted(cache, "File too small");
return 0;
}
cache->hdr = hdr = cache->mmap_base;
if (cache->hdr->indexid != cache->index->indexid) {
/* index id changed */
if (cache->hdr->indexid != 0)
mail_cache_set_corrupted(cache, "indexid changed");
return 0;
}
if (cache->trans_ctx != NULL) {
/* we've updated used_file_size, do nothing */
return 1;
}
cache->used_file_size = nbo_to_uint32(hdr->used_file_size);
/* only check the header if we're locked */
if (cache->locks == 0)
return 1;
if (cache->used_file_size < sizeof(struct mail_cache_header)) {
mail_cache_set_corrupted(cache, "used_file_size too small");
return 0;
}
if ((cache->used_file_size % sizeof(uint32_t)) != 0) {
mail_cache_set_corrupted(cache, "used_file_size not aligned");
return 0;
}
if (cache->used_file_size > cache->mmap_length) {
/* maybe a crash truncated the file - just fix it */
hdr->used_file_size = uint32_to_nbo(cache->mmap_length & ~3);
if (msync(cache->mmap_base, sizeof(*hdr), MS_SYNC) < 0) {
mail_cache_set_syscall_error(cache, "msync()");
return -1;
}
}
return 1;
}
static int mmap_update_nocheck(struct mail_cache *cache,
size_t offset, size_t size)
{
struct stat st;
/* if sequence has changed, the file has to be reopened.
note that if main index isn't locked, it may change again */
if (cache->hdr->file_seq != cache->index->hdr->cache_file_seq &&
cache->mmap_base != NULL) {
if (!mail_cache_file_reopen(cache))
return -1;
}
if (offset < cache->mmap_length &&
size <= cache->mmap_length - offset &&
!cache->mmap_refresh) {
/* already mapped */
if (size != 0 || cache->anon_mmap)
return 1;
/* requesting the whole file - see if we need to
re-mmap */
if (fstat(cache->fd, &st) < 0) {
mail_cache_set_syscall_error(cache, "fstat()");
return -1;
}
if ((uoff_t)st.st_size == cache->mmap_length)
return 1;
}
cache->mmap_refresh = FALSE;
if (cache->anon_mmap)
return 1;
if (cache->mmap_base != NULL) {
if (cache->locks != 0) {
/* in the middle of transaction - write the changes */
if (msync(cache->mmap_base, cache->mmap_length,
MS_SYNC) < 0) {
mail_cache_set_syscall_error(cache, "msync()");
return -1;
}
}
if (munmap(cache->mmap_base, cache->mmap_length) < 0)
mail_cache_set_syscall_error(cache, "munmap()");
}
i_assert(cache->fd != -1);
/* map the whole file */
cache->hdr = NULL;
cache->mmap_length = 0;
cache->mmap_base = mmap_rw_file(cache->fd, &cache->mmap_length);
if (cache->mmap_base == MAP_FAILED) {
cache->mmap_base = NULL;
mail_cache_set_syscall_error(cache, "mmap()");
return -1;
}
/* re-mmaped, check header */
return 0;
}
static int mmap_update(struct mail_cache *cache, size_t offset, size_t size)
{
int synced, ret;
for (synced = FALSE;; synced = TRUE) {
ret = mmap_update_nocheck(cache, offset, size);
if (ret > 0)
return TRUE;
if (ret < 0)
return FALSE;
if (!mmap_verify_header(cache))
return FALSE;
/* see if cache file was rebuilt - do it only once to avoid
infinite looping */
if (cache->hdr->sync_id == cache->index->cache_sync_id ||
synced)
break;
if (!mail_cache_file_reopen(cache))
return FALSE;
}
return TRUE;
}
static int mail_cache_open_and_verify(struct mail_cache *cache, int silent)
{
struct stat st;
mail_cache_file_close(cache);
cache->fd = open(cache->filepath, O_RDWR);
if (cache->fd == -1) {
if (errno == ENOENT)
return 0;
mail_cache_set_syscall_error(cache, "open()");
return -1;
}
if (fstat(cache->fd, &st) < 0) {
mail_cache_set_syscall_error(cache, "fstat()");
return -1;
}
if (st.st_size < sizeof(struct mail_cache_header))
return 0;
cache->mmap_refresh = TRUE;
if (mmap_update_nocheck(cache, 0, sizeof(struct mail_cache_header)) < 0)
return -1;
/* verify that this really is the cache for wanted index */
cache->silent = silent;
if (!mmap_verify_header(cache)) {
cache->silent = FALSE;
return 0;
}
cache->silent = FALSE;
return 1;
}
static int mail_cache_open_or_create_file(struct mail_cache *cache,
struct mail_cache_header *hdr)
{
int ret, fd;
cache->filepath = i_strconcat(cache->index->filepath,
MAIL_CACHE_FILE_PREFIX, NULL);
ret = mail_cache_open_and_verify(cache, FALSE);
if (ret != 0)
return ret > 0;
/* we'll have to clear cache_offsets which requires exclusive lock */
if (!mail_index_set_lock(cache->index, MAIL_LOCK_EXCLUSIVE))
return FALSE;
/* maybe a rebuild.. */
fd = file_dotlock_open(cache->filepath, NULL, MAIL_CACHE_LOCK_TIMEOUT,
MAIL_CACHE_LOCK_CHANGE_TIMEOUT,
MAIL_CACHE_LOCK_IMMEDIATE_TIMEOUT, NULL, NULL);
if (fd == -1) {
mail_cache_set_syscall_error(cache, "file_dotlock_open()");
return FALSE;
}
/* see if someone else just created the cache file */
ret = mail_cache_open_and_verify(cache, TRUE);
if (ret != 0) {
(void)file_dotlock_delete(cache->filepath, fd);
return ret > 0;
}
/* rebuild then */
if (write_full(fd, hdr, sizeof(*hdr)) < 0) {
mail_cache_set_syscall_error(cache, "write_full()");
(void)file_dotlock_delete(cache->filepath, fd);
return FALSE;
}
if (file_set_size(fd, MAIL_CACHE_INITIAL_SIZE) < 0) {
mail_cache_set_syscall_error(cache, "file_set_size()");
(void)file_dotlock_delete(cache->filepath, fd);
return FALSE;
}
if (cache->index->hdr.cache_file_seq != 0) {
// FIXME: recreate index file with cache_offsets cleared
}
mail_cache_file_close(cache);
cache->fd = dup(fd);
if (file_dotlock_replace(cache->filepath, fd, FALSE) < 0) {
mail_cache_set_syscall_error(cache, "file_dotlock_replace()");
return FALSE;
}
if (!mmap_update(cache, 0, sizeof(struct mail_cache_header)))
return FALSE;
return TRUE;
}
int mail_cache_open_or_create(struct mail_index *index)
{
struct mail_cache_header hdr;
struct mail_cache *cache;
memset(&hdr, 0, sizeof(hdr));
hdr.indexid = index->indexid;
hdr.sync_id = index->hdr->cache_file_seq; // FIXME
hdr.used_file_size = uint32_to_nbo(sizeof(hdr));
cache = i_new(struct mail_cache, 1);
cache->index = index;
cache->fd = -1;
cache->split_header_pool = pool_alloconly_create("Headers", 512);
index->cache = cache;
/* we'll do anon-mmaping only if initially requested. if we fail
because of out of disk space, we'll just let the main index code
know it and fail. */
if (!mail_cache_open_or_create_file(cache, &hdr)) {
mail_cache_free(cache);
return FALSE;
}
return TRUE;
}
void mail_cache_free(struct mail_cache *cache)
{
i_assert(cache->trans_ctx == NULL);
cache->index->cache = NULL;
mail_cache_file_close(cache);
pool_unref(cache->split_header_pool);
i_free(cache->filepath);
i_free(cache);
}
void mail_cache_set_defaults(struct mail_cache *cache,
enum mail_cache_field default_cache_fields,
enum mail_cache_field never_cache_fields)
{
cache->default_cache_fields = default_cache_fields;
cache->never_cache_fields = never_cache_fields;
}
int mail_cache_reset(struct mail_cache *cache)
{
struct mail_cache_header hdr;
int ret, fd;
i_assert(cache->index->lock_type == MAIL_LOCK_EXCLUSIVE);
memset(&hdr, 0, sizeof(hdr));
hdr.indexid = cache->index->indexid;
hdr.sync_id = cache->sync_id = cache->index->cache_sync_id =
++cache->index->hdr->cache_sync_id;
hdr.used_file_size = uint32_to_nbo(sizeof(hdr));
cache->used_file_size = sizeof(hdr);
fd = file_dotlock_open(cache->filepath, NULL, MAIL_CACHE_LOCK_TIMEOUT,
MAIL_CACHE_LOCK_CHANGE_TIMEOUT,
MAIL_CACHE_LOCK_IMMEDIATE_TIMEOUT, NULL, NULL);
if (fd == -1) {
mail_cache_set_syscall_error(cache, "file_dotlock_open()");
return -1;
}
if (write_full(fd, &hdr, sizeof(hdr)) < 0) {
mail_cache_set_syscall_error(cache, "write_full()");
(void)file_dotlock_delete(cache->filepath, fd);
return -1;
}
if (file_set_size(fd, MAIL_CACHE_INITIAL_SIZE) < 0) {
mail_cache_set_syscall_error(cache, "file_set_size()");
(void)file_dotlock_delete(cache->filepath, fd);
return -1;
}
mail_cache_file_close(cache);
cache->fd = dup(fd);
if (file_dotlock_replace(cache->filepath, fd, FALSE) < 0) {
mail_cache_set_syscall_error(cache, "file_dotlock_replace()");
return -1;
}
cache->mmap_refresh = TRUE;
if (!mmap_update(cache, 0, sizeof(struct mail_cache_header)))
return -1;
return 0;
}
int mail_cache_lock(struct mail_cache *cache, int nonblock)
{
int ret;
if (cache->locks++ != 0)
return TRUE;
if (cache->anon_mmap)
return TRUE;
if (nonblock) {
ret = file_try_lock(cache->fd, F_WRLCK);
if (ret < 0)
mail_cache_set_syscall_error(cache, "file_try_lock()");
} else {
ret = file_wait_lock(cache->fd, F_WRLCK);
if (ret <= 0)
mail_cache_set_syscall_error(cache, "file_wait_lock()");
}
if (ret > 0) {
if (!mmap_update(cache, 0, 0)) {
(void)mail_cache_unlock(cache);
return -1;
}
if (cache->sync_id != cache->index->cache_sync_id) {
/* we have the cache file locked and sync_id still
doesn't match. it means we crashed between updating
cache file and updating sync_id in index header.
just update the sync_ids so they match. */
i_warning("Updating broken sync_id in cache file %s",
cache->filepath);
cache->sync_id = cache->hdr->sync_id =
cache->index->cache_sync_id;
}
}
return ret;
}
int mail_cache_unlock(struct mail_cache *cache)
{
if (--cache->locks > 0)
return TRUE;
if (cache->anon_mmap)
return TRUE;
if (file_wait_lock(cache->fd, F_UNLCK) <= 0) {
mail_cache_set_syscall_error(cache, "file_wait_lock(F_UNLCK)");
return FALSE;
}
return TRUE;
}
int mail_cache_is_locked(struct mail_cache *cache)
{
return cache->locks > 0;
}
struct mail_cache_view *
mail_cache_view_open(struct mail_cache *cache, struct mail_index_view *view)
{
struct mail_cache_view *view;
view = i_new(struct mail_cache_view, 1);
view->cache = cache;
view->view = view;
return view;
}
void mail_cache_view_close(struct mail_cache_view *view)
{
i_free(view);
}
static const char *
mail_cache_get_header_fields_str(struct mail_cache *cache, unsigned int idx)
{
uint32_t offset, data_size;
unsigned char *buf;
offset = offset_to_uint32(cache->hdr->header_offsets[idx]);
if (offset == 0)
return NULL;
if (!mmap_update(cache, offset, 1024))
return NULL;
if (offset + sizeof(data_size) > cache->mmap_length) {
mail_cache_set_corrupted(cache, "Header %u points outside file",
idx);
return NULL;
}
buf = cache->mmap_base;
memcpy(&data_size, buf + offset, sizeof(data_size));
data_size = nbo_to_uint32(data_size);
offset += sizeof(data_size);
if (data_size == 0) {
mail_cache_set_corrupted(cache,
"Header %u points to empty string", idx);
return NULL;
}
if (!mmap_update(cache, offset, data_size))
return NULL;
if (offset + data_size > cache->mmap_length) {
mail_cache_set_corrupted(cache, "Header %u points outside file",
idx);
return NULL;
}
buf = cache->mmap_base;
if (buf[offset + data_size - 1] != '\0') {
mail_cache_set_corrupted(cache,
"Header %u points to invalid string", idx);
return NULL;
}
return buf + offset;
}
static const char *const *
split_header(struct mail_cache *cache, const char *header)
{
const char *const *arr, *const *tmp;
const char *null = NULL;
char *str;
buffer_t *buf;
if (header == NULL)
return NULL;
arr = t_strsplit(header, "\n");
buf = buffer_create_dynamic(cache->split_header_pool, 32, (size_t)-1);
for (tmp = arr; *tmp != NULL; tmp++) {
str = p_strdup(cache->split_header_pool, *tmp);
buffer_append(buf, &str, sizeof(str));
}
buffer_append(buf, &null, sizeof(null));
return buffer_get_data(buf, NULL);
}
const char *const *mail_cache_get_header_fields(struct mail_cache *cache,
unsigned int idx)
{
const char *str;
int i;
i_assert(idx < MAIL_CACHE_HEADERS_COUNT);
/* t_strsplit() is a bit slow, so we cache it */
if (cache->hdr->header_offsets[idx] != cache->split_offsets[idx]) {
p_clear(cache->split_header_pool);
t_push();
for (i = 0; i < MAIL_CACHE_HEADERS_COUNT; i++) {
cache->split_offsets[i] =
cache->hdr->header_offsets[i];
str = mail_cache_get_header_fields_str(cache, i);
cache->split_headers[i] = split_header(cache, str);
}
t_pop();
}
return cache->split_headers[idx];
}
static const char *write_header_string(const char *const headers[],
uint32_t *size_r)
{
buffer_t *buffer;
size_t size;
buffer = buffer_create_dynamic(pool_datastack_create(),
512, (size_t)-1);
while (*headers != NULL) {
if (buffer_get_used_size(buffer) != 0)
buffer_append(buffer, "\n", 1);
buffer_append(buffer, *headers, strlen(*headers));
headers++;
}
buffer_append(buffer, null4, 1);
size = buffer_get_used_size(buffer);
if ((size & 3) != 0) {
buffer_append(buffer, null4, 4 - (size & 3));
size += 4 - (size & 3);
}
*size_r = size;
return buffer_get_data(buffer, NULL);
}
int mail_cache_set_header_fields(struct mail_cache_transaction_ctx *ctx,
unsigned int idx, const char *const headers[])
{
struct mail_cache *cache = ctx->cache;
uint32_t offset, update_offset, size;
const char *header_str, *prev_str;
i_assert(*headers != NULL);
i_assert(idx < MAIL_CACHE_HEADERS_COUNT);
i_assert(idx >= ctx->next_unused_header_lowwater);
i_assert(offset_to_uint32(cache->hdr->header_offsets[idx]) == 0);
t_push();
header_str = write_header_string(headers, &size);
if (idx != 0) {
prev_str = mail_cache_get_header_fields_str(cache, idx-1);
if (prev_str == NULL) {
t_pop();
return FALSE;
}
i_assert(strcmp(header_str, prev_str) != 0);
}
offset = mail_cache_append_space(ctx, size + sizeof(uint32_t));
if (offset != 0) {
memcpy((char *) cache->mmap_base + offset + sizeof(uint32_t),
header_str, size);
size = uint32_to_nbo(size);
memcpy((char *) cache->mmap_base + offset,
&size, sizeof(uint32_t));
/* update cached headers */
cache->split_offsets[idx] = cache->hdr->header_offsets[idx];
cache->split_headers[idx] = split_header(cache, header_str);
/* mark used-bit to be updated later. not really needed for
read-safety, but if transaction get rolled back we can't let
this point to invalid location. */
update_offset = (char *) &cache->hdr->header_offsets[idx] -
(char *) cache->mmap_base;
mark_update(&ctx->cache_marks, update_offset,
uint32_to_offset(offset));
/* make sure get_header_fields() still works for this header
while the transaction isn't yet committed. */
ctx->next_unused_header_lowwater = idx + 1;
}
t_pop();
return offset > 0;
}
static struct mail_cache_record *
cache_get_record(struct mail_cache *cache, uint32_t offset)
{
#define CACHE_PREFETCH 1024
struct mail_cache_record *cache_rec;
size_t size;
offset = offset_to_uint32(offset);
if (offset == 0)
return NULL;
if (!mmap_update(cache, offset, sizeof(*cache_rec) + CACHE_PREFETCH))
return NULL;
if (offset + sizeof(*cache_rec) > cache->mmap_length) {
mail_cache_set_corrupted(cache, "record points outside file");
return NULL;
}
cache_rec = CACHE_RECORD(cache, offset);
size = nbo_to_uint32(cache_rec->size);
if (size < sizeof(*cache_rec)) {
mail_cache_set_corrupted(cache, "invalid record size");
return NULL;
}
if (size > CACHE_PREFETCH) {
if (!mmap_update(cache, offset, size))
return NULL;
}
if (offset + size > cache->mmap_length) {
mail_cache_set_corrupted(cache, "record points outside file");
return NULL;
}
return cache_rec;
}
static struct mail_cache_record *
cache_get_next_record(struct mail_cache *cache, struct mail_cache_record *rec)
{
struct mail_cache_record *next;
next = cache_get_record(cache, rec->next_offset);
if (next != NULL && next <= rec) {
mail_cache_set_corrupted(cache, "next_offset points backwards");
return NULL;
}
return next;
}
static struct mail_cache_record *
mail_cache_lookup(struct mail_cache *cache, const struct mail_index_record *rec,
enum mail_cache_field fields)
{
struct mail_cache_record *cache_rec;
unsigned int idx;
if (cache->trans_ctx != NULL &&
cache->trans_ctx->first_uid <= rec->uid &&
cache->trans_ctx->last_uid >= rec->uid &&
(cache->trans_ctx->prev_uid != rec->uid || fields == 0 ||
(cache->trans_ctx->prev_fields & fields) != 0)) {
/* we have to auto-commit since we're not capable of looking
into uncommitted records. it would be possible by checking
index_marks and cache_marks, but it's just more trouble
than worth. */
idx = INDEX_RECORD_INDEX(cache->index, rec);
if (cache->trans_ctx->last_idx == idx) {
if (!mail_cache_write(cache->trans_ctx))
return NULL;
}
if (!mail_cache_transaction_commit(cache->trans_ctx))
return NULL;
}
cache_rec = cache_get_record(cache, rec->cache_offset);
if (cache_rec == NULL)
return NULL;
return cache_rec;
}
enum mail_cache_field
mail_cache_get_fields(struct mail_cache *cache,
const struct mail_index_record *rec)
{
struct mail_cache_record *cache_rec;
enum mail_cache_field fields = 0;
cache_rec = mail_cache_lookup(cache, rec, 0);
while (cache_rec != NULL) {
fields |= cache_rec->fields;
cache_rec = cache_get_next_record(cache, cache_rec);
}
return fields;
}
static int cache_get_field(struct mail_cache *cache,
struct mail_cache_record *cache_rec,
enum mail_cache_field field,
void **data_r, size_t *size_r)
{
unsigned char *buf;
unsigned int mask;
uint32_t rec_size, data_size;
size_t offset, next_offset;
int i;
rec_size = nbo_to_uint32(cache_rec->size);
buf = (unsigned char *) cache_rec;
offset = sizeof(*cache_rec);
for (i = 0, mask = 1; i < 31; i++, mask <<= 1) {
if ((cache_rec->fields & mask) == 0)
continue;
/* all records are at least 32bit. we have to check this
before getting data_size. */
if (offset + sizeof(uint32_t) > rec_size) {
mail_cache_set_corrupted(cache,
"Record continues outside it's allocated size");
return FALSE;
}
if ((mask & MAIL_CACHE_FIXED_MASK) != 0)
data_size = mail_cache_field_sizes[i];
else {
memcpy(&data_size, buf + offset, sizeof(data_size));
data_size = nbo_to_uint32(data_size);
offset += sizeof(data_size);
}
next_offset = offset + ((data_size + 3) & ~3);
if (next_offset > rec_size) {
mail_cache_set_corrupted(cache,
"Record continues outside it's allocated size");
return FALSE;
}
if (field == mask) {
if (data_size == 0) {
mail_cache_set_corrupted(cache,
"Field size is 0");
return FALSE;
}
*data_r = buf + offset;
*size_r = data_size;
return TRUE;
}
offset = next_offset;
}
i_unreached();
return FALSE;
}
static int cache_lookup_field(struct mail_cache *cache,
const struct mail_index_record *rec,
enum mail_cache_field field,
void **data_r, size_t *size_r)
{
struct mail_cache_record *cache_rec;
cache_rec = mail_cache_lookup(cache, rec, field);
while (cache_rec != NULL) {
if ((cache_rec->fields & field) != 0) {
return cache_get_field(cache, cache_rec, field,
data_r, size_r);
}
cache_rec = cache_get_next_record(cache, cache_rec);
}
return FALSE;
}
int mail_cache_lookup_field(struct mail_cache *cache,
const struct mail_index_record *rec,
enum mail_cache_field field,
const void **data_r, size_t *size_r)
{
void *data;
if (!cache_lookup_field(cache, rec, field, &data, size_r))
return FALSE;
*data_r = data;
return TRUE;
}
const char *mail_cache_lookup_string_field(struct mail_cache *cache,
const struct mail_index_record *rec,
enum mail_cache_field field)
{
const void *data;
size_t size;
i_assert((field & MAIL_CACHE_STRING_MASK) != 0);
if (!mail_cache_lookup_field(cache, rec, field, &data, &size))
return NULL;
if (((const char *) data)[size-1] != '\0') {
mail_cache_set_corrupted(cache,
"String field %x doesn't end with NUL", field);
return NULL;
}
return data;
}
int mail_cache_copy_fixed_field(struct mail_cache *cache,
const struct mail_index_record *rec,
enum mail_cache_field field,
void *buffer, size_t buffer_size)
{
const void *data;
size_t size;
i_assert((field & MAIL_CACHE_FIXED_MASK) != 0);
if (!mail_cache_lookup_field(cache, rec, field, &data, &size))
return FALSE;
if (buffer_size != size) {
i_panic("cache: fixed field %x wrong size "
"(%"PRIuSIZE_T" vs %"PRIuSIZE_T")",
field, size, buffer_size);
}
memcpy(buffer, data, buffer_size);
return TRUE;
}
void mail_cache_mark_missing(struct mail_cache *cache,
enum mail_cache_field fields)
{
// FIXME: count these
}
enum mail_index_record_flag
mail_cache_get_index_flags(struct mail_cache *cache,
const struct mail_index_record *rec)
{
enum mail_index_record_flag flags;
if (!mail_cache_copy_fixed_field(cache, rec, MAIL_CACHE_INDEX_FLAGS,
&flags, sizeof(flags)))
return 0;
return flags;
}
int mail_cache_update_index_flags(struct mail_cache *cache,
const struct mail_index_record *rec,
enum mail_index_record_flag flags)
{
void *data;
size_t size;
i_assert(cache->locks > 0);
if (!cache_lookup_field(cache, rec, MAIL_CACHE_INDEX_FLAGS,
&data, &size)) {
mail_cache_set_corrupted(cache,
"Missing index flags for record %u", rec->uid);
return FALSE;
}
memcpy(data, &flags, sizeof(flags));
return TRUE;
}
int mail_cache_update_location_offset(struct mail_cache *cache,
const struct mail_index_record *rec,
uoff_t offset)
{
void *data;
size_t size;
i_assert(cache->locks > 0);
if (!cache_lookup_field(cache, rec, MAIL_CACHE_LOCATION_OFFSET,
&data, &size)) {
mail_cache_set_corrupted(cache,
"Missing location offset for record %u", rec->uid);
return FALSE;
}
memcpy(data, &offset, sizeof(offset));
return TRUE;
}
void *mail_cache_get_mmaped(struct mail_cache *cache, size_t *size)
{
if (!mmap_update(cache, 0, 0))
return NULL;
*size = cache->mmap_length;
return cache->mmap_base;
}
--- NEW FILE: mail-cache-private.h ---
#ifndef __MAIL_CACHE_PRIVATE_H
#define __MAIL_CACHE_PRIVATE_H
#include "mail-index-private.h"
#include "mail-cache.h"
/* Never compress the file if it's smaller than this */
#define COMPRESS_MIN_SIZE (1024*50)
/* Compress the file when deleted space reaches n% of total size */
#define COMPRESS_PERCENTAGE 20
/* Compress the file when n% of rows contain continued rows.
200% means that there's 2 continued rows per record. */
#define COMPRESS_CONTINUED_PERCENTAGE 200
/* Initial size for the file */
#define MAIL_CACHE_INITIAL_SIZE (sizeof(struct mail_cache_header) + 10240)
/* When more space is needed, grow the file n% larger than the previous size */
#define MAIL_CACHE_GROW_PERCENTAGE 10
#define MAIL_CACHE_LOCK_TIMEOUT 120
#define MAIL_CACHE_LOCK_CHANGE_TIMEOUT 60
#define MAIL_CACHE_LOCK_IMMEDIATE_TIMEOUT (5*60)
#define CACHE_RECORD(cache, offset) \
((struct mail_cache_record *) ((char *) (cache)->mmap_base + offset))
struct mail_cache_header {
uint32_t indexid;
uint32_t file_seq;
uint32_t continued_record_count;
uint32_t used_file_size;
uint32_t deleted_space;
uint32_t used_fields; /* enum mail_cache_field */
uint32_t field_usage_start; /* time_t */
uint32_t field_usage_counts[32];
uint32_t header_offsets[MAIL_CACHE_HEADERS_COUNT];
};
struct mail_cache_record {
uint32_t fields; /* enum mail_cache_field */
uint32_t next_offset;
uint32_t size; /* full record size, including this header */
};
struct mail_cache {
struct mail_index *index;
char *filepath;
int fd;
void *mmap_base;
size_t mmap_length;
uint32_t used_file_size;
struct mail_cache_header *hdr;
pool_t split_header_pool;
uint32_t split_offsets[MAIL_CACHE_HEADERS_COUNT];
const char *const *split_headers[MAIL_CACHE_HEADERS_COUNT];
enum mail_cache_field default_cache_fields;
enum mail_cache_field never_cache_fields;
struct mail_cache_transaction_ctx *trans_ctx;
unsigned int locks;
unsigned int mmap_refresh:1;
unsigned int silent:1;
};
struct mail_cache_view {
struct mail_cache *cache;
struct mail_index_view *view;
unsigned int broken:1;
};
extern unsigned int mail_cache_field_sizes[32];
extern enum mail_cache_field mail_cache_header_fields[MAIL_CACHE_HEADERS_COUNT];
uint32_t mail_cache_uint32_to_offset(uint32_t offset);
uint32_t mail_cache_offset_to_uint32(uint32_t offset);
const char *
mail_cache_get_header_fields_str(struct mail_cache *cache, unsigned int idx);
const char *const *
mail_cache_split_header(struct mail_cache *cache, const char *header);
struct mail_cache_record *
mail_cache_get_record(struct mail_cache *cache, uint32_t offset);
struct mail_cache_record *
mail_cache_get_next_record(struct mail_cache *cache,
struct mail_cache_record *rec);
struct mail_cache_record *
mail_cache_lookup(struct mail_cache_view *view, uint32_t seq,
enum mail_cache_field fields);
int
mail_cache_transaction_autocommit(struct mail_cache_view *view,
uint32_t seq, enum mail_cache_field fields);
void mail_cache_set_syscall_error(struct mail_cache *cache,
const char *function);
#endif
--- NEW FILE: mail-cache-transaction.c ---
/* Copyright (C) 2003-2004 Timo Sirainen */
#include "lib.h"
#include "buffer.h"
#include "byteorder.h"
#include "file-set-size.h"
#include "mmap-util.h"
#include "mail-cache-private.h"
#include <sys/stat.h>
#if 0
struct mail_cache_transaction_ctx {
struct mail_cache *cache;
struct mail_cache_view *view;
struct mail_index_transaction *trans;
unsigned int next_unused_header_lowwater;
struct mail_cache_record cache_rec;
buffer_t *cache_data;
uint32_t first_seq, last_seq, prev_seq;
enum mail_cache_field prev_fields;
buffer_t *cache_marks;
};
static const unsigned char *null4[] = { 0, 0, 0, 0 };
int mail_cache_transaction_begin(struct mail_cache_view *view, int nonblock,
struct mail_index_transaction *t,
struct mail_cache_transaction_ctx **ctx_r)
{
struct mail_cache_transaction_ctx *ctx;
int ret;
i_assert(view->cache->trans_ctx == NULL);
ret = mail_cache_lock(view->cache, nonblock);
if (ret <= 0)
return ret;
ctx = i_new(struct mail_cache_transaction_ctx, 1);
ctx->cache = view->cache;
ctx->view = view;
ctx->trans = t;
ctx->cache_data = buffer_create_dynamic(system_pool, 8192, (size_t)-1);
view->cache->trans_ctx = ctx;
*ctx_r = ctx;
return 1;
}
int mail_cache_transaction_end(struct mail_cache_transaction_ctx *ctx)
{
int ret = 0;
i_assert(ctx->cache->trans_ctx != NULL);
(void)mail_cache_transaction_rollback(ctx);
if (mail_cache_unlock(ctx->cache) < 0)
ret = -1;
ctx->cache->trans_ctx = NULL;
if (ctx->cache_marks != NULL)
buffer_free(ctx->cache_marks);
buffer_free(ctx->cache_data);
i_free(ctx);
return ret;
}
static void mail_cache_transaction_flush(struct mail_cache_transaction_ctx *ctx)
{
memset(&ctx->cache_rec, 0, sizeof(ctx->cache_rec));
ctx->next_unused_header_lowwater = 0;
ctx->first_seq = ctx->last_seq = ctx->prev_seq = 0;
ctx->prev_fields = 0;
if (ctx->cache_marks != NULL)
buffer_set_used_size(ctx->cache_marks, 0);
buffer_set_used_size(ctx->cache_data, 0);
}
static void mark_update(buffer_t **buf, uint32_t offset, uint32_t data)
{
if (*buf == NULL)
*buf = buffer_create_dynamic(system_pool, 1024, (size_t)-1);
buffer_append(*buf, &offset, sizeof(offset));
buffer_append(*buf, &data, sizeof(data));
}
static int write_mark_updates(struct mail_cache *cache)
{
const uint32_t *data, *end;
size_t size;
data = buffer_get_data(cache->trans_ctx->cache_marks, &size);
end = data + size/sizeof(uint32_t);
while (data < end) {
if (pwrite(cache->fd, data+1, sizeof(*data), data[0]) < 0) {
mail_cache_set_syscall_error(cache, "pwrite()");
return -1;
}
data += 2;
}
return 0;
}
static int commit_all_changes(struct mail_cache_transaction_ctx *ctx)
{
struct mail_cache *cache = ctx->cache;
uint32_t cont;
/* write everything to disk */
if (msync(cache->mmap_base, cache->mmap_length, MS_SYNC) < 0) {
mail_cache_set_syscall_error(cache, "msync()");
return -1;
}
if (fdatasync(cache->fd) < 0) {
mail_cache_set_syscall_error(cache, "fdatasync()");
return -1;
}
if (ctx->cache_marks == NULL ||
buffer_get_used_size(ctx->cache_marks) == 0)
return 0;
/* now that we're sure it's written, set on all the used-bits */
if (write_mark_updates(cache) < 0)
return -1;
/* update continued records count */
cont = nbo_to_uint32(cache->hdr->continued_record_count);
cont += buffer_get_used_size(ctx->cache_marks) /
(sizeof(uint32_t) * 2);
if (cont * 100 / cache->index->hdr->messages_count >=
COMPRESS_CONTINUED_PERCENTAGE &&
cache->used_file_size >= COMPRESS_MIN_SIZE) {
/* too many continued rows, compress */
//FIXME:cache->index->set_flags |= MAIL_INDEX_HDR_FLAG_COMPRESS_CACHE;
}
cache->hdr->continued_record_count = uint32_to_nbo(cont);
return 0;
}
static int mail_cache_grow(struct mail_cache *cache, uint32_t size)
{
struct stat st;
uoff_t grow_size, new_fsize;
new_fsize = cache->used_file_size + size;
grow_size = new_fsize / 100 * MAIL_CACHE_GROW_PERCENTAGE;
if (grow_size < 16384)
grow_size = 16384;
new_fsize += grow_size;
new_fsize &= ~1023;
if (fstat(cache->fd, &st) < 0) {
mail_cache_set_syscall_error(cache, "fstat()");
return -1;
}
if (cache->used_file_size + size <= (uoff_t)st.st_size) {
/* no need to grow, just update mmap */
if (mmap_update(cache, 0, 0) < 0)
return -1;
i_assert(cache->mmap_length >= (uoff_t)st.st_size);
return 0;
}
if (file_set_size(cache->fd, (off_t)new_fsize) < 0) {
mail_cache_set_syscall_error(cache, "file_set_size()");
return -1;
}
return mmap_update(cache, 0, 0);
}
static uint32_t mail_cache_append_space(struct mail_cache_transaction_ctx *ctx,
uint32_t size)
{
/* NOTE: must be done within transaction or rollback would break it */
uint32_t offset;
i_assert((size & 3) == 0);
offset = ctx->cache->used_file_size;
if (offset >= 0x40000000) {
mail_index_set_error(ctx->cache->index,
"Cache file too large: %s",
ctx->cache->filepath);
return 0;
}
if (offset + size > ctx->cache->mmap_length) {
if (mail_cache_grow(ctx->cache, size) < 0)
return 0;
}
ctx->cache->used_file_size += size;
return offset;
}
static int mail_cache_write(struct mail_cache_transaction_ctx *ctx)
{
struct mail_cache *cache = ctx->cache;
struct mail_cache_record *cache_rec, *next;
const struct mail_index_record *rec;
uint32_t write_offset, update_offset;
const void *buf;
size_t size, buf_size;
buf = buffer_get_data(ctx->cache_data, &buf_size);
size = sizeof(*cache_rec) + buf_size;
ctx->cache_rec.size = uint32_to_nbo(size);
write_offset = mail_cache_append_space(ctx, size);
if (write_offset == 0)
return -1;
// FIXME: check cache_offset in transaction
if (mail_index_lookup_latest(ctx->view->view, ctx->prev_seq, &rec) < 0)
return -1;
cache_rec = mail_cache_get_record(cache, rec->cache_offset);
if (cache_rec == NULL) {
/* first cache record - update offset in index file */
mail_index_update_cache(ctx->trans, ctx->prev_seq,
write_offset);
} else {
/* find the last cache record */
while ((next = mail_cache_get_next_record(cache,
cache_rec)) != NULL)
cache_rec = next;
/* mark next_offset to be updated later */
update_offset = (char *) &cache_rec->next_offset -
(char *) cache->mmap_base;
mark_update(&ctx->cache_marks, update_offset,
mail_cache_uint32_to_offset(write_offset));
}
ctx->prev_seq = 0;
ctx->prev_fields = 0;
memcpy((char *) cache->mmap_base + write_offset,
&ctx->cache_rec, sizeof(ctx->cache_rec));
memcpy((char *) cache->mmap_base + write_offset +
sizeof(ctx->cache_rec), buf, buf_size);
/* reset the write context */
memset(&ctx->cache_rec, 0, sizeof(ctx->cache_rec));
buffer_set_used_size(ctx->cache_data, 0);
return 0;
}
int mail_cache_transaction_commit(struct mail_cache_transaction_ctx *ctx)
{
int ret = 0;
if (ctx->prev_seq != 0) {
if (mail_cache_write(ctx) < 0)
return -1;
}
ctx->cache->hdr->used_file_size =
uint32_to_nbo(ctx->cache->used_file_size);
if (commit_all_changes(ctx) < 0)
ret = -1;
if (ctx->next_unused_header_lowwater == MAIL_CACHE_HEADERS_COUNT) {
/* they're all used - compress the cache to get more */
/* FIXME: ctx->cache->index->set_flags |=
MAIL_INDEX_HDR_FLAG_COMPRESS_CACHE;*/
}
mail_cache_transaction_flush(ctx);
return ret;
}
void mail_cache_transaction_rollback(struct mail_cache_transaction_ctx *ctx)
{
struct mail_cache *cache = ctx->cache;
unsigned int i;
/* no need to actually modify the file - we just didn't update
used_file_size */
cache->used_file_size = nbo_to_uint32(cache->hdr->used_file_size);
/* make sure we don't cache the headers */
for (i = 0; i < ctx->next_unused_header_lowwater; i++) {
uint32_t offset = cache->hdr->header_offsets[i];
if (mail_cache_offset_to_uint32(offset) == 0)
cache->split_offsets[i] = 1;
}
mail_cache_transaction_flush(ctx);
}
static const char *write_header_string(const char *const headers[],
uint32_t *size_r)
{
buffer_t *buffer;
size_t size;
buffer = buffer_create_dynamic(pool_datastack_create(),
512, (size_t)-1);
while (*headers != NULL) {
if (buffer_get_used_size(buffer) != 0)
buffer_append(buffer, "\n", 1);
buffer_append(buffer, *headers, strlen(*headers));
headers++;
}
buffer_append(buffer, null4, 1);
size = buffer_get_used_size(buffer);
if ((size & 3) != 0) {
buffer_append(buffer, null4, 4 - (size & 3));
size += 4 - (size & 3);
}
*size_r = size;
return buffer_get_data(buffer, NULL);
}
int mail_cache_set_header_fields(struct mail_cache_transaction_ctx *ctx,
unsigned int idx, const char *const headers[])
{
struct mail_cache *cache = ctx->cache;
uint32_t offset, update_offset, size;
const char *header_str, *prev_str;
i_assert(*headers != NULL);
i_assert(idx < MAIL_CACHE_HEADERS_COUNT);
i_assert(idx >= ctx->next_unused_header_lowwater);
i_assert(mail_cache_offset_to_uint32(cache->hdr->
header_offsets[idx]) == 0);
t_push();
header_str = write_header_string(headers, &size);
if (idx != 0) {
prev_str = mail_cache_get_header_fields_str(cache, idx-1);
if (prev_str == NULL) {
t_pop();
return FALSE;
}
i_assert(strcmp(header_str, prev_str) != 0);
}
offset = mail_cache_append_space(ctx, size + sizeof(uint32_t));
if (offset != 0) {
memcpy((char *) cache->mmap_base + offset + sizeof(uint32_t),
header_str, size);
size = uint32_to_nbo(size);
memcpy((char *) cache->mmap_base + offset,
&size, sizeof(uint32_t));
/* update cached headers */
cache->split_offsets[idx] = cache->hdr->header_offsets[idx];
cache->split_headers[idx] =
mail_cache_split_header(cache, header_str);
/* mark used-bit to be updated later. not really needed for
read-safety, but if transaction get rolled back we can't let
this point to invalid location. */
update_offset = (char *) &cache->hdr->header_offsets[idx] -
(char *) cache->mmap_base;
mark_update(&ctx->cache_marks, update_offset,
mail_cache_uint32_to_offset(offset));
/* make sure get_header_fields() still works for this header
while the transaction isn't yet committed. */
ctx->next_unused_header_lowwater = idx + 1;
}
t_pop();
return offset > 0;
}
static size_t get_insert_offset(struct mail_cache_transaction_ctx *ctx,
enum mail_cache_field field)
{
const unsigned char *buf;
unsigned int mask;
uint32_t data_size;
size_t offset = 0;
int i;
buf = buffer_get_data(ctx->cache_data, NULL);
for (i = 0, mask = 1; i < 31; i++, mask <<= 1) {
if ((field & mask) != 0)
return offset;
if ((ctx->cache_rec.fields & mask) != 0) {
if ((mask & MAIL_CACHE_FIXED_MASK) != 0)
data_size = mail_cache_field_sizes[i];
else {
memcpy(&data_size, buf + offset,
sizeof(data_size));
data_size = nbo_to_uint32(data_size);
offset += sizeof(data_size);
}
offset += (data_size + 3) & ~3;
}
}
i_unreached();
return offset;
}
static int get_field_num(enum mail_cache_field field)
{
unsigned int mask;
int i;
for (i = 0, mask = 1; i < 31; i++, mask <<= 1) {
if ((field & mask) != 0)
return i;
}
return -1;
}
int mail_cache_add(struct mail_cache_transaction_ctx *ctx, uint32_t seq,
enum mail_cache_field field,
const void *data, size_t data_size)
{
uint32_t nb_data_size;
size_t full_size, offset;
unsigned char *buf;
int field_num;
i_assert(data_size > 0);
i_assert(data_size < (uint32_t)-1);
nb_data_size = uint32_to_nbo((uint32_t)data_size);
if ((field & MAIL_CACHE_FIXED_MASK) != 0) {
field_num = get_field_num(field);
i_assert(field_num != -1);
i_assert(mail_cache_field_sizes[field_num] == data_size);
} else if ((field & MAIL_CACHE_STRING_MASK) != 0) {
i_assert(((char *) data)[data_size-1] == '\0');
}
if (ctx->prev_seq != seq && ctx->prev_seq != 0) {
if (mail_cache_write(ctx) < 0)
return -1;
}
ctx->prev_seq = seq;
i_assert((ctx->cache_rec.fields & field) == 0);
full_size = (data_size + 3) & ~3;
if ((field & MAIL_CACHE_FIXED_MASK) == 0)
full_size += sizeof(nb_data_size);
/* fields must be ordered. find where to insert it. */
if (field > ctx->cache_rec.fields)
buf = buffer_append_space_unsafe(ctx->cache_data, full_size);
else {
offset = get_insert_offset(ctx, field);
buffer_copy(ctx->cache_data, offset + full_size,
ctx->cache_data, offset, (size_t)-1);
buf = buffer_get_space_unsafe(ctx->cache_data,
offset, full_size);
}
ctx->cache_rec.fields |= field;
/* @UNSAFE */
if ((field & MAIL_CACHE_FIXED_MASK) == 0) {
memcpy(buf, &nb_data_size, sizeof(nb_data_size));
buf += sizeof(nb_data_size);
}
memcpy(buf, data, data_size); buf += data_size;
if ((data_size & 3) != 0)
memset(buf, 0, 4 - (data_size & 3));
/* remember the transaction uid range */
if (seq < ctx->first_seq || ctx->first_seq == 0)
ctx->first_seq = seq;
if (seq > ctx->last_seq)
ctx->last_seq = seq;
ctx->prev_fields |= field;
return 0;
}
int mail_cache_delete(struct mail_cache_transaction_ctx *ctx, uint32_t seq)
{
struct mail_cache *cache = ctx->cache;
struct mail_cache_record *cache_rec;
uint32_t deleted_space;
uoff_t max_del_space;
cache_rec = mail_cache_lookup(ctx->view, seq, 0);
if (cache_rec == NULL)
return 0;
/* we'll only update the deleted_space in header. we can't really
do any actual deleting as other processes might still be using
the data. also it's actually useful as some index views are still
able to ask cached data from messages that have already been
expunged. */
deleted_space = nbo_to_uint32(cache->hdr->deleted_space);
do {
deleted_space -= nbo_to_uint32(cache_rec->size);
cache_rec = mail_cache_get_next_record(cache, cache_rec);
} while (cache_rec != NULL);
/* see if we've reached the max. deleted space in file */
max_del_space = cache->used_file_size / 100 * COMPRESS_PERCENTAGE;
if (deleted_space >= max_del_space &&
cache->used_file_size >= COMPRESS_MIN_SIZE) {
//FIXME:cache->index->set_flags |= MAIL_INDEX_HDR_FLAG_COMPRESS_CACHE;
}
cache->hdr->deleted_space = uint32_to_nbo(deleted_space);
return 0;
}
int
mail_cache_transaction_autocommit(struct mail_cache_view *view,
uint32_t seq, enum mail_cache_field fields)
{
struct mail_cache *cache = view->cache;
if (cache->trans_ctx != NULL &&
cache->trans_ctx->first_seq <= seq &&
cache->trans_ctx->last_seq >= seq &&
(cache->trans_ctx->prev_seq != seq || fields == 0 ||
(cache->trans_ctx->prev_fields & fields) != 0)) {
/* write non-index changes */
if (cache->trans_ctx->prev_seq == seq) {
if (mail_cache_write(cache->trans_ctx) < 0)
return -1;
}
if (mail_cache_transaction_commit(cache->trans_ctx) < 0)
return -1;
}
return 0;
}
#else
#endif
--- NEW FILE: mail-index-lock.c ---
/* Copyright (C) 2003-2004 Timo Sirainen */
/*
Locking is meant to be as transparent as possible. Anything that locks
the index must either keep it only a short time, or be prepared that the
lock is lost.
Lock is lost in only one situation: when we try to get an exclusive lock
but we already have a shared lock. Then we'll drop all shared locks and
get the exclusive lock.
Locking should never fail or timeout. Exclusive locks must be kept as short
time as possible. Shared locks can be long living, so if can't get exclusive
lock directly within 2 seconds, we'll replace the index file with a copy of
it. That means the shared lock holders can keep using the old file while
we're modifying the new file.
lock_id is used to figure out if acquired lock is still valid. Shared
locks have even numbers, exclusive locks have odd numbers. The number is
increased by two every time the lock is dropped.
mail_index_lock_shared() -> lock_id=2
mail_index_lock_shared() -> lock_id=2
mail_index_lock_exclusive() -> lock_id=5 (had to drop shared locks)
mail_index_lock_shared() -> lock_id=4
Only 4 and 5 locks are valid at this time.
*/
#include "lib.h"
#include "file-lock.h"
#include "write-full.h"
#include "mail-index-private.h"
#include <stdio.h>
#include <sys/stat.h>
static int mail_index_reopen(struct mail_index *index, int fd)
{
int ret;
mail_index_unmap(index, index->map);
index->map = NULL;
if (close(index->fd) < 0)
mail_index_set_syscall_error(index, "close()");
index->fd = fd;
ret = fd < 0 ? mail_index_try_open(index) :
mail_index_map(index, FALSE);
if (ret <= 0) {
// FIXME: serious problem, we'll just crash later..
return -1;
}
return 0;
}
static int mail_index_has_changed(struct mail_index *index)
{
struct stat st1, st2;
if (fstat(index->fd, &st1) < 0)
return mail_index_set_syscall_error(index, "fstat()");
if (stat(index->filepath, &st2) < 0)
return mail_index_set_syscall_error(index, "stat()");
if (st1.st_ino != st2.st_ino ||
!CMP_DEV_T(st1.st_dev, st2.st_dev)) {
if (mail_index_reopen(index, -1) < 0)
return -1;
return 1;
} else {
return 0;
}
}
static int mail_index_lock(struct mail_index *index, int lock_type,
unsigned int timeout_secs, int update_index,
unsigned int *lock_id_r)
{
// FIXME: mprotect() the index to make sure we don't access it unlocked!
int ret;
i_assert(lock_type == F_RDLCK || lock_type == F_WRLCK);
if (lock_type == F_WRLCK && index->lock_type == F_RDLCK) {
/* drop shared locks */
i_assert(index->excl_lock_count == 0);
if (file_wait_lock(index->fd, F_UNLCK) < 0)
mail_index_set_syscall_error(index, "file_wait_lock()");
index->shared_lock_count = 0;
index->lock_type = F_UNLCK;
index->lock_id += 2; /* make sure failures below work right */
}
if (index->excl_lock_count > 0 || index->shared_lock_count > 0) {
i_assert(lock_type == F_RDLCK || index->excl_lock_count > 0);
if (lock_type == F_RDLCK) {
index->shared_lock_count++;
*lock_id_r = index->lock_id;
} else {
index->excl_lock_count++;
*lock_id_r = index->lock_id + 1;
}
return 1;
}
i_assert(index->lock_type == F_UNLCK);
if (update_index && lock_type != F_WRLCK) {
if (mail_index_has_changed(index) < 0)
return -1;
}
do {
ret = file_wait_lock_full(index->fd, lock_type, timeout_secs,
NULL, NULL);
if (ret <= 0) {
if (ret == 0)
return 0;
mail_index_set_syscall_error(index, "file_wait_lock()");
return -1;
}
if (lock_type == F_WRLCK) {
/* we need to have the latest index file locked -
check if it's been updated. */
if ((ret = mail_index_has_changed(index)) < 0) {
(void)file_wait_lock(index->fd, F_UNLCK);
return -1;
}
if (ret > 0)
continue;
}
} while (0);
index->lock_type = lock_type;
index->lock_id += 2;
if (lock_type == F_RDLCK) {
index->shared_lock_count++;
*lock_id_r = index->lock_id;
} else {
index->excl_lock_count++;
*lock_id_r = index->lock_id + 1;
}
return 1;
}
int mail_index_lock_shared(struct mail_index *index, int update_index,
unsigned int *lock_id_r)
{
int ret;
ret = mail_index_lock(index, F_RDLCK, DEFAULT_LOCK_TIMEOUT,
update_index, lock_id_r);
if (ret > 0)
return 0;
if (ret < 0)
return -1;
mail_index_set_error(index, "Timeout while waiting for release of "
"shared fcntl() lock for index file %s",
index->filepath);
index->index_lock_timeout = TRUE;
return -1;
}
static int mail_index_copy(struct mail_index *index)
{
const char *path;
int ret, fd;
fd = mail_index_create_tmp_file(index, &path);
if (fd == -1)
return -1;
ret = 0;
if (write_full(fd, index->map->mmap_base,
index->map->mmap_used_size) < 0) {
mail_index_file_set_syscall_error(index, path, "write_full()");
(void)close(fd);
(void)unlink(path);
return -1;
}
i_assert(index->copy_lock_path == NULL);
index->copy_lock_path = i_strdup(path);
return fd;
}
static int mail_index_need_lock(struct mail_index *index,
uint32_t log_file_seq, uoff_t log_file_offset)
{
if (mail_index_map(index, FALSE) <= 0)
return 1;
if (log_file_seq != 0 &&
(index->hdr->log_file_seq > log_file_seq ||
(index->hdr->log_file_seq == log_file_seq &&
index->hdr->log_file_offset >= log_file_offset))) {
/* already synced */
return 0;
}
return 1;
}
int mail_index_lock_exclusive(struct mail_index *index,
uint32_t log_file_seq, uoff_t log_file_offset,
unsigned int *lock_id_r)
{
unsigned int lock_id;
int ret;
/* exclusive transaction log lock protects exclusive locking
for the main index file */
i_assert(index->log_locked);
/* wait two seconds for exclusive lock */
ret = mail_index_lock(index, F_WRLCK, 2, TRUE, lock_id_r);
if (ret > 0) {
if (mail_index_need_lock(index, log_file_seq, log_file_offset))
return 1;
mail_index_unlock(index, *lock_id_r);
return 0;
}
if (ret < 0)
return -1;
/* Grab shared lock to make sure it's not already being
exclusively locked */
if (mail_index_lock_shared(index, TRUE, &lock_id) < 0)
return -1;
if (log_file_seq != 0) {
/* check first if we really need to recreate it */
ret = mail_index_need_lock(index, log_file_seq,
log_file_offset);
if (ret == 0) {
mail_index_unlock(index, lock_id);
return 0;
}
}
mail_index_unlock(index, lock_id);
*lock_id_r = 0;
return mail_index_lock_exclusive_copy(index);
}
int mail_index_lock_exclusive_copy(struct mail_index *index)
{
int fd;
if (index->copy_lock_path != NULL) {
index->excl_lock_count++;
return 1;
}
/* copy the index to index.tmp and use it. when */
fd = mail_index_copy(index);
if (fd == -1)
return -1;
if (mail_index_reopen(index, fd) < 0) {
(void)mail_index_reopen(index, -1);
i_free(index->copy_lock_path);
index->copy_lock_path = NULL;
return -1;
}
index->lock_type = F_WRLCK;
index->excl_lock_count++;
return 1;
}
static void mail_index_copy_lock_finish(struct mail_index *index)
{
if (fsync(index->fd) < 0) {
mail_index_file_set_syscall_error(index, index->copy_lock_path,
"fsync()");
}
if (rename(index->copy_lock_path, index->filepath) < 0) {
mail_index_set_error(index, "rename(%s, %s) failed: %m",
index->copy_lock_path, index->filepath);
// FIXME: this isn't good
}
i_free(index->copy_lock_path);
index->copy_lock_path = NULL;
index->shared_lock_count = 0;
index->lock_id += 2;
index->lock_type = F_UNLCK;
}
void mail_index_unlock(struct mail_index *index, unsigned int lock_id)
{
if (index->copy_lock_path != NULL) {
i_assert(index->log_locked);
i_assert(index->excl_lock_count > 0);
if (--index->excl_lock_count == 0)
mail_index_copy_lock_finish(index);
return;
}
if ((lock_id & 1) == 0) {
/* shared lock */
if (mail_index_is_locked(index, lock_id)) {
i_assert(index->shared_lock_count > 0);
index->shared_lock_count--;
}
} else {
/* exclusive lock */
i_assert(lock_id == index->lock_id+1);
i_assert(index->excl_lock_count > 0);
index->excl_lock_count--;
}
if (index->shared_lock_count == 0 && index->excl_lock_count == 0) {
index->lock_id += 2;
index->lock_type = F_UNLCK;
if (file_wait_lock(index->fd, F_UNLCK) < 0)
mail_index_set_syscall_error(index, "file_wait_lock()");
}
}
int mail_index_is_locked(struct mail_index *index, unsigned int lock_id)
{
return (index->lock_id ^ lock_id) <= 1;
}
--- NEW FILE: mail-index-private.h ---
#ifndef __MAIL_INDEX_PRIVATE_H
#define __MAIL_INDEX_PRIVATE_H
#include "mail-index.h"
struct mail_transaction_header;
/* number of records to always keep allocated in index file,
either used or unused */
#define INDEX_MIN_RECORDS_COUNT 64
/* when empty space in index file gets full, grow the file n% larger */
#define INDEX_GROW_PERCENTAGE 10
/* ftruncate() the index file when only n% of it is in use */
#define INDEX_TRUNCATE_PERCENTAGE 30
/* don't truncate whole file anyway, keep n% of the empty space */
#define INDEX_TRUNCATE_KEEP_PERCENTAGE 10
/* Compress the file when deleted space reaches n% of total size */
#define INDEX_COMPRESS_PERCENTAGE 50
/* Compress the file when searching deleted records tree has to go this deep */
#define INDEX_COMPRESS_DEPTH 10
enum mail_index_mail_flags {
MAIL_INDEX_MAIL_FLAG_DIRTY = 0x80,
MAIL_INDEX_MAIL_FLAG_EXPUNGED = 0x40,
MAIL_INDEX_MAIL_FLAG_NONRECENT = MAIL_RECENT
};
#define MAIL_INDEX_MAP_IS_IN_MEMORY(map) \
((map)->buffer != NULL)
struct mail_index_map {
int refcount;
const struct mail_index_header *hdr;
struct mail_index_record *records;
unsigned int records_count;
void *mmap_base;
size_t mmap_size, mmap_used_size;
buffer_t *buffer;
struct mail_index_header hdr_copy;
};
struct mail_index {
char *dir, *prefix;
struct mail_cache *cache;
struct mail_transaction_log *log;
mode_t mode;
gid_t gid;
char *filepath;
int fd;
struct mail_index_map *map;
const struct mail_index_header *hdr;
uint32_t indexid;
int lock_type, shared_lock_count, excl_lock_count;
unsigned int lock_id, copy_lock_id;
char *copy_lock_path;
char *error;
unsigned int nodiskspace:1;
unsigned int index_lock_timeout:1;
unsigned int opened:1;
unsigned int log_locked:1;
unsigned int use_mmap:1;
unsigned int readonly:1;
unsigned int fsck:1;
};
void mail_index_header_init(struct mail_index_header *hdr);
int mail_index_write_header(struct mail_index *index,
const struct mail_index_header *hdr);
int mail_index_create(struct mail_index *index, struct mail_index_header *hdr);
int mail_index_try_open(struct mail_index *index);
int mail_index_create_tmp_file(struct mail_index *index, const char **path_r);
/* Returns 0 = ok, -1 = error. If update_index is TRUE, reopens the index
file if needed to get later version of it (not necessarily latest due to
races, unless transaction log is exclusively locked). */
int mail_index_lock_shared(struct mail_index *index, int update_index,
unsigned int *lock_id_r);
/* Returns 1 = ok, 0 = already synced up to given log_file_offset, -1 = error */
int mail_index_lock_exclusive(struct mail_index *index,
uint32_t log_file_seq, uoff_t log_file_offset,
unsigned int *lock_id_r);
int mail_index_lock_exclusive_copy(struct mail_index *index);
void mail_index_unlock(struct mail_index *index, unsigned int lock_id);
/* Returns 1 if given lock_id is valid, 0 if not. */
int mail_index_is_locked(struct mail_index *index, unsigned int lock_id);
/* Map index file to memory, replacing the previous mapping for index.
Returns 1 = ok, 0 = corrupted, -1 = error. If index needs fscking, it
returns 1 but sets index->fsck = TRUE. */
int mail_index_map(struct mail_index *index, int force);
/* Unreference given mapping and unmap it if it's dropped to zero. */
void mail_index_unmap(struct mail_index *index, struct mail_index_map *map);
struct mail_index_map *mail_index_map_to_memory(struct mail_index_map *map);
void mail_index_update_cache(struct mail_index_transaction *t,
uint32_t seq, uint32_t offset);
int mail_index_fix_header(struct mail_index *index, struct mail_index_map *map,
struct mail_index_header *hdr, const char **error_r);
void mail_index_view_transaction_ref(struct mail_index_view *view);
void mail_index_view_transaction_unref(struct mail_index_view *view);
int mail_index_sync_get_rec(struct mail_index_view *view,
struct mail_index_sync_rec *rec,
const struct mail_transaction_header *hdr,
const void *data, size_t *data_offset);
int mail_index_mark_corrupted(struct mail_index *index);
int mail_index_set_error(struct mail_index *index, const char *fmt, ...)
__attr_format__(2, 3);
/* "%s failed with index file %s: %m" */
int mail_index_set_syscall_error(struct mail_index *index,
const char *function);
/* "%s failed with file %s: %m" */
int mail_index_file_set_syscall_error(struct mail_index *index,
const char *filepath,
const char *function);
void mail_index_reset_error(struct mail_index *index);
#endif
--- NEW FILE: mail-index-reset.c ---
/* Copyright (C) 2004 Timo Sirainen */
#include "lib.h"
#include "mmap-util.h"
#include "write-full.h"
#include "mail-index-private.h"
#include "mail-transaction-log.h"
int mail_index_reset(struct mail_index *index)
{
struct mail_index_header hdr;
/* this invalidates all views even if we fail later */
index->indexid = 0;
if (mail_index_mark_corrupted(index) < 0)
return -1;
mail_index_header_init(&hdr);
if (hdr.indexid == index->indexid)
hdr.indexid++;
// FIXME: close it? ..
if (mail_index_create(index, &hdr) < 0)
return -1;
/* reopen transaction log - FIXME: doesn't work, we have log views
open.. */
mail_transaction_log_close(index->log);
index->log = mail_transaction_log_open_or_create(index);
if (index->log == NULL) {
/* FIXME: creates potential crashes.. */
return -1;
}
return 0;
}
--- NEW FILE: mail-index-sync-private.h ---
#ifndef __MAIL_INDEX_SYNC_PRIVATE_H
#define __MAIL_INDEX_SYNC_PRIVATE_H
struct mail_index_sync_ctx {
struct mail_index *index;
struct mail_index_view *view;
buffer_t *expunges_buf, *updates_buf, *appends_buf;
const struct mail_transaction_expunge *expunges;
const struct mail_transaction_flag_update *updates;
size_t expunges_count, updates_count;
const struct mail_transaction_header *hdr;
const void *data;
size_t expunge_idx, update_idx;
uint32_t next_seq;
unsigned int lock_id;
unsigned int sync_appends:1;
};
int mail_index_sync_update_index(struct mail_index_sync_ctx *sync_ctx);
void mail_index_header_update_counts(struct mail_index_header *hdr,
uint8_t old_flags, uint8_t new_flags);
void mail_index_header_update_lowwaters(struct mail_index_header *hdr,
const struct mail_index_record *rec);
#endif
--- NEW FILE: mail-index-sync-update.c ---
/* Copyright (C) 2004 Timo Sirainen */
#include "lib.h"
#include "buffer.h"
#include "file-set-size.h"
#include "mmap-util.h"
#include "mail-index-view-private.h"
#include "mail-index-sync-private.h"
#include "mail-transaction-log.h"
struct mail_index_update_ctx {
struct mail_index *index;
struct mail_index_header hdr;
struct mail_transaction_log_view *log_view;
};
void mail_index_header_update_counts(struct mail_index_header *hdr,
uint8_t old_flags, uint8_t new_flags)
{
if (((old_flags ^ new_flags) & MAIL_SEEN) != 0) {
/* different seen-flag */
if ((old_flags & MAIL_SEEN) == 0)
hdr->seen_messages_count++;
else
hdr->seen_messages_count--;
}
if (((old_flags ^ new_flags) & MAIL_DELETED) != 0) {
/* different deleted-flag */
if ((old_flags & MAIL_DELETED) == 0)
hdr->deleted_messages_count++;
else
hdr->deleted_messages_count--;
}
}
void mail_index_header_update_lowwaters(struct mail_index_header *hdr,
const struct mail_index_record *rec)
{
if ((rec->flags & MAIL_RECENT) != 0 &&
rec->uid < hdr->first_recent_uid_lowwater)
hdr->first_recent_uid_lowwater = rec->uid;
if ((rec->flags & MAIL_SEEN) == 0 &&
rec->uid < hdr->first_unseen_uid_lowwater)
hdr->first_unseen_uid_lowwater = rec->uid;
if ((rec->flags & MAIL_DELETED) != 0 &&
rec->uid < hdr->first_deleted_uid_lowwater)
hdr->first_deleted_uid_lowwater = rec->uid;
}
static void mail_index_sync_update_expunges(struct mail_index_update_ctx *ctx,
uint32_t seq1, uint32_t seq2)
{
struct mail_index_record *rec;
rec = &ctx->index->map->records[seq1-1];
for (; seq1 <= seq2; seq1++, rec++)
mail_index_header_update_counts(&ctx->hdr, rec->flags, 0);
}
static void mail_index_sync_update_flags(struct mail_index_update_ctx *ctx,
struct mail_index_sync_rec *syncrec)
{
struct mail_index_record *rec, *end;
uint8_t flag_mask, old_flags;
custom_flags_mask_t custom_mask;
int i, update_custom;
update_custom = FALSE;
for (i = 0; i < INDEX_CUSTOM_FLAGS_BYTE_COUNT; i++) {
if (syncrec->add_custom_flags[i] != 0)
update_custom = TRUE;
if (syncrec->remove_custom_flags[i] != 0)
update_custom = TRUE;
custom_mask[i] = ~syncrec->remove_custom_flags[i];
}
flag_mask = ~syncrec->remove_flags;
rec = &ctx->index->map->records[syncrec->seq1-1];
end = rec + (syncrec->seq2 - syncrec->seq1) + 1;
for (; rec != end; rec++) {
old_flags = rec->flags;
rec->flags = (rec->flags & flag_mask) | syncrec->add_flags;
if (update_custom) {
for (i = 0; i < INDEX_CUSTOM_FLAGS_BYTE_COUNT; i++) {
rec->custom_flags[i] =
(rec->custom_flags[i]&custom_mask[i]) |
syncrec->add_custom_flags[i];
}
}
mail_index_header_update_counts(&ctx->hdr,
old_flags, rec->flags);
mail_index_header_update_lowwaters(&ctx->hdr, rec);
}
}
static int mail_index_grow(struct mail_index *index, unsigned int count)
{
size_t size, mmap_used_size;
unsigned int records_count;
// FIXME: grow exponentially
size = index->map->mmap_used_size +
count * sizeof(struct mail_index_record);
if (file_set_size(index->fd, (off_t)size) < 0)
return mail_index_set_syscall_error(index, "file_set_size()");
records_count = index->map->records_count;
mmap_used_size = index->map->mmap_used_size;
if (mail_index_map(index, TRUE) <= 0)
return -1;
i_assert(index->map->mmap_size >= size);
index->map->records_count = records_count;
index->map->mmap_used_size = mmap_used_size;
return 0;
}
static int mail_index_sync_appends(struct mail_index_update_ctx *ctx,
const struct mail_index_record *appends,
unsigned int count)
{
struct mail_index_map *map = ctx->index->map;
unsigned int i;
size_t space;
uint32_t next_uid;
if (!ctx->index->use_mmap) {
// FIXME
}
space = (map->mmap_size - map->mmap_used_size) / sizeof(*appends);
if (space < count) {
if (mail_index_grow(ctx->index, count) < 0)
return -1;
if (mprotect(map->mmap_base, map->mmap_size,
PROT_READ|PROT_WRITE) < 0) {
mail_index_set_syscall_error(ctx->index, "mprotect()");
return -1;
}
}
next_uid = ctx->hdr.next_uid;
for (i = 0; i < count; i++) {
mail_index_header_update_counts(&ctx->hdr, 0, appends[i].flags);
mail_index_header_update_lowwaters(&ctx->hdr, &appends[i]);
if (appends[i].uid < next_uid) {
/* FIXME: should we rather just update the record?
this can actually happen if append was written to
transaction log but index wasn't updated, then
another sync wrote it again.. */
mail_transaction_log_view_set_corrupted(ctx->log_view,
"Append with UID %u, but next_uid = %u",
appends[i].uid, next_uid);
return -1;
}
next_uid = appends[i].uid+1;
}
ctx->hdr.next_uid = next_uid;
memcpy(map->records + map->records_count, appends,
count * sizeof(*appends));
map->records_count += count;
map->mmap_used_size += count * sizeof(struct mail_index_record);
return 0;
}
int mail_index_sync_update_index(struct mail_index_sync_ctx *sync_ctx)
{
struct mail_index *index = sync_ctx->index;
struct mail_index_map *map = index->map;
struct mail_index_update_ctx ctx;
struct mail_index_sync_rec rec;
const struct mail_index_record *appends;
unsigned int append_count;
uint32_t count, file_seq, src_idx, dest_idx;
uoff_t file_offset;
int ret, locked = FALSE;
if (mprotect(map->mmap_base, map->mmap_size, PROT_READ|PROT_WRITE) < 0)
return mail_index_set_syscall_error(index, "mprotect()");
/* rewind */
sync_ctx->update_idx = sync_ctx->expunge_idx = 0;
sync_ctx->sync_appends =
buffer_get_used_size(sync_ctx->appends_buf) != 0;
memset(&ctx, 0, sizeof(ctx));
ctx.index = index;
ctx.hdr = *index->hdr;
ctx.log_view = sync_ctx->view->log_view;
src_idx = dest_idx = 0;
append_count = 0; appends = NULL;
while (mail_index_sync_next(sync_ctx, &rec) > 0) {
switch (rec.type) {
case MAIL_INDEX_SYNC_TYPE_APPEND:
i_assert(appends == NULL);
append_count = rec.seq2 - rec.seq1 + 1;
appends = rec.appends;
break;
case MAIL_INDEX_SYNC_TYPE_EXPUNGE:
if (src_idx != 0) {
count = (rec.seq1-1) - src_idx;
memmove(map->records + dest_idx,
map->records + src_idx,
count * sizeof(*map->records));
dest_idx += count;
} else {
dest_idx = rec.seq1-1;
if (mail_index_lock_exclusive_copy(index) <= 0)
return -1;
map = index->map;
if (mprotect(map->mmap_base, map->mmap_size,
PROT_READ|PROT_WRITE) < 0) {
mail_index_set_syscall_error(index,
"mprotect()");
return -1;
}
locked = TRUE;
}
mail_index_sync_update_expunges(&ctx, rec.seq1,
rec.seq2);
src_idx = rec.seq2;
break;
case MAIL_INDEX_SYNC_TYPE_FLAGS:
mail_index_sync_update_flags(&ctx, &rec);
break;
}
}
if (src_idx != 0) {
count = map->records_count - src_idx;
memmove(map->records + dest_idx,
map->records + src_idx,
count * sizeof(*map->records));
dest_idx += count;
map->records_count = dest_idx;
map->mmap_used_size = index->hdr->header_size +
map->records_count * sizeof(struct mail_index_record);
}
ret = 0;
if (append_count > 0)
ret = mail_index_sync_appends(&ctx, appends, append_count);
mail_transaction_log_get_head(index->log, &file_seq, &file_offset);
ctx.hdr.messages_count = map->records_count;
ctx.hdr.log_file_seq = file_seq;
ctx.hdr.log_file_offset = file_offset;
if (index->use_mmap) {
memcpy(map->mmap_base, &ctx.hdr, sizeof(ctx.hdr));
if (msync(map->mmap_base, map->mmap_used_size, MS_SYNC) < 0)
return mail_index_set_syscall_error(index, "msync()");
} else {
// FIXME
}
if (mprotect(map->mmap_base, map->mmap_size, PROT_READ) < 0)
mail_index_set_syscall_error(index, "mprotect()");
if (locked)
mail_index_unlock(index, 0);
return ret;
}
--- NEW FILE: mail-index-sync.c ---
/* Copyright (C) 2003-2004 Timo Sirainen */
#include "lib.h"
#include "buffer.h"
#include "mail-index-view-private.h"
#include "mail-index-sync-private.h"
#include "mail-transaction-log.h"
#include "mail-transaction-util.h"
#include <stdlib.h>
static void mail_index_sync_sort_flags(struct mail_index_sync_ctx *ctx)
{
const struct mail_transaction_flag_update *src, *src_end;
const struct mail_transaction_flag_update *dest;
struct mail_transaction_flag_update new_update;
struct mail_transaction_expunge_traverse_ctx *exp_ctx;
uint32_t last;
size_t i, dest_count;
src = ctx->data;
src_end = PTR_OFFSET(src, ctx->hdr->size);
dest = buffer_get_data(ctx->updates_buf, &dest_count);
dest_count /= sizeof(*dest);
exp_ctx = mail_transaction_expunge_traverse_init(ctx->expunges_buf);
for (i = 0; src != src_end; src++) {
new_update = *src;
/* find seq1 */
new_update.seq1 +=
mail_transaction_expunge_traverse_to(exp_ctx,
src->seq1);
/* find seq2 */
new_update.seq2 +=
mail_transaction_expunge_traverse_to(exp_ctx,
src->seq2);
/* insert it into buffer, split it in multiple parts if needed
to make sure the ordering stays the same */
for (; i < dest_count; i++) {
if (dest[i].seq1 <= new_update.seq1)
continue;
if (dest[i].seq1 > new_update.seq2)
break;
/* partial */
last = new_update.seq2;
new_update.seq2 = dest[i].seq1-1;
buffer_insert(ctx->updates_buf, i * sizeof(new_update),
&new_update, sizeof(new_update));
dest = buffer_get_data(ctx->updates_buf, NULL);
dest_count++;
new_update.seq1 = new_update.seq2+1;
new_update.seq2 = last;
}
buffer_insert(ctx->updates_buf, i * sizeof(new_update),
&new_update, sizeof(new_update));
dest = buffer_get_data(ctx->updates_buf, NULL);
dest_count++;
}
mail_transaction_expunge_traverse_deinit(exp_ctx);
}
static void mail_index_sync_sort_transaction(struct mail_index_sync_ctx *ctx)
{
switch (ctx->hdr->type & MAIL_TRANSACTION_TYPE_MASK) {
case MAIL_TRANSACTION_EXPUNGE:
if (buffer_get_used_size(ctx->expunges_buf) == 0) {
buffer_append(ctx->expunges_buf, ctx->data,
ctx->hdr->size);
} else {
mail_transaction_log_sort_expunges(ctx->expunges_buf,
ctx->data,
ctx->hdr->size);
}
break;
case MAIL_TRANSACTION_FLAG_UPDATE:
if (buffer_get_used_size(ctx->expunges_buf) == 0 &&
buffer_get_used_size(ctx->updates_buf) == 0) {
buffer_append(ctx->updates_buf, ctx->data,
ctx->hdr->size);
} else {
mail_index_sync_sort_flags(ctx);
}
break;
case MAIL_TRANSACTION_APPEND:
buffer_append(ctx->appends_buf, ctx->data, ctx->hdr->size);
ctx->sync_appends = TRUE;
break;
}
}
static int mail_index_sync_read_and_sort(struct mail_index_sync_ctx *ctx,
int external)
{
enum mail_transaction_type flag;
int ret;
flag = external ? MAIL_TRANSACTION_EXTERNAL : 0;
while ((ret = mail_transaction_log_view_next(ctx->view->log_view,
&ctx->hdr,
&ctx->data, NULL)) > 0) {
if ((ctx->hdr->type & MAIL_TRANSACTION_EXTERNAL) == flag)
mail_index_sync_sort_transaction(ctx);
}
return ret;
}
int mail_index_sync_begin(struct mail_index *index,
struct mail_index_sync_ctx **ctx_r,
struct mail_index_view **view_r,
uint32_t log_file_seq, uoff_t log_file_offset)
{
struct mail_index_sync_ctx *ctx;
uint32_t seq;
uoff_t offset;
size_t size;
unsigned int lock_id;
int ret;
if (mail_transaction_log_sync_lock(index->log, &seq, &offset) < 0)
return -1;
/* FIXME: really needed yet? If there are readers, the index file
is copied even if there are no changes.. */
ret = mail_index_lock_exclusive(index, log_file_seq,
log_file_offset, &lock_id);
if (ret <= 0) {
mail_transaction_log_sync_unlock(index->log);
return ret;
}
if (mail_index_map(index, FALSE) <= 0) {
mail_transaction_log_sync_unlock(index->log);
return -1;
}
ctx = i_new(struct mail_index_sync_ctx, 1);
ctx->index = index;
ctx->lock_id = lock_id;
ctx->view = mail_index_view_open(index);
ctx->view->external = TRUE;
if (mail_transaction_log_view_set(ctx->view->log_view,
index->hdr->log_file_seq,
index->hdr->log_file_offset,
seq, offset,
MAIL_TRANSACTION_TYPE_MASK) < 0) {
mail_index_sync_end(ctx);
return -1;
}
/* we need to have all the transactions sorted to optimize
caller's mailbox access patterns */
ctx->expunges_buf = buffer_create_dynamic(default_pool,
1024, (size_t)-1);
ctx->updates_buf = buffer_create_dynamic(default_pool,
1024, (size_t)-1);
ctx->appends_buf = buffer_create_dynamic(default_pool,
1024, (size_t)-1);
if (mail_index_sync_read_and_sort(ctx, FALSE) < 0) {
mail_index_sync_end(ctx);
return -1;
}
ctx->expunges = buffer_get_data(ctx->expunges_buf, &size);
ctx->expunges_count = size / sizeof(*ctx->expunges);
ctx->updates = buffer_get_data(ctx->updates_buf, &size);
ctx->updates_count = size / sizeof(*ctx->updates);
*ctx_r = ctx;
*view_r = ctx->view;
return 1;
}
static void
mail_index_sync_get_expunge(struct mail_index_sync_rec *rec,
const struct mail_transaction_expunge *exp)
{
rec->type = MAIL_INDEX_SYNC_TYPE_EXPUNGE;
rec->seq1 = exp->seq1;
rec->seq2 = exp->seq2;
}
static void
mail_index_sync_get_update(struct mail_index_sync_rec *rec,
const struct mail_transaction_flag_update *update)
{
rec->type = MAIL_INDEX_SYNC_TYPE_FLAGS;
rec->seq1 = update->seq1;
rec->seq2 = update->seq2;
rec->add_flags = update->add_flags;
memcpy(rec->add_custom_flags, update->add_custom_flags,
sizeof(rec->add_custom_flags));
rec->remove_flags = update->remove_flags;
memcpy(rec->remove_custom_flags, update->remove_custom_flags,
sizeof(rec->remove_custom_flags));
}
static int mail_index_sync_rec_check(struct mail_index_view *view,
struct mail_index_sync_rec *rec)
{
uint32_t message_count;
switch (rec->type) {
case MAIL_INDEX_SYNC_TYPE_EXPUNGE:
case MAIL_INDEX_SYNC_TYPE_FLAGS:
if (rec->seq1 > rec->seq2 || rec->seq1 == 0) {
mail_transaction_log_view_set_corrupted(view->log_view,
"Broken sequence: %u..%u",
rec->seq1, rec->seq2);
return FALSE;
}
message_count = mail_index_view_get_message_count(view);
if (rec->seq2 > message_count) {
mail_transaction_log_view_set_corrupted(view->log_view,
"Sequence out of range: %u > %u",
rec->seq2, message_count);
return FALSE;
}
break;
case MAIL_INDEX_SYNC_TYPE_APPEND:
break;
}
return TRUE;
}
int mail_index_sync_get_rec(struct mail_index_view *view,
struct mail_index_sync_rec *rec,
const struct mail_transaction_header *hdr,
const void *data, size_t *data_offset)
{
switch (hdr->type & MAIL_TRANSACTION_TYPE_MASK) {
case MAIL_TRANSACTION_APPEND: {
rec->type = MAIL_INDEX_SYNC_TYPE_APPEND;
rec->seq1 = view->index->map->records_count + 1;
rec->seq2 = rec->seq1 + hdr->size /
sizeof(struct mail_index_record) - 1;
rec->appends = NULL;
*data_offset += hdr->size;
break;
}
case MAIL_TRANSACTION_EXPUNGE: {
const struct mail_transaction_expunge *exp =
CONST_PTR_OFFSET(data, *data_offset);
*data_offset += sizeof(*exp);
mail_index_sync_get_expunge(rec, exp);
break;
}
case MAIL_TRANSACTION_FLAG_UPDATE: {
const struct mail_transaction_flag_update *update =
CONST_PTR_OFFSET(data, *data_offset);
*data_offset += sizeof(*update);
mail_index_sync_get_update(rec, update);
break;
}
default:
i_unreached();
}
return mail_index_sync_rec_check(view, rec);
}
int mail_index_sync_next(struct mail_index_sync_ctx *ctx,
struct mail_index_sync_rec *sync_rec)
{
const struct mail_transaction_expunge *next_exp;
const struct mail_transaction_flag_update *next_update;
next_exp = ctx->expunge_idx == ctx->expunges_count ? NULL :
&ctx->expunges[ctx->expunge_idx];
next_update = ctx->update_idx == ctx->updates_count ? NULL :
&ctx->updates[ctx->update_idx];
/* the ugliness here is to avoid returning overlapping expunge
and update areas. For example:
updates[] = A { 1, 7 }, B { 1, 3 }
expunges[] = { 5, 6 }
will make us return
update A: 1, 4
update B: 1, 3
expunge : 5, 6
update A: 7, 7
*/
while (next_update != NULL &&
(next_exp == NULL || next_update->seq1 < next_exp->seq1)) {
if (next_update->seq2 >= ctx->next_seq) {
mail_index_sync_get_update(sync_rec, next_update);
if (next_exp != NULL &&
next_exp->seq1 <= next_update->seq2) {
/* it's overlapping.. */
sync_rec->seq2 = next_exp->seq1-1;
}
if (sync_rec->seq1 < ctx->next_seq)
sync_rec->seq1 = ctx->next_seq;
i_assert(sync_rec->seq1 <= sync_rec->seq2);
ctx->update_idx++;
return mail_index_sync_rec_check(ctx->view, sync_rec);
}
if (++ctx->update_idx == ctx->updates_count)
break;
next_update++;
}
if (next_exp != NULL) {
/* a few sanity checks here, we really don't ever want to
accidentally expunge a message. If sequence and UID matches,
it's quite unlikely this expunge was caused by some bug. */
uint32_t uid1, uid2;
if (mail_index_lookup_uid(ctx->view, next_exp->seq1, &uid1) < 0)
return -1;
if (mail_index_lookup_uid(ctx->view, next_exp->seq2, &uid2) < 0)
return -1;
if (next_exp->uid1 != uid1 || next_exp->uid2 != uid2) {
mail_transaction_log_view_set_corrupted(
ctx->view->log_view, "Expunge range %u..%u: "
"UIDs %u..%u doesn't match real UIDs %u..%u",
next_exp->seq1, next_exp->seq2,
next_exp->uid1, next_exp->uid2, uid1, uid2);
return -1;
}
mail_index_sync_get_expunge(sync_rec, next_exp);
ctx->expunge_idx++;
/* scan updates again from the beginning */
ctx->update_idx = 0;
ctx->next_seq = next_exp->seq2;
return mail_index_sync_rec_check(ctx->view, sync_rec);
}
if (ctx->sync_appends) {
ctx->sync_appends = FALSE;
sync_rec->type = MAIL_INDEX_SYNC_TYPE_APPEND;
sync_rec->seq1 = ctx->index->map->records_count+1;
sync_rec->seq2 = sync_rec->seq1-1 +
buffer_get_used_size(ctx->appends_buf) /
sizeof(struct mail_index_record);
sync_rec->appends = buffer_get_data(ctx->appends_buf, NULL);
return 1;
}
return 0;
}
int mail_index_sync_end(struct mail_index_sync_ctx *ctx)
{
uint32_t seq;
uoff_t offset;
int ret = 0;
if (mail_transaction_log_view_is_corrupted(ctx->view->log_view))
ret = -1;
mail_transaction_log_get_head(ctx->index->log, &seq, &offset);
if (mail_transaction_log_view_set(ctx->view->log_view,
ctx->index->hdr->log_file_seq,
ctx->index->hdr->log_file_offset,
seq, offset,
MAIL_TRANSACTION_TYPE_MASK) < 0)
ret = -1;
if (ret == 0) {
mail_index_sync_read_and_sort(ctx, TRUE);
if (mail_index_sync_update_index(ctx) < 0)
ret = -1;
}
mail_index_unlock(ctx->index, ctx->lock_id);
mail_transaction_log_sync_unlock(ctx->index->log);
if (ctx->view != NULL)
mail_index_view_close(ctx->view);
if (ctx->expunges_buf != NULL)
buffer_free(ctx->expunges_buf);
if (ctx->updates_buf != NULL)
buffer_free(ctx->updates_buf);
if (ctx->appends_buf != NULL)
buffer_free(ctx->appends_buf);
i_free(ctx);
return ret;
}
void mail_index_sync_flags_apply(const struct mail_index_sync_rec *sync_rec,
uint8_t *flags,
custom_flags_mask_t custom_flags)
{
int i;
i_assert(sync_rec->type == MAIL_INDEX_SYNC_TYPE_FLAGS);
*flags = (*flags & ~sync_rec->remove_flags) | sync_rec->add_flags;
for (i = 0; i < INDEX_CUSTOM_FLAGS_BYTE_COUNT; i++) {
custom_flags[i] =
(custom_flags[i] & ~sync_rec->remove_custom_flags[i]) |
sync_rec->add_custom_flags[i];
}
}
--- NEW FILE: mail-index-transaction-private.h ---
#ifndef __MAIL_INDEX_TRANSACTION_PRIVATE_H
#define __MAIL_INDEX_TRANSACTION_PRIVATE_H
struct mail_index_transaction {
struct mail_index_view *view;
buffer_t *appends;
uint32_t first_new_seq, last_new_seq, next_uid;
buffer_t *expunges;
buffer_t *updates;
struct mail_transaction_flag_update last_update;
enum modify_type last_update_modify_type;
buffer_t *cache_updates;
unsigned int hide_transaction:1;
};
#endif
--- NEW FILE: mail-index-transaction.c ---
/* Copyright (C) 2003-2004 Timo Sirainen */
#include "lib.h"
#include "buffer.h"
#include "mail-index-view-private.h"
#include "mail-transaction-log.h"
#include "mail-index-transaction-private.h"
static void mail_index_transaction_add_last(struct mail_index_transaction *t);
struct mail_index_transaction *
mail_index_transaction_begin(struct mail_index_view *view, int hide)
{
struct mail_index_transaction *t;
/* don't allow syncing view while there's ongoing transactions */
mail_index_view_transaction_ref(view);
t = i_new(struct mail_index_transaction, 1);
t->view = view;
t->hide_transaction = hide;
t->next_uid = view->index->hdr->next_uid;
return t;
}
static void mail_index_transaction_free(struct mail_index_transaction *t)
{
mail_index_view_transaction_unref(t->view);
if (t->appends != NULL)
buffer_free(t->appends);
if (t->expunges != NULL)
buffer_free(t->expunges);
if (t->updates != NULL)
buffer_free(t->updates);
i_free(t);
}
static void
mail_index_transaction_expunge_updates(struct mail_index_transaction *t)
{
/* FIXME: is this useful? do we even want this? */
const struct mail_transaction_expunge *expunges, *last_expunge;
struct mail_transaction_flag_update *updates;
size_t expunge_size, update_count, i, dest;
uint32_t seq1, seq2;
int cut;
expunges = buffer_get_data(t->expunges, &expunge_size);
last_expunge = CONST_PTR_OFFSET(expunges, expunge_size);
if (expunge_size == 0)
return;
updates = buffer_get_modifyable_data(t->updates, &update_count);
update_count /= sizeof(*updates);
/* Cut off the updates that contain expunged messages. However if
the cutting would require creating another flag update entry
(eg. updates=1..3, expunge=2), don't do it. */
for (i = 0, dest = 0; i < update_count; i++) {
while (expunges->seq2 < updates[i].seq1) {
if (++expunges == last_expunge)
break;
}
cut = FALSE;
if (expunges->seq1 <= updates[i].seq2) {
/* they're overlapping at least partially */
seq1 = I_MIN(expunges->seq1, updates[i].seq1);
seq2 = I_MAX(expunges->seq2, updates[i].seq2);
if (seq1 == expunges->seq1 && seq2 == expunges->seq2) {
/* cut it off completely */
cut = TRUE;
} else if (seq1 == expunges->seq1) {
/* cut the beginning */
updates[i].seq1 = expunges->seq2+1;
} else if (seq2 == expunges->seq2) {
/* cut the end */
updates[i].seq2 = expunges->seq1-1;
} else {
/* expunge range is in the middle -
don't bother cutting it */
}
}
if (!cut) {
if (i != dest)
updates[dest] = updates[i];
dest++;
}
}
if (i != dest)
buffer_set_used_size(t->updates, dest * sizeof(*updates));
}
int mail_index_transaction_commit(struct mail_index_transaction *t,
uint32_t *log_file_seq_r,
uoff_t *log_file_offset_r)
{
int ret;
if (mail_index_view_is_inconsistent(t->view)) {
mail_index_transaction_free(t);
return -1;
}
if (t->last_update.seq1 != 0)
mail_index_transaction_add_last(t);
if (t->updates != NULL && t->expunges != NULL)
mail_index_transaction_expunge_updates(t);
ret = mail_transaction_log_append(t, log_file_seq_r, log_file_offset_r);
mail_index_transaction_free(t);
return ret;
}
void mail_index_transaction_rollback(struct mail_index_transaction *t)
{
mail_index_transaction_free(t);
}
void mail_index_append(struct mail_index_transaction *t, uint32_t uid,
uint32_t *seq_r)
{
struct mail_index_record *rec;
i_assert(uid >= t->next_uid);
if (t->appends == NULL) {
t->appends = buffer_create_dynamic(default_pool,
4096, (size_t)-1);
}
/* sequence number is visible only inside given view,
so let it generate it */
if (t->last_new_seq != 0)
*seq_r = ++t->last_new_seq;
else {
*seq_r = t->first_new_seq = t->last_new_seq =
mail_index_view_get_message_count(t->view)+1;
}
rec = buffer_append_space_unsafe(t->appends, sizeof(*rec));
memset(rec, 0, sizeof(*rec));
rec->uid = uid;
t->next_uid = uid+1;
}
void mail_index_expunge(struct mail_index_transaction *t, uint32_t seq)
{
struct mail_transaction_expunge exp, *data;
unsigned int idx, left_idx, right_idx;
uint32_t uid;
size_t size;
i_assert(seq > 0 && seq <= mail_index_view_get_message_count(t->view));
uid = t->view->map->records[seq-1].uid;
exp.seq1 = exp.seq2 = seq;
exp.uid1 = exp.uid2 = uid;
/* expunges is a sorted array of {seq1, seq2, ..}, .. */
if (t->expunges == NULL) {
t->expunges = buffer_create_dynamic(default_pool,
1024, (size_t)-1);
buffer_append(t->expunges, &exp, sizeof(exp));
return;
}
data = buffer_get_modifyable_data(t->expunges, &size);
size /= sizeof(*data);
i_assert(size > 0);
/* quick checks */
if (data[size-1].seq2 == seq-1) {
/* grow last range */
data[size-1].seq2 = seq;
data[size-1].uid2 = uid;
return;
}
if (data[size-1].seq2 < seq) {
buffer_append(t->expunges, &exp, sizeof(exp));
return;
}
if (data[0].seq1 == seq+1) {
/* grow down first range */
data[0].seq1 = seq;
data[0].uid1 = uid;
return;
}
if (data[0].seq1 > seq) {
buffer_insert(t->expunges, 0, &exp, sizeof(exp));
return;
}
/* somewhere in the middle, array is sorted so find it with
binary search */
idx = 0; left_idx = 0; right_idx = size;
while (left_idx < right_idx) {
idx = (left_idx + right_idx) / 2;
if (data[idx].seq1 < seq)
left_idx = idx+1;
else if (data[idx].seq1 > seq)
right_idx = idx;
else
break;
}
if (data[idx].seq2 < seq)
idx++;
/* idx == size couldn't happen because we already handle it above */
i_assert(idx < size && data[idx].seq1 >= seq);
if (data[idx].seq1 <= seq && data[idx].seq2 >= seq) {
/* already expunged */
return;
}
if (data[idx].seq1 == seq+1) {
data[idx].seq1 = seq;
data[idx].uid1 = uid;
if (idx > 0 && data[idx-1].seq2 == seq-1) {
/* merge */
data[idx-1].seq2 = data[idx].seq2;
data[idx-1].uid2 = data[idx].uid2;
buffer_delete(t->expunges, idx * sizeof(*data),
sizeof(*data));
}
} else if (data[idx].seq2 == seq-1) {
i_assert(idx+1 < size); /* already handled above */
data[idx].seq2 = seq;
data[idx].uid2 = uid;
if (data[idx+1].seq1 == seq+1) {
/* merge */
data[idx+1].seq1 = data[idx].seq1;
data[idx+1].uid1 = data[idx].uid1;
buffer_delete(t->expunges, idx * sizeof(*data),
sizeof(*data));
}
} else {
buffer_insert(t->expunges, idx * sizeof(*data),
&exp, sizeof(exp));
}
}
static void mail_index_record_modify_flags(struct mail_index_record *rec,
enum modify_type modify_type,
enum mail_flags flags,
custom_flags_mask_t custom_flags)
{
int i;
switch (modify_type) {
case MODIFY_REPLACE:
rec->flags = flags;
memcpy(rec->custom_flags, custom_flags,
INDEX_CUSTOM_FLAGS_BYTE_COUNT);
break;
case MODIFY_ADD:
rec->flags |= flags;
for (i = 0; i < INDEX_CUSTOM_FLAGS_BYTE_COUNT; i++)
rec->custom_flags[i] |= custom_flags[i];
break;
case MODIFY_REMOVE:
rec->flags &= ~flags;
for (i = 0; i < INDEX_CUSTOM_FLAGS_BYTE_COUNT; i++)
rec->custom_flags[i] &= ~custom_flags[i];
break;
}
}
#define IS_COMPATIBLE_UPDATE(t, modify_type, flags, custom_flags) \
((t)->last_update_modify_type == (modify_type) && \
(t)->last_update.add_flags == (flags) && \
memcmp((t)->last_update.add_custom_flags, custom_flags, \
INDEX_CUSTOM_FLAGS_BYTE_COUNT) == 0)
void mail_index_update_flags(struct mail_index_transaction *t, uint32_t seq,
enum modify_type modify_type,
enum mail_flags flags,
custom_flags_mask_t custom_flags)
{
struct mail_index_record *rec;
size_t pos;
if (t->first_new_seq != 0 && seq >= t->first_new_seq) {
/* just appended message, modify it directly */
i_assert(seq > 0 && seq <= t->last_new_seq);
pos = (seq - t->first_new_seq) * sizeof(*rec);
rec = buffer_get_space_unsafe(t->appends, pos, sizeof(*rec));
mail_index_record_modify_flags(rec, modify_type,
flags, custom_flags);
return;
}
i_assert(seq > 0 && seq <= mail_index_view_get_message_count(t->view));
/* first get group updates into same structure. this allows faster
updates if same mails have multiple flag updates during same
transaction (eg. 1:10 +seen, 1:10 +deleted) */
if (t->last_update.seq2 == seq-1) {
if (t->last_update.seq1 != 0 &&
IS_COMPATIBLE_UPDATE(t, modify_type, flags, custom_flags)) {
t->last_update.seq2 = seq;
return;
}
} else if (t->last_update.seq1 == seq+1) {
if (t->last_update.seq1 != 0 &&
IS_COMPATIBLE_UPDATE(t, modify_type, flags, custom_flags)) {
t->last_update.seq1 = seq;
return;
}
}
if (t->last_update.seq1 != 0)
mail_index_transaction_add_last(t);
t->last_update_modify_type = modify_type;
t->last_update.seq1 = t->last_update.seq2 = seq;
t->last_update.add_flags = flags;
memcpy(t->last_update.add_custom_flags, custom_flags,
INDEX_CUSTOM_FLAGS_BYTE_COUNT);
}
static void
mail_index_transaction_get_last(struct mail_index_transaction *t,
struct mail_transaction_flag_update *update)
{
int i;
*update = t->last_update;
switch (t->last_update_modify_type) {
case MODIFY_REPLACE:
/* remove_flags = ~add_flags */
update->remove_flags =
~update->add_flags & MAIL_INDEX_FLAGS_MASK;
for (i = 0; i < INDEX_CUSTOM_FLAGS_BYTE_COUNT; i++) {
update->remove_custom_flags[i] =
~update->add_custom_flags[i];
}
break;
case MODIFY_ADD:
/* already in add_flags */
break;
case MODIFY_REMOVE:
/* add_flags -> remove_flags */
update->remove_flags = update->add_flags;
memcpy(&update->remove_custom_flags, &update->add_custom_flags,
INDEX_CUSTOM_FLAGS_BYTE_COUNT);
update->add_flags = 0;
memset(&update->add_custom_flags, 0,
INDEX_CUSTOM_FLAGS_BYTE_COUNT);
break;
}
}
static void mail_index_transaction_add_last(struct mail_index_transaction *t)
{
struct mail_transaction_flag_update update, *data;
unsigned int idx, left_idx, right_idx;
uint32_t last;
size_t size;
mail_index_transaction_get_last(t, &update);
if (t->updates == NULL) {
t->updates = buffer_create_dynamic(default_pool,
4096, (size_t)-1);
}
data = buffer_get_modifyable_data(t->updates, &size);
size /= sizeof(*data);
/* find the nearest sequence from existing updates */
idx = 0; left_idx = 0; right_idx = size;
while (left_idx < right_idx) {
idx = (left_idx + right_idx) / 2;
if (data[idx].seq1 < update.seq1)
left_idx = idx+1;
else if (data[idx].seq1 > update.seq1)
right_idx = idx;
else
break;
}
if (idx < size && data[idx].seq2 < update.seq1)
idx++;
i_assert(idx == size || data[idx].seq1 < update.seq1);
/* insert it into buffer, split it in multiple parts if needed
to make sure the ordering stays the same */
for (; idx < size; idx++) {
if (data[idx].seq1 > update.seq2)
break;
/* partial */
last = update.seq2;
update.seq2 = data[idx].seq1-1;
buffer_insert(t->updates, idx * sizeof(update),
&update, sizeof(update));
data = buffer_get_modifyable_data(t->updates, NULL);
size++;
update.seq1 = update.seq2+1;
update.seq2 = last;
}
buffer_insert(t->updates, idx * sizeof(update),
&update, sizeof(update));
}
void mail_index_update_cache(struct mail_index_transaction *t,
uint32_t seq, uint32_t offset)
{
struct mail_transaction_cache_update *data, update;
unsigned int idx, left_idx, right_idx;
size_t size;
if (t->cache_updates == NULL) {
t->cache_updates = buffer_create_dynamic(default_pool,
1024, (size_t)-1);
}
data = buffer_get_modifyable_data(t->cache_updates, &size);
size /= sizeof(*data);
/* we're probably appending it, check */
if (size == 0 || data[size-1].seq < seq)
idx = size;
else {
idx = 0; left_idx = 0; right_idx = size;
while (left_idx < right_idx) {
idx = (left_idx + right_idx) / 2;
if (data[idx].seq < seq)
left_idx = idx+1;
else if (data[idx].seq > seq)
right_idx = idx;
else {
/* already there, update */
data[idx].cache_offset = offset;
return;
}
}
}
update.seq = seq;
update.cache_offset = offset;
buffer_insert(t->updates, idx * sizeof(update),
&update, sizeof(update));
}
--- NEW FILE: mail-index-view-private.h ---
#ifndef __MAIL_INDEX_VIEW_PRIVATE_H
#define __MAIL_INDEX_VIEW_PRIVATE_H
#include "mail-index-private.h"
struct mail_index_view {
struct mail_index *index;
struct mail_transaction_log_view *log_view;
struct mail_index_map *map;
uint32_t log_file_seq;
uoff_t log_file_offset;
buffer_t *log_syncs;
int transactions;
unsigned int lock_id;
unsigned int inconsistent:1;
unsigned int syncing:1;
unsigned int external:1;
};
int mail_index_view_lock(struct mail_index_view *view, int update_index);
void mail_index_view_add_synced_transaction(struct mail_index_view *view,
uint32_t log_file_seq,
uoff_t log_file_offset);
#endif
--- NEW FILE: mail-index-view-sync.c ---
/* Copyright (C) 2003-2004 Timo Sirainen */
#include "lib.h"
#include "buffer.h"
#include "mail-index-view-private.h"
#include "mail-index-sync-private.h"
#include "mail-transaction-log.h"
#include "mail-transaction-util.h"
struct mail_index_view_sync_ctx {
struct mail_index_view *view;
enum mail_index_sync_type sync_mask;
struct mail_index_map *sync_map;
buffer_t *expunges;
const struct mail_transaction_header *hdr;
const void *data;
size_t data_offset;
unsigned int skipped:1;
unsigned int last_read:1;
};
static int
view_sync_get_expunges(struct mail_index_view *view, buffer_t **expunges_r)
{
const struct mail_transaction_expunge *exp, *end;
buffer_t *expunges;
size_t size;
/* with mask 0 we don't get anything, we'll just read the expunges
while seeking to end */
if (mail_transaction_log_view_set(view->log_view,
view->log_file_seq,
view->log_file_offset,
view->index->hdr->log_file_seq,
view->index->hdr->log_file_offset,
0) < 0)
return -1;
if (mail_transaction_log_view_next(view->log_view,
NULL, NULL, NULL) < 0)
return -1;
expunges = mail_transaction_log_view_get_expunges(view->log_view);
exp = buffer_get_data(expunges, &size);
end = CONST_PTR_OFFSET(exp, size);
*expunges_r = buffer_create_dynamic(default_pool, size, (size_t)-1);
for (; exp != end; exp++) {
buffer_append(*expunges_r, &exp->seq1, sizeof(exp->seq1));
buffer_append(*expunges_r, &exp->seq2, sizeof(exp->seq2));
}
return 0;
}
int mail_index_view_sync_begin(struct mail_index_view *view,
enum mail_index_sync_type sync_mask,
struct mail_index_view_sync_ctx **ctx_r)
{
const struct mail_index_header *hdr;
struct mail_index_view_sync_ctx *ctx;
struct mail_index_map *map;
enum mail_transaction_type mask;
buffer_t *expunges = NULL;
/* We must sync flags as long as view is mmap()ed, as the flags may
have already changed under us. */
i_assert((sync_mask & MAIL_INDEX_SYNC_TYPE_FLAGS) != 0);
i_assert(view->transactions == 0);
i_assert(!view->syncing);
if (mail_index_view_lock(view, TRUE) < 0)
return -1;
hdr = view->index->hdr;
if ((sync_mask & MAIL_INDEX_SYNC_TYPE_EXPUNGE) != 0) {
/* get list of all expunges first */
if (view_sync_get_expunges(view, &expunges) < 0)
return -1;
}
mask = mail_transaction_type_mask_get(sync_mask);
if (mail_transaction_log_view_set(view->log_view,
view->log_file_seq,
view->log_file_offset,
hdr->log_file_seq,
hdr->log_file_offset, mask) < 0) {
if (expunges != NULL)
buffer_free(expunges);
return -1;
}
if (sync_mask == MAIL_INDEX_SYNC_MASK_ALL) {
map = view->index->map;
map->refcount++;
} else {
map = mail_index_map_to_memory(view->map);
}
view->syncing = TRUE;
ctx = i_new(struct mail_index_view_sync_ctx, 1);
ctx->view = view;
ctx->sync_mask = sync_mask;
ctx->sync_map = map;
ctx->expunges = expunges;
*ctx_r = ctx;
return 0;
}
static int view_is_transaction_synced(struct mail_index_view *view,
uint32_t seq, uoff_t offset)
{
const unsigned char *data, *end;
size_t size;
if (view->log_syncs == NULL)
return 0;
data = buffer_get_data(view->log_syncs, &size);
end = data + size;
for (; data < end; ) {
if (*((const uoff_t *)data) == offset &&
*((const uint32_t *)(data + sizeof(uoff_t))) == seq)
return 1;
data += sizeof(uoff_t) + sizeof(uint32_t);
}
return 0;
}
static int sync_expunge(const struct mail_transaction_expunge *e, void *context)
{
struct mail_index_map *map = context;
unsigned int idx, count;
for (idx = e->seq1-1; idx < e->seq2; idx++) {
mail_index_header_update_counts(&map->hdr_copy,
map->records[idx].flags, 0);
}
count = e->seq2 - e->seq1 + 1;
buffer_delete(map->buffer,
(e->seq1-1) * sizeof(struct mail_index_record),
count * sizeof(struct mail_index_record));
map->records = buffer_get_modifyable_data(map->buffer, NULL);
map->records_count -= count;
map->hdr_copy.messages_count -= count;
return 1;
}
static int sync_append(const struct mail_index_record *rec, void *context)
{
struct mail_index_map *map = context;
buffer_append(map->buffer, rec, sizeof(*rec));
map->records = buffer_get_modifyable_data(map->buffer, NULL);
map->records_count++;
map->hdr_copy.messages_count++;
mail_index_header_update_counts(&map->hdr_copy, 0, rec->flags);
mail_index_header_update_lowwaters(&map->hdr_copy, rec);
return 1;
}
static int sync_flag_update(const struct mail_transaction_flag_update *u,
void *context)
{
struct mail_index_map *map = context;
struct mail_index_record *rec;
unsigned int i, idx;
uint8_t old_flags;
for (idx = u->seq1-1; idx < u->seq2; idx++) {
rec = &map->records[idx];
old_flags = rec->flags;
rec->flags = (rec->flags & ~u->remove_flags) | u->add_flags;
for (i = 0; i < INDEX_CUSTOM_FLAGS_BYTE_COUNT; i++) {
rec->custom_flags[i] =
(rec->custom_flags[i] &
~u->remove_custom_flags[i]) |
u->add_custom_flags[i];
}
mail_index_header_update_counts(&map->hdr_copy, old_flags,
rec->flags);
mail_index_header_update_lowwaters(&map->hdr_copy, rec);
}
return 1;
}
static int sync_cache_update(const struct mail_transaction_cache_update *u,
void *context)
{
struct mail_index_map *map = context;
map->records[u->seq-1].cache_offset = u->cache_offset;
return 1;
}
static int mail_index_view_sync_map(struct mail_index_view_sync_ctx *ctx)
{
static struct mail_transaction_map_functions map_funcs = {
sync_expunge, sync_append, sync_flag_update, sync_cache_update
};
return mail_transaction_map(ctx->hdr, ctx->data,
&map_funcs, ctx->sync_map);
}
static int mail_index_view_sync_next_trans(struct mail_index_view_sync_ctx *ctx,
uint32_t *seq_r, uoff_t *offset_r)
{
struct mail_transaction_log_view *log_view = ctx->view->log_view;
struct mail_index_view *view = ctx->view;
int ret, skipped;
ret = mail_transaction_log_view_next(log_view, &ctx->hdr, &ctx->data,
&skipped);
if (ret <= 0) {
if (ret < 0)
return -1;
ctx->last_read = TRUE;
return 1;
}
if (skipped)
ctx->skipped = TRUE;
mail_transaction_log_view_get_prev_pos(log_view, seq_r, offset_r);
/* skip flag changes that we committed ourself or have already synced */
if (view_is_transaction_synced(view, *seq_r, *offset_r))
return 0;
if (ctx->sync_mask != MAIL_INDEX_SYNC_MASK_ALL) {
if (mail_index_view_sync_map(ctx) < 0)
return -1;
}
return 1;
}
int mail_index_view_sync_next(struct mail_index_view_sync_ctx *ctx,
struct mail_index_sync_rec *sync_rec)
{
struct mail_index_view *view = ctx->view;
uint32_t seq;
uoff_t offset;
int ret;
if (ctx->hdr == NULL || ctx->data_offset == ctx->hdr->size) {
ctx->data_offset = 0;
do {
ret = mail_index_view_sync_next_trans(ctx, &seq,
&offset);
if (ret < 0)
return -1;
if (ctx->last_read)
return 0;
if (!ctx->skipped) {
view->log_file_seq = seq;
view->log_file_offset = offset +
sizeof(*ctx->hdr) + ctx->hdr->size;
}
} while (ret == 0);
if (ctx->skipped) {
mail_index_view_add_synced_transaction(view, seq,
offset);
}
}
if (!mail_index_sync_get_rec(view, sync_rec, ctx->hdr, ctx->data,
&ctx->data_offset))
return -1;
return 1;
}
const uint32_t *
mail_index_view_sync_get_expunges(struct mail_index_view_sync_ctx *ctx,
size_t *count_r)
{
const uint32_t *data;
size_t size;
data = buffer_get_data(ctx->expunges, &size);
*count_r = size / (sizeof(uint32_t)*2);
return data;
}
void mail_index_view_sync_end(struct mail_index_view_sync_ctx *ctx)
{
struct mail_index_view *view = ctx->view;
i_assert(view->syncing);
if (view->log_syncs != NULL && !ctx->skipped)
buffer_set_used_size(view->log_syncs, 0);
if (!ctx->last_read && ctx->hdr != NULL &&
ctx->data_offset != ctx->hdr->size) {
/* we didn't sync everything */
view->inconsistent = TRUE;
}
mail_index_unmap(view->index, view->map);
view->map = ctx->sync_map;
if (ctx->expunges != NULL)
buffer_free(ctx->expunges);
view->syncing = FALSE;
i_free(ctx);
}
void mail_index_view_add_synced_transaction(struct mail_index_view *view,
uint32_t log_file_seq,
uoff_t log_file_offset)
{
if (view->log_syncs == NULL) {
view->log_syncs = buffer_create_dynamic(default_pool,
128, (size_t)-1);
}
buffer_append(view->log_syncs, &log_file_offset,
sizeof(log_file_offset));
buffer_append(view->log_syncs, &log_file_seq, sizeof(log_file_seq));
}
--- NEW FILE: mail-index-view.c ---
/* Copyright (C) 2003-2004 Timo Sirainen */
#include "lib.h"
#include "buffer.h"
#include "file-lock.h"
#include "mail-index-view-private.h"
#include "mail-transaction-log.h"
struct mail_index_view *mail_index_view_open(struct mail_index *index)
{
struct mail_index_view *view;
view = i_new(struct mail_index_view, 1);
view->index = index;
view->log_view = mail_transaction_log_view_open(index->log);
view->map = index->map;
view->map->refcount++;
view->log_file_seq = view->index->hdr->log_file_seq;
view->log_file_offset = view->index->hdr->log_file_offset;
return view;
}
void mail_index_view_close(struct mail_index_view *view)
{
mail_index_view_unlock(view);
mail_transaction_log_view_close(view->log_view);
if (view->log_syncs != NULL)
buffer_free(view->log_syncs);
mail_index_unmap(view->index, view->map);
i_free(view);
}
static int
mail_index_view_lock_head(struct mail_index_view *view, int update_index)
{
if (!mail_index_is_locked(view->index, view->lock_id)) {
if (view->index->indexid != view->map->hdr->indexid) {
/* index was rebuilt */
view->inconsistent = TRUE;
return -1;
}
if (mail_index_lock_shared(view->index, update_index,
&view->lock_id) < 0)
return -1;
if (mail_index_map(view->index, FALSE) <= 0) {
view->inconsistent = TRUE;
return -1;
}
}
return 0;
}
int mail_index_view_lock(struct mail_index_view *view, int update_index)
{
if (view->inconsistent)
return -1;
if (view->map != view->index->map) {
/* not head mapping, no need to lock */
return 0;
}
return mail_index_view_lock_head(view, update_index);
}
void mail_index_view_unlock(struct mail_index_view *view)
{
if (view->lock_id != 0) {
mail_index_unlock(view->index, view->lock_id);
view->lock_id = 0;
}
}
uint32_t mail_index_view_get_message_count(struct mail_index_view *view)
{
return view->map->records_count;
}
int mail_index_view_is_inconsistent(struct mail_index_view *view)
{
return view->inconsistent;
}
struct mail_index *mail_index_view_get_index(struct mail_index_view *view)
{
return view->index;
}
void mail_index_view_transaction_ref(struct mail_index_view *view)
{
view->transactions++;
}
void mail_index_view_transaction_unref(struct mail_index_view *view)
{
i_assert(view->transactions > 0);
view->transactions--;
}
const struct mail_index_header *
mail_index_get_header(struct mail_index_view *view)
{
return view->map->hdr;
}
int mail_index_lookup(struct mail_index_view *view, uint32_t seq,
const struct mail_index_record **rec_r)
{
struct mail_index_map *map;
const struct mail_index_record *rec;
uint32_t uid;
i_assert(seq > 0);
i_assert(seq <= view->map->records_count);
if (mail_index_view_lock(view, FALSE) < 0)
return -1;
rec = &view->map->records[seq-1];
if (view->map == view->index->map) {
*rec_r = rec;
return 0;
}
if (mail_index_view_lock_head(view, FALSE) < 0)
return -1;
/* look for it in the head mapping */
uid = rec->uid;
if (seq > view->index->hdr->messages_count)
seq = view->index->hdr->messages_count;
map = view->index->map;
while (seq > 0) {
// FIXME: we could be skipping more by uid diff
if (map->records[--seq].uid <= uid)
break;
}
*rec_r = map->records[seq].uid == uid ?
&map->records[seq] : rec;
return 0;
}
int mail_index_lookup_uid(struct mail_index_view *view, uint32_t seq,
uint32_t *uid_r)
{
i_assert(seq > 0);
i_assert(seq <= view->map->records_count);
if (mail_index_view_lock(view, FALSE) < 0)
return -1;
*uid_r = view->map->records[seq-1].uid;
return 0;
}
static uint32_t mail_index_bsearch_uid(struct mail_index_view *view,
uint32_t uid, uint32_t *left_idx_p,
int nearest_side)
{
const struct mail_index_record *rec;
uint32_t idx, left_idx, right_idx;
rec = view->map->records;
idx = 0;
left_idx = *left_idx_p;
right_idx = view->map->records_count;
while (left_idx < right_idx) {
idx = (left_idx + right_idx) / 2;
if (rec[idx].uid < uid)
left_idx = idx+1;
else if (rec[idx].uid > uid)
right_idx = idx;
else
break;
}
*left_idx_p = left_idx;
if (rec[idx].uid != uid) {
if (nearest_side > 0) {
/* we want uid or larger */
return rec[idx].uid > uid ? idx+1 :
idx == view->map->records_count-1 ? 0 : idx+2;
} else {
/* we want uid or smaller */
return rec[idx].uid < uid ? idx + 1 : idx;
}
}
return idx+1;
}
int mail_index_lookup_uid_range(struct mail_index_view *view,
uint32_t first_uid, uint32_t last_uid,
uint32_t *first_seq_r, uint32_t *last_seq_r)
{
uint32_t left_idx;
i_assert(first_uid > 0);
i_assert(first_uid <= last_uid);
if (mail_index_view_lock(view, FALSE) < 0)
return -1;
left_idx = 0;
*first_seq_r = mail_index_bsearch_uid(view, first_uid, &left_idx, 1);
if (*first_seq_r == 0 ||
view->map->records[*first_seq_r-1].uid > last_uid) {
*first_seq_r = 0;
*last_seq_r = 0;
return 0;
}
if (first_uid == last_uid) {
*last_seq_r = *first_seq_r;
return 0;
}
/* optimization - binary lookup only from right side: */
*last_seq_r = mail_index_bsearch_uid(view, last_uid, &left_idx, -1);
i_assert(*last_seq_r >= *first_seq_r);
return 0;
}
int mail_index_lookup_first(struct mail_index_view *view, enum mail_flags flags,
uint8_t flags_mask, uint32_t *seq_r)
{
#define LOW_UPDATE(x) \
STMT_START { if ((x) > low_uid) low_uid = x; } STMT_END
const struct mail_index_record *rec;
uint32_t seq, low_uid = 1;
*seq_r = 0;
if (mail_index_view_lock(view, FALSE) < 0)
return -1;
if ((flags_mask & MAIL_RECENT) != 0 && (flags & MAIL_RECENT) != 0)
LOW_UPDATE(view->map->hdr->first_recent_uid_lowwater);
if ((flags_mask & MAIL_SEEN) != 0 && (flags & MAIL_SEEN) == 0)
LOW_UPDATE(view->map->hdr->first_unseen_uid_lowwater);
if ((flags_mask & MAIL_DELETED) != 0 && (flags & MAIL_DELETED) != 0)
LOW_UPDATE(view->map->hdr->first_deleted_uid_lowwater);
if (low_uid == 1)
seq = 1;
else {
if (mail_index_lookup_uid_range(view, low_uid, low_uid,
&seq, &seq) < 0)
return -1;
if (seq == 0)
return 0;
}
rec = &view->map->records[seq-1];
for (; seq <= view->map->records_count; seq++, rec++) {
if ((rec->flags & flags_mask) == (uint8_t)flags) {
*seq_r = seq;
break;
}
}
return 0;
}
--- NEW FILE: mail-transaction-log-private.h ---
#ifndef __MAIL_TRANSACTION_LOG_VIEW_H
#define __MAIL_TRANSACTION_LOG_VIEW_H
#include "mail-transaction-log.h"
struct mail_transaction_log_file {
struct mail_transaction_log *log;
struct mail_transaction_log_file *next;
int refcount;
char *filepath;
int fd;
int lock_type;
ino_t st_ino;
dev_t st_dev;
buffer_t *buffer;
uoff_t buffer_offset;
size_t buffer_size;
void *mmap_base;
size_t mmap_size;
struct mail_transaction_log_header hdr;
};
struct mail_transaction_log {
struct mail_index *index;
struct mail_transaction_log_view *views;
struct mail_transaction_log_file *head, *tail;
};
void
mail_transaction_log_file_set_corrupted(struct mail_transaction_log_file *file,
const char *fmt, ...);
int mail_transaction_log_file_find(struct mail_transaction_log *log,
uint32_t file_seq,
struct mail_transaction_log_file **file_r);
int mail_transaction_log_file_map(struct mail_transaction_log_file *file,
uoff_t start_offset, uoff_t end_offset);
void mail_transaction_logs_clean(struct mail_transaction_log *log);
#endif
--- NEW FILE: mail-transaction-log-view.c ---
/* Copyright (C) 2003-2004 Timo Sirainen */
#include "lib.h"
#include "buffer.h"
#include "mail-index-private.h"
#include "mail-transaction-log-private.h"
#include "mail-transaction-util.h"
struct mail_transaction_log_view {
struct mail_transaction_log *log;
struct mail_transaction_log_view *next;
uint32_t min_file_seq, max_file_seq;
uoff_t min_file_offset, max_file_offset;
enum mail_transaction_type type_mask;
buffer_t *expunges_buf, *data_buf;
struct mail_transaction_expunge_traverse_ctx *exp_ctx;
struct mail_transaction_header tmp_hdr;
struct mail_transaction_log_file *file;
uoff_t file_offset;
uint32_t prev_file_seq;
uoff_t prev_file_offset;
unsigned int broken:1;
};
struct mail_transaction_log_view *
mail_transaction_log_view_open(struct mail_transaction_log *log)
{
struct mail_transaction_log_view *view;
view = i_new(struct mail_transaction_log_view, 1);
view->log = log;
view->expunges_buf =
buffer_create_dynamic(default_pool, 512, (size_t)-1);
view->next = log->views;
log->views = view;
return view;
}
static void
mail_transaction_log_view_close_files(struct mail_transaction_log_view *view)
{
struct mail_transaction_log_file *file;
for (file = view->log->tail; file != NULL; file = file->next) {
if (file->hdr.file_seq > view->max_file_seq)
break;
if (file->hdr.file_seq >= view->min_file_seq)
file->refcount--;
}
mail_transaction_logs_clean(view->log);
}
void mail_transaction_log_view_close(struct mail_transaction_log_view *view)
{
mail_transaction_log_view_close_files(view);
if (view->data_buf != NULL)
buffer_free(view->data_buf);
buffer_free(view->expunges_buf);
i_free(view);
}
int
mail_transaction_log_view_set(struct mail_transaction_log_view *view,
uint32_t min_file_seq, uoff_t min_file_offset,
uint32_t max_file_seq, uoff_t max_file_offset,
enum mail_transaction_type type_mask)
{
/* FIXME: error handling for "not found" case is bad.. should the
caller after all check it and handle as it sees best..? */
struct mail_transaction_log_file *file, *first;
uint32_t seq;
uoff_t end_offset;
int ret;
i_assert(min_file_seq <= max_file_seq);
i_assert(min_file_offset >= sizeof(struct mail_transaction_log_header));
i_assert(max_file_offset >= sizeof(struct mail_transaction_log_header));
view->broken = TRUE;
mail_transaction_log_view_close_files(view);
ret = mail_transaction_log_file_find(view->log, min_file_seq, &file);
if (ret <= 0)
return -1;
end_offset = min_file_seq == max_file_seq ?
max_file_offset : (uoff_t)-1;
ret = mail_transaction_log_file_map(file, min_file_offset, end_offset);
if (ret <= 0)
return -1;
first = file;
for (seq = min_file_seq+1; seq <= max_file_seq; seq++) {
file = file->next;
if (file == NULL || file->hdr.file_seq != seq)
return -1;
end_offset = file->hdr.file_seq == max_file_seq ?
max_file_offset : (uoff_t)-1;
ret = mail_transaction_log_file_map(file,
sizeof(struct mail_transaction_log_header),
end_offset);
if (ret <= 0)
return -1;
}
i_assert(max_file_offset <= file->hdr.used_size);
/* we have it all, refcount the files */
for (file = first, seq = min_file_seq; seq <= max_file_seq; seq++) {
file->refcount++;
file = file->next;
}
buffer_set_used_size(view->expunges_buf, 0);
view->prev_file_seq = 0;
view->prev_file_offset = 0;
view->file = first;
view->file_offset = min_file_offset;
view->min_file_seq = min_file_seq;
view->min_file_offset = min_file_offset;
view->max_file_seq = max_file_seq;
view->max_file_offset = max_file_offset;
view->type_mask = type_mask;
view->broken = FALSE;
return 0;
}
void
mail_transaction_log_view_get_prev_pos(struct mail_transaction_log_view *view,
uint32_t *file_seq_r,
uoff_t *file_offset_r)
{
*file_seq_r = view->prev_file_seq;
*file_offset_r = view->prev_file_offset;
}
void
mail_transaction_log_view_set_corrupted(struct mail_transaction_log_view *view,
const char *fmt, ...)
{
va_list va;
i_assert(view->file != NULL);
view->broken = TRUE;
va_start(va, fmt);
t_push();
mail_transaction_log_file_set_corrupted(view->file, "%s",
t_strdup_vprintf(fmt, va));
t_pop();
va_end(va);
}
int
mail_transaction_log_view_is_corrupted(struct mail_transaction_log_view *view)
{
return view->broken;
}
static int log_view_get_next(struct mail_transaction_log_view *view,
const struct mail_transaction_header **hdr_r,
const void **data_r)
{
const struct mail_transaction_header *hdr;
struct mail_transaction_log_file *file = view->file;
const struct mail_transaction_type_map *type_rec;
const void *data;
unsigned int record_size;
size_t size;
view->prev_file_seq = file->hdr.file_seq;
view->prev_file_offset = view->file_offset;
if (view->file_offset == file->hdr.used_size) {
view->file = file->next;
view->file_offset = sizeof(struct mail_transaction_log_header);
return 0;
}
data = buffer_get_data(file->buffer, &size);
if (view->file_offset + sizeof(*hdr) > file->hdr.used_size) {
mail_transaction_log_file_set_corrupted(file,
"offset points outside file (%u + %"PRIuSIZE_T" > %u)",
view->file_offset, sizeof(*hdr), size);
return -1;
}
hdr = CONST_PTR_OFFSET(data, view->file_offset - file->buffer_offset);
view->file_offset += sizeof(*hdr);
if (file->hdr.used_size - view->file_offset < hdr->size) {
mail_transaction_log_file_set_corrupted(file,
"record size too large "
"(type=0x%x, offset=%u, size=%u, end=%u)",
hdr->type & MAIL_TRANSACTION_TYPE_MASK,
view->file_offset, hdr->size, file->hdr.used_size);
view->file_offset = file->hdr.used_size;
return -1;
}
type_rec = mail_transaction_type_lookup(hdr->type);
if (type_rec != NULL)
record_size = type_rec->record_size;
else {
mail_transaction_log_file_set_corrupted(file,
"unknown record type 0x%x",
hdr->type & MAIL_TRANSACTION_TYPE_MASK);
view->file_offset = file->hdr.used_size;
return -1;
}
if ((hdr->type & MAIL_TRANSACTION_EXPUNGE) != 0) {
if ((hdr->type & MAIL_TRANSACTION_TYPE_MASK) !=
(MAIL_TRANSACTION_EXPUNGE|MAIL_TRANSACTION_EXPUNGE_PROT)) {
mail_transaction_log_file_set_corrupted(file,
"found expunge without protection mask");
return -1;
}
} else if ((hdr->type & MAIL_TRANSACTION_TYPE_MASK) != type_rec->type) {
mail_transaction_log_file_set_corrupted(file,
"extra bits in header type: 0x%x",
hdr->type & MAIL_TRANSACTION_TYPE_MASK);
return -1;
}
if (hdr->size % record_size != 0) {
mail_transaction_log_file_set_corrupted(file,
"record size wrong (type 0x%x, %u %% %u != 0)",
hdr->type & MAIL_TRANSACTION_TYPE_MASK,
hdr->size, record_size);
view->file_offset = file->hdr.used_size;
return -1;
}
*hdr_r = hdr;
*data_r = CONST_PTR_OFFSET(data, view->file_offset -
file->buffer_offset);
view->file_offset += hdr->size;
return 1;
}
static int seqfix_expunge(const struct mail_transaction_expunge *e,
void *context)
{
struct mail_transaction_log_view *view = context;
struct mail_transaction_expunge new_e;
uint32_t expunges_before;
expunges_before = mail_transaction_expunge_traverse_to(view->exp_ctx,
e->seq2);
if (expunges_before == 0) {
buffer_append(view->data_buf, e, sizeof(*e));
return 1;
}
/* FIXME: if there's expunges in the middle of the
range, we'd have to split this to multiple records */
new_e = *e;
new_e.seq2 += expunges_before;
new_e.seq1 += mail_transaction_expunge_traverse_to(view->exp_ctx,
new_e.seq1);
buffer_append(view->data_buf, &new_e, sizeof(new_e));
return 1;
}
static int seqfix_flag_update(const struct mail_transaction_flag_update *u,
void *context)
{
struct mail_transaction_log_view *view = context;
struct mail_transaction_flag_update new_u;
uint32_t expunges_before;
expunges_before = mail_transaction_expunge_traverse_to(view->exp_ctx,
u->seq2);
if (expunges_before == 0) {
buffer_append(view->data_buf, u, sizeof(*u));
return 1;
}
/* FIXME: if there's expunges in the middle of the
range, we'd have to split this to multiple records */
new_u = *u;
new_u.seq2 += expunges_before;
new_u.seq1 += mail_transaction_expunge_traverse_to(view->exp_ctx,
new_u.seq1);
buffer_append(view->data_buf, &new_u, sizeof(new_u));
return 1;
}
static int seqfix_cache_update(const struct mail_transaction_cache_update *u,
void *context)
{
struct mail_transaction_log_view *view = context;
struct mail_transaction_cache_update new_u;
uint32_t expunges_before;
expunges_before = mail_transaction_expunge_traverse_to(view->exp_ctx,
u->seq);
if (expunges_before != 0) {
new_u = *u;
new_u.seq += expunges_before;
u = &new_u;
}
buffer_append(view->data_buf, u, sizeof(*u));
return 1;
}
int mail_transaction_log_view_next(struct mail_transaction_log_view *view,
const struct mail_transaction_header **hdr_r,
const void **data_r, int *skipped_r)
{
struct mail_transaction_map_functions seqfix_funcs = {
seqfix_expunge, NULL, seqfix_flag_update, seqfix_cache_update
};
const struct mail_transaction_header *hdr;
const void *data;
int ret = 0;
if (skipped_r != NULL)
*skipped_r = FALSE;
if (view->broken)
return -1;
while ((ret = log_view_get_next(view, &hdr, &data)) > 0) {
if ((view->type_mask & hdr->type) != 0)
break;
/* we don't want this record */
if (skipped_r != NULL)
*skipped_r = TRUE;
if ((hdr->type & MAIL_TRANSACTION_EXPUNGE) != 0) {
mail_transaction_log_sort_expunges(view->expunges_buf,
data, hdr->size);
}
/* FIXME: hide flag/cache updates for appends if
append isn't in mask */
}
if (ret <= 0)
return ret;
*hdr_r = hdr;
*data_r = data;
if (buffer_get_used_size(view->expunges_buf) > 0) {
/* we have to fix sequences in the data */
if (view->data_buf == NULL) {
view->data_buf =
buffer_create_dynamic(default_pool,
hdr->size, (size_t)-1);
} else {
buffer_set_used_size(view->data_buf, 0);
}
view->exp_ctx = mail_transaction_expunge_traverse_init(
view->expunges_buf);
ret = mail_transaction_map(hdr, data, &seqfix_funcs, view);
mail_transaction_expunge_traverse_deinit(view->exp_ctx);
i_assert(buffer_get_used_size(view->data_buf) == hdr->size);
*data_r = buffer_get_data(view->data_buf, NULL);
}
if ((hdr->type & MAIL_TRANSACTION_EXPUNGE) != 0) {
mail_transaction_log_sort_expunges(view->expunges_buf,
data, hdr->size);
/* hide expunge protection */
view->tmp_hdr = *hdr;
view->tmp_hdr.type &= ~MAIL_TRANSACTION_EXPUNGE_PROT;
*hdr_r = &view->tmp_hdr;
}
return 1;
}
buffer_t *
mail_transaction_log_view_get_expunges(struct mail_transaction_log_view *view)
{
return view->expunges_buf;
}
--- NEW FILE: mail-transaction-log.c ---
/* Copyright (C) 2003-2004 Timo Sirainen */
#include "lib.h"
#include "buffer.h"
#include "file-lock.h"
#include "file-dotlock.h"
#include "read-full.h"
#include "write-full.h"
#include "mmap-util.h"
#include "mail-index-private.h"
#include "mail-index-view-private.h"
#include "mail-transaction-log-private.h"
#include "mail-transaction-util.h"
#include "mail-index-transaction-private.h"
#include <stddef.h>
#include <sys/stat.h>
struct mail_transaction_add_ctx {
struct mail_transaction_log *log;
struct mail_index_view *view;
buffer_t *appends, *expunges;
buffer_t *flag_updates, *cache_updates;
};
static struct mail_transaction_log_file *
mail_transaction_log_file_open_or_create(struct mail_transaction_log *log,
const char *path);
static int mail_transaction_log_rotate(struct mail_transaction_log *log);
static int
mail_transaction_log_file_lock(struct mail_transaction_log_file *file,
int lock_type);
static int mail_transaction_log_lock_head(struct mail_transaction_log *log);
void
mail_transaction_log_file_set_corrupted(struct mail_transaction_log_file *file,
const char *fmt, ...)
{
va_list va;
file->hdr.indexid = 0;
if (pwrite_full(file->fd, &file->hdr.indexid,
sizeof(file->hdr.indexid), 0) < 0) {
mail_index_file_set_syscall_error(file->log->index,
file->filepath, "pwrite()");
}
va_start(va, fmt);
t_push();
mail_index_set_error(file->log->index,
"Corrupted transaction log file %s: %s",
file->filepath, t_strdup_vprintf(fmt, va));
t_pop();
va_end(va);
}
static int mail_transaction_log_check_file_seq(struct mail_transaction_log *log)
{
struct mail_index *index = log->index;
struct mail_transaction_log_file *file;
unsigned int lock_id;
int ret;
if (mail_transaction_log_lock_head(log) < 0)
return -1;
file = log->head;
ret = mail_index_lock_shared(index, TRUE, &lock_id);
if (ret == 0) {
ret = mail_index_map(index, FALSE);
if (ret <= 0)
ret = -1;
else if (file->hdr.file_seq != index->hdr->log_file_seq) {
/* broken - fix it by creating a new log file */
ret = mail_transaction_log_rotate(log);
}
}
(void)mail_transaction_log_file_lock(file, F_UNLCK);
return ret;
}
struct mail_transaction_log *
mail_transaction_log_open_or_create(struct mail_index *index)
{
struct mail_transaction_log *log;
const char *path;
log = i_new(struct mail_transaction_log, 1);
log->index = index;
path = t_strconcat(log->index->filepath,
MAIL_TRANSACTION_LOG_PREFIX, NULL);
log->head = mail_transaction_log_file_open_or_create(log, path);
if (log->head == NULL) {
i_free(log);
return NULL;
}
if (index->fd != -1 &&
log->head->hdr.file_seq != index->hdr->log_file_seq) {
/* head log file isn't same as head index file -
shouldn't happen except in race conditions. lock them and
check again - FIXME: missing error handling */
(void)mail_transaction_log_check_file_seq(log);
}
return log;
}
void mail_transaction_log_close(struct mail_transaction_log *log)
{
i_assert(log->views == NULL);
i_free(log);
}
static int
mail_transaction_log_file_lock(struct mail_transaction_log_file *file,
int lock_type)
{
int ret;
if (lock_type == F_UNLCK) {
i_assert(file->lock_type != F_UNLCK);
} else {
i_assert(file->lock_type == F_UNLCK);
}
ret = file_wait_lock_full(file->fd, lock_type, DEFAULT_LOCK_TIMEOUT,
NULL, NULL);
if (ret > 0) {
file->lock_type = lock_type;
return 0;
}
if (ret < 0) {
mail_index_file_set_syscall_error(file->log->index,
file->filepath,
"file_wait_lock()");
return -1;
}
mail_index_set_error(file->log->index,
"Timeout while waiting for release of "
"%s fcntl() lock for transaction log file %s",
lock_type == F_WRLCK ? "exclusive" : "shared",
file->filepath);
file->log->index->index_lock_timeout = TRUE;
return -1;
}
static void
mail_transaction_log_file_close(struct mail_transaction_log_file *file)
{
if (close(file->fd) < 0) {
mail_index_file_set_syscall_error(file->log->index,
file->filepath, "close()");
}
i_free(file->filepath);
i_free(file);
}
static int
mail_transaction_log_file_read_hdr(struct mail_transaction_log_file *file,
struct stat *st)
{
int ret;
uint32_t old_size = file->hdr.used_size;
if (file->lock_type != F_UNLCK)
ret = pread_full(file->fd, &file->hdr, sizeof(file->hdr), 0);
else {
if (mail_transaction_log_file_lock(file, F_RDLCK) < 0)
return -1;
ret = pread_full(file->fd, &file->hdr, sizeof(file->hdr), 0);
(void)mail_transaction_log_file_lock(file, F_UNLCK);
}
if (ret < 0) {
mail_index_file_set_syscall_error(file->log->index,
file->filepath, "pread()");
return -1;
}
if (ret == 0) {
mail_transaction_log_file_set_corrupted(file,
"unexpected end of file while reading header");
return 0;
}
if (file->hdr.indexid == 0) {
/* corrupted */
mail_index_set_error(file->log->index,
"Transaction log file %s: marked corrupted",
file->filepath);
return 0;
}
if (file->hdr.indexid != file->log->index->indexid &&
file->log->index->indexid != 0) {
/* either index was just recreated, or transaction has wrong
indexid. we don't know here which one is the case, so we'll
just fail. If index->indexid == 0, we're rebuilding it and
we just want to lock the transaction log. */
mail_index_set_error(file->log->index,
"Transaction log file %s: invalid indexid",
file->filepath);
return 0;
}
if (file->hdr.used_size > st->st_size) {
mail_transaction_log_file_set_corrupted(file,
"used_size (%u) > file size (%"PRIuUOFF_T")",
file->hdr.used_size, (uoff_t)st->st_size);
return 0;
}
if (file->hdr.used_size < old_size) {
mail_transaction_log_file_set_corrupted(file,
"used_size (%u) < old_size (%u)",
file->hdr.used_size, old_size);
return 0;
}
return 1;
}
static int mail_transaction_log_file_create(struct mail_transaction_log *log,
const char *path,
dev_t dev, ino_t ino)
{
struct mail_index *index = log->index;
struct mail_transaction_log_header hdr;
struct stat st;
unsigned int lock_id;
int fd, fd2, ret;
/* this lock should never exist for a long time.. */
fd = file_dotlock_open(path, NULL, 30, 0, 120, NULL, NULL);
if (fd == -1) {
mail_index_file_set_syscall_error(index, path,
"file_dotlock_open()");
return -1;
}
/* log creation is locked now - see if someone already created it */
fd2 = open(path, O_RDWR);
if (fd2 != -1) {
if ((ret = fstat(fd2, &st)) < 0) {
mail_index_file_set_syscall_error(index, path,
"fstat()");
} else if (st.st_dev == dev && st.st_ino == ino) {
/* same file, still broken */
} else {
(void)file_dotlock_delete(path, fd2);
return fd2;
}
(void)close(fd2);
fd2 = -1;
if (ret < 0)
return -1;
} else if (errno != ENOENT) {
mail_index_file_set_syscall_error(index, path, "open()");
return -1;
}
memset(&hdr, 0, sizeof(hdr));
hdr.indexid = index->indexid;
hdr.used_size = sizeof(hdr);
if (index->fd != -1) {
index->log_locked = TRUE; /* kludging around assert.. */
if (mail_index_lock_exclusive(index, 0, 0, &lock_id) < 0) {
(void)file_dotlock_delete(path, fd);
index->log_locked = FALSE;
return -1;
}
index->log_locked = FALSE;
ret = mail_index_map(index, FALSE);
if (ret > 0) {
/* update log_file_* fields in header */
struct mail_index_header idx_hdr;
idx_hdr = *index->hdr;
idx_hdr.log_file_seq++;
idx_hdr.log_file_offset = sizeof(hdr);
if (mail_index_write_header(index, &idx_hdr) < 0)
ret = -1;
}
mail_index_unlock(index, lock_id);
if (ret <= 0) {
(void)file_dotlock_delete(path, fd);
return -1;
}
hdr.file_seq = index->hdr->log_file_seq;
} else {
/* creating new index file */
hdr.file_seq = index->hdr->log_file_seq+1;
}
if (write_full(fd, &hdr, sizeof(hdr)) < 0) {
mail_index_file_set_syscall_error(index, path, "write_full()");
(void)file_dotlock_delete(path, fd);
return -1;
}
fd2 = dup(fd);
if (fd2 < 0) {
mail_index_file_set_syscall_error(index, path, "dup()");
(void)file_dotlock_delete(path, fd);
return -1;
}
if (file_dotlock_replace(path, fd, FALSE) <= 0)
return -1;
/* success */
return fd2;
}
static struct mail_transaction_log_file *
mail_transaction_log_file_fd_open(struct mail_transaction_log *log,
const char *path, int fd)
{
struct mail_transaction_log_file **p;
struct mail_transaction_log_file *file;
struct stat st;
int ret;
if (fstat(fd, &st) < 0) {
mail_index_file_set_syscall_error(log->index, path, "stat()");
return NULL;
}
file = i_new(struct mail_transaction_log_file, 1);
file->refcount = 1;
file->log = log;
file->filepath = i_strdup(path);
file->fd = fd;
file->lock_type = F_UNLCK;
file->st_dev = st.st_dev;
file->st_ino = st.st_ino;
ret = mail_transaction_log_file_read_hdr(file, &st);
if (ret == 0) {
/* corrupted header */
fd = mail_transaction_log_file_create(log, path,
st.st_dev, st.st_ino);
if (fstat(fd, &st) < 0) {
mail_index_file_set_syscall_error(log->index, path,
"stat()");
(void)close(fd);
fd = -1;
ret = -1;
}
if (fd != -1) {
(void)close(file->fd);
file->fd = fd;
file->st_dev = st.st_dev;
file->st_ino = st.st_ino;
memset(&file->hdr, 0, sizeof(file->hdr));
ret = mail_transaction_log_file_read_hdr(file, &st);
}
}
if (ret <= 0) {
mail_transaction_log_file_close(file);
return NULL;
}
for (p = &log->tail; *p != NULL; p = &(*p)->next)
;
*p = file;
return file;
}
static struct mail_transaction_log_file *
mail_transaction_log_file_open_or_create(struct mail_transaction_log *log,
const char *path)
{
int fd;
fd = open(path, O_RDWR);
if (fd == -1) {
if (errno != ENOENT) {
mail_index_file_set_syscall_error(log->index, path,
"open()");
return NULL;
}
fd = mail_transaction_log_file_create(log, path, 0, 0);
if (fd == -1)
return NULL;
}
return mail_transaction_log_file_fd_open(log, path, fd);
}
void mail_transaction_logs_clean(struct mail_transaction_log *log)
{
struct mail_transaction_log_file **p;
for (p = &log->tail; *p != NULL; ) {
if ((*p)->refcount != 0)
p = &(*p)->next;
else {
mail_transaction_log_file_close(*p);
*p = (*p)->next;
}
}
}
static int mail_transaction_log_rotate(struct mail_transaction_log *log)
{
struct mail_transaction_log_file *file;
struct stat st;
int fd;
if (fstat(log->head->fd, &st) < 0) {
mail_index_file_set_syscall_error(log->index,
log->head->filepath,
"fstat()");
return -1;
}
fd = mail_transaction_log_file_create(log, log->head->filepath,
st.st_dev, st.st_ino);
if (fd == -1)
return 0;
file = mail_transaction_log_file_fd_open(log, log->head->filepath, fd);
if (file == NULL)
return -1;
if (log->head != NULL) {
if (--log->head->refcount == 0)
mail_transaction_logs_clean(log);
}
log->head = file;
return 0;
}
static int mail_transaction_log_refresh(struct mail_transaction_log *log)
{
struct mail_transaction_log_file *file;
struct stat st;
const char *path;
int ret;
path = t_strconcat(log->index->filepath,
MAIL_TRANSACTION_LOG_PREFIX, NULL);
if (stat(path, &st) < 0) {
mail_index_file_set_syscall_error(log->index, path, "stat()");
return -1;
}
if (log->head != NULL &&
log->head->st_ino == st.st_ino &&
log->head->st_dev == st.st_dev) {
/* same file */
ret = mail_transaction_log_file_read_hdr(log->head, &st);
return ret <= 0 ? -1 : 0;
}
file = mail_transaction_log_file_open_or_create(log, path);
if (file == NULL)
return -1;
if (log->head != NULL) {
if (--log->head->refcount == 0)
mail_transaction_logs_clean(log);
}
log->head = file;
return 0;
}
int mail_transaction_log_file_find(struct mail_transaction_log *log,
uint32_t file_seq,
struct mail_transaction_log_file **file_r)
{
struct mail_transaction_log_file *file;
if (file_seq > log->head->hdr.file_seq) {
if (mail_transaction_log_refresh(log) < 0)
return -1;
}
for (file = log->tail; file != NULL; file = file->next) {
if (file->hdr.file_seq == file_seq) {
*file_r = file;
return 1;
}
}
return 0;
}
static int
mail_transaction_log_file_read(struct mail_transaction_log_file *file,
uoff_t offset)
{
void *data;
size_t size;
int ret;
i_assert(file->mmap_base == NULL);
i_assert(offset <= file->hdr.used_size);
if (file->buffer != NULL && file->buffer_offset > offset) {
/* we have to insert missing data to beginning of buffer */
size = file->buffer_offset - offset;
buffer_copy(file->buffer, size, file->buffer, 0, (size_t)-1);
file->buffer_offset = offset;
data = buffer_get_modifyable_data(file->buffer, NULL);
ret = pread(file->fd, data, size, offset);
if (ret < 0 && errno == ESTALE) {
/* log file was deleted in NFS server, fail silently */
ret = 0;
}
if (ret <= 0)
return ret;
}
if (file->buffer == NULL) {
size = file->hdr.used_size - offset;
file->buffer = buffer_create_dynamic(default_pool,
size, (size_t)-1);
file->buffer_offset = offset;
size = 0;
} else {
size = buffer_get_used_size(file->buffer);
if (file->buffer_offset + size >= file->hdr.used_size) {
/* caller should have checked this.. */
return 1;
}
}
size = file->hdr.used_size - file->buffer_offset - size;
data = buffer_append_space_unsafe(file->buffer, size);
ret = pread(file->fd, data, size, offset);
if (ret < 0 && errno == ESTALE) {
/* log file was deleted in NFS server, fail silently */
ret = 0;
}
return ret;
}
int mail_transaction_log_file_map(struct mail_transaction_log_file *file,
uoff_t start_offset, uoff_t end_offset)
{
size_t size;
struct stat st;
int ret;
i_assert(start_offset <= end_offset);
if (file->hdr.indexid == 0) {
/* corrupted */
return 0;
}
if (file->buffer != NULL && file->buffer_offset <= start_offset) {
/* see if we already have it */
size = buffer_get_used_size(file->buffer);
if (file->buffer_offset + size >= end_offset)
return 1;
}
if (fstat(file->fd, &st) < 0) {
mail_index_file_set_syscall_error(file->log->index,
file->filepath, "fstat()");
return -1;
}
if (st.st_size == file->hdr.used_size && end_offset == (uoff_t)-1) {
/* we've seen the whole file.. do we have all of it mapped? */
size = buffer_get_used_size(file->buffer);
if (file->buffer_offset + size == file->hdr.used_size)
return 1;
}
if (file->buffer != NULL &&
(file->mmap_base != NULL || file->log->index->use_mmap)) {
buffer_free(file->buffer);
file->buffer = NULL;
}
if (file->mmap_base != NULL) {
if (munmap(file->mmap_base, file->mmap_size) < 0) {
mail_index_file_set_syscall_error(file->log->index,
file->filepath,
"munmap()");
}
file->mmap_base = NULL;
}
if (mail_transaction_log_file_read_hdr(file, &st) <= 0)
return -1;
if (end_offset == (uoff_t)-1)
end_offset = file->hdr.used_size;
if (start_offset < sizeof(file->hdr)) {
mail_transaction_log_file_set_corrupted(file,
"offset (%"PRIuUOFF_T"u) < header size (%"PRIuSIZE_T")",
start_offset, sizeof(file->hdr));
return -1;
}
if (end_offset > file->hdr.used_size) {
mail_transaction_log_file_set_corrupted(file,
"offset (%"PRIuUOFF_T"u) > used_size (%u)",
end_offset, file->hdr.used_size);
return -1;
}
if (!file->log->index->use_mmap) {
ret = mail_transaction_log_file_read(file, start_offset);
if (ret <= 0) {
/* make sure we don't leave ourself in
inconsistent state */
if (file->buffer != NULL) {
buffer_free(file->buffer);
file->buffer = NULL;
}
file->buffer_size = 0;
} else {
file->buffer_size = buffer_get_used_size(file->buffer);
}
return ret;
}
file->mmap_size = file->hdr.used_size;
file->mmap_base = mmap(NULL, file->mmap_size, PROT_READ,
MAP_SHARED, file->fd, 0);
if (file->mmap_base == MAP_FAILED) {
file->mmap_base = NULL;
mail_index_file_set_syscall_error(file->log->index,
file->filepath, "mmap()");
return -1;
}
file->buffer = buffer_create_const_data(default_pool, file->mmap_base,
file->mmap_size);
file->buffer_size = buffer_get_used_size(file->buffer);
return 1;
}
static int mail_transaction_log_lock_head(struct mail_transaction_log *log)
{
struct mail_transaction_log_file *file;
int ret = 0;
/* we want to get the head file locked. this is a bit racy,
since by the time we have it locked a new log file may have been
created.
creating new log file requires locking the head file, so if we
can lock it and don't see another file, we can be sure no-one is
creating a new log at the moment */
for (;;) {
file = log->head;
if (mail_transaction_log_file_lock(file, F_WRLCK) < 0)
return -1;
file->refcount++;
ret = mail_transaction_log_refresh(log);
if (--file->refcount == 0) {
mail_transaction_logs_clean(log);
file = NULL;
}
if (ret == 0 && log->head == file) {
/* success */
break;
}
if (file != NULL) {
if (mail_transaction_log_file_lock(file, F_UNLCK) < 0)
return -1;
}
if (ret < 0)
break;
/* try again */
}
return ret;
}
static int get_expunge_buf(struct mail_transaction_log *log,
struct mail_index_view *view, buffer_t *expunges)
{
struct mail_transaction_log_view *sync_view;
const struct mail_transaction_header *hdr;
const void *data;
int ret;
sync_view = mail_transaction_log_view_open(log);
ret = mail_transaction_log_view_set(sync_view, view->log_file_seq,
view->log_file_offset,
log->head->hdr.file_seq,
log->head->hdr.used_size,
MAIL_TRANSACTION_TYPE_MASK);
while ((ret = mail_transaction_log_view_next(sync_view,
&hdr, &data, NULL)) == 1) {
if ((hdr->type & MAIL_TRANSACTION_TYPE_MASK) ==
MAIL_TRANSACTION_EXPUNGE) {
mail_transaction_log_sort_expunges(expunges,
data, hdr->size);
}
}
mail_transaction_log_view_close(sync_view);
return ret;
}
static void
log_view_fix_sequences(struct mail_index_view *view, buffer_t *view_expunges,
buffer_t *buf, size_t record_size, int two, int uids)
{
// FIXME: make sure this function works correctly
const struct mail_transaction_expunge *exp, *exp_end, *exp2;
unsigned char *data;
uint32_t *seq, expunges_before, count;
size_t src_idx, dest_idx, size;
if (buf == NULL)
return;
exp = buffer_get_data(view_expunges, &size);
exp_end = exp + (size / sizeof(*exp));
if (exp == exp_end)
return;
data = buffer_get_modifyable_data(buf, &size);
expunges_before = 0;
for (src_idx = dest_idx = 0; src_idx < size; src_idx += record_size) {
seq = (uint32_t *)&data[src_idx];
while (exp != exp_end && exp->seq1 < seq[0]) {
expunges_before += exp->seq2 - exp->seq1 + 1;
exp++;
}
if (exp != exp_end && exp->seq1 == seq[0]) {
/* this sequence was expunged */
if (!two)
continue;
/* we point to next non-expunged message */
}
if (expunges_before != 0) {
if (uids) {
(void)mail_index_lookup_uid(view, seq[0],
&seq[2]);
}
seq[0] -= expunges_before;
}
if (two) {
exp2 = exp;
count = expunges_before;
while (exp2 != exp_end && exp2->seq1 <= seq[1]) {
count += exp->seq2 - exp->seq1 + 1;
exp2++;
}
if (seq[1] < count || seq[1]-count < seq[0]) {
/* whole range is expunged */
continue;
}
if (count != 0) {
if (uids) {
(void)mail_index_lookup_uid(view,
seq[1],
&seq[3]);
}
seq[1] -= count;
}
}
if (src_idx != dest_idx)
memcpy(&data[dest_idx], &data[src_idx], record_size);
dest_idx += record_size;
}
buffer_set_used_size(buf, dest_idx);
}
static int
mail_transaction_log_fix_sequences(struct mail_transaction_log *log,
struct mail_index_transaction *t)
{
buffer_t *view_expunges;
if (t->updates == NULL && t->cache_updates == NULL &&
t->expunges == NULL)
return 0;
/* all sequences are currently relative to given view. we have to
find out all the expunges since then, even the ones that aren't
yet synchronized to index file. */
view_expunges = buffer_create_dynamic(default_pool, 1024, (size_t)-1);
if (get_expunge_buf(log, t->view, view_expunges) < 0) {
buffer_free(view_expunges);
return -1;
}
log_view_fix_sequences(t->view, view_expunges, t->updates,
sizeof(struct mail_transaction_flag_update),
TRUE, FALSE);
log_view_fix_sequences(t->view, view_expunges, t->cache_updates,
sizeof(struct mail_transaction_cache_update),
FALSE, FALSE);
log_view_fix_sequences(t->view, view_expunges, t->expunges,
sizeof(struct mail_transaction_expunge),
TRUE, TRUE);
buffer_free(view_expunges);
return 0;
}
static int
log_append_buffer(struct mail_transaction_log_file *file, const buffer_t *buf,
enum mail_transaction_type type, int external)
{
struct mail_transaction_header hdr;
const void *data;
size_t size;
i_assert((type & MAIL_TRANSACTION_TYPE_MASK) != 0);
if (buf != NULL) {
data = buffer_get_data(buf, &size);
if (size == 0)
return 0;
} else {
/* write only the header */
data = NULL;
size = 0;
}
hdr.type = type;
if (type == MAIL_TRANSACTION_EXPUNGE)
hdr.type |= MAIL_TRANSACTION_EXPUNGE_PROT;
if (external)
hdr.type |= MAIL_TRANSACTION_EXTERNAL;
hdr.size = size;
if (pwrite_full(file->fd, &hdr, sizeof(hdr), file->hdr.used_size) < 0)
return -1;
file->hdr.used_size += sizeof(hdr);
if (size != 0) {
if (pwrite_full(file->fd, data, size, file->hdr.used_size) < 0)
return -1;
file->hdr.used_size += size;
}
return 0;
}
int mail_transaction_log_append(struct mail_index_transaction *t,
uint32_t *log_file_seq_r,
uoff_t *log_file_offset_r)
{
struct mail_index_view *view = t->view;
struct mail_transaction_log *log;
struct mail_transaction_log_file *file;
size_t offset;
uoff_t append_offset;
int ret;
if (t->updates == NULL && t->cache_updates == NULL &&
t->expunges == NULL && t->appends == NULL) {
/* nothing to append */
return 0;
}
log = mail_index_view_get_index(view)->log;
if (log->index->log_locked) {
i_assert(view->external);
} else {
if (mail_transaction_log_lock_head(log) < 0)
return -1;
}
file = log->head;
append_offset = file->hdr.used_size;
if (mail_transaction_log_fix_sequences(log, t) < 0) {
if (!log->index->log_locked)
(void)mail_transaction_log_file_lock(file, F_UNLCK);
return -1;
}
ret = 0;
if (t->appends != NULL) {
ret = log_append_buffer(file, t->appends,
MAIL_TRANSACTION_APPEND,
view->external);
}
if (t->updates != NULL && ret == 0) {
ret = log_append_buffer(file, t->updates,
MAIL_TRANSACTION_FLAG_UPDATE,
view->external);
}
if (t->cache_updates != NULL && ret == 0) {
ret = log_append_buffer(file, t->cache_updates,
MAIL_TRANSACTION_CACHE_UPDATE,
view->external);
}
if (t->expunges != NULL && ret == 0) {
ret = log_append_buffer(file, t->expunges,
MAIL_TRANSACTION_EXPUNGE,
view->external);
}
if (ret == 0) {
/* rewrite used_size */
offset = offsetof(struct mail_transaction_log_header,
used_size);
ret = pwrite_full(file->fd, &file->hdr.used_size,
sizeof(file->hdr.used_size), offset);
}
if (ret == 0 && (t->updates != NULL || t->appends != NULL) &&
t->hide_transaction) {
mail_index_view_add_synced_transaction(view, file->hdr.file_seq,
append_offset);
}
if (ret < 0) {
file->hdr.used_size = append_offset;
mail_index_file_set_syscall_error(log->index, file->filepath,
"pwrite()");
} else if (fsync(file->fd) < 0) {
/* we don't know how much of it got written,
it may be corrupted now.. */
mail_index_file_set_syscall_error(log->index, file->filepath,
"fsync()");
ret = -1;
}
*log_file_seq_r = file->hdr.file_seq;
*log_file_offset_r = file->hdr.used_size;
if (!log->index->log_locked)
(void)mail_transaction_log_file_lock(file, F_UNLCK);
return ret;
}
int mail_transaction_log_sync_lock(struct mail_transaction_log *log,
uint32_t *file_seq_r, uoff_t *file_offset_r)
{
i_assert(!log->index->log_locked);
if (mail_transaction_log_lock_head(log) < 0)
return -1;
log->index->log_locked = TRUE;
*file_seq_r = log->head->hdr.file_seq;
*file_offset_r = log->head->hdr.used_size;
return 0;
}
void mail_transaction_log_sync_unlock(struct mail_transaction_log *log)
{
i_assert(log->index->log_locked);
log->index->log_locked = FALSE;
(void)mail_transaction_log_file_lock(log->head, F_UNLCK);
}
void mail_transaction_log_get_head(struct mail_transaction_log *log,
uint32_t *file_seq_r, uoff_t *file_offset_r)
{
i_assert(log->index->log_locked);
*file_seq_r = log->head->hdr.file_seq;
*file_offset_r = log->head->hdr.used_size;
}
--- NEW FILE: mail-transaction-log.h ---
#ifndef __MAIL_TRANSACTION_LOG_H
#define __MAIL_TRANSACTION_LOG_H
#define MAIL_TRANSACTION_LOG_PREFIX ".log"
struct mail_transaction_log_header {
uint32_t indexid;
uint32_t file_seq;
uint32_t used_size;
};
enum mail_transaction_type {
MAIL_TRANSACTION_EXPUNGE = 0x00000001,
MAIL_TRANSACTION_APPEND = 0x00000002,
MAIL_TRANSACTION_FLAG_UPDATE = 0x00000004,
MAIL_TRANSACTION_CACHE_UPDATE = 0x00000008,
MAIL_TRANSACTION_TYPE_MASK = 0x0000ffff,
/* since we'll expunge mails based on data read from transaction log,
try to avoid the possibility of corrupted transaction log expunging
messages. this value is ORed to the actual MAIL_TRANSACTION_EXPUNGE
flag. if it's not present, assume corrupted log. */
MAIL_TRANSACTION_EXPUNGE_PROT = 0x0000cd90,
/* Mailbox synchronization noticed this change. */
MAIL_TRANSACTION_EXTERNAL = 0x10000000
};
struct mail_transaction_header {
uint32_t size;
uint32_t type; /* enum mail_transaction_type */
};
struct mail_transaction_expunge {
uint32_t seq1, seq2;
uint32_t uid1, uid2; /* only to avoid accidental expunges due to bugs */
};
struct mail_transaction_cache_update {
uint32_t seq;
uint32_t cache_offset;
};
struct mail_transaction_flag_update {
uint32_t seq1, seq2;
uint8_t add_flags;
custom_flags_mask_t add_custom_flags;
uint8_t remove_flags;
custom_flags_mask_t remove_custom_flags;
};
struct mail_transaction_log *
mail_transaction_log_open_or_create(struct mail_index *index);
void mail_transaction_log_close(struct mail_transaction_log *log);
struct mail_transaction_log_view *
mail_transaction_log_view_open(struct mail_transaction_log *log);
void mail_transaction_log_view_close(struct mail_transaction_log_view *view);
/* Set view boundaries. Returns -1 if error, 0 if ok. */
int
mail_transaction_log_view_set(struct mail_transaction_log_view *view,
uint32_t min_file_seq, uoff_t min_file_offset,
uint32_t max_file_seq, uoff_t max_file_offset,
enum mail_transaction_type type_mask);
/* Read next transaction record from current position. The position is updated.
Returns -1 if error, 0 if we're at end of the view, 1 if ok. */
int mail_transaction_log_view_next(struct mail_transaction_log_view *view,
const struct mail_transaction_header **hdr_r,
const void **data_r, int *skipped_r);
/* Returns the position of the record returned previously with
mail_transaction_log_view_next() */
void
mail_transaction_log_view_get_prev_pos(struct mail_transaction_log_view *view,
uint32_t *file_seq_r,
uoff_t *file_offset_r);
buffer_t *
mail_transaction_log_view_get_expunges(struct mail_transaction_log_view *view);
/* Marks the log file in current position to be corrupted. */
void
mail_transaction_log_view_set_corrupted(struct mail_transaction_log_view *view,
const char *fmt, ...)
__attr_format__(2, 3);
int
mail_transaction_log_view_is_corrupted(struct mail_transaction_log_view *view);
/* Write data to transaction log. This is atomic operation. Sequences in
updates[] and expunges[] are relative to given view, they're modified
to real ones. */
int mail_transaction_log_append(struct mail_index_transaction *t,
uint32_t *log_file_seq_r,
uoff_t *log_file_offset_r);
/* Lock transaction log for index synchronization. Log cannot be read or
written to while it's locked. Returns end offset. */
int mail_transaction_log_sync_lock(struct mail_transaction_log *log,
uint32_t *file_seq_r, uoff_t *file_offset_r);
void mail_transaction_log_sync_unlock(struct mail_transaction_log *log);
/* Returns the current head. Works only when log is locked. */
void mail_transaction_log_get_head(struct mail_transaction_log *log,
uint32_t *file_seq_r, uoff_t *file_offset_r);
#endif
--- NEW FILE: mail-transaction-util.c ---
/* Copyright (C) 2004 Timo Sirainen */
#include "lib.h"
#include "buffer.h"
#include "mail-index-private.h"
#include "mail-transaction-log.h"
#include "mail-transaction-util.h"
struct mail_transaction_expunge_traverse_ctx {
const struct mail_transaction_expunge *expunges;
size_t expunges_count, cur_idx, old_idx;
uint32_t cur_seq, expunges_before;
uint32_t old_seq, old_expunges_before;
};
const struct mail_transaction_type_map mail_transaction_type_map[] = {
{ MAIL_TRANSACTION_APPEND, MAIL_INDEX_SYNC_TYPE_APPEND,
sizeof(struct mail_index_record) },
{ MAIL_TRANSACTION_EXPUNGE, MAIL_INDEX_SYNC_TYPE_EXPUNGE,
sizeof(struct mail_transaction_expunge) },
{ MAIL_TRANSACTION_FLAG_UPDATE, MAIL_INDEX_SYNC_TYPE_FLAGS,
sizeof(struct mail_transaction_flag_update) },
{ MAIL_TRANSACTION_CACHE_UPDATE, 0,
sizeof(struct mail_transaction_cache_update) },
{ 0, 0, 0 }
};
const struct mail_transaction_type_map *
mail_transaction_type_lookup(enum mail_transaction_type type)
{
int i;
for (i = 0; mail_transaction_type_map[i].type != 0; i++) {
if ((mail_transaction_type_map[i].type & type) != 0)
return &mail_transaction_type_map[i];
}
return NULL;
}
enum mail_transaction_type
mail_transaction_type_mask_get(enum mail_index_sync_type sync_type)
{
enum mail_transaction_type type = 0;
int i;
for (i = 0; mail_transaction_type_map[i].type != 0; i++) {
if ((mail_transaction_type_map[i].sync_type & sync_type) != 0)
type |= mail_transaction_type_map[i].type;
}
return type;
}
int mail_transaction_map(const struct mail_transaction_header *hdr,
const void *data,
struct mail_transaction_map_functions *map,
void *context)
{
int ret = 0;
switch (hdr->type & MAIL_TRANSACTION_TYPE_MASK) {
case MAIL_TRANSACTION_APPEND: {
const struct mail_index_record *rec, *end;
if (map->append == NULL)
break;
end = CONST_PTR_OFFSET(data, hdr->size);
for (rec = data; rec != end; rec++) {
ret = map->append(rec, context);
if (ret <= 0)
break;
}
break;
}
case MAIL_TRANSACTION_EXPUNGE: {
const struct mail_transaction_expunge *rec, *end;
if (map->expunge == NULL)
break;
end = CONST_PTR_OFFSET(data, hdr->size);
for (rec = data; rec != end; rec++) {
ret = map->expunge(rec, context);
if (ret <= 0)
break;
}
break;
}
case MAIL_TRANSACTION_FLAG_UPDATE: {
const struct mail_transaction_flag_update *rec, *end;
if (map->flag_update == NULL)
break;
end = CONST_PTR_OFFSET(data, hdr->size);
for (rec = data; rec != end; rec++) {
ret = map->flag_update(rec, context);
if (ret <= 0)
break;
}
break;
}
case MAIL_TRANSACTION_CACHE_UPDATE: {
const struct mail_transaction_cache_update *rec, *end;
if (map->cache_update == NULL)
break;
end = CONST_PTR_OFFSET(data, hdr->size);
for (rec = data; rec != end; rec++) {
ret = map->cache_update(rec, context);
if (ret <= 0)
break;
}
break;
}
default:
i_unreached();
}
return ret;
}
void
mail_transaction_log_sort_expunges(buffer_t *expunges_buf,
const struct mail_transaction_expunge *src,
size_t src_buf_size)
{
const struct mail_transaction_expunge *src_end;
struct mail_transaction_expunge *dest;
struct mail_transaction_expunge new_exp;
uint32_t cur_seq, prev_seq, expunges_before, count;
size_t first, i, dest_count;
i_assert(src_buf_size % sizeof(*src) == 0);
src_end = CONST_PTR_OFFSET(src, src_buf_size);
/* @UNSAFE */
dest = buffer_get_modifyable_data(expunges_buf, &dest_count);
dest_count /= sizeof(*dest);
cur_seq = prev_seq = 1; expunges_before = 0;
for (i = 0; src != src_end; src++) {
for (; i < dest_count; i++) {
count = dest[i].seq1 - prev_seq;
if (cur_seq + count > src->seq1)
break;
cur_seq += count;
expunges_before += dest[i].seq2 - dest[i].seq1 + 1;
prev_seq = dest[i].seq2+1;
}
new_exp.seq1 = src->seq1 + expunges_before;
new_exp.seq2 = src->seq2 + expunges_before;
/* if src[] is in format {1,2}{1,2} rather than {1,2}{3,4}:
expunges_before += new_exp.seq2 - new_exp.seq1 + 1;*/
first = i;
while (i < dest_count && new_exp.seq2 >= dest[i].seq1-1) {
/* we can/must merge with next record */
count = dest[i].seq2 - dest[i].seq1 + 1;
expunges_before += count;
new_exp.seq2 += count;
i++;
}
if (first > 0 && new_exp.seq1 == dest[first-1].seq2+1) {
/* continue previous record */
dest[first-1].seq2 = new_exp.seq2;
} else if (i == first) {
buffer_insert(expunges_buf, i * sizeof(new_exp),
&new_exp, sizeof(new_exp));
i++; first++;
dest = buffer_get_modifyable_data(expunges_buf, NULL);
dest_count++;
} else {
/* use next record */
dest[first].seq1 = new_exp.seq1;
dest[first].seq2 = new_exp.seq2;
first++;
}
if (i > first) {
buffer_delete(expunges_buf, first * sizeof(new_exp),
(i - first) * sizeof(new_exp));
dest = buffer_get_modifyable_data(expunges_buf, NULL);
dest_count -= i - first;
i = first + 1;
}
}
}
struct mail_transaction_expunge_traverse_ctx *
mail_transaction_expunge_traverse_init(const buffer_t *expunges_buf)
{
struct mail_transaction_expunge_traverse_ctx *ctx;
ctx = i_new(struct mail_transaction_expunge_traverse_ctx, 1);
ctx->cur_seq = 1;
ctx->old_seq = 1;
if (expunges_buf != NULL) {
ctx->expunges =
buffer_get_data(expunges_buf, &ctx->expunges_count);
ctx->expunges_count /= sizeof(*ctx->expunges);
}
return ctx;
}
void mail_transaction_expunge_traverse_deinit(
struct mail_transaction_expunge_traverse_ctx *ctx)
{
i_free(ctx);
}
uint32_t mail_transaction_expunge_traverse_to(
struct mail_transaction_expunge_traverse_ctx *ctx, uint32_t seq)
{
uint32_t idx, count, last_seq;
if (seq < ctx->cur_seq) {
/* allow seeking one back */
ctx->cur_idx = ctx->old_idx;
ctx->cur_seq = ctx->old_seq;
ctx->expunges_before = ctx->old_expunges_before;
} else {
ctx->old_idx = ctx->cur_idx;
ctx->old_seq = ctx->cur_seq;
ctx->old_expunges_before = ctx->expunges_before;
}
i_assert(seq >= ctx->cur_seq);
idx = ctx->cur_idx;
last_seq = idx == 0 ? 1 : ctx->expunges[idx-1].seq2 + 1;
for (; idx < ctx->expunges_count; idx++) {
count = ctx->expunges[idx].seq1 - last_seq;
if (ctx->cur_seq + count > seq)
break;
ctx->cur_seq += count;
ctx->expunges_before += ctx->expunges[idx].seq2 -
ctx->expunges[idx].seq1 + 1;
last_seq = ctx->expunges[idx].seq2+1;
}
ctx->cur_idx = idx;
return ctx->expunges_before;
}
--- NEW FILE: mail-transaction-util.h ---
#ifndef __MAIL_TRANSACTION_UTIL_H
#define __MAIL_TRANSACTION_UTIL_H
struct mail_transaction_type_map {
enum mail_transaction_type type;
enum mail_index_sync_type sync_type;
size_t record_size;
};
extern const struct mail_transaction_type_map mail_transaction_type_map[];
struct mail_transaction_map_functions {
int (*expunge)(const struct mail_transaction_expunge *e, void *context);
int (*append)(const struct mail_index_record *rec, void *context);
int (*flag_update)(const struct mail_transaction_flag_update *u,
void *context);
int (*cache_update)(const struct mail_transaction_cache_update *u,
void *context);
};
const struct mail_transaction_type_map *
mail_transaction_type_lookup(enum mail_transaction_type type);
enum mail_transaction_type
mail_transaction_type_mask_get(enum mail_index_sync_type sync_type);
int mail_transaction_map(const struct mail_transaction_header *hdr,
const void *data,
struct mail_transaction_map_functions *map,
void *context);
void
mail_transaction_log_sort_expunges(buffer_t *expunges_buf,
const struct mail_transaction_expunge *src,
size_t src_buf_size);;
struct mail_transaction_expunge_traverse_ctx *
mail_transaction_expunge_traverse_init(const buffer_t *expunges_buf);
void mail_transaction_expunge_traverse_deinit(
struct mail_transaction_expunge_traverse_ctx *ctx);
uint32_t mail_transaction_expunge_traverse_to(
struct mail_transaction_expunge_traverse_ctx *ctx, uint32_t seq);
#endif
Index: Makefile.am
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/Makefile.am,v
retrieving revision 1.15
retrieving revision 1.16
diff -u -d -r1.15 -r1.16
--- Makefile.am 11 Aug 2003 02:40:40 -0000 1.15
+++ Makefile.am 27 Apr 2004 20:25:53 -0000 1.16
@@ -1,26 +1,34 @@
-SUBDIRS = maildir mbox
-
noinst_LIBRARIES = libindex.a
INCLUDES = \
-I$(top_srcdir)/src/lib \
- -I$(top_srcdir)/src/lib-mail \
- -I$(top_srcdir)/src/lib-imap
+ -I$(top_srcdir)/src/lib-mail
libindex_a_SOURCES = \
- mail-cache.c \
- mail-custom-flags.c \
+ mail-cache.c \
+ mail-cache-lookup.c \
+ mail-cache-transaction.c \
mail-index.c \
- mail-index-file.c \
mail-index-fsck.c \
- mail-index-open.c \
- mail-index-rebuild.c \
- mail-index-util.c \
- mail-modifylog.c
+ mail-index-lock.c \
+ mail-index-transaction.c \
+ mail-index-reset.c \
+ mail-index-sync.c \
+ mail-index-sync-update.c \
+ mail-index-view.c \
+ mail-index-view-sync.c \
+ mail-transaction-log.c \
+ mail-transaction-log-view.c \
+ mail-transaction-util.c
noinst_HEADERS = \
- mail-cache.h \
- mail-custom-flags.h \
+ mail-cache.h \
+ mail-cache-private.h \
mail-index.h \
- mail-index-util.h \
- mail-modifylog.h
+ mail-index-private.h \
+ mail-index-sync-private.h \
+ mail-index-transaction-private.h \
+ mail-index-view-private.h \
+ mail-transaction-log.h \
+ mail-transaction-log-private.h \
+ mail-transaction-util.h
More information about the dovecot-cvs
mailing list