[dovecot-cvs] dovecot/src/lib-storage/index/dbox dbox-file.c, 1.10,
1.11 dbox-storage.h, 1.8, 1.9 dbox-sync-expunge.c, 1.10,
1.11 dbox-sync.c, 1.10, 1.11 dbox-uidlist.c, 1.19,
1.20 dbox-uidlist.h, 1.4, 1.5
cras at dovecot.org
cras at dovecot.org
Sat Apr 1 00:21:24 EEST 2006
Update of /var/lib/cvs/dovecot/src/lib-storage/index/dbox
In directory talvi:/tmp/cvs-serv21180
Modified Files:
dbox-file.c dbox-storage.h dbox-sync-expunge.c dbox-sync.c
dbox-uidlist.c dbox-uidlist.h
Log Message:
Fixes and cleanups.
Index: dbox-file.c
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-storage/index/dbox/dbox-file.c,v
retrieving revision 1.10
retrieving revision 1.11
diff -u -d -r1.10 -r1.11
--- dbox-file.c 26 Feb 2006 10:05:17 -0000 1.10
+++ dbox-file.c 31 Mar 2006 21:21:19 -0000 1.11
@@ -201,6 +201,8 @@
int dbox_file_header_parse(struct dbox_mailbox *mbox, struct dbox_file *file,
const struct dbox_file_header *hdr)
{
+ file->hdr = *hdr;
+
file->base_header_size = hex2dec(hdr->base_header_size_hex,
sizeof(hdr->base_header_size_hex));
file->header_size = hex2dec(hdr->header_size_hex,
Index: dbox-storage.h
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-storage/index/dbox/dbox-storage.h,v
retrieving revision 1.8
retrieving revision 1.9
diff -u -d -r1.8 -r1.9
--- dbox-storage.h 24 Feb 2006 16:06:51 -0000 1.8
+++ dbox-storage.h 31 Mar 2006 21:21:20 -0000 1.9
@@ -46,6 +46,9 @@
/* Offset for the keyword list inside the file header. */
unsigned char keyword_list_offset_hex[8];
+ /* Non-zero if some mails have been marked as expunged in the file. */
+ unsigned char have_expunged_mails;
+
/* space reserved for keyword list and possible other future
extensions. */
/* unsigned char [header_size - header_base_size]; */
@@ -96,6 +99,7 @@
uint64_t keyword_list_offset;
uint32_t keyword_list_size_alloc;
uint32_t keyword_list_size_used;
+ struct dbox_file_header hdr;
uoff_t seeked_offset;
uoff_t seeked_mail_size;
Index: dbox-sync-expunge.c
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-storage/index/dbox/dbox-sync-expunge.c,v
retrieving revision 1.10
retrieving revision 1.11
diff -u -d -r1.10 -r1.11
--- dbox-sync-expunge.c 30 Mar 2006 10:31:28 -0000 1.10
+++ dbox-sync-expunge.c 31 Mar 2006 21:21:20 -0000 1.11
@@ -73,7 +73,7 @@
const struct dbox_sync_file_entry *sync_entry,
unsigned int sync_idx,
uint32_t first_nonexpunged_uid,
- struct dbox_uidlist_entry *orig_entry,
+ const struct dbox_uidlist_entry *orig_entry,
uoff_t orig_offset)
{
struct dbox_mailbox *mbox = ctx->mbox;
@@ -90,8 +90,16 @@
uoff_t full_size;
off_t bytes;
- /* skip mails until we find the first we don't want expunged */
ret = dbox_file_seek(mbox, orig_entry->file_seq, orig_offset);
+
+ if (ret >= 0 && mbox->file->hdr.have_expunged_mails != '0') {
+ /* there are some expunged mails in the file, go through all
+ of the mails. */
+ ret = dbox_file_seek(mbox, orig_entry->file_seq,
+ mbox->file->header_size);
+ }
+
+ /* skip mails until we find the first we don't want expunged */
while (ret > 0) {
ret = dbox_file_seek_next_nonexpunged(mbox);
if (mbox->file->seeked_uid >= first_nonexpunged_uid)
@@ -271,6 +279,7 @@
const struct dbox_sync_rec *sync_recs;
struct dbox_uidlist_entry *entry;
struct seq_range *range;
+ const char *path;
unsigned int i, count, sync_count;
uint32_t file_seq, uid, uid1, uid2, first_expunged_uid;
uoff_t offset;
@@ -281,6 +290,7 @@
if (dbox_sync_get_file_offset(ctx, sync_recs[sync_idx].seq1,
&file_seq, &offset) < 0)
return -1;
+ i_assert(file_seq == sync_entry->file_seq);
entry = dbox_uidlist_entry_lookup(mbox->uidlist, sync_entry->file_seq);
if (entry == NULL) {
@@ -307,7 +317,7 @@
}
while (uid <= range[i].seq2) {
- if (uid < uid1) {
+ if (uid < uid1 || uid1 == 0) {
/* non-expunged mails exist in this file */
break;
}
@@ -351,12 +361,21 @@
if (!skipped_expunges) {
/* all mails expunged from file, unlink it. */
- return dbox_uidlist_sync_unlink(ctx->uidlist_sync_ctx,
- entry->file_seq);
+ path = t_strdup_printf("%s/"DBOX_MAILDIR_NAME"/"
+ DBOX_MAIL_FILE_FORMAT,
+ mbox->path, entry->file_seq);
+ if (unlink(path) < 0) {
+ mail_storage_set_critical(STORAGE(mbox->storage),
+ "unlink(%s) failed: %m", path);
+ return -1;
+ }
+
+ dbox_uidlist_sync_unlink(ctx->uidlist_sync_ctx,
+ entry->file_seq);
+ return 0;
}
/* mails expunged from the end of file, ftruncate() it */
-
ret = dbox_file_seek(mbox, entry->file_seq, offset);
if (ret <= 0) {
if (ret < 0)
@@ -378,6 +397,19 @@
"ftruncate(%s) failed: %m", mbox->path);
return -1;
}
+
+ if (mbox->file->hdr.have_expunged_mails != '0') {
+ /* all mails in the file are expunged now */
+ if (pwrite_full(mbox->file->fd, "0", 1,
+ offsetof(struct dbox_file_header,
+ have_expunged_mails)) < 0) {
+ mail_storage_set_critical(
+ STORAGE(mbox->storage),
+ "pwrite_full(%s) failed: %m",
+ mbox->path);
+ return -1;
+ }
+ }
}
/* remove from uidlist entry */
@@ -386,7 +418,7 @@
break;
}
array_delete(&entry->uid_list, i, count-i);
- if (i > 0)
+ if (i > 0 && range[i-1].seq2 >= first_expunged_uid)
range[i-1].seq2 = first_expunged_uid-1;
/* file can no longer be written to */
@@ -397,25 +429,27 @@
}
int dbox_sync_expunge(struct dbox_sync_context *ctx,
- const struct dbox_sync_file_entry *entry,
+ const struct dbox_sync_file_entry *sync_entry,
unsigned int sync_idx)
{
+ struct dbox_mailbox *mbox = ctx->mbox;
const struct dbox_sync_rec *sync_rec;
+ struct dbox_uidlist_entry *entry;
struct dotlock *dotlock;
const char *path;
int ret;
- if (ctx->dotlock_failed_file_seq != entry->file_seq) {
+ if (ctx->dotlock_failed_file_seq != sync_entry->file_seq && 0) {
/* we need to have the file locked in case another process is
appending there already. */
path = t_strdup_printf("%s/"DBOX_MAILDIR_NAME"/"
DBOX_MAIL_FILE_FORMAT,
- ctx->mbox->path, entry->file_seq);
+ mbox->path, sync_entry->file_seq);
ret = file_dotlock_create(&new_file_dotlock_set, path,
DOTLOCK_CREATE_FLAG_NONBLOCK,
&dotlock);
if (ret < 0) {
- mail_storage_set_critical(STORAGE(ctx->mbox->storage),
+ mail_storage_set_critical(STORAGE(mbox->storage),
"file_dotlock_create(%s) failed: %m", path);
return -1;
}
@@ -423,20 +457,50 @@
if (ret > 0) {
/* locked - copy the non-expunged mails after the
expunged mail to new file */
- ret = dbox_sync_expunge_file(ctx, entry, sync_idx);
+ ret = dbox_sync_expunge_file(ctx, sync_entry, sync_idx);
file_dotlock_delete(&dotlock);
return ret < 0 ? -1 : 1;
}
/* remember that we failed, so we don't waste time trying to
lock the file multiple times within same sync. */
- ctx->dotlock_failed_file_seq = entry->file_seq;
+ ctx->dotlock_failed_file_seq = sync_entry->file_seq;
}
/* couldn't lock it, someone's appending. we have no other
choice but to just mark the mail expunged. otherwise we'd
deadlock (appending process waits for uidlist lock which
we have, we wait for file lock which append process has) */
- sync_rec = array_idx(&entry->sync_recs, sync_idx);
- return dbox_sync_update_flags(ctx, sync_rec);
+ sync_rec = array_idx(&sync_entry->sync_recs, sync_idx);
+ if (dbox_sync_update_flags(ctx, sync_rec) < 0)
+ return -1;
+
+ /* mark in the header that the file contains expunged messages */
+ if (pwrite_full(mbox->file->fd, "1", 1,
+ offsetof(struct dbox_file_header,
+ have_expunged_mails)) < 0) {
+ mail_storage_set_critical(STORAGE(mbox->storage),
+ "pwrite(%s) failed: %m", mbox->file->path);
+ return -1;
+ }
+
+ /* remove UIDs from the uidlist entry */
+ entry = dbox_uidlist_entry_lookup(mbox->uidlist, sync_entry->file_seq);
+ if (entry != NULL) {
+ const struct dbox_sync_rec *recs;
+ unsigned int i, count, seq;
+
+ recs = array_get(&sync_entry->sync_recs, &count);
+ for (i = 0; i < count; i++) {
+ for (seq = recs[i].seq1; seq <= recs[i].seq2; seq++)
+ seq_range_array_remove(&entry->uid_list, seq);
+ }
+ if (array_count(&entry->uid_list) == 0) {
+ dbox_uidlist_sync_unlink(ctx->uidlist_sync_ctx,
+ entry->file_seq);
+ }
+ dbox_uidlist_sync_set_modified(ctx->uidlist_sync_ctx);
+ }
+
+ return 0;
}
Index: dbox-sync.c
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-storage/index/dbox/dbox-sync.c,v
retrieving revision 1.10
retrieving revision 1.11
diff -u -d -r1.10 -r1.11
--- dbox-sync.c 30 Mar 2006 10:31:28 -0000 1.10
+++ dbox-sync.c 31 Mar 2006 21:21:20 -0000 1.11
@@ -65,7 +65,7 @@
new_sync_rec = *sync_rec;
new_sync_rec.seq1 = seq;
- array_append(&entry->sync_recs, sync_rec, 1);
+ array_append(&entry->sync_recs, &new_sync_rec, 1);
return 0;
}
@@ -139,7 +139,7 @@
return -1;
if (mail_index_lookup_uid(ctx->sync_view, sync_rec->seq2, &uid2) < 0) {
- mail_storage_set_index_error(&ctx->mbox->ibox);
+ mail_storage_set_index_error(&mbox->ibox);
return -1;
}
@@ -159,7 +159,7 @@
break;
i++;
}
- ret = pwrite_full(ctx->mbox->file->fd,
+ ret = pwrite_full(mbox->file->fd,
array + start, i - start,
offset + first_flag_offset + start);
if (ret < 0) {
Index: dbox-uidlist.c
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-storage/index/dbox/dbox-uidlist.c,v
retrieving revision 1.19
retrieving revision 1.20
diff -u -d -r1.19 -r1.20
--- dbox-uidlist.c 30 Mar 2006 10:31:28 -0000 1.19
+++ dbox-uidlist.c 31 Mar 2006 21:21:20 -0000 1.20
@@ -68,6 +68,7 @@
pool_t pool;
struct dbox_uidlist *uidlist;
+ time_t min_usable_timestamp;
unsigned int mail_count;
array_t ARRAY_DEFINE(files, struct dbox_save_file *);
@@ -466,6 +467,26 @@
return dbox_uidlist_entry_lookup_int(uidlist, file_seq, &idx);
}
+static time_t get_min_timestamp(unsigned int days)
+{
+ struct tm tm;
+ time_t stamp;
+
+ if (days == 0)
+ return 0;
+
+ /* get beginning of today */
+ tm = *localtime(&ioloop_time);
+ tm.tm_hour = 0;
+ tm.tm_min = 0;
+ tm.tm_sec = 0;
+ stamp = mktime(&tm);
+ if (stamp == (time_t)-1)
+ i_panic("mktime(today) failed");
+
+ return stamp - (3600*24 * (days-1));
+}
+
struct dbox_uidlist_append_ctx *
dbox_uidlist_append_init(struct dbox_uidlist *uidlist)
{
@@ -478,6 +499,8 @@
ctx = p_new(pool, struct dbox_uidlist_append_ctx, 1);
ctx->pool = pool;
ctx->uidlist = uidlist;
+ ctx->min_usable_timestamp =
+ get_min_timestamp(uidlist->mbox->rotate_days);
ARRAY_CREATE(&ctx->files, pool, struct dbox_save_file *, 16);
return ctx;
}
@@ -829,75 +852,88 @@
return FALSE;
}
-static time_t get_min_timestamp(unsigned int days)
+#define DBOX_CAN_APPEND(ctx, create_time, file_size) \
+ (((create_time) >= (ctx)->min_usable_timestamp && \
+ (file_size) < (ctx)->uidlist->mbox->rotate_size) || \
+ (file_size) < (ctx)->uidlist->mbox->rotate_min_size)
+
+static int
+dbox_file_append(struct dbox_uidlist_append_ctx *ctx,
+ const char *path, struct dbox_uidlist_entry *entry,
+ struct dbox_file **file_r)
{
- struct tm tm;
- time_t stamp;
+ struct dbox_mailbox *mbox = ctx->uidlist->mbox;
+ struct dbox_file *file;
+ struct stat st;
+ int fd;
- if (days == 0)
- return 0;
+ *file_r = NULL;
- /* get beginning of today */
- tm = *localtime(&ioloop_time);
- tm.tm_hour = 0;
- tm.tm_min = 0;
- tm.tm_sec = 0;
- stamp = mktime(&tm);
- if (stamp == (time_t)-1)
- i_panic("mktime(today) failed");
+ fd = open(path, O_CREAT | O_RDWR, 0600);
+ if (fd == -1) {
+ mail_storage_set_critical(STORAGE(mbox->storage),
+ "open(%s) failed: %m", path);
+ return -1;
+ }
- return stamp - (3600*24 * (days-1));
-}
+ if (fstat(fd, &st) < 0) {
+ mail_storage_set_critical(STORAGE(mbox->storage),
+ "fstat(%s) failed: %m", path);
+ (void)close(fd);
+ return -1;
+ }
-#define DBOX_CAN_APPEND(mbox, create_time, file_size, min_usable_timestamp) \
- (((create_time) >= (min_usable_timestamp) && \
- (file_size) < (mbox)->rotate_size) || \
- (file_size) < (mbox)->rotate_min_size)
+ file = i_new(struct dbox_file, 1);
+ file->path = i_strdup(path);
+ file->fd = fd;
-int dbox_uidlist_append_locked(struct dbox_uidlist_append_ctx *ctx,
- struct dbox_file **file_r)
+ file->input = i_stream_create_file(file->fd, default_pool,
+ 65536, FALSE);
+ file->output = o_stream_create_file(file->fd, default_pool, 0, FALSE);
+ if ((uoff_t)st.st_size < sizeof(struct dbox_file_header)) {
+ if (dbox_file_write_header(mbox, file) < 0) {
+ dbox_file_close(file);
+ return -1;
+ }
+ } else {
+ if (dbox_file_read_header(mbox, file) < 0) {
+ dbox_file_close(file);
+ return -1;
+ }
+
+ if (entry != NULL) {
+ entry->create_time = file->create_time;
+ entry->file_size = file->append_offset;
+ }
+
+ if (!DBOX_CAN_APPEND(ctx, file->create_time,
+ file->append_offset)) {
+ dbox_file_close(file);
+ return 0;
+ }
+ }
+
+ *file_r = file;
+ return 1;
+}
+
+static int
+dbox_file_append_lock(struct dbox_uidlist_append_ctx *ctx, string_t *path,
+ uint32_t *file_seq_r, struct dbox_uidlist_entry **entry_r,
+ struct dotlock **dotlock_r)
{
struct dbox_mailbox *mbox = ctx->uidlist->mbox;
- struct dbox_save_file *const *files, *save_file;
struct dbox_uidlist_entry *const *entries;
- struct dbox_file *file;
- struct dotlock *dotlock;
- struct ostream *output;
- string_t *str;
unsigned int i, count;
- struct stat st;
uint32_t file_seq;
- time_t min_usable_timestamp;
int ret;
- min_usable_timestamp = get_min_timestamp(mbox->rotate_days);
-
- /* check first from already opened files */
- files = array_get(&ctx->files, &count);
- for (i = 0; i < count; i++) {
- if (DBOX_CAN_APPEND(mbox, files[i]->file->create_time,
- files[i]->append_offset,
- min_usable_timestamp)) {
- if (dbox_reopen_file(ctx, files[i]) < 0)
- return -1;
-
- *file_r = file = files[i]->file;
- o_stream_seek(file->output, file->append_offset);
- return 0;
- }
- }
-
- /* check from other existing files. use uidlist's file_size field.
- it's not completely trustworthy though. */
- str = str_new(ctx->pool, 64);
entries = array_get(&ctx->uidlist->entries, &count);
-__again:
for (i = 0;; i++) {
- file_seq = 0;
+ file_seq = 0;
for (; i < count; i++) {
- if (DBOX_CAN_APPEND(mbox, entries[i]->create_time,
- entries[i]->file_size,
- min_usable_timestamp) &&
+ if (DBOX_CAN_APPEND(ctx, entries[i]->create_time,
+ entries[i]->file_size) &&
!dbox_uidlist_files_lookup(ctx,
entries[i]->file_seq)) {
file_seq = entries[i]->file_seq;
@@ -911,12 +947,12 @@
}
/* try locking the file. */
- str_truncate(str, 0);
- str_printfa(str, "%s/"DBOX_MAILDIR_NAME"/"
+ str_truncate(path, 0);
+ str_printfa(path, "%s/"DBOX_MAILDIR_NAME"/"
DBOX_MAIL_FILE_FORMAT, mbox->path, file_seq);
- ret = file_dotlock_create(&dbox_file_dotlock_set, str_c(str),
+ ret = file_dotlock_create(&dbox_file_dotlock_set, str_c(path),
DOTLOCK_CREATE_FLAG_NONBLOCK,
- &dotlock);
+ dotlock_r);
if (ret > 0) {
/* success */
break;
@@ -924,57 +960,63 @@
if (ret < 0) {
mail_storage_set_critical(STORAGE(mbox->storage),
"file_dotlock_create(%s) failed: %m",
- str_c(str));
+ str_c(path));
return -1;
}
/* lock already exists, try next file */
}
- file = i_new(struct dbox_file, 1);
- file->file_seq = file_seq;
- file->path = i_strdup(str_c(str));
+ *file_seq_r = file_seq;
+ *entry_r = i < count ? entries[i] : NULL;
+ return 0;
+}
- file->fd = open(file->path, O_CREAT | O_RDWR, 0600);
- if (file->fd == -1) {
- mail_storage_set_critical(STORAGE(mbox->storage),
- "open(%s) failed: %m", file->path);
- return -1;
- }
+int dbox_uidlist_append_locked(struct dbox_uidlist_append_ctx *ctx,
+ struct dbox_file **file_r)
+{
+ struct dbox_save_file *const *files, *save_file;
+ struct dbox_uidlist_entry *entry;
+ struct dbox_file *file = NULL;
+ struct dotlock *dotlock = NULL;
+ struct ostream *output;
+ string_t *path;
+ unsigned int i, count;
+ struct stat st;
+ uint32_t file_seq;
+ int ret;
- if (fstat(file->fd, &st) < 0) {
- mail_storage_set_critical(STORAGE(mbox->storage),
- "fstat(%s) failed: %m", file->path);
- (void)close(file->fd);
- return -1;
- }
+ /* check first from already opened files */
+ files = array_get(&ctx->files, &count);
+ for (i = 0; i < count; i++) {
+ if (DBOX_CAN_APPEND(ctx, files[i]->file->create_time,
+ files[i]->append_offset)) {
+ if (dbox_reopen_file(ctx, files[i]) < 0)
+ return -1;
- file->input = i_stream_create_file(file->fd, default_pool,
- 65536, FALSE);
- file->output = o_stream_create_file(file->fd, default_pool, 0, FALSE);
- if ((uoff_t)st.st_size < sizeof(struct dbox_file_header)) {
- if (dbox_file_write_header(mbox, file) < 0) {
- dbox_file_close(file);
- return -1;
- }
- } else {
- if (dbox_file_read_header(mbox, file) < 0) {
- dbox_file_close(file);
- return -1;
+ *file_r = file = files[i]->file;
+ o_stream_seek(file->output, file->append_offset);
+ return 0;
}
+ }
- if (i < count) {
- entries[i]->create_time = file->create_time;
- entries[i]->file_size = file->append_offset;
- }
+ /* check from other existing files. use uidlist's file_size field.
+ it's not completely trustworthy though. */
+ path = str_new(ctx->pool, 64);
+ do {
+ if (dotlock != NULL)
+ file_dotlock_delete(&dotlock);
+ if (dbox_file_append_lock(ctx, path, &file_seq,
+ &entry, &dotlock) < 0)
+ return -1;
+ } while ((ret = dbox_file_append(ctx, str_c(path), entry, &file)) == 0);
- if (!DBOX_CAN_APPEND(mbox, file->create_time,
- file->append_offset,
- min_usable_timestamp)) {
- dbox_file_close(file);
- goto __again;
- }
+ if (ret < 0) {
+ file_dotlock_delete(&dotlock);
+ dbox_file_close(file);
+ return -1;
}
+ file->file_seq = file_seq;
/* we'll always use CRLF linefeeds for mails (but not the header,
so don't do this before dbox_file_write_header()) */
@@ -1165,28 +1207,18 @@
}
}
-int dbox_uidlist_sync_unlink(struct dbox_uidlist_sync_ctx *ctx,
- uint32_t file_seq)
+void dbox_uidlist_sync_unlink(struct dbox_uidlist_sync_ctx *ctx,
+ uint32_t file_seq)
{
struct dbox_uidlist_entry *entry;
- const char *path;
unsigned int idx;
entry = dbox_uidlist_entry_lookup_int(ctx->uidlist, file_seq, &idx);
i_assert(entry != NULL);
- path = t_strdup_printf("%s/"DBOX_MAILDIR_NAME"/"
- DBOX_MAIL_FILE_FORMAT,
- ctx->uidlist->mbox->path, entry->file_seq);
- if (unlink(path) < 0) {
- mail_storage_set_critical(STORAGE(ctx->uidlist->mbox->storage),
- "unlink(%s) failed: %m", path);
- return -1;
- }
array_delete(&ctx->uidlist->entries, idx, 1);
dbox_uidlist_sync_set_modified(ctx);
- return 0;
}
uint32_t dbox_uidlist_sync_get_uid_validity(struct dbox_uidlist_sync_ctx *ctx)
Index: dbox-uidlist.h
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-storage/index/dbox/dbox-uidlist.h,v
retrieving revision 1.4
retrieving revision 1.5
diff -u -d -r1.4 -r1.5
--- dbox-uidlist.h 29 Jan 2006 09:54:32 -0000 1.4
+++ dbox-uidlist.h 31 Mar 2006 21:21:20 -0000 1.5
@@ -55,8 +55,8 @@
void dbox_uidlist_sync_append(struct dbox_uidlist_sync_ctx *ctx,
const struct dbox_uidlist_entry *entry);
-int dbox_uidlist_sync_unlink(struct dbox_uidlist_sync_ctx *ctx,
- uint32_t file_seq);
+void dbox_uidlist_sync_unlink(struct dbox_uidlist_sync_ctx *ctx,
+ uint32_t file_seq);
uint32_t dbox_uidlist_sync_get_uid_validity(struct dbox_uidlist_sync_ctx *ctx);
uint32_t dbox_uidlist_sync_get_next_uid(struct dbox_uidlist_sync_ctx *ctx);
More information about the dovecot-cvs
mailing list