[dovecot-cvs] dovecot/src/lib-index mail-index-private.h, 1.32,
1.33 mail-index-sync-private.h, 1.13,
1.14 mail-index-sync-update.c, 1.51, 1.52 mail-index-sync.c,
1.36, 1.37 mail-index-view-sync.c, 1.25,
1.26 mail-index-view.c, 1.25, 1.26 mail-index.c, 1.154,
1.155 mail-index.h, 1.132, 1.133 mail-transaction-log-view.c,
1.32, 1.33 mail-transaction-log.c, 1.68, 1.69
cras at dovecot.org
cras at dovecot.org
Mon Oct 25 20:42:54 EEST 2004
Update of /var/lib/cvs/dovecot/src/lib-index
In directory talvi:/tmp/cvs-serv29099/lib-index
Modified Files:
mail-index-private.h mail-index-sync-private.h
mail-index-sync-update.c mail-index-sync.c
mail-index-view-sync.c mail-index-view.c mail-index.c
mail-index.h mail-transaction-log-view.c
mail-transaction-log.c
Log Message:
Use separate sync offsets for internal/external transactions. Pending external
transactions are committed into index at the beginning of syncing, internal
ones aren't.
Index: mail-index-private.h
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-index/mail-index-private.h,v
retrieving revision 1.32
retrieving revision 1.33
diff -u -d -r1.32 -r1.33
--- mail-index-private.h 17 Oct 2004 16:42:20 -0000 1.32
+++ mail-index-private.h 25 Oct 2004 17:42:51 -0000 1.33
@@ -66,7 +66,8 @@
buffer_t *buffer;
uint32_t log_file_seq;
- uoff_t log_file_offset;
+ uoff_t log_file_int_offset;
+ uoff_t log_file_ext_offset;
buffer_t *hdr_copy_buf;
uint32_t base_header_size; /* so we don't need lock to access it */
Index: mail-index-sync-private.h
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-index/mail-index-sync-private.h,v
retrieving revision 1.13
retrieving revision 1.14
diff -u -d -r1.13 -r1.14
--- mail-index-sync-private.h 17 Oct 2004 18:24:21 -0000 1.13
+++ mail-index-sync-private.h 25 Oct 2004 17:42:51 -0000 1.14
@@ -35,7 +35,8 @@
extern struct mail_transaction_map_functions mail_index_map_sync_funcs;
-int mail_index_sync_update_index(struct mail_index_sync_ctx *sync_ctx);
+int mail_index_sync_update_index(struct mail_index_sync_ctx *sync_ctx,
+ int sync_only_external);
int mail_index_sync_record(struct mail_index_sync_map_ctx *ctx,
const struct mail_transaction_header *hdr,
Index: mail-index-sync-update.c
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-index/mail-index-sync-update.c,v
retrieving revision 1.51
retrieving revision 1.52
diff -u -d -r1.51 -r1.52
--- mail-index-sync-update.c 17 Oct 2004 18:24:21 -0000 1.51
+++ mail-index-sync-update.c 25 Oct 2004 17:42:51 -0000 1.52
@@ -897,7 +897,8 @@
return ret;
}
-int mail_index_sync_update_index(struct mail_index_sync_ctx *sync_ctx)
+int mail_index_sync_update_index(struct mail_index_sync_ctx *sync_ctx,
+ int sync_only_external)
{
struct mail_index *index = sync_ctx->index;
struct mail_index_view *view = sync_ctx->view;
@@ -909,7 +910,7 @@
unsigned int count, old_lock_id;
uint32_t seq, i, first_append_uid;
uoff_t offset;
- int ret, had_dirty, skipped;
+ int ret, had_dirty, skipped, check_ext_offsets;
memset(&sync_map_ctx, 0, sizeof(sync_map_ctx));
sync_map_ctx.view = view;
@@ -946,9 +947,29 @@
if (had_dirty)
tmphdr->flags &= ~MAIL_INDEX_HDR_FLAG_HAVE_DIRTY;
- first_append_uid = 0;
+ first_append_uid = 0;
+ check_ext_offsets = TRUE;
while ((ret = mail_transaction_log_view_next(view->log_view, &thdr,
&data, &skipped)) > 0) {
+ if ((thdr->type & MAIL_TRANSACTION_EXTERNAL) == 0) {
+ if (sync_only_external) {
+ /* we're syncing only external changes. */
+ continue;
+ }
+ } else if (check_ext_offsets) {
+ uint32_t prev_seq;
+ uoff_t prev_offset;
+
+ mail_transaction_log_view_get_prev_pos(view->log_view,
+ &prev_seq,
+ &prev_offset);
+ if (prev_offset < index->hdr->log_file_ext_offset) {
+ /* we have already synced this change */
+ continue;
+ }
+ check_ext_offsets = FALSE;
+ }
+
if ((thdr->type & MAIL_TRANSACTION_APPEND) != 0) {
const struct mail_index_record *rec = data;
@@ -994,7 +1015,9 @@
/* hdr pointer may have changed, update it */
tmphdr = buffer_get_modifyable_data(map->hdr_copy_buf, NULL);
tmphdr->log_file_seq = seq;
- tmphdr->log_file_offset = offset;
+ if (!sync_only_external)
+ tmphdr->log_file_int_offset = offset;
+ tmphdr->log_file_ext_offset = offset;
if (first_append_uid != 0)
mail_index_update_day_headers(tmphdr, first_append_uid);
Index: mail-index-sync.c
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-index/mail-index-sync.c,v
retrieving revision 1.36
retrieving revision 1.37
diff -u -d -r1.36 -r1.37
--- mail-index-sync.c 17 Oct 2004 16:39:58 -0000 1.36
+++ mail-index-sync.c 25 Oct 2004 17:42:51 -0000 1.37
@@ -211,12 +211,12 @@
static int
mail_index_sync_read_and_sort(struct mail_index_sync_ctx *ctx, int sync_recent,
- int *update_index_now_r)
+ int *seen_external_r)
{
size_t size;
int ret;
- *update_index_now_r = FALSE;
+ *seen_external_r = FALSE;
if ((ctx->view->map->hdr->flags & MAIL_INDEX_HDR_FLAG_HAVE_DIRTY) &&
ctx->sync_dirty) {
@@ -233,21 +233,10 @@
while ((ret = mail_transaction_log_view_next(ctx->view->log_view,
&ctx->hdr,
&ctx->data, NULL)) > 0) {
- if ((ctx->hdr->type & MAIL_TRANSACTION_EXTERNAL) != 0) {
- /* last sync was written to transaction log,
- but it wasn't committed to index. do it now so
- the next sync won't do things wrong (especially
- duplicate appends). */
- *update_index_now_r = TRUE;
- } else if ((ctx->hdr->type & MAIL_TRANSACTION_TYPE_MASK) ==
- MAIL_TRANSACTION_APPEND) {
- /* we appended new message, and now we're committing
- it into indexes. do it immediately so that we don't
- break if we have to sync the mailbox too */
- *update_index_now_r = TRUE;
- } else {
+ if ((ctx->hdr->type & MAIL_TRANSACTION_EXTERNAL) != 0)
+ *seen_external_r = TRUE;
+ else
mail_index_sync_sort_transaction(ctx);
- }
}
ctx->expunges = buffer_get_data(ctx->expunges_buf, &size);
@@ -266,7 +255,8 @@
if (index->hdr->log_file_seq > log_file_seq ||
(index->hdr->log_file_seq == log_file_seq &&
- index->hdr->log_file_offset >= log_file_offset)) {
+ index->hdr->log_file_int_offset >= log_file_offset &&
+ index->hdr->log_file_ext_offset >= log_file_offset)) {
/* already synced */
return mail_cache_need_compress(index->cache);
}
@@ -274,6 +264,32 @@
return 1;
}
+static int mail_index_sync_commit_external(struct mail_index_sync_ctx *ctx,
+ uint32_t seq, uoff_t offset)
+{
+ int ret;
+
+ while ((ret = mail_transaction_log_view_next(ctx->view->log_view,
+ &ctx->hdr, &ctx->data,
+ NULL)) > 0) {
+ if ((ctx->hdr->type & MAIL_TRANSACTION_EXTERNAL) != 0)
+ break;
+ }
+ if (ret < 0)
+ return -1;
+
+ if (ret > 0) {
+ if (mail_transaction_log_view_set(ctx->view->log_view,
+ ctx->index->hdr->log_file_seq,
+ ctx->index->hdr->log_file_ext_offset,
+ seq, offset, MAIL_TRANSACTION_TYPE_MASK) < 0)
+ return -1;
+ if (mail_index_sync_update_index(ctx, TRUE) < 0)
+ return -1;
+ }
+ return 0;
+}
+
int mail_index_sync_begin(struct mail_index *index,
struct mail_index_sync_ctx **ctx_r,
struct mail_index_view **view_r,
@@ -284,7 +300,7 @@
uint32_t seq;
uoff_t offset;
unsigned int lock_id;
- int update_now;
+ int seen_external;
if (mail_transaction_log_sync_lock(index->log, &seq, &offset) < 0)
return -1;
@@ -317,36 +333,51 @@
if (mail_transaction_log_view_set(ctx->view->log_view,
index->hdr->log_file_seq,
- index->hdr->log_file_offset,
+ index->hdr->log_file_int_offset,
seq, offset,
MAIL_TRANSACTION_TYPE_MASK) < 0) {
mail_index_sync_rollback(ctx);
return -1;
}
- /* we need to have all the transactions sorted to optimize
- caller's mailbox access patterns */
- ctx->expunges_buf = buffer_create_dynamic(default_pool, 1024);
- ctx->updates_buf = buffer_create_dynamic(default_pool, 1024);
- if (mail_index_sync_read_and_sort(ctx, sync_recent, &update_now) < 0) {
- mail_index_sync_rollback(ctx);
- return -1;
- }
+ /* See if there are some external transactions which were
+ written to transaction log, but weren't yet committed to
+ index. commit them first to avoid conflicts with another
+ external sync.
- if (update_now) {
- if (mail_transaction_log_view_set(ctx->view->log_view,
- index->hdr->log_file_seq,
- index->hdr->log_file_offset,
- seq, offset, MAIL_TRANSACTION_TYPE_MASK) < 0) {
+ This is mostly needed to make sure there won't be multiple
+ appends with same UIDs, because those would cause
+ transaction log to be marked corrupted.
+
+ Note that any internal transactions must not be committed
+ yet. They need to be synced with the real mailbox first. */
+ if (seq != index->hdr->log_file_seq ||
+ offset != index->hdr->log_file_ext_offset) {
+ if (mail_index_sync_commit_external(ctx, seq, offset) < 0) {
mail_index_sync_rollback(ctx);
return -1;
}
- if (mail_index_sync_update_index(ctx) < 0) {
+
+ if (mail_transaction_log_view_set(ctx->view->log_view,
+ index->hdr->log_file_seq,
+ index->hdr->log_file_int_offset,
+ seq, offset,
+ MAIL_TRANSACTION_TYPE_MASK) < 0) {
mail_index_sync_rollback(ctx);
return -1;
}
}
+ /* we need to have all the transactions sorted to optimize
+ caller's mailbox access patterns */
+ ctx->expunges_buf = buffer_create_dynamic(default_pool, 1024);
+ ctx->updates_buf = buffer_create_dynamic(default_pool, 1024);
+ if (mail_index_sync_read_and_sort(ctx, sync_recent,
+ &seen_external) < 0) {
+ mail_index_sync_rollback(ctx);
+ return -1;
+ }
+
*ctx_r = ctx;
*view_r = ctx->view;
return 1;
@@ -482,16 +513,20 @@
if (mail_transaction_log_view_is_corrupted(ctx->view->log_view))
ret = -1;
+ /* we have had the transaction log locked since the beginning of sync,
+ so only external changes could have been committed. write them to
+ the index here as well. */
mail_transaction_log_get_head(ctx->index->log, &seq, &offset);
hdr = ctx->index->hdr;
if (ret == 0 && (hdr->log_file_seq != seq ||
- hdr->log_file_offset != offset)) {
+ hdr->log_file_int_offset != offset ||
+ hdr->log_file_ext_offset != offset)) {
if (mail_transaction_log_view_set(ctx->view->log_view,
- hdr->log_file_seq, hdr->log_file_offset,
+ hdr->log_file_seq, hdr->log_file_int_offset,
seq, offset, MAIL_TRANSACTION_TYPE_MASK) < 0)
ret = -1;
- else if (mail_index_sync_update_index(ctx) < 0)
+ else if (mail_index_sync_update_index(ctx, FALSE) < 0)
ret = -1;
}
@@ -506,7 +541,7 @@
seq, offset, seq2, offset2,
MAIL_TRANSACTION_TYPE_MASK) < 0)
ret = -1;
- else if (mail_index_sync_update_index(ctx) < 0)
+ else if (mail_index_sync_update_index(ctx, FALSE) < 0)
ret = -1;
}
}
Index: mail-index-view-sync.c
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-index/mail-index-view-sync.c,v
retrieving revision 1.25
retrieving revision 1.26
diff -u -d -r1.25 -r1.26
--- mail-index-view-sync.c 21 Oct 2004 17:16:32 -0000 1.25
+++ mail-index-view-sync.c 25 Oct 2004 17:42:51 -0000 1.26
@@ -39,7 +39,7 @@
view->log_file_seq,
view->log_file_offset,
view->index->hdr->log_file_seq,
- view->index->hdr->log_file_offset,
+ view->index->hdr->log_file_int_offset,
MAIL_TRANSACTION_EXPUNGE) < 0)
return -1;
while ((ret = mail_transaction_log_view_next(view->log_view,
@@ -112,7 +112,7 @@
view->log_file_seq,
view->log_file_offset,
hdr->log_file_seq,
- hdr->log_file_offset, mask) < 0) {
+ hdr->log_file_int_offset, mask) < 0) {
if (expunges != NULL)
buffer_free(expunges);
return -1;
Index: mail-index-view.c
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-index/mail-index-view.c,v
retrieving revision 1.25
retrieving revision 1.26
diff -u -d -r1.25 -r1.26
--- mail-index-view.c 17 Oct 2004 16:42:20 -0000 1.25
+++ mail-index-view.c 25 Oct 2004 17:42:51 -0000 1.26
@@ -483,6 +483,8 @@
view->messages_count = view->map->records_count;
view->log_file_seq = view->map->log_file_seq;
- view->log_file_offset = view->map->log_file_offset;
+ view->log_file_offset =
+ I_MIN(view->map->log_file_int_offset,
+ view->map->log_file_ext_offset);
return view;
}
Index: mail-index.c
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-index/mail-index.c,v
retrieving revision 1.154
retrieving revision 1.155
diff -u -d -r1.154 -r1.155
--- mail-index.c 17 Oct 2004 16:42:20 -0000 1.154
+++ mail-index.c 25 Oct 2004 17:42:51 -0000 1.155
@@ -618,7 +618,8 @@
index->fsck = TRUE;
map->log_file_seq = map->hdr->log_file_seq;
- map->log_file_offset = map->hdr->log_file_offset;
+ map->log_file_int_offset = map->hdr->log_file_int_offset;
+ map->log_file_ext_offset = map->hdr->log_file_ext_offset;
map->base_header_size = map->hdr->base_header_size;
index->hdr = map->hdr;
@@ -691,7 +692,8 @@
}
mem_map->log_file_seq = mem_map->hdr->log_file_seq;
- mem_map->log_file_offset = mem_map->hdr->log_file_offset;
+ mem_map->log_file_int_offset = mem_map->hdr->log_file_int_offset;
+ mem_map->log_file_ext_offset = mem_map->hdr->log_file_ext_offset;
mem_map->base_header_size = mem_map->hdr->base_header_size;
return mem_map;
Index: mail-index.h
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-index/mail-index.h,v
retrieving revision 1.132
retrieving revision 1.133
diff -u -d -r1.132 -r1.133
--- mail-index.h 10 Oct 2004 17:25:44 -0000 1.132
+++ mail-index.h 25 Oct 2004 17:42:51 -0000 1.133
@@ -3,10 +3,10 @@
#include "mail-types.h"
-#define MAIL_INDEX_MAJOR_VERSION 5
+#define MAIL_INDEX_MAJOR_VERSION 6
#define MAIL_INDEX_MINOR_VERSION 0
-#define MAIL_INDEX_HEADER_MIN_SIZE 120
+#define MAIL_INDEX_HEADER_MIN_SIZE 124
/* Number of keywords in mail_index_record. */
#define INDEX_KEYWORDS_COUNT (3*8)
@@ -94,11 +94,14 @@
uint32_t first_unseen_uid_lowwater;
uint32_t first_deleted_uid_lowwater;
+ /* We have internal and external sync offsets. External changes are
+ synced into index somewhat more often, so int_offset <= ext_offset */
uint32_t log_file_seq;
- uint32_t log_file_offset;
+ uint32_t log_file_int_offset;
+ uint32_t log_file_ext_offset;
- uint32_t sync_stamp;
uint64_t sync_size;
+ uint32_t sync_stamp;
uint32_t cache_file_seq;
Index: mail-transaction-log-view.c
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-index/mail-transaction-log-view.c,v
retrieving revision 1.32
retrieving revision 1.33
diff -u -d -r1.32 -r1.33
--- mail-transaction-log-view.c 7 Oct 2004 21:19:05 -0000 1.32
+++ mail-transaction-log-view.c 25 Oct 2004 17:42:51 -0000 1.33
@@ -97,6 +97,11 @@
max_file_seq = min_file_seq;
max_file_offset = min_file_offset;
}
+ } else if (min_file_offset == 0) {
+ /* this could happen if internal transactions haven't yet been
+ committed but external are. just assume we're at the
+ beginning. */
+ min_file_offset = sizeof(struct mail_transaction_log_header);
}
if (min_file_seq == view->log->tail->hdr.prev_file_seq &&
Index: mail-transaction-log.c
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-index/mail-transaction-log.c,v
retrieving revision 1.68
retrieving revision 1.69
diff -u -d -r1.68 -r1.69
--- mail-transaction-log.c 24 Oct 2004 00:21:50 -0000 1.68
+++ mail-transaction-log.c 25 Oct 2004 17:42:51 -0000 1.69
@@ -177,10 +177,10 @@
#define INDEX_HAS_MISSING_LOGS(index, file) \
!(((file)->hdr.file_seq == (index)->hdr->log_file_seq && \
- (index)->hdr->log_file_offset >= \
+ (index)->hdr->log_file_int_offset >= \
sizeof(struct mail_transaction_log_header)) || \
((file)->hdr.prev_file_seq == (index)->hdr->log_file_seq && \
- (file)->hdr.prev_file_offset == (index)->hdr->log_file_offset))
+ (file)->hdr.prev_file_offset == (index)->hdr->log_file_int_offset))
static int mail_transaction_log_check_file_seq(struct mail_transaction_log *log)
{
@@ -362,7 +362,7 @@
if (mail_index_lock_shared(index, TRUE, &lock_id) < 0)
return -1;
hdr.prev_file_seq = index->hdr->log_file_seq;
- hdr.prev_file_offset = index->hdr->log_file_offset;
+ hdr.prev_file_offset = index->hdr->log_file_int_offset;
}
hdr.file_seq = index->hdr->log_file_seq+1;
@@ -478,11 +478,11 @@
if (log->index->map != NULL &&
file->hdr.file_seq == log->index->map->log_file_seq &&
- log->index->map->log_file_offset != 0) {
+ log->index->map->log_file_int_offset != 0) {
/* we can get a valid log offset from index file. initialize
sync_offset from it so we don't have to read the whole log
file from beginning. */
- file->sync_offset = log->index->map->log_file_offset;
+ file->sync_offset = log->index->map->log_file_int_offset;
}
for (p = &log->tail; *p != NULL; p = &(*p)->next) {
@@ -1219,7 +1219,8 @@
/* we might want to rotate, but check first that everything is
synced in index. */
if (log->head->hdr.file_seq == idx_hdr.log_file_seq &&
- log->head->sync_offset == idx_hdr.log_file_offset) {
+ log->head->sync_offset == idx_hdr.log_file_int_offset &&
+ log->head->sync_offset == idx_hdr.log_file_ext_offset) {
if (mail_transaction_log_rotate(log, TRUE) < 0) {
/* that didn't work. well, try to continue
anyway */
More information about the dovecot-cvs
mailing list