[dovecot-cvs] dovecot/src/lib-index mail-index-lock.c, 1.60, 1.61 mail-index-sync-update.c, 1.115, 1.116 mail-index-sync.c, 1.82, 1.83 mail-index.c, 1.267, 1.268 mail-transaction-log-append.c, 1.22, 1.23 mail-transaction-log-view.c, 1.52, 1.53 mail-transaction-log.h, 1.32, 1.33

[tss at dovecot.org]

Update of /var/lib/cvs/dovecot/src/lib-index
In directory talvi:/tmp/cvs-serv4709

Modified Files:
	mail-index-lock.c mail-index-sync-update.c mail-index-sync.c 
	mail-index.c mail-transaction-log-append.c 
	mail-transaction-log-view.c mail-transaction-log.h 
Log Message:
If we run out of disk space, move to in-memory indexes.



Index: mail-index-lock.c
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-index/mail-index-lock.c,v
retrieving revision 1.60
retrieving revision 1.61
diff -u -d -r1.60 -r1.61
--- mail-index-lock.c	17 Jan 2007 16:17:42 -0000	1.60
+++ mail-index-lock.c	11 Mar 2007 16:10:41 -0000	1.61
@@ -217,8 +217,16 @@
 
 	/* copy the index to index.tmp and use it */
 	fd = mail_index_copy(index);
-	if (fd == -1)
-		return -1;
+	if (fd == -1) {
+		if (!index->nodiskspace)
+			return -1;
+
+		if (mail_index_move_to_memory(index) < 0)
+			return -1;
+		index->lock_type = F_WRLCK;
+		index->excl_lock_count++;
+		return 0;
+	}
 
 	old_lock_type = index->lock_type;
 	index->lock_type = F_WRLCK;
@@ -261,7 +269,6 @@
 		if (ret < 0)
 			return -1;
 	}
-
 	if (mail_index_lock_exclusive_copy(index) < 0)
 		return -1;
 	*lock_id_r = index->lock_id + 1;
@@ -329,10 +336,27 @@
 	return 0;
 }
 
+static int mail_index_copy_and_reopen(struct mail_index *index)
+{
+	int fd;
+
+	fd = mail_index_copy(index);
+	if (fd == -1) {
+		if (!index->nodiskspace)
+			return -1;
+		return mail_index_move_to_memory(index);
+	}
+
+	if (mail_index_reopen(index, fd) < 0) {
+		(void)close(fd);
+		return -1;
+	}
+	return 0;
+}
+
 static void mail_index_write_map(struct mail_index *index)
 {
 	struct mail_index_map *map = index->map;
-	int fd;
 
 	if (map->write_atomic || index->copy_lock_path != NULL ||
 	    index->fd == -1) {
@@ -347,13 +371,10 @@
 		}
 
 		if (!MAIL_INDEX_IS_IN_MEMORY(index)) {
-			fd = mail_index_copy(index);
-			if (fd == -1 || mail_index_reopen(index, fd) < 0) {
-				if (fd != -1)
-					(void)close(fd);
+			if (mail_index_copy_and_reopen(index) < 0)
 				mail_index_set_inconsistent(index);
-			}
 		}
+
 	} else {
 		/* write the modified parts. header is small enough to be
 		   always written, write_seq_* specifies the record range. */

Index: mail-index-sync-update.c
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-index/mail-index-sync-update.c,v
retrieving revision 1.115
retrieving revision 1.116
diff -u -d -r1.115 -r1.116
--- mail-index-sync-update.c	10 Mar 2007 18:48:49 -0000	1.115
+++ mail-index-sync-update.c	11 Mar 2007 16:10:41 -0000	1.116
@@ -489,7 +489,7 @@
 	size_t size, hdr_copy_size;
 
 	if (MAIL_INDEX_MAP_IS_IN_MEMORY(map))
-		return 0;
+		return 1;
 
 	i_assert(map == index->map);
 	i_assert(!index->mapping); /* mail_index_sync_from_transactions() */
@@ -497,7 +497,7 @@
 	size = map->hdr.header_size +
 		(map->records_count + count) * map->hdr.record_size;
 	if (size <= map->mmap_size)
-		return 0;
+		return 1;
 
 	/* when we grow fast, do it exponentially */
 	if (count < index->last_grow_count)
@@ -508,8 +508,11 @@
 
 	size = map->hdr.header_size +
 		(map->records_count + count) * map->hdr.record_size;
-	if (file_set_size(index->fd, (off_t)size) < 0)
-		return mail_index_set_syscall_error(index, "file_set_size()");
+	if (file_set_size(index->fd, (off_t)size) < 0) {
+		mail_index_set_syscall_error(index, "file_set_size()");
+		return !ENOSPACE(errno) ? -1 :
+			mail_index_move_to_memory(index);
+	}
 
 	/* we only wish to grow the file, but mail_index_map() updates the
 	   headers as well and may break our modified hdr_copy. so, take
@@ -535,8 +538,9 @@
 	map->records_count = map->hdr.messages_count;
 
 	i_assert(map->mmap_size >= size);
+
 	t_pop();
-	return 0;
+	return 1;
 }
 
 static void
@@ -782,6 +786,16 @@
 	map->hdr.first_recent_uid_lowwater = map->hdr.next_uid;
 }
 
+static void log_view_seek_back(struct mail_transaction_log_view *log_view)
+{
+	uint32_t prev_seq;
+	uoff_t prev_offset;
+
+	mail_transaction_log_view_get_prev_pos(log_view, &prev_seq,
+					       &prev_offset);
+	mail_transaction_log_view_seek(log_view, prev_seq, prev_offset);
+}
+
 int mail_index_sync_update_index(struct mail_index_sync_ctx *sync_ctx,
 				 bool sync_only_external)
 {
@@ -862,25 +876,27 @@
 
 			map = view->map;
 			count = thdr->size / sizeof(*rec);
-			if (mail_index_grow(index, map, count) < 0) {
-				ret = -1;
+			if ((ret = mail_index_grow(index, map, count)) < 0)
 				break;
-			}
+
 			if (map != index->map) {
 				index->map->refcount++;
 				mail_index_sync_replace_map(&sync_map_ctx,
 							    index->map);
 			}
+
+			if (ret == 0) {
+				/* moved to memory. data pointer is invalid,
+				   seek back and do this append again. */
+				log_view_seek_back(view->log_view);
+				continue;
+			}
 		}
 
 		if (mail_index_sync_record(&sync_map_ctx, thdr, data) < 0) {
 			ret = -1;
 			break;
 		}
-
-		/* mail_index_sync_record() might have changed map to anything.
-		   make sure we don't accidentally try to use it. */
-		map = NULL;
 	}
 
 	if (ret == 0) {

Index: mail-index-sync.c
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-index/mail-index-sync.c,v
retrieving revision 1.82
retrieving revision 1.83
diff -u -d -r1.82 -r1.83
--- mail-index-sync.c	23 Jan 2007 16:30:54 -0000	1.82
+++ mail-index-sync.c	11 Mar 2007 16:10:41 -0000	1.83
@@ -389,7 +389,9 @@
 	     				index->hdr->log_file_seq,
 					index->hdr->log_file_ext_offset))) {
 		/* broken sync positions. fix them. */
-		mail_index_set_error(index, "broken sync positions");
+		mail_index_set_error(index,
+			"broken sync positions in index file %s",
+			index->filepath);
 		if (mail_index_fsck(index) <= 0) {
 			mail_index_unlock(index, lock_id);
 			mail_transaction_log_sync_unlock(index->log);

Index: mail-index.c
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-index/mail-index.c,v
retrieving revision 1.267
retrieving revision 1.268
diff -u -d -r1.267 -r1.268
--- mail-index.c	10 Mar 2007 19:30:26 -0000	1.267
+++ mail-index.c	11 Mar 2007 16:10:42 -0000	1.268
@@ -1579,7 +1579,7 @@
 		return -1;
 	}
 
-	if (index->fd == -1) {
+	if (index->map == NULL) {
 		mail_index_header_init(&hdr);
 		index->hdr = &hdr;
 
@@ -1912,10 +1912,13 @@
 	}
 
 	/* move index map to memory */
-	map = mail_index_map_clone(index->map, index->map->hdr.record_size);
-	mail_index_unmap(index, &index->map);
-	index->map = map;
-	index->hdr = &map->hdr;
+	if (!MAIL_INDEX_MAP_IS_IN_MEMORY(index->map)) {
+		map = mail_index_map_clone(index->map,
+					   index->map->hdr.record_size);
+		mail_index_unmap(index, &index->map);
+		index->map = map;
+		index->hdr = &map->hdr;
+	}
 
 	if (index->log != NULL) {
 		/* move transaction log to memory */

Index: mail-transaction-log-append.c
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-index/mail-transaction-log-append.c,v
retrieving revision 1.22
retrieving revision 1.23
diff -u -d -r1.22 -r1.23
--- mail-transaction-log-append.c	9 Mar 2007 21:21:02 -0000	1.22
+++ mail-transaction-log-append.c	11 Mar 2007 16:10:42 -0000	1.23
@@ -81,7 +81,16 @@
 		if (!ENOSPACE(errno))
 			return -1;
 
-		/* not enough space. fallback to in-memory indexes. */
+		/* not enough space. fallback to in-memory indexes. first
+		   we need to truncate this latest write so that log syncing
+		   doesn't break */
+		if (ftruncate(file->fd, file->sync_offset) < 0) {
+			mail_index_file_set_syscall_error(file->log->index,
+							  file->filepath,
+							  "ftruncate()");
+			return -1;
+		}
+
 		if (mail_index_move_to_memory(file->log->index) < 0)
 			return -1;
 		i_assert(MAIL_TRANSACTION_LOG_FILE_IN_MEMORY(file));

Index: mail-transaction-log-view.c
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-index/mail-transaction-log-view.c,v
retrieving revision 1.52
retrieving revision 1.53
diff -u -d -r1.52 -r1.53
--- mail-transaction-log-view.c	9 Mar 2007 21:21:02 -0000	1.52
+++ mail-transaction-log-view.c	11 Mar 2007 16:10:42 -0000	1.53
@@ -453,3 +453,25 @@
 	*data_r = data;
 	return 1;
 }
+
+void mail_transaction_log_view_seek(struct mail_transaction_log_view *view,
+				    uint32_t seq, uoff_t offset)
+{
+	struct mail_transaction_log_file *file;
+
+	i_assert(seq >= view->min_file_seq && seq <= view->max_file_seq);
+	i_assert(seq != view->min_file_seq || offset >= view->min_file_offset);
+	i_assert(seq != view->max_file_seq || offset < view->max_file_offset);
+
+	if (view->cur == NULL || seq != view->cur->hdr.file_seq) {
+		for (file = view->tail; file != NULL; file = file->next) {
+			if (file->hdr.file_seq == seq)
+				break;
+		}
+		i_assert(file != NULL);
+
+		view->cur = file;
+	}
+
+	view->cur_offset = offset;
+}

Index: mail-transaction-log.h
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-index/mail-transaction-log.h,v
retrieving revision 1.32
retrieving revision 1.33
diff -u -d -r1.32 -r1.33
--- mail-transaction-log.h	8 Mar 2007 20:02:37 -0000	1.32
+++ mail-transaction-log.h	11 Mar 2007 16:10:42 -0000	1.33
@@ -129,6 +129,9 @@
 int mail_transaction_log_view_next(struct mail_transaction_log_view *view,
 				   const struct mail_transaction_header **hdr_r,
 				   const void **data_r, bool *skipped_r);
+/* Seek to given position within view. Must be inside the view's range. */
+void mail_transaction_log_view_seek(struct mail_transaction_log_view *view,
+				    uint32_t seq, uoff_t offset);
 
 /* Returns the position of the record returned previously with
    mail_transaction_log_view_next() */