[dovecot-cvs] dovecot/src/imap imap-fetch-body-section.c,NONE,1.1 imap-fetch.c,NONE,1.1 imap-fetch.h,NONE,1.1 imap-search.c,NONE,1.1 imap-search.h,NONE,1.1 imap-sort.c,NONE,1.1 imap-sort.h,NONE,1.1 imap-thread.c,NONE,1.1 imap-thread.h,NONE,1.1 Makefile.am,1.10,1.11 Message-Id: <20030120145253.72E4723837@danu.procontrol.fi>

cras at procontrol.fi cras at procontrol.fi
Mon Jan 20 16:52:53 EET 2003


Update of /home/cvs/dovecot/src/imap
In directory danu:/tmp/cvs-serv7093/imap

Modified Files:
	Makefile.am cmd-append.c cmd-fetch.c cmd-search.c cmd-sort.c 
	cmd-store.c cmd-thread.c commands-util.c commands-util.h 
Added Files:
	imap-fetch-body-section.c imap-fetch.c imap-fetch.h 
	imap-search.c imap-search.h imap-sort.c imap-sort.h 
	imap-thread.c imap-thread.h 
Log Message:
mail-storage.h interface changes, affects pretty much everything.
FETCH, SEARCH, SORT and THREAD handling were pretty much moved from
lib-storage/ to imap/ so adding non-index storages would be much easier now.
Also POP3 server can now be easily implemented with lib-storage.

Not too well tested, and at least one major problem: partial fetching is
_slow_.



--- NEW FILE: imap-fetch-body-section.c ---
/* Copyright (C) 2002 Timo Sirainen */

#include "common.h"
#include "str.h"
#include "istream.h"
#include "ostream.h"
#include "message-parser.h"
#include "message-send.h"
#include "mail-storage.h"
#include "imap-fetch.h"

#include <ctype.h>
#include <unistd.h>

/* For FETCH[HEADER.FIELDS*] we need to modify the header data before sending
   it. We can either save it in memory and then send it, or we can parse it
   twice, first calculating the size and then send it. This value specifies
   the maximum amount of memory we allow to allocate before using
   double-parsing. */
#define MAX_HEADER_BUFFER_SIZE (32*1024)

#define UNSIGNED_CRLF (const unsigned char *) "\r\n"

struct fetch_header_field_context {
	string_t *dest;
	struct ostream *output;
	uoff_t dest_size;

	uoff_t skip, max_size;
	const char *const *fields;
	int (*match_func) (const char *const *, const unsigned char *, size_t);
};

/* fetch BODY[] or BODY[TEXT] */
static int fetch_body(struct imap_fetch_context *ctx,
		      const struct imap_fetch_body_data *body,
		      struct mail *mail, int fetch_header)
{
	struct message_size hdr_size, body_size;
	struct istream *stream;
	const char *str;

	stream = mail->get_stream(mail, &hdr_size, &body_size);
	if (stream == NULL)
		return FALSE;

	if (!fetch_header)
		i_stream_seek(stream, hdr_size.physical_size);
	else
		message_size_add(&body_size, &hdr_size);

	str = t_strdup_printf("%s {%"PRIuUOFF_T"}\r\n",
			      ctx->prefix, body_size.virtual_size);
	if (o_stream_send_str(ctx->output, str) < 0)
		return FALSE;

	/* FIXME: SLOW! we need some cache for this. */
	return message_send(ctx->output, stream, &body_size,
			    body->skip, body->max_size);
}

static const char **get_fields_array(const char *fields)
{
	const char **field_list, **field;

	while (*fields == ' ')
		fields++;
	if (*fields == '(')
		fields++;

	field_list = t_strsplit(fields, " )");

	/* array ends at ")" element */
	for (field = field_list; *field != NULL; field++) {
		if (strcmp(*field, ")") == 0)
			*field = NULL;
	}

	return field_list;
}

static int header_match(const char *const *fields,
			const unsigned char *name, size_t size)
{
	const unsigned char *name_start, *name_end;
	const char *field;

	if (size == 0)
		return FALSE;

	name_start = name;
	name_end = name + size;

	for (; *fields != NULL; fields++) {
		field = *fields;
		if (*field == '\0')
			continue;

		for (name = name_start; name != name_end; name++) {
			/* field has been uppercased long time ago while
			   parsing FETCH command */
			if (i_toupper(*name) != *field)
				break;

			field++;
			if (*field == '\0') {
				if (name+1 == name_end)
					return TRUE;
				break;
			}
		}
	}

	return FALSE;
}

static int header_match_not(const char *const *fields,
			    const unsigned char *name, size_t size)
{
	return !header_match(fields, name, size);
}

static int header_match_mime(const char *const *fields __attr_unused__,
			     const unsigned char *name, size_t size)
{
	if (size > 8 && memcasecmp(name, "Content-", 8) == 0)
		return TRUE;

	if (size == 12 && memcasecmp(name, "Mime-Version", 12) == 0)
		return TRUE;

	return FALSE;
}

static int fetch_header_append(struct fetch_header_field_context *ctx,
			       const unsigned char *str, size_t size)
{
	if (ctx->skip > 0) {
		if (ctx->skip >= size) {
			ctx->skip -= size;
			return TRUE;
		}

		str += ctx->skip;
		size -= ctx->skip;
		ctx->skip = 0;
	}

	if (ctx->dest_size + size > ctx->max_size) {
		i_assert(ctx->dest_size <= ctx->max_size);
		size = ctx->max_size - ctx->dest_size;
	}

	if (ctx->dest != NULL)
		str_append_n(ctx->dest, str, size);
	ctx->dest_size += size;

	if (ctx->output != NULL) {
		if (o_stream_send(ctx->output, str, size) < 0)
			return FALSE;
	}
	return ctx->dest_size < ctx->max_size;
}

static void fetch_header_field(struct message_part *part __attr_unused__,
			       const unsigned char *name, size_t name_len,
			       const unsigned char *value __attr_unused__,
			       size_t value_len __attr_unused__,
			       void *context)
{
	struct fetch_header_field_context *ctx = context;
	const unsigned char *field_start, *field_end, *cr, *p;

	/* see if we want this field. */
	if (!ctx->match_func(ctx->fields, name, name_len) || name_len == 0)
		return;

	/* add the field, inserting CRs when needed. FIXME: is this too
	   kludgy? we assume name continues with ": value". but otherwise
	   we wouldn't reply with correct LWSP around ":". */
	field_start = name;
	field_end = value + value_len;

	cr = NULL;
	for (p = field_start; p != field_end; p++) {
		if (*p == '\r')
			cr = p;
		else if (*p == '\n' && cr != p-1) {
			/* missing CR */
			if (!fetch_header_append(ctx, field_start,
						 (size_t) (p-field_start)))
				return;
			if (!fetch_header_append(ctx, UNSIGNED_CRLF, 2))
				return;

			field_start = p+1;
		}
	}

	if (field_start != field_end) {
		if (!fetch_header_append(ctx, field_start,
					 (size_t) (field_end-field_start)))
			return;
	}

	(void)fetch_header_append(ctx, UNSIGNED_CRLF, 2);
}

static int fetch_header_fields(struct istream *input, const char *section,
			       struct fetch_header_field_context *ctx)
{
	if (strncmp(section, "HEADER.FIELDS ", 14) == 0) {
		ctx->fields = get_fields_array(section + 14);
		ctx->match_func = header_match;
	} else if (strncmp(section, "HEADER.FIELDS.NOT ", 18) == 0) {
		ctx->fields = get_fields_array(section + 18);
		ctx->match_func = header_match_not;
	} else if (strcmp(section, "MIME") == 0) {
		/* Mime-Version + Content-* fields */
		ctx->match_func = header_match_mime;
	} else {
		i_warning("BUG: Accepted invalid section from user: '%s'",
			  section);
		return FALSE;
	}

	ctx->dest_size = 0;
	message_parse_header(NULL, input, NULL, fetch_header_field, ctx);

	/* FIXME: The blank line must not be filtered, says RFC. However, we
	   shouldn't add it if it wasn't there in the first place. Not very
	   easy to know currently so we'll just do it always, it'll be present
	   in all sane messages anyway.. */
	(void)fetch_header_append(ctx, UNSIGNED_CRLF, 2);

	i_assert(ctx->dest_size <= ctx->max_size);
	i_assert(ctx->dest == NULL || str_len(ctx->dest) == ctx->dest_size);
	return TRUE;
}

/* fetch wanted headers from given data */
static int fetch_header_from(struct imap_fetch_context *ctx,
			     struct istream *input,
			     const struct message_size *size,
			     const struct imap_fetch_body_data *body)
{
	struct fetch_header_field_context hdr_ctx;
	const char *str;
	uoff_t start_offset;
	int failed;

	/* HEADER, MIME, HEADER.FIELDS (list), HEADER.FIELDS.NOT (list) */

	if (strcmp(body->section, "HEADER") == 0) {
		/* all headers */
		str = t_strdup_printf("%s {%"PRIuUOFF_T"}\r\n",
				      ctx->prefix, size->virtual_size);
		if (o_stream_send_str(ctx->output, str) < 0)
			return FALSE;
		return message_send(ctx->output, input, size,
				    body->skip, body->max_size);
	}

	/* partial headers - copy the wanted fields into memory, inserting
	   missing CRs on the way. If the header is too large, calculate 
	   the size first and then send the data directly to output stream. */

	memset(&hdr_ctx, 0, sizeof(hdr_ctx));
	hdr_ctx.skip = body->skip;
	hdr_ctx.max_size = body->max_size;

	failed = FALSE;
	start_offset = input->v_offset;

	t_push();

	/* first pass, we need at least the size */
	if (size->virtual_size > MAX_HEADER_BUFFER_SIZE &&
	    body->max_size > MAX_HEADER_BUFFER_SIZE) {
		if (!fetch_header_fields(input, body->section, &hdr_ctx))
			failed = TRUE;

		i_assert(hdr_ctx.dest_size <= size->virtual_size);
	} else {
		hdr_ctx.dest = t_str_new(size->virtual_size < 8192 ?
					 size->virtual_size : 8192);
		if (!fetch_header_fields(input, body->section, &hdr_ctx))
			failed = TRUE;
	}

	if (!failed) {
		str = t_strdup_printf("%s {%"PRIuUOFF_T"}\r\n",
				      ctx->prefix, hdr_ctx.dest_size);
		if (o_stream_send_str(ctx->output, str) < 0)
			failed = TRUE;
	}

	if (!failed) {
		if (hdr_ctx.dest == NULL) {
			/* second pass, write the data to output stream */
			uoff_t first_size = hdr_ctx.dest_size;

			hdr_ctx.output = ctx->output;
			i_stream_seek(input, start_offset);

			if (!failed &&
			    !fetch_header_fields(input, body->section,
						 &hdr_ctx))
				failed = TRUE;

			i_assert(first_size == hdr_ctx.dest_size);
		} else {
			if (o_stream_send(ctx->output, str_data(hdr_ctx.dest),
					  str_len(hdr_ctx.dest)) < 0)
				failed = TRUE;
		}
	}

	t_pop();
	return !failed;
}

static int fetch_header(struct imap_fetch_context *ctx, struct mail *mail,
			const struct imap_fetch_body_data *body)
{
	struct istream *stream;
	struct message_size hdr_size;

	stream = mail->get_stream(mail, &hdr_size, NULL);
	if (stream == NULL)
		return FALSE;

	return fetch_header_from(ctx, stream, &hdr_size, body);
}

/* Find message_part for section (eg. 1.3.4) */
static const struct message_part *
part_find(struct mail *mail, const struct imap_fetch_body_data *body,
	  const char **section)
{
	const struct message_part *part;
	const char *path;
	unsigned int num;

	part = mail->get_parts(mail);
	if (part == NULL)
		return NULL;

	path = body->section;
	while (*path >= '0' && *path <= '9' && part != NULL) {
		/* get part number */
		num = 0;
		while (*path != '\0' && *path != '.') {
			if (*path < '0' || *path > '9')
				return NULL;
			num = num*10 + (*path - '0');
			path++;
		}

		if (*path == '.')
			path++;

		if (part->flags & MESSAGE_PART_FLAG_MULTIPART) {
			/* find the part */
			part = part->children;
			for (; num > 1 && part != NULL; num--)
				part = part->next;
		} else {
			/* only 1 allowed with non-multipart messages */
			if (num != 1)
				return NULL;
		}

		if (part != NULL &&
		    (part->flags & MESSAGE_PART_FLAG_MESSAGE_RFC822)) {
			/* skip the message/rfc822 part */
			part = part->children;
		}
	}

	*section = path;
	return part;
}

/* fetch BODY[1.2] or BODY[1.2.TEXT] */
static int fetch_part_body(struct imap_fetch_context *ctx,
			   struct istream *stream,
			   const struct imap_fetch_body_data *body,
			   const struct message_part *part)
{
	const char *str;

	/* jump to beginning of part body */
	i_stream_seek(stream, part->physical_pos +
		      part->header_size.physical_size);

	str = t_strdup_printf("%s {%"PRIuUOFF_T"}\r\n",
			      ctx->prefix, part->body_size.virtual_size);
	if (o_stream_send_str(ctx->output, str) < 0)
		return FALSE;

	/* FIXME: potential performance problem with big messages:
	   FETCH BODY[1]<100000..1024>, hopefully no clients do this */
	return message_send(ctx->output, stream, &part->body_size,
			    body->skip, body->max_size);
}

static int fetch_part(struct imap_fetch_context *ctx, struct mail *mail,
		      const struct imap_fetch_body_data *body)
{
	struct istream *stream;
	const struct message_part *part;
	const char *section;

	part = part_find(mail, body, &section);
	if (part == NULL)
		return FALSE;

	stream = mail->get_stream(mail, NULL, NULL);
	if (stream == NULL)
		return FALSE;

	if (*section == '\0' || strcmp(section, "TEXT") == 0)
		return fetch_part_body(ctx, stream, body, part);

	if (strncmp(section, "HEADER", 6) == 0 ||
	    strcmp(section, "MIME") == 0) {
		i_stream_seek(stream, part->physical_pos);
		return fetch_header_from(ctx, stream, &part->header_size, body);
	}

	i_warning("BUG: Accepted invalid section from user: '%s'",
		  body->section);
	return FALSE;
}

int imap_fetch_body_section(struct imap_fetch_context *ctx,
			    const struct imap_fetch_body_data *body,
			    struct mail *mail)
{
	ctx->prefix = !body->skip_set ?
		t_strdup_printf(" BODY[%s]", body->section) :
		t_strdup_printf(" BODY[%s]<%"PRIuUOFF_T">",
				body->section, body->skip);
	if (ctx->first) {
		ctx->prefix++; ctx->first = FALSE;
	}

	if (*body->section == '\0')
		return fetch_body(ctx, body, mail, TRUE);
	if (strcmp(body->section, "TEXT") == 0)
		return fetch_body(ctx, body, mail, FALSE);
	if (strncmp(body->section, "HEADER", 6) == 0)
		return fetch_header(ctx, mail, body);
	if (*body->section >= '0' && *body->section <= '9')
		return fetch_part(ctx, mail, body);

	i_warning("BUG: Accepted invalid section from user: '%s'",
		  body->section);
	return FALSE;
}

--- NEW FILE: imap-fetch.c ---
/* Copyright (C) 2002 Timo Sirainen */

#include "common.h"
#include "istream.h"
#include "ostream.h"
#include "str.h"
#include "message-send.h"
#include "message-size.h"
#include "imap-date.h"
#include "commands.h"
#include "imap-fetch.h"

#include <unistd.h>

static void fetch_uid(struct imap_fetch_context *ctx, struct mail *mail)
{
	str_printfa(ctx->str, "UID %u ", mail->uid);
}

static int fetch_flags(struct imap_fetch_context *ctx, struct mail *mail)
{
	const struct mail_full_flags *flags;

	flags = mail->get_flags(mail);
	if (flags == NULL)
		return FALSE;

	str_printfa(ctx->str, "FLAGS (%s) ",
		    imap_write_flags(flags->flags, flags->custom_flags,
				     flags->custom_flags_count));
	return TRUE;
}

static int fetch_internaldate(struct imap_fetch_context *ctx, struct mail *mail)
{
	time_t time;

	time = mail->get_received_date(mail);
	if (time == (time_t)-1)
		return FALSE;

	str_printfa(ctx->str, "INTERNALDATE \"%s\" ", imap_to_datetime(time));
	return TRUE;
}

static int fetch_rfc822_size(struct imap_fetch_context *ctx, struct mail *mail)
{
	uoff_t size;

	size = mail->get_size(mail);
	if (size == (uoff_t)-1)
		return FALSE;

	str_printfa(ctx->str, "RFC822.SIZE %"PRIuUOFF_T" ", size);
	return TRUE;
}

static int fetch_body(struct imap_fetch_context *ctx, struct mail *mail)
{
	const char *body;

	body = mail->get_special(mail, MAIL_FETCH_IMAP_BODY);
	if (body == NULL)
		return FALSE;

	str_printfa(ctx->str, "BODY (%s) ", body);
	return TRUE;
}

static int fetch_bodystructure(struct imap_fetch_context *ctx,
			       struct mail *mail)
{
	const char *bodystructure;

	bodystructure = mail->get_special(mail, MAIL_FETCH_IMAP_BODYSTRUCTURE);
	if (bodystructure == NULL)
		return FALSE;

	str_printfa(ctx->str, "BODYSTRUCTURE (%s) ", bodystructure);
	return TRUE;
}

static int fetch_envelope(struct imap_fetch_context *ctx, struct mail *mail)
{
	const char *envelope;

	envelope = mail->get_special(mail, MAIL_FETCH_IMAP_ENVELOPE);
	if (envelope == NULL)
		return FALSE;

	str_printfa(ctx->str, "ENVELOPE (%s) ", envelope);
	return TRUE;
}

static int fetch_send_rfc822(struct imap_fetch_context *ctx, struct mail *mail)
{
	struct message_size hdr_size, body_size;
	struct istream *stream;
	const char *str;

	stream = mail->get_stream(mail, &hdr_size, &body_size);
	if (stream == NULL)
		return FALSE;

	message_size_add(&body_size, &hdr_size);

	str = t_strdup_printf(" RFC822 {%"PRIuUOFF_T"}\r\n",
			      body_size.virtual_size);
	if (ctx->first) {
		str++; ctx->first = FALSE;
	}
	if (o_stream_send_str(ctx->output, str) < 0)
		return FALSE;

	return message_send(ctx->output, stream, &body_size, 0, (uoff_t)-1);
}

static int fetch_send_rfc822_header(struct imap_fetch_context *ctx,
				    struct mail *mail)
{
	struct message_size hdr_size;
	struct istream *stream;
	const char *str;

	stream = mail->get_stream(mail, &hdr_size, NULL);
	if (stream == NULL)
		return FALSE;

	str = t_strdup_printf(" RFC822.HEADER {%"PRIuUOFF_T"}\r\n",
			      hdr_size.virtual_size);
	if (ctx->first) {
		str++; ctx->first = FALSE;
	}
	if (o_stream_send_str(ctx->output, str) < 0)
		return FALSE;

	return message_send(ctx->output, stream, &hdr_size, 0, (uoff_t)-1);
}

static int fetch_send_rfc822_text(struct imap_fetch_context *ctx,
				  struct mail *mail)
{
	struct message_size hdr_size, body_size;
	struct istream *stream;
	const char *str;

	stream = mail->get_stream(mail, &hdr_size, &body_size);
	if (stream == NULL)
		return FALSE;

	str = t_strdup_printf(" RFC822.TEXT {%"PRIuUOFF_T"}\r\n",
			      body_size.virtual_size);
	if (ctx->first) {
		str++; ctx->first = FALSE;
	}
	if (o_stream_send_str(ctx->output, str) < 0)
		return FALSE;

	i_stream_seek(stream, hdr_size.physical_size);
	return message_send(ctx->output, stream, &body_size, 0, (uoff_t)-1);
}

static int fetch_mail(struct imap_fetch_context *ctx, struct mail *mail)
{
	struct imap_fetch_body_data *body;
	size_t len, orig_len;
	int failed, data_written;

	str_truncate(ctx->str, 0);
	str_printfa(ctx->str, "* %u FETCH (", mail->seq);
	orig_len = str_len(ctx->str);

	failed = TRUE;
	data_written = FALSE;
	do {
		/* write the data into temp string */
		if (ctx->imap_data & IMAP_FETCH_UID)
			fetch_uid(ctx, mail);
		if ((ctx->fetch_data & MAIL_FETCH_FLAGS) || mail->seen_updated)
			fetch_flags(ctx, mail);
		if (ctx->fetch_data & MAIL_FETCH_RECEIVED_DATE)
			fetch_internaldate(ctx, mail);
		if (ctx->fetch_data & MAIL_FETCH_SIZE)
			fetch_rfc822_size(ctx, mail);
		if (ctx->fetch_data & MAIL_FETCH_IMAP_BODY)
			fetch_body(ctx, mail);
		if (ctx->fetch_data & MAIL_FETCH_IMAP_BODYSTRUCTURE)
			fetch_bodystructure(ctx, mail);
		if (ctx->fetch_data & MAIL_FETCH_IMAP_ENVELOPE)
			fetch_envelope(ctx, mail);

		/* send the data written into temp string */
		len = str_len(ctx->str);
		ctx->first = len == orig_len;

		if (!ctx->first)
			str_truncate(ctx->str, --len);
		if (o_stream_send(ctx->output, str_data(ctx->str), len) < 0)
			break;

		data_written = TRUE;

		/* large data */
		if (ctx->imap_data & IMAP_FETCH_RFC822)
			if (!fetch_send_rfc822(ctx, mail))
				break;
		if (ctx->imap_data & IMAP_FETCH_RFC822_HEADER)
			if (!fetch_send_rfc822_header(ctx, mail))
				break;
		if (ctx->imap_data & IMAP_FETCH_RFC822_TEXT)
			if (!fetch_send_rfc822_text(ctx, mail))
				break;

		for (body = ctx->bodies; body != NULL; body = body->next) {
			if (!imap_fetch_body_section(ctx, body, mail))
				break;
		}

		failed = FALSE;
	} while (0);

	if (data_written) {
		if (o_stream_send(ctx->output, ")\r\n", 3) < 0)
			failed = TRUE;
	}

	return !failed;
}

int imap_fetch(struct client *client,
	       enum mail_fetch_field fetch_data,
	       enum imap_fetch_field imap_data,
	       struct imap_fetch_body_data *bodies,
	       const char *messageset, int uidset)
{
	struct imap_fetch_context ctx;
	struct mail *mail;
	int all_found, update_seen = FALSE;

	if (!client->mailbox->readonly) {
		/* If we have any BODY[..] sections, \Seen flag is added for
		   all messages */
		struct imap_fetch_body_data *body;

		for (body = bodies; body != NULL; body = body->next) {
			if (!body->peek) {
				update_seen = TRUE;
				break;
			}
		}

		if (imap_data & (IMAP_FETCH_RFC822|IMAP_FETCH_RFC822_TEXT))
			update_seen = TRUE;
	}

	memset(&ctx, 0, sizeof(ctx));
	ctx.fetch_data = fetch_data;
	ctx.imap_data = imap_data;
	ctx.bodies = bodies;
	ctx.output = client->output;
	ctx.str = t_str_new(8192);

	ctx.fetch_ctx = client->mailbox->
		fetch_init(client->mailbox, fetch_data, &update_seen,
			   messageset, uidset);
	if (ctx.fetch_ctx == NULL)
		return -1;

	while ((mail = client->mailbox->fetch_next(ctx.fetch_ctx)) != NULL) {
		if (!fetch_mail(&ctx, mail)) {
			ctx.failed = TRUE;
			break;
		}
	}

	if (!client->mailbox->fetch_deinit(ctx.fetch_ctx, &all_found))
		return -1;
	return ctx.failed ? -1 : all_found;
}

--- NEW FILE: imap-fetch.h ---
#ifndef __IMAP_FETCH_H
#define __IMAP_FETCH_H

enum imap_fetch_field {
	IMAP_FETCH_UID			= 0x01,
	IMAP_FETCH_RFC822		= 0x02,
	IMAP_FETCH_RFC822_HEADER	= 0x04,
	IMAP_FETCH_RFC822_TEXT		= 0x08
};

struct imap_fetch_body_data {
	struct imap_fetch_body_data *next;

	const char *section; /* NOTE: always uppercased */
	uoff_t skip, max_size; /* if you don't want max_size,
	                          set it to (uoff_t)-1 */
	unsigned int skip_set:1;
	unsigned int peek:1;
};

struct imap_fetch_context {
	struct mail_fetch_context *fetch_ctx;

	enum mail_fetch_field fetch_data;
	enum imap_fetch_field imap_data;
	struct imap_fetch_body_data *bodies;

	string_t *str;
	struct ostream *output;
	const char *prefix;

	int first, failed;
};

int imap_fetch(struct client *client,
	       enum mail_fetch_field fetch_data,
	       enum imap_fetch_field imap_data,
	       struct imap_fetch_body_data *bodies,
	       const char *messageset, int uidset);

int imap_fetch_body_section(struct imap_fetch_context *ctx,
			    const struct imap_fetch_body_data *body,
			    struct mail *mail);

#endif

--- NEW FILE: imap-search.c ---
/* Copyright (C) 2002 Timo Sirainen */

#include "common.h"
#include "mail-search.h"
#include "imap-search.h"

struct search_build_data {
	pool_t pool;
	const char *error;
};

static struct mail_search_arg *
search_arg_new(pool_t pool, enum mail_search_arg_type type)
{
	struct mail_search_arg *arg;

	arg = p_new(pool, struct mail_search_arg, 1);
	arg->type = type;

	return arg;
}

#define ARG_NEW(type, value) \
	arg_new(data, args, next_sarg, type, value)

static int arg_new(struct search_build_data *data, struct imap_arg **args,
		   struct mail_search_arg **next_sarg,
		   enum mail_search_arg_type type, int value)
{
	struct mail_search_arg *sarg;

	*next_sarg = sarg = search_arg_new(data->pool, type);
	if (value == 0)
		return TRUE;

	/* first arg */
	if ((*args)->type == IMAP_ARG_EOL) {
		data->error = "Missing parameter for argument";
		return FALSE;
	}

	if ((*args)->type != IMAP_ARG_ATOM &&
	    (*args)->type != IMAP_ARG_STRING) {
		data->error = "Invalid parameter for argument";
		return FALSE;
	}

	sarg->value.str = str_ucase(IMAP_ARG_STR(*args));
	*args += 1;

	/* second arg */
	if (value == 2) {
		if ((*args)->type == IMAP_ARG_EOL) {
			data->error = "Missing parameter for argument";
			return FALSE;
		}

		if ((*args)->type != IMAP_ARG_ATOM &&
		    (*args)->type != IMAP_ARG_STRING) {
			data->error = "Invalid parameter for argument";
			return FALSE;
		}

                sarg->hdr_field_name = sarg->value.str;
		sarg->value.str = str_ucase(IMAP_ARG_STR(*args));
		*args += 1;
	}

	return TRUE;
}

static int search_arg_build(struct search_build_data *data,
			    struct imap_arg **args,
			    struct mail_search_arg **next_sarg)
{
	struct mail_search_arg **subargs;
	struct imap_arg *arg;
	char *str;

	if ((*args)->type == IMAP_ARG_EOL) {
		data->error = "Missing argument";
		return FALSE;
	}

	arg = *args;

	if (arg->type == IMAP_ARG_NIL) {
		/* NIL not allowed */
		data->error = "NIL not allowed";
		return FALSE;
	}

	if (arg->type == IMAP_ARG_LIST) {
		struct imap_arg *listargs = IMAP_ARG_LIST(arg)->args;

		*next_sarg = search_arg_new(data->pool, SEARCH_SUB);
		subargs = &(*next_sarg)->value.subargs;
		while (listargs->type != IMAP_ARG_EOL) {
			if (!search_arg_build(data, &listargs, subargs))
				return FALSE;
			subargs = &(*subargs)->next;
		}

		*args += 1;
		return TRUE;
	}

	i_assert(arg->type == IMAP_ARG_ATOM ||
		 arg->type == IMAP_ARG_STRING);

	/* string argument - get the name and jump to next */
	str = IMAP_ARG_STR(arg);
	*args += 1;
	str_ucase(str);

	switch (*str) {
	case 'A':
		if (strcmp(str, "ANSWERED") == 0)
			return ARG_NEW(SEARCH_ANSWERED, 0);
		else if (strcmp(str, "ALL") == 0)
			return ARG_NEW(SEARCH_ALL, 0);
		break;
	case 'B':
		if (strcmp(str, "BODY") == 0) {
			/* <string> */
			return ARG_NEW(SEARCH_BODY, 1);
		} else if (strcmp(str, "BEFORE") == 0) {
			/* <date> */
			return ARG_NEW(SEARCH_BEFORE, 1);
		} else if (strcmp(str, "BCC") == 0) {
			/* <string> */
			return ARG_NEW(SEARCH_BCC, 1);
		}
		break;
	case 'C':
		if (strcmp(str, "CC") == 0) {
			/* <string> */
			return ARG_NEW(SEARCH_CC, 1);
		}
		break;
	case 'D':
		if (strcmp(str, "DELETED") == 0)
			return ARG_NEW(SEARCH_DELETED, 0);
		else if (strcmp(str, "DRAFT") == 0)
			return ARG_NEW(SEARCH_DRAFT, 0);
		break;
	case 'F':
		if (strcmp(str, "FLAGGED") == 0)
			return ARG_NEW(SEARCH_FLAGGED, 0);
		else if (strcmp(str, "FROM") == 0) {
			/* <string> */
			return ARG_NEW(SEARCH_FROM, 1);
		}
		break;
	case 'H':
		if (strcmp(str, "HEADER") == 0) {
			/* <field-name> <string> */
			const char *key;

			if ((*args)->type == IMAP_ARG_EOL) {
				data->error = "Missing parameter for HEADER";
				return FALSE;
			}
			if ((*args)->type != IMAP_ARG_ATOM &&
			    (*args)->type != IMAP_ARG_STRING) {
				data->error = "Invalid parameter for HEADER";
				return FALSE;
			}

			key = str_ucase(IMAP_ARG_STR(*args));

			if (strcmp(key, "FROM") == 0) {
				*args += 1;
				return ARG_NEW(SEARCH_FROM, 1);
			} else if (strcmp(key, "TO") == 0) {
				*args += 1;
				return ARG_NEW(SEARCH_TO, 1);
			} else if (strcmp(key, "CC") == 0) {
				*args += 1;
				return ARG_NEW(SEARCH_CC, 1);
			} else if (strcmp(key, "BCC") == 0) {
				*args += 1;
				return ARG_NEW(SEARCH_BCC, 1);
			} else if (strcmp(key, "SUBJECT") == 0) {
				*args += 1;
				return ARG_NEW(SEARCH_SUBJECT, 1);
			} else if (strcmp(key, "IN-REPLY-TO") == 0) {
				*args += 1;
				return ARG_NEW(SEARCH_IN_REPLY_TO, 1);
			} else if (strcmp(key, "MESSAGE-ID") == 0) {
				*args += 1;
				return ARG_NEW(SEARCH_MESSAGE_ID, 1);
			} else {
				return ARG_NEW(SEARCH_HEADER, 2);
			}
		}
		break;
	case 'K':
		if (strcmp(str, "KEYWORD") == 0) {
			/* <flag> */
			return ARG_NEW(SEARCH_KEYWORD, 1);
		}
		break;
	case 'L':
		if (strcmp(str, "LARGER") == 0) {
			/* <n> */
			return ARG_NEW(SEARCH_LARGER, 1);
		}
		break;
	case 'N':
		if (strcmp(str, "NOT") == 0) {
			if (!search_arg_build(data, args, next_sarg))
				return FALSE;
			(*next_sarg)->not = !(*next_sarg)->not;
			return TRUE;
		} else if (strcmp(str, "NEW") == 0) {
			/* NEW == (RECENT UNSEEN) */
			*next_sarg = search_arg_new(data->pool, SEARCH_SUB);

			subargs = &(*next_sarg)->value.subargs;
			*subargs = search_arg_new(data->pool, SEARCH_RECENT);
			(*subargs)->next = search_arg_new(data->pool,
							  SEARCH_SEEN);
			(*subargs)->next->not = TRUE;
			return TRUE;
		}
		break;
	case 'O':
		if (strcmp(str, "OR") == 0) {
			/* <search-key1> <search-key2> */
			*next_sarg = search_arg_new(data->pool, SEARCH_OR);

			subargs = &(*next_sarg)->value.subargs;
			for (;;) {
				if (!search_arg_build(data, args, subargs))
					return FALSE;

				subargs = &(*subargs)->next;

				/* <key> OR <key> OR ... <key> - put them all
				   under one SEARCH_OR list. */
				if ((*args)->type == IMAP_ARG_EOL)
					break;

				if ((*args)->type != IMAP_ARG_ATOM ||
				    strcasecmp(IMAP_ARG_STR(*args), "OR") != 0)
					break;

				*args += 1;
			}

			if (!search_arg_build(data, args, subargs))
				return FALSE;
			return TRUE;
		} if (strcmp(str, "ON") == 0) {
			/* <date> */
			return ARG_NEW(SEARCH_ON, 1);
		} if (strcmp(str, "OLD") == 0) {
			/* OLD == NOT RECENT */
			if (!ARG_NEW(SEARCH_RECENT, 0))
				return FALSE;

			(*next_sarg)->not = TRUE;
			return TRUE;
		}
		break;
	case 'R':
		if (strcmp(str, "RECENT") == 0)
			return ARG_NEW(SEARCH_RECENT, 0);
		break;
	case 'S':
		if (strcmp(str, "SEEN") == 0)
			return ARG_NEW(SEARCH_SEEN, 0);
		else if (strcmp(str, "SUBJECT") == 0) {
			/* <string> */
			return ARG_NEW(SEARCH_SUBJECT, 1);
		} else if (strcmp(str, "SENTBEFORE") == 0) {
			/* <date> */
			return ARG_NEW(SEARCH_SENTBEFORE, 1);
		} else if (strcmp(str, "SENTON") == 0) {
			/* <date> */
			return ARG_NEW(SEARCH_SENTON, 1);
		} else if (strcmp(str, "SENTSINCE") == 0) {
			/* <date> */
			return ARG_NEW(SEARCH_SENTSINCE, 1);
		} else if (strcmp(str, "SINCE") == 0) {
			/* <date> */
			return ARG_NEW(SEARCH_SINCE, 1);
		} else if (strcmp(str, "SMALLER") == 0) {
			/* <n> */
			return ARG_NEW(SEARCH_SMALLER, 1);
		}
		break;
	case 'T':
		if (strcmp(str, "TEXT") == 0) {
			/* <string> */
			return ARG_NEW(SEARCH_TEXT, 1);
		} else if (strcmp(str, "TO") == 0) {
			/* <string> */
			return ARG_NEW(SEARCH_TO, 1);
		}
		break;
	case 'U':
		if (strcmp(str, "UID") == 0) {
			/* <message set> */
			return ARG_NEW(SEARCH_UID, 1);
		} else if (strcmp(str, "UNANSWERED") == 0) {
			if (!ARG_NEW(SEARCH_ANSWERED, 0))
				return FALSE;
			(*next_sarg)->not = TRUE;
			return TRUE;
		} else if (strcmp(str, "UNDELETED") == 0) {
			if (!ARG_NEW(SEARCH_DELETED, 0))
				return FALSE;
			(*next_sarg)->not = TRUE;
			return TRUE;
		} else if (strcmp(str, "UNDRAFT") == 0) {
			if (!ARG_NEW(SEARCH_DRAFT, 0))
				return FALSE;
			(*next_sarg)->not = TRUE;
			return TRUE;
		} else if (strcmp(str, "UNFLAGGED") == 0) {
			if (!ARG_NEW(SEARCH_FLAGGED, 0))
				return FALSE;
			(*next_sarg)->not = TRUE;
			return TRUE;
		} else if (strcmp(str, "UNKEYWORD") == 0) {
			if (!ARG_NEW(SEARCH_KEYWORD, 0))
				return FALSE;
			(*next_sarg)->not = TRUE;
			return TRUE;
		} else if (strcmp(str, "UNSEEN") == 0) {
			if (!ARG_NEW(SEARCH_SEEN, 0))
				return FALSE;
			(*next_sarg)->not = TRUE;
			return TRUE;
		}
		break;
	default:
		if (*str == '*' || (*str >= '0' && *str <= '9')) {
			/* <message-set> */
			if (!ARG_NEW(SEARCH_SET, 0))
				return FALSE;

			(*next_sarg)->value.str = str;
			return TRUE;
		}
		break;
	}

	data->error = t_strconcat("Unknown argument ", str, NULL);
	return FALSE;
}

struct mail_search_arg *
imap_search_args_build(pool_t pool, struct imap_arg *args, const char **error)
{
        struct search_build_data data;
	struct mail_search_arg *first_sarg, **sargs;

	data.pool = pool;
	data.error = NULL;

	/* get the first arg */
	first_sarg = NULL; sargs = &first_sarg;
	while (args->type != IMAP_ARG_EOL) {
		if (!search_arg_build(&data, &args, sargs)) {
			*error = data.error;
			return NULL;
		}
		sargs = &(*sargs)->next;
	}

	*error = NULL;
	return first_sarg;
}


--- NEW FILE: imap-search.h ---
#ifndef __IMAP_SEARCH_H
#define __IMAP_SEARCH_H

/* Builds search arguments based on IMAP arguments. */
struct mail_search_arg *
imap_search_args_build(pool_t pool, struct imap_arg *args, const char **error);

#endif

--- NEW FILE: imap-sort.c ---
/* Copyright (C) 2002 Timo Sirainen */

/* Implementation of draft-ietf-imapext-sort-10 sorting algorithm.
   Pretty messy code actually, adding any sort types requires care.
   This is pretty fast however and takes only as much memory as needed to be
   reasonably fast. */

#include "common.h"
#include "buffer.h"
#include "hash.h"
#include "ostream.h"
#include "str.h"
#include "imap-base-subject.h"
#include "imap-sort.h"

#include <stdlib.h>

#define MAX_WANTED_HEADERS 10
#define STRBUF_SIZE 1024

#define IS_SORT_STRING(type) \
	((type) == MAIL_SORT_CC || (type) == MAIL_SORT_FROM || \
	 (type) == MAIL_SORT_SUBJECT || (type) == MAIL_SORT_TO)

#define IS_SORT_TIME(type) \
	((type) == MAIL_SORT_ARRIVAL || (type) == MAIL_SORT_DATE)

struct sort_context {
	struct mail_search_context *search_ctx;

	enum mail_sort_type sort_program[MAX_SORT_PROGRAM_SIZE];
	enum mail_sort_type common_mask, cache_mask;

	struct mailbox *box;
	struct ostream *output;
	string_t *str;

	buffer_t *sort_buffer;
	size_t sort_element_size;

	pool_t temp_pool, str_pool;
	struct hash_table *string_table;

	time_t last_arrival, last_date;
	uoff_t last_size;
	char *last_cc, *last_from, *last_subject, *last_to;

	int written, id_is_uid;
};

static void mail_sort_input(struct sort_context *ctx, struct mail *mail);
static void mail_sort_flush(struct sort_context *ctx);

static enum mail_sort_type
mail_sort_normalize(const enum mail_sort_type *input, buffer_t *output)
{
        enum mail_sort_type type, mask = 0;
	int pos, reverse;

	reverse = FALSE;
	for (pos = 0; *input != MAIL_SORT_END; input++) {
		if (*input == MAIL_SORT_REVERSE)
			reverse = !reverse;
		else {
			if ((mask & *input) == 0) {
				if (reverse) {
					type = MAIL_SORT_REVERSE;
					buffer_append(output,
						      &type, sizeof(type));
				}

				buffer_append(output, input, sizeof(*input));
				mask |= *input;
			}

			reverse = FALSE;
		}
	}

	type = MAIL_SORT_END;
	buffer_append(output, &type, sizeof(type));

	return mask;
}

static enum mail_sort_type
mail_sort_get_common_mask(const enum mail_sort_type *sort1,
			  const enum mail_sort_type *sort2,
			  unsigned int *count)
{
	enum mail_sort_type mask = 0;

	*count = 0;
	while (*sort1 == *sort2 && *sort1 != MAIL_SORT_END) {
		if (*sort1 != MAIL_SORT_REVERSE)
			mask |= *sort1;
		sort1++; sort2++; (*count)++;
	}

	return mask;
}

static enum mail_fetch_field
init_sort_elements(struct sort_context *ctx,
		   const char *wanted_headers[MAX_WANTED_HEADERS])
{
	unsigned int i;
        enum mail_fetch_field fields;

	/* figure out what data we'd like to cache */
	ctx->sort_element_size = sizeof(unsigned int);
	ctx->cache_mask = 0;

	for (i = 0; ctx->sort_program[i] != MAIL_SORT_END; i++) {
		enum mail_sort_type type = ctx->sort_program[i];

		if (IS_SORT_STRING(type)) {
			ctx->sort_element_size += sizeof(const char *);

			/* cache the second rule as well, if available */
			if (ctx->cache_mask != 0) {
				ctx->cache_mask |= type;
				break;
			}
			ctx->cache_mask |= type;
		} else if (IS_SORT_TIME(type)) {
			ctx->sort_element_size += sizeof(time_t);
			ctx->cache_mask |= type;
			break;
		} else if (type == MAIL_SORT_SIZE) {
			ctx->sort_element_size += sizeof(uoff_t);
			ctx->cache_mask |= type;
			break;
		}
	}

	fields = 0;
	if (ctx->cache_mask & MAIL_SORT_ARRIVAL)
		fields |= MAIL_FETCH_RECEIVED_DATE;
	if (ctx->cache_mask & MAIL_SORT_DATE)
		fields |= MAIL_FETCH_DATE;
	if (ctx->cache_mask & MAIL_SORT_SIZE)
		fields |= MAIL_FETCH_SIZE;

	/* @UNSAFE */
	i_assert(MAX_WANTED_HEADERS > 4);
	i = 0;
	if (ctx->cache_mask & MAIL_SORT_CC)
		wanted_headers[i++] = "cc";
	if (ctx->cache_mask & MAIL_SORT_FROM)
		wanted_headers[i++] = "from";
	if (ctx->cache_mask & MAIL_SORT_TO)
		wanted_headers[i++] = "to";
	if (ctx->cache_mask & MAIL_SORT_SUBJECT)
		wanted_headers[i++] = "subject";
	wanted_headers[i] = NULL;

	if ((ctx->cache_mask & MAIL_SORT_CC) ||
	    (ctx->cache_mask & MAIL_SORT_FROM) ||
	    (ctx->cache_mask & MAIL_SORT_TO) ||
	    (ctx->cache_mask & MAIL_SORT_SUBJECT)) {
		ctx->str_pool = pool_alloconly_create("sort str", 8192);
		ctx->string_table = hash_create(default_pool, ctx->str_pool,
						0, str_hash,
						(hash_cmp_callback_t)strcmp);
	}

	return fields;
}

static void mail_sort_deinit(struct sort_context *ctx)
{
	mail_sort_flush(ctx);

	if (ctx->string_table != NULL)
		hash_destroy(ctx->string_table);
	if (ctx->str_pool != NULL)
		pool_unref(ctx->str_pool);
	buffer_free(ctx->sort_buffer);
	pool_unref(ctx->temp_pool);

	i_free(ctx->last_cc);
	i_free(ctx->last_from);
	i_free(ctx->last_subject);
	i_free(ctx->last_to);
}

int imap_sort(struct client *client, const char *charset,
	      struct mail_search_arg *args,
	      const enum mail_sort_type *sort_program)
{
	enum mail_sort_type norm_prog[MAX_SORT_PROGRAM_SIZE];
        enum mail_fetch_field wanted_fields;
	const char *wanted_headers[MAX_WANTED_HEADERS];
	struct sort_context *ctx;
	struct mail *mail;
	buffer_t *buf;
	unsigned int count;
	int ret;

	ctx = t_new(struct sort_context, 1);

	/* normalize sorting program */
	buf = buffer_create_data(data_stack_pool, norm_prog, sizeof(norm_prog));
	mail_sort_normalize(sort_program, buf);
	memcpy(ctx->sort_program, norm_prog, sizeof(ctx->sort_program));

	/* remove the common part from sort program, we already know input is
	   sorted that much so we don't have to worry about it. */
	if (!client->mailbox->search_get_sorting(client->mailbox, norm_prog))
		return FALSE;
	ctx->common_mask = mail_sort_get_common_mask(ctx->sort_program,
						     norm_prog, &count);
	if (count > 0) {
		memmove(ctx->sort_program, ctx->sort_program + count,
			sizeof(ctx->sort_program) -
			sizeof(ctx->sort_program[0]) * count);
	}

	memset(wanted_headers, 0, sizeof(wanted_headers));
	wanted_fields = init_sort_elements(ctx, wanted_headers);

	/* initialize searching */
	ctx->search_ctx = client->mailbox->
		search_init(client->mailbox, charset, args, norm_prog,
			    wanted_fields, wanted_headers);
	if (ctx->search_ctx == NULL)
		return FALSE;

	ctx->box = client->mailbox;
	ctx->output = client->output;
	ctx->temp_pool = pool_alloconly_create("sort temp", 8192);
	ctx->sort_buffer = buffer_create_dynamic(system_pool,
						 128 * ctx->sort_element_size,
						 (size_t)-1);

	ctx->str = t_str_new(STRBUF_SIZE);
	str_append(ctx->str, "* SORT");

        ctx->id_is_uid = client->cmd_uid;

	while ((mail = client->mailbox->search_next(ctx->search_ctx)) != NULL)
		mail_sort_input(ctx, mail);

	mail_sort_flush(ctx);
	ret = client->mailbox->search_deinit(ctx->search_ctx);

	if (ctx->written || ret) {
		str_append(ctx->str, "\r\n");
		o_stream_send(client->output, str_data(ctx->str),
			      str_len(ctx->str));
	}

        mail_sort_deinit(ctx);
	return ret;
}

static const char *string_table_get(struct sort_context *ctx, const char *str)
{
	char *value;

	if (str == NULL)
		return NULL;
	if (*str == '\0')
		return "";

	value = hash_lookup(ctx->string_table, str);
	if (value == NULL) {
		value = p_strdup(ctx->str_pool, str);
		hash_insert(ctx->string_table, value, value);
	}

	return value;
}

static void mail_sort_check_flush(struct sort_context *ctx, struct mail *mail)
{
	const char *str;
	time_t t;
	uoff_t size;
	int changed = FALSE;

	if (ctx->common_mask & MAIL_SORT_ARRIVAL) {
		t = mail->get_received_date(mail);
		if (t != ctx->last_arrival) {
			ctx->last_arrival = t;
			changed = TRUE;
		}
	}

	if (ctx->common_mask & MAIL_SORT_CC) {
		str = mail->get_first_mailbox(mail, "cc");
		if (str != NULL)
			str = str_ucase(t_strdup_noconst(str));

		if (null_strcmp(str, ctx->last_cc) != 0) {
			i_free(ctx->last_cc);
			ctx->last_cc = i_strdup(str);
			changed = TRUE;
		}
	}

	if (ctx->common_mask & MAIL_SORT_DATE) {
		t = mail->get_date(mail, NULL);
		if (t != ctx->last_date) {
			ctx->last_date = t;
			changed = TRUE;
		}
	}

	if (ctx->common_mask & MAIL_SORT_FROM) {
		str = mail->get_first_mailbox(mail, "from");
		if (str != NULL)
			str = str_ucase(t_strdup_noconst(str));

		if (null_strcmp(str, ctx->last_from) != 0) {
			i_free(ctx->last_from);
			ctx->last_from = i_strdup(str);
			changed = TRUE;
		}
	}

	if (ctx->common_mask & MAIL_SORT_SIZE) {
		size = mail->get_size(mail);
		if (size != ctx->last_size) {
			ctx->last_size = size;
			changed = TRUE;
		}
	}

	if (ctx->common_mask & MAIL_SORT_SUBJECT) {
		str = mail->get_header(mail, "subject");
		if (str != NULL) {
			p_clear(ctx->temp_pool);
			str = imap_get_base_subject_cased(ctx->temp_pool,
							  str, NULL);
		}

		if (null_strcmp(str, ctx->last_subject) != 0) {
			i_free(ctx->last_subject);
			ctx->last_subject = i_strdup(str);
			changed = TRUE;
		}
	}

	if (ctx->common_mask & MAIL_SORT_TO) {
		str = mail->get_first_mailbox(mail, "to");
		if (str != NULL)
			str = str_ucase(t_strdup_noconst(str));

		if (null_strcmp(str, ctx->last_to) != 0) {
			i_free(ctx->last_to);
			ctx->last_to = i_strdup(str);
			changed = TRUE;
		}
	}

	if (changed)
		mail_sort_flush(ctx);
}

static void mail_sort_input(struct sort_context *ctx, struct mail *mail)
{
	/* @UNSAFE */
	unsigned char *buf;
	unsigned int id;
	time_t t;
	uoff_t size;
	const char *str;
	size_t pos;

	t_push();
	if (ctx->common_mask != 0)
		mail_sort_check_flush(ctx, mail);

	buf = buffer_append_space(ctx->sort_buffer, ctx->sort_element_size);
	id = ctx->id_is_uid ? mail->uid : mail->seq;
	memcpy(buf, &id, sizeof(id)); pos = sizeof(id);

	if (ctx->cache_mask & MAIL_SORT_ARRIVAL) {
		if (ctx->common_mask & MAIL_SORT_ARRIVAL)
			t = ctx->last_arrival;
		else
			t = mail->get_received_date(mail);
		memcpy(buf + pos, &t, sizeof(t)); pos += sizeof(t);
	}

	if (ctx->cache_mask & MAIL_SORT_DATE) {
		if (ctx->common_mask & MAIL_SORT_DATE)
			t = ctx->last_date;
		else
			t = mail->get_date(mail, NULL);
		memcpy(buf + pos, &t, sizeof(t)); pos += sizeof(t);
	}

	if (ctx->cache_mask & MAIL_SORT_SIZE) {
		if (ctx->common_mask & MAIL_SORT_SIZE)
			size = ctx->last_size;
		else
			size = mail->get_size(mail);

		memcpy(buf + pos, &size, sizeof(size)); pos += sizeof(size);
	}

	if (ctx->cache_mask & MAIL_SORT_CC) {
		if (ctx->common_mask & MAIL_SORT_CC)
			str = ctx->last_cc;
		else {
			str = mail->get_first_mailbox(mail, "cc");
			if (str != NULL)
				str = str_ucase(t_strdup_noconst(str));
		}
		str = string_table_get(ctx, str);

		memcpy(buf + pos, &str, sizeof(const char *));
		pos += sizeof(const char *);
	}

	if (ctx->cache_mask & MAIL_SORT_FROM) {
		if (ctx->common_mask & MAIL_SORT_FROM)
			str = ctx->last_from;
		else {
			str = mail->get_first_mailbox(mail, "from");
			if (str != NULL)
				str = str_ucase(t_strdup_noconst(str));
		}
		str = string_table_get(ctx, str);

		memcpy(buf + pos, &str, sizeof(const char *));
		pos += sizeof(const char *);
	}

	if (ctx->cache_mask & MAIL_SORT_TO) {
		if (ctx->common_mask & MAIL_SORT_TO)
			str = ctx->last_to;
		else {
			str = mail->get_first_mailbox(mail, "to");
			if (str != NULL)
				str = str_ucase(t_strdup_noconst(str));
		}
		str = string_table_get(ctx, str);

		memcpy(buf + pos, &str, sizeof(const char *));
		pos += sizeof(const char *);
	}

	if (ctx->cache_mask & MAIL_SORT_SUBJECT) {
		if (ctx->common_mask & MAIL_SORT_SUBJECT)
			str = ctx->last_subject;
		else {
			str = mail->get_header(mail, "subject");

			if (str != NULL) {
				p_clear(ctx->temp_pool);
				str = imap_get_base_subject_cased(
					ctx->temp_pool, str, NULL);
			}
		}
		str = string_table_get(ctx, str);

		memcpy(buf + pos, &str, sizeof(const char *));
		pos += sizeof(const char *);
	}

	i_assert(pos == ctx->sort_element_size);

	t_pop();
}

static struct sort_context *qsort_context;

static struct mail *get_mail(struct sort_context *ctx, const unsigned char *buf)
{
	unsigned int id = *((unsigned int *) buf);

	if (ctx->id_is_uid)
		return ctx->box->fetch_uid(ctx->box, id, 0);
	else
		return ctx->box->fetch_seq(ctx->box, id, 0);

}

static time_t get_time(enum mail_sort_type type, const unsigned char *buf,
		       struct sort_context *ctx)
{
	time_t t;

	if ((ctx->cache_mask & type) == 0) {
		struct mail *mail = get_mail(ctx, buf);

		if (mail == NULL)
			return 0;

		switch (type) {
		case MAIL_SORT_ARRIVAL:
			return mail->get_received_date(mail);
		case MAIL_SORT_DATE:
			t = mail->get_date(mail, NULL);
			if (t == (time_t)-1)
				t = 0;
			return t;
		default:
			i_unreached();
			return 0;
		}
	}

	/* use memcpy() to avoid any alignment problems */
	memcpy(&t, buf + sizeof(unsigned int), sizeof(t));
	return t;
}

static time_t get_uofft(enum mail_sort_type type, const unsigned char *buf,
			struct sort_context *ctx)
{
	uoff_t size;

	if ((ctx->cache_mask & type) == 0) {
		struct mail *mail = get_mail(ctx, buf);

		if (mail == NULL)
			return 0;

		i_assert(type == MAIL_SORT_SIZE);

		return mail->get_size(mail);
	}

	/* use memcpy() to avoid any alignment problems */
	memcpy(&size, buf + sizeof(unsigned int), sizeof(size));
	return size;
}

static const char *get_str(enum mail_sort_type type, const unsigned char *buf,
			   struct sort_context *ctx)
{
	const char *str;
	enum mail_sort_type type2;
	int pos;

	if ((ctx->cache_mask & type) == 0) {
		struct mail *mail = get_mail(ctx, buf);

		if (mail == NULL)
			return NULL;

		switch (type) {
		case MAIL_SORT_SUBJECT:
			str = mail->get_header(mail, "subject");
			if (str == NULL)
				return NULL;

			p_clear(ctx->temp_pool);
			return imap_get_base_subject_cased(ctx->temp_pool,
							   str, NULL);
		case MAIL_SORT_CC:
			str = mail->get_first_mailbox(mail, "cc");
			break;
		case MAIL_SORT_FROM:
			str = mail->get_first_mailbox(mail, "from");
			break;
		case MAIL_SORT_TO:
			str = mail->get_first_mailbox(mail, "to");
			break;
		default:
			i_unreached();
		}

		if (str != NULL)
			str = str_ucase(t_strdup_noconst(str));
		return str;
	}

	/* figure out where it is. pretty ugly. */
	type2 = (ctx->cache_mask & ~type);

	if (type2 == 0)
		pos = 0;
	else if (IS_SORT_TIME(type2))
		pos = sizeof(time_t);
	else if (type2 == MAIL_SORT_SIZE)
		pos = sizeof(uoff_t);
	else {
		if (type == MAIL_SORT_SUBJECT)
			pos = sizeof(const char *);
		else if (type2 != MAIL_SORT_SUBJECT && type > type2)
			pos = sizeof(const char *);
		else
			pos = 0;
	}

	/* use memcpy() to avoid any alignment problems */
	memcpy(&str, buf + pos + sizeof(unsigned int), sizeof(const char *));
	return str;
}

static int mail_sort_qsort_func(const void *p1, const void *p2)
{
	enum mail_sort_type *sorting;
	int ret, reverse = FALSE;

	sorting = qsort_context->sort_program;

	t_push();

	ret = 0;
	for (; *sorting != MAIL_SORT_END && ret == 0; sorting++) {
		if (*sorting == MAIL_SORT_REVERSE) {
			reverse = !reverse;
			continue;
		}

		switch (*sorting) {
		case MAIL_SORT_ARRIVAL:
		case MAIL_SORT_DATE: {
			time_t r1, r2;

			r1 = get_time(*sorting, p1, qsort_context);
			r2 = get_time(*sorting, p2, qsort_context);
			ret = r1 < r2 ? -1 : r1 > r2 ? 1 : 0;
			break;
		}
		case MAIL_SORT_SIZE: {
			uoff_t r1, r2;

			r1 = get_uofft(*sorting, p1, qsort_context);
			r2 = get_uofft(*sorting, p2, qsort_context);
			ret = r1 < r2 ? -1 : r1 > r2 ? 1 : 0;
			break;
		}
		case MAIL_SORT_CC:
		case MAIL_SORT_FROM:
		case MAIL_SORT_TO:
		case MAIL_SORT_SUBJECT:
			ret = null_strcmp(get_str(*sorting, p1, qsort_context),
					  get_str(*sorting, p2, qsort_context));
			break;
		default:
			i_unreached();
		}

		if (reverse) {
			if (ret > 0)
				ret = -1;
			else if (ret < 0)
				ret = 1;
		}

		reverse = FALSE;
	}

	t_pop();

	return ret != 0 ? ret :
		(*((unsigned int *) p1) < *((unsigned int *) p2) ? -1 : 1);
}

static void mail_sort_flush(struct sort_context *ctx)
{
	unsigned char *arr;
	size_t i, count;

	qsort_context = ctx;

	arr = buffer_get_modifyable_data(ctx->sort_buffer, NULL);
	count = buffer_get_used_size(ctx->sort_buffer) / ctx->sort_element_size;
	if (count == 0)
		return;

	qsort(arr, count, ctx->sort_element_size, mail_sort_qsort_func);

	for (i = 0; i < count; i++, arr += ctx->sort_element_size) {
		if (str_len(ctx->str) >= STRBUF_SIZE-MAX_INT_STRLEN) {
			/* flush */
			o_stream_send(ctx->output,
				      str_data(ctx->str), str_len(ctx->str));
			str_truncate(ctx->str, 0);
			ctx->written = TRUE;
		}

		str_printfa(ctx->str, " %u", *((unsigned int *) arr));
	}

	buffer_set_used_size(ctx->sort_buffer, 0);

	if (ctx->string_table != NULL) {
		hash_clear(ctx->string_table, TRUE);
		p_clear(ctx->str_pool);
	}
}

--- NEW FILE: imap-sort.h ---
#ifndef __IMAP_SORT_H
#define __IMAP_SORT_H

int imap_sort(struct client *client, const char *charset,
	      struct mail_search_arg *args, enum mail_sort_type *sorting);

#endif

--- NEW FILE: imap-thread.c ---
/* Copyright (C) 2002 Timo Sirainen */

/*
 * Merge sort code in sort_nodes() is copyright 2001 Simon Tatham.
 * 
 * Permission is hereby granted, free of charge, to any person
 * obtaining a copy of this software and associated documentation
 * files (the "Software"), to deal in the Software without
 * restriction, including without limitation the rights to use,
 * copy, modify, merge, publish, distribute, sublicense, and/or
 * sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following
 * conditions:
 * 
 * The above copyright notice and this permission notice shall be
 * included in all copies or substantial portions of the Software.
 * 
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
 * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 * NONINFRINGEMENT.  IN NO EVENT SHALL SIMON TATHAM BE LIABLE FOR
 * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF
 * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 */

/* Implementation of draft-ietf-imapext-thread-12 threading algorithm */

#include "common.h"
#include "hash.h"
#include "ostream.h"
#include "str.h"
#include "message-tokenize.h"
#include "imap-base-subject.h"
#include "imap-thread.h"

#include <stdlib.h>

/* how much memory to allocate initially. these are very rough
   approximations. */
#define APPROX_MSG_COUNT 128
#define APPROX_MSGID_SIZE 45

/* Try to buffer this much data before sending it to output stream. */
#define OUTPUT_BUF_SIZE 2048

#define NODE_IS_DUMMY(node) ((node)->id == 0)
#define NODE_HAS_PARENT(ctx, node) \
	((node)->parent != NULL && (node)->parent != &(ctx)->root_node)

struct root_info {
	char *base_subject;
	unsigned int reply:1;
	unsigned int sorted:1;
};

struct node {
	struct node *parent, *first_child, *next;

	unsigned int id;
	time_t sent_date;

	union {
		char *msgid;
		struct root_info *info;
	} u;
};

struct thread_context {
	struct mail_search_context *search_ctx;
	struct mailbox *box;
	struct ostream *output;

	pool_t pool;
	pool_t temp_pool;

	struct hash_table *msgid_hash;
	struct hash_table *subject_hash;

	struct node root_node;
	size_t root_count; /* not exact after prune_dummy_messages() */

	int id_is_uid;
};

static void mail_thread_input(struct thread_context *ctx, struct mail *mail);
static void mail_thread_finish(struct thread_context *ctx);

static void mail_thread_deinit(struct thread_context *ctx)
{
	if (ctx->msgid_hash != NULL)
		hash_destroy(ctx->msgid_hash);
	if (ctx->subject_hash != NULL)
		hash_destroy(ctx->subject_hash);

	pool_unref(ctx->temp_pool);
	pool_unref(ctx->pool);
}

int imap_thread(struct client *client, const char *charset,
		struct mail_search_arg *args, enum mail_thread_type type)
{
	static const char *wanted_headers[] = {
		"message-id", "in-reply-to", "references",
		NULL
	};
	struct thread_context *ctx;
	struct mail *mail;
	int ret;

	if (type != MAIL_THREAD_REFERENCES)
		i_fatal("Only REFERENCES threading supported");

	ctx = t_new(struct thread_context, 1);

	/* initialize searching */
	ctx->search_ctx = client->mailbox->
		search_init(client->mailbox, charset, args, NULL,
			    MAIL_FETCH_DATE, wanted_headers);
	if (ctx->search_ctx == NULL)
		return FALSE;

	ctx->box = client->mailbox;
	ctx->output = client->output;
	ctx->pool = pool_alloconly_create("thread_context",
					  sizeof(struct node) *
					  APPROX_MSG_COUNT);
	ctx->temp_pool = pool_alloconly_create("thread_context temp",
					       APPROX_MSG_COUNT *
					       APPROX_MSGID_SIZE);
	ctx->msgid_hash = hash_create(default_pool, ctx->temp_pool,
				      APPROX_MSG_COUNT*2, str_hash,
				      (hash_cmp_callback_t)strcmp);

	ctx->id_is_uid = client->cmd_uid;
	while ((mail = client->mailbox->search_next(ctx->search_ctx)) != NULL)
		mail_thread_input(ctx, mail);

	o_stream_send_str(client->output, "* THREAD");
	mail_thread_finish(ctx);
	o_stream_send_str(client->output, "\r\n");

	ret = client->mailbox->search_deinit(ctx->search_ctx);
        mail_thread_deinit(ctx);
	return ret;
}

static void add_root(struct thread_context *ctx, struct node *node)
{
	node->parent = &ctx->root_node;
	node->next = ctx->root_node.first_child;
	ctx->root_node.first_child = node;

	ctx->root_count++;
}

static struct node *create_node(struct thread_context *ctx, const char *msgid)
{
	struct node *node;

	node = p_new(ctx->pool, struct node, 1);
	node->u.msgid = p_strdup(ctx->temp_pool, msgid);

	hash_insert(ctx->msgid_hash, node->u.msgid, node);
	return node;
}

static struct node *create_id_node(struct thread_context *ctx,
				   unsigned int id, time_t sent_date)
{
	struct node *node;

	node = p_new(ctx->pool, struct node, 1);
	node->id = id;
	node->sent_date = sent_date;

	add_root(ctx, node);
	return node;
}

static struct node *update_message(struct thread_context *ctx,
				   const char *msgid, time_t sent_date,
				   unsigned int id)
{
	struct node *node;

	if (msgid == NULL)
		return create_id_node(ctx, id, sent_date);

	node = hash_lookup(ctx->msgid_hash, msgid);
	if (node == NULL) {
		/* first time we see this message */
		node = create_node(ctx, msgid);
		node->id = id;
		node->sent_date = sent_date;
		return node;
	}

	if (node->id == 0) {
		/* seen before in references */
		node->id = id;
		node->sent_date = sent_date;
	} else {
		/* duplicate */
		node = create_id_node(ctx, id, sent_date);
	}

	return node;
}

static int get_untokenized_msgid(const char **msgid_p, string_t *msgid)
{
	static const enum message_token stop_tokens[] = { '>', TOKEN_LAST };
	struct message_tokenizer *tok;
	int valid_end;

	tok = message_tokenize_init((const unsigned char *) *msgid_p,
				    (size_t)-1, NULL, NULL);
	message_tokenize_dot_token(tok, FALSE); /* just a minor speedup */

	message_tokenize_get_string(tok, msgid, NULL, stop_tokens);
	valid_end = message_tokenize_get(tok) == '>';

	*msgid_p += message_tokenize_get_parse_position(tok);
	message_tokenize_deinit(tok);

	if (valid_end) {
		if (strchr(str_c(msgid), '@') != NULL) {
			/* <xx at xx> - valid message ID found */
			return TRUE;
		}
	}

	return FALSE;
}

static void strip_lwsp(char *str)
{
	/* @UNSAFE */
	char *dest;

	/* find the first lwsp */
	while (*str != ' ' && *str != '\t' && *str != '\r' && *str != '\n') {
		if (*str == '\0')
			return;
		str++;
	}

	for (dest = str; *str != '\0'; str++) {
		if (*str != ' ' && *str != '\t' && *str != '\r' && *str != '\n')
			*dest++ = *str;
	}
	*dest = '\0';
}

static const char *get_msgid(const char **msgid_p)
{
	const char *msgid = *msgid_p;
	const char *p;
	string_t *str = NULL;
	int found_at;

	if (*msgid_p == NULL)
		return NULL;

	for (;;) {
		/* skip until '<' */
		while (*msgid != '<') {
			if (*msgid == '\0') {
				*msgid_p = msgid;
				return NULL;
			}
			msgid++;
		}
		msgid++;

		/* check it through quickly to see if it's already normalized */
		p = msgid; found_at = FALSE;
		for (;; p++) {
			if ((unsigned char)*p >= 'A') /* matches most */
				continue;

			if (*p == '@')
				found_at = TRUE;
			if (*p == '>' || *p == '"' || *p == '(')
				break;

			if (*p == '\0') {
				*msgid_p = p;
				return NULL;
			}
		}

		if (*p == '>') {
			*msgid_p = p+1;
			if (found_at) {
				char *s;

				s = p_strdup_until(data_stack_pool, msgid, p);
				strip_lwsp(s);
				return s;
			}
		} else {
			/* ok, do it the slow way */
			*msgid_p = msgid;

			if (str == NULL) {
				/* allocate only once, so we don't leak
				   with multiple invalid message IDs */
				str = t_str_new(256);
			}
			if (get_untokenized_msgid(msgid_p, str))
				return str_c(str);
		}

		/* invalid message id, see if there's another valid one */
		msgid = *msgid_p;
	}
}

static void unlink_child(struct thread_context *ctx,
			 struct node *child, int add_to_root)
{
	struct node **node;

        node = &child->parent->first_child;
	for (; *node != NULL; node = &(*node)->next) {
		if (*node == child) {
			*node = child->next;
			break;
		}
	}

	child->next = NULL;
	if (!add_to_root)
		child->parent = NULL;
	else
		add_root(ctx, child);
}

static int find_child(struct node *node, struct node *child)
{
	do {
		if (node == child)
			return TRUE;

		if (node->first_child != NULL) {
			if (find_child(node->first_child, child))
				return TRUE;
		}

		node = node->next;
	} while (node != NULL);

	return FALSE;
}

static void link_node(struct thread_context *ctx, const char *parent_msgid,
		      struct node *child, int replace)
{
	struct node *parent, **node;

	if (NODE_HAS_PARENT(ctx, child) && !replace) {
		/* already got a parent, don't want to replace it */
		return;
	}

	parent = hash_lookup(ctx->msgid_hash, parent_msgid);
	if (parent == NULL)
		parent = create_node(ctx, parent_msgid);

	if (child->parent == parent) {
		/* already have this parent, ignore */
		return;
	}

	if (find_child(child, parent)) {
		/* this would create a loop, not allowed */
		return;
	}

	if (child->parent != NULL)
		unlink_child(ctx, child, FALSE);

	/* link them */
	child->parent = parent;

	node = &parent->first_child;
	while (*node != NULL)
		node = &(*node)->next;
	*node = child;
}

static void link_message(struct thread_context *ctx,
			 const char *parent_msgid, const char *child_msgid,
			 int replace)
{
	struct node *child;

	child = hash_lookup(ctx->msgid_hash, child_msgid);
	if (child == NULL)
		child = create_node(ctx, child_msgid);

	link_node(ctx, parent_msgid, child, replace);
}

static int link_references(struct thread_context *ctx,
			   struct node *node, const char *references)
{
	const char *parent_id, *child_id;

	parent_id = get_msgid(&references);
	if (parent_id == NULL)
		return FALSE;

	while ((child_id = get_msgid(&references)) != NULL) {
		link_message(ctx, parent_id, child_id, FALSE);
		parent_id = child_id;
	}

	/* link the last message to us */
	link_node(ctx, parent_id, node, TRUE);
	return TRUE;
}

static void mail_thread_input(struct thread_context *ctx, struct mail *mail)
{
	const char *refid, *message_id, *in_reply_to, *references;
	struct node *node;
	time_t sent_date;

	t_push();

	sent_date = mail->get_date(mail, NULL);
	if (sent_date == (time_t)-1)
		sent_date = 0;

	message_id = mail->get_header(mail, "message-id");
	node = update_message(ctx, get_msgid(&message_id), sent_date,
			      ctx->id_is_uid ? mail->uid : mail->seq);

	/* link references */
	references = mail->get_header(mail, "references");
	if (!link_references(ctx, node, references)) {
		in_reply_to = mail->get_header(mail, "in-reply-to");
		refid = in_reply_to == NULL ? NULL : get_msgid(&in_reply_to);

		if (refid != NULL)
			link_node(ctx, refid, node, TRUE);
		else {
			/* no references, make sure it's not linked */
			if (node != NULL && NODE_HAS_PARENT(ctx, node))
				unlink_child(ctx, node, TRUE);
		}
	}

	t_pop();
}

static struct node *find_last_child(struct node *node)
{
	node = node->first_child;
	while (node->next != NULL)
		node = node->next;

	return node;
}

static struct node **promote_children(struct node **parent)
{
	struct node *new_parent, *old_parent, *child;

	old_parent = *parent;
	new_parent = old_parent->parent;

	child = old_parent->first_child;
	*parent = child;

	for (;;) {
		child->parent = new_parent;
		if (child->next == NULL)
			break;
		child = child->next;
	}

	child->next = old_parent->next;
	return &child->next;
}

static void prune_dummy_messages(struct thread_context *ctx,
				 struct node **node_p)
{
	struct node **a;

	a = node_p;
	while (*node_p != NULL) {
		if ((*node_p)->first_child != NULL)
			prune_dummy_messages(ctx, &(*node_p)->first_child);

		if (NODE_IS_DUMMY(*node_p)) {
			if ((*node_p)->first_child == NULL) {
				/* no children -> delete */
				*node_p = (*node_p)->next;
				continue;
			} else if (NODE_HAS_PARENT(ctx, *node_p) ||
				   (*node_p)->first_child->next == NULL) {
				/* promote children to our level,
				   deleting the dummy node */
				node_p = promote_children(node_p);
				continue;
			}
		}

                node_p = &(*node_p)->next;
	}
}

static int node_cmp(struct node *a, struct node *b)
{
	time_t date_a, date_b;
	unsigned int id_a, id_b;

	date_a = a->id != 0 ? a->sent_date : a->first_child->sent_date;
	date_b = b->id != 0 ? b->sent_date : b->first_child->sent_date;

	if (date_a != date_b && date_a != 0 && date_b != 0)
		return date_a < date_b ? -1 : 1;

	id_a = a->id != 0 ? a->id : a->first_child->id;
	id_b = b->id != 0 ? b->id : b->first_child->id;
	return id_a < id_b ? -1 : 1;
}

static struct node *sort_nodes(struct node *list)
{
	struct node *p, *q, *e, *tail;
	size_t insize, nmerges, psize, qsize, i;

	i_assert(list != NULL);

	if (list->next == NULL)
		return list; /* just one node */

	insize = 1;

	for (;;) {
		p = list;
		list = NULL;
		tail = NULL;

		nmerges = 0;  /* count number of merges we do in this pass */
		while (p != 0) {
			nmerges++;  /* there exists a merge to be done */

			/* step `insize' places along from p */
			q = p;
			psize = 0;
			for (i = 0; i < insize; i++) {
				psize++;
				q = q->next;
				if (q == NULL) break;
			}

			/* if q hasn't fallen off end, we have two lists to
			   merge */
			qsize = insize;

			/* now we have two lists; merge them */
			while (psize > 0 || (qsize > 0 && q != NULL)) {
				/* decide whether next element of merge comes
				   from p or q */
				if (psize == 0) {
					/* p is empty; e must come from q. */
					e = q; q = q->next; qsize--;
				} else if (qsize == 0 || !q) {
					/* q is empty; e must come from p. */
					e = p; p = p->next; psize--;
				} else if (node_cmp(p, q) <= 0) {
					/* First element of p is lower
					   (or same); e must come from p. */
					e = p; p = p->next; psize--;
				} else {
					/* First element of q is lower;
					   e must come from q. */
					e = q; q = q->next; qsize--;
				}

				/* add the next element to the merged list */
				if (tail)
					tail->next = e;
				else
					list = e;
				tail = e;
			}

			/* now p has stepped `insize' places along,
			   and q has too */
			p = q;
		}
		tail->next = NULL;

		/* If we have done only one merge, we're finished. */
		if (nmerges <= 1) {
                        /* allow for nmerges == 0, the empty list case */
			return list;
		}

		/* Otherwise repeat, merging lists twice the size */
		insize *= 2;
	}
}

static void add_base_subject(struct thread_context *ctx,
			     const char *subject, struct node *node)
{
	struct node *hash_node;
	char *hash_subject;
	void *key, *value;
	int is_reply_or_forward;

	if (subject == NULL)
		return;

	subject = imap_get_base_subject_cased(data_stack_pool, subject,
					      &is_reply_or_forward);
	if (*subject == '\0')
		return;

	if (!hash_lookup_full(ctx->subject_hash, subject, &key, &value)) {
		hash_subject = p_strdup(ctx->temp_pool, subject);
		hash_insert(ctx->subject_hash, hash_subject, node);
	} else {
		hash_subject = key;
		hash_node = value;

		if (!NODE_IS_DUMMY(hash_node) &&
		    (NODE_IS_DUMMY(node) ||
		     (hash_node->u.info->reply && !is_reply_or_forward)))
			hash_update(ctx->subject_hash, hash_subject, node);
	}

	node->u.info->base_subject = hash_subject;
	node->u.info->reply = is_reply_or_forward;
}

static void gather_base_subjects(struct thread_context *ctx)
{
	struct mail *mail;
	struct node *node;
	unsigned int id;

	ctx->subject_hash =
		hash_create(default_pool, ctx->temp_pool, ctx->root_count * 2,
			    str_hash, (hash_cmp_callback_t)strcmp);

	node = ctx->root_node.first_child;
	for (; node != NULL; node = node->next) {
		if (!NODE_IS_DUMMY(node))
			id = node->id;
		else {
			/* sort children, use the first one's id */
			node->first_child = sort_nodes(node->first_child);
			id = node->first_child->id;

			node->u.info->sorted = TRUE;
		}

		if (ctx->id_is_uid)
			mail = ctx->box->fetch_uid(ctx->box, id, 0);
		else
			mail = ctx->box->fetch_seq(ctx->box, id, 0);

		if (mail != NULL) {
			t_push();
			add_base_subject(ctx, mail->get_header(mail, "subject"),
					 node);
			t_pop();
		}
	}
}

static void reset_children_parent(struct node *parent)
{
	struct node *node;

	for (node = parent->first_child; node != NULL; node = node->next)
		node->parent = parent;
}

static void merge_subject_threads(struct thread_context *ctx)
{
	struct node **node_p, *node, *hash_node;
	char *base_subject;

	for (node_p = &ctx->root_node.first_child; *node_p != NULL; ) {
		node = *node_p;

		if (node->u.info == NULL) {
			/* deleted node */
			*node_p = node->next;
			continue;
		}

		/* (ii) If the thread subject is empty, skip this message. */
		base_subject = node->u.info->base_subject;
		if (base_subject == NULL) {
			node_p = &node->next;
			continue;
		}

		/* (iii) Lookup the message associated with this thread
		   subject in the subject table. */
		hash_node = hash_lookup(ctx->subject_hash, base_subject);
		i_assert(hash_node != NULL);

		/* (iv) If the message in the subject table is the current
		   message, skip this message. */
		if (hash_node == node) {
			node_p = &node->next;
			continue;
		}

		/* Otherwise, merge the current message with the one in the
		   subject table using the following rules: */

		if (NODE_IS_DUMMY(node) &&
		    NODE_IS_DUMMY(hash_node)) {
			/* If both messages are dummies, append the current
			   message's children to the children of the message in
			   the subject table (the children of both messages
			   become siblings), and then delete the current
			   message. */
			find_last_child(hash_node)->next = node->first_child;

			*node_p = node->next;
			hash_node->u.info->sorted = FALSE;
		} else if (NODE_IS_DUMMY(hash_node) ||
			   (node->u.info->reply && !hash_node->u.info->reply)) {
			/* If the message in the subject table is a dummy
			   and the current message is not, make the current
			   message a child of the message in the subject table
			   (a sibling of its children).

			   If the current message is a reply or forward and
			   the message in the subject table is not, make the
			   current message a child of the message in the
			   subject table (a sibling of its children). */
			*node_p = node->next;

			node->parent = hash_node;
			node->next = hash_node->first_child;
			hash_node->first_child = node;

			hash_node->u.info->sorted = FALSE;
		} else {
			/* Otherwise, create a new dummy message and make both
			   the current message and the message in the subject
			   table children of the dummy.  Then replace the
			   message in the subject table with the dummy
			   message. */

			/* create new nodes for the children - reusing
			   existing ones have problems since the other one
			   might have been handled already and we'd introduce
			   loops..

			   current node will be destroyed, hash_node will be
			   the dummy so we don't need to update hash */
			struct node *node1, *node2;

			node1 = p_new(ctx->pool, struct node, 1);
			node2 = p_new(ctx->pool, struct node, 1);

			memcpy(node1, node, sizeof(struct node));
			memcpy(node2, hash_node, sizeof(struct node));

			node1->parent = hash_node;
			node2->parent = hash_node;
			node1->next = node2;
			node2->next = NULL;

			reset_children_parent(node1);
			reset_children_parent(node2);

			hash_node->id = 0;
			hash_node->first_child = node1;
			hash_node->u.info->reply = FALSE;
			hash_node->u.info->sorted = FALSE;

			node->first_child = NULL;
			node->u.info = NULL;
			*node_p = node->next;
		}
	}
}

static void sort_root_nodes(struct thread_context *ctx)
{
	struct node *node;

	/* sort the children first, they're needed to sort dummy root nodes */
        node = ctx->root_node.first_child;
	for (; node != NULL; node = node->next) {
		if (node->u.info == NULL)
			continue;

		if (NODE_IS_DUMMY(node) && !node->u.info->sorted &&
		    node->first_child != NULL)
			node->first_child = sort_nodes(node->first_child);
	}

	ctx->root_node.first_child = sort_nodes(ctx->root_node.first_child);
}

static int send_nodes(struct thread_context *ctx,
		      string_t *str, struct node *node)
{
	if (node->next == NULL && NODE_HAS_PARENT(ctx, node)) {
		/* no siblings - special case to avoid extra paranthesis */
		if (node->first_child == NULL)
			str_printfa(str, "%u", node->id);
		else {
			str_printfa(str, "%u ", node->id);
			send_nodes(ctx, str, sort_nodes(node->first_child));
		}
		return TRUE;
	}

	while (node != NULL) {
		if (str_len(str) + MAX_INT_STRLEN*2 + 3 >= OUTPUT_BUF_SIZE) {
			/* string getting full, flush it */
			if (!o_stream_send(ctx->output,
					   str_data(str), str_len(str)))
				return FALSE;
			str_truncate(str, 0);
		}

		if (node->first_child == NULL)
			str_printfa(str, "(%u)", node->id);
		else {
			str_printfa(str, "(%u ", node->id);
			send_nodes(ctx, str, sort_nodes(node->first_child));
			str_append_c(str, ')');
		}

		node = node->next;
	}
	return TRUE;
}

static void send_roots(struct thread_context *ctx)
{
	struct node *node;
	string_t *str;

	str = t_str_new(OUTPUT_BUF_SIZE);
	str_append_c(str, ' ');

	/* sort root nodes again, they have been modified since the last time */
	sort_root_nodes(ctx);

        node = ctx->root_node.first_child;
	for (; node != NULL; node = node->next) {
		if (node->u.info == NULL)
			continue;

		if (str_len(str) + MAX_INT_STRLEN*2 + 3 >= OUTPUT_BUF_SIZE) {
			/* string getting full, flush it */
			if (!o_stream_send(ctx->output,
					   str_data(str), str_len(str)))
				return;
			str_truncate(str, 0);
		}

		str_append_c(str, '(');
		if (!NODE_IS_DUMMY(node))
			str_printfa(str, "%u", node->id);

		if (node->first_child != NULL) {
			if (!NODE_IS_DUMMY(node))
				str_append_c(str, ' ');

			if (!node->u.info->sorted) {
				node->first_child =
					sort_nodes(node->first_child);
			}

			if (!send_nodes(ctx, str, node->first_child))
				return;
		}

		str_append_c(str, ')');
	}

	(void)o_stream_send(ctx->output, str_data(str), str_len(str));
}

static void save_root_cb(void *key __attr_unused__, void *value, void *context)
{
	struct thread_context *ctx = context;
	struct node *node = value;

	if (node->parent == NULL)
		add_root(ctx, node);
}

static void mail_thread_finish(struct thread_context *ctx)
{
	struct node *node;

	/* (2) save root nodes and drop the msgids */
	hash_foreach(ctx->msgid_hash, save_root_cb, ctx);

	/* drop the memory allocated for message-IDs and msgid_hash,
	   reuse their memory for base subjects */
	hash_destroy(ctx->msgid_hash);
	ctx->msgid_hash = NULL;

	p_clear(ctx->temp_pool);

	if (ctx->root_node.first_child == NULL) {
		/* no messages */
		mail_thread_deinit(ctx);
		return;
	}

	/* (3) */
	prune_dummy_messages(ctx, &ctx->root_node.first_child);

	/* initialize the node->u.info for all root nodes */
        node = ctx->root_node.first_child;
	for (; node != NULL; node = node->next)
		node->u.info = p_new(ctx->pool, struct root_info, 1);

	/* (4) */
	sort_root_nodes(ctx);

	/* (5) Gather together messages under the root that have the same
	   base subject text. */
	gather_base_subjects(ctx);

	/* (5.C) Merge threads with the same thread subject. */
	merge_subject_threads(ctx);

	/* (6) Sort and send replies */
	t_push();
	send_roots(ctx);
	t_pop();
}

--- NEW FILE: imap-thread.h ---
#ifndef __IMAP_THREAD_H
#define __IMAP_THREAD_H

int imap_thread(struct client *client, const char *charset,
		struct mail_search_arg *args, enum mail_thread_type type);

#endif

Index: Makefile.am
===================================================================
RCS file: /home/cvs/dovecot/src/imap/Makefile.am,v
retrieving revision 1.10
retrieving revision 1.11
diff -u -d -r1.10 -r1.11
--- Makefile.am	8 Jan 2003 20:49:52 -0000	1.10
+++ Makefile.am	20 Jan 2003 14:52:51 -0000	1.11
@@ -56,13 +56,23 @@
 	client.c \
 	commands.c \
 	commands-util.c \
+	imap-fetch.c \
+	imap-fetch-body-section.c \
+	imap-search.c \
+	imap-sort.c \
+	imap-thread.c \
 	mail-storage-callbacks.c \
 	main.c \
 	rawlog.c
 
+
 noinst_HEADERS = \
 	client.h \
 	commands.h \
 	commands-util.h \
 	common.h \
+	imap-fetch.h \
+	imap-search.h \
+	imap-sort.h \
+	imap-thread.h \
 	rawlog.h

Index: cmd-append.c
===================================================================
RCS file: /home/cvs/dovecot/src/imap/cmd-append.c,v
retrieving revision 1.18
retrieving revision 1.19
diff -u -d -r1.18 -r1.19
--- cmd-append.c	5 Jan 2003 13:09:51 -0000	1.18
+++ cmd-append.c	20 Jan 2003 14:52:51 -0000	1.19
@@ -76,9 +76,8 @@
 {
 	struct imap_arg_list *flags_list;
 	struct mailbox *box;
-	enum mail_flags flags;
+	struct mail_full_flags flags;
 	time_t internal_date;
-	const char *custom_flags[MAIL_CUSTOM_FLAGS_COUNT];
 	const char *mailbox, *internal_date_str;
 	uoff_t msg_size;
 	int failed, timezone_offset;
@@ -97,13 +96,10 @@
 
 	if (flags_list != NULL) {
 		if (!client_parse_mail_flags(client, flags_list->args,
-					     flags_list->size,
-					     &flags, custom_flags))
+					     &flags))
 			return TRUE;
 	} else {
-		if (!client_parse_mail_flags(client, NULL, 0,
-					     &flags, custom_flags))
-			return TRUE;
+		memset(&flags, 0, sizeof(flags));
 	}
 
 	if (internal_date_str == NULL) {
@@ -131,8 +127,7 @@
 	o_stream_flush(client->output);
 
 	/* save the mail */
-	failed = !box->save(box, flags, custom_flags,
-			    internal_date, timezone_offset,
+	failed = !box->save(box, &flags, internal_date, timezone_offset,
 			    client->input, msg_size);
 	box->close(box);
 

Index: cmd-fetch.c
===================================================================
RCS file: /home/cvs/dovecot/src/imap/cmd-fetch.c,v
retrieving revision 1.11
retrieving revision 1.12
diff -u -d -r1.11 -r1.12
--- cmd-fetch.c	5 Jan 2003 13:09:51 -0000	1.11
+++ cmd-fetch.c	20 Jan 2003 14:52:51 -0000	1.12
@@ -2,6 +2,7 @@
 
 #include "common.h"
 #include "commands.h"
+#include "imap-fetch.h"
 
 /* Parse next digits in string into integer. Returns FALSE if the integer
    becomes too big and wraps. */
@@ -23,23 +24,96 @@
 	return TRUE;
 }
 
+static int check_header_section(const char *section)
+{
+	/* HEADER, HEADER.FIELDS (list), HEADER.FIELDS.NOT (list) */
+	if (*section == '\0')
+		return TRUE;
+
+	if (strncmp(section, ".FIELDS", 7) != 0)
+		return FALSE;
+
+	section += 7;
+	if (strncmp(section, ".NOT", 4) == 0)
+		section += 4;
+
+	while (*section == ' ') section++;
+	if (*section++ != '(')
+		return FALSE;
+
+	while (*section != '\0' && *section != ')') {
+		if (*section == '(')
+			return FALSE;
+		section++;
+	}
+
+	if (*section++ != ')')
+		return FALSE;
+
+	if (*section != '\0')
+		return FALSE;
+	return TRUE;
+}
+
+static int check_section(struct client *client, const char *section,
+			 enum mail_fetch_field *fetch_data)
+{
+	if (*section == '\0') {
+		*fetch_data |= MAIL_FETCH_STREAM_HEADER |
+			MAIL_FETCH_STREAM_BODY;
+		return TRUE;
+	}
+
+	if (strcmp(section, "TEXT") == 0) {
+		*fetch_data |= MAIL_FETCH_STREAM_BODY;
+		return TRUE;
+	}
+
+	if (strncmp(section, "HEADER", 6) == 0) {
+		*fetch_data |= MAIL_FETCH_STREAM_HEADER;
+		if (check_header_section(section+6))
+			return TRUE;
+	} else if (*section >= '0' && *section <= '9') {
+		*fetch_data |= MAIL_FETCH_STREAM_BODY |
+			MAIL_FETCH_MESSAGE_PARTS;
+
+		while ((*section >= '0' && *section <= '9') ||
+		       *section == '.') section++;
+
+		if (*section == '\0')
+			return TRUE;
+		if (strcmp(section, "MIME") == 0 ||
+		    strcmp(section, "TEXT") == 0)
+			return TRUE;
+
+		if (strncmp(section, "HEADER", 6) == 0 &&
+		    check_header_section(section+6))
+			return TRUE;
+	}
+
+	client_send_tagline(client, t_strconcat(
+		"BAD Invalid BODY[] section: ", section, NULL));
+	return FALSE;
+}
+
 /* BODY[] and BODY.PEEK[] items. item points to next character after '[' */
-static int parse_body_section(struct client *client, const char *item,
-			      struct mail_fetch_data *data, int peek)
+static int parse_body_section(struct client *client, const char *item, int peek,
+			      enum mail_fetch_field *fetch_data,
+			      struct imap_fetch_body_data ***bodies)
 {
-	struct mail_fetch_body_data *body;
+	/* @UNSAFE */
+	struct imap_fetch_body_data *body;
 	uoff_t num;
-	const char *section;
 	char *p;
 
-	body = t_new(struct mail_fetch_body_data, 1);
+	body = t_new(struct imap_fetch_body_data, 1);
 	body->peek = peek;
 
 	p = t_strdup_noconst(item);
 
 	/* read section */
 	body->section = p;
-	for (section = p; *p != ']'; p++) {
+	for (; *p != ']'; p++) {
 		if (*p == '\0') {
 			client_send_tagline(client, t_strconcat(
 				"BAD Missing ']' with ", item, NULL));
@@ -48,6 +122,9 @@
 	}
 	*p++ = '\0';
 
+	if (!check_section(client, body->section, fetch_data))
+		return FALSE;
+
 	/* <start.end> */
 	body->skip = 0;
 	body->max_size = (uoff_t)-1;
@@ -89,14 +166,15 @@
 		}
 	}
 
-	body->next = data->body_sections;
-	data->body_sections = body;
-
+	**bodies = body;
+	*bodies = &body->next;
 	return TRUE;
 }
 
 static int parse_arg(struct client *client, struct imap_arg *arg,
-		     struct mail_fetch_data *data)
+		     enum mail_fetch_field *fetch_data,
+		     enum imap_fetch_field *imap_data,
+		     struct imap_fetch_body_data ***bodies)
 {
 	char *item;
 
@@ -111,10 +189,10 @@
 	switch (*item) {
 	case 'A':
 		if (strcmp(item, "ALL") == 0) {
-			data->flags = TRUE;
-			data->internaldate = TRUE;
-			data->rfc822_size = TRUE;
-			data->envelope = TRUE;
+			*fetch_data |= MAIL_FETCH_FLAGS |
+				MAIL_FETCH_RECEIVED_DATE |
+				MAIL_FETCH_SIZE |
+				MAIL_FETCH_IMAP_ENVELOPE;
 		} else
 			item = NULL;
 		break;
@@ -128,46 +206,48 @@
 
 		if (*item == '\0') {
 			/* BODY */
-			data->body = TRUE;
+			*fetch_data |= MAIL_FETCH_IMAP_BODY;
 		} else if (*item == '[') {
 			/* BODY[...] */
-			if (!parse_body_section(client, item+1, data, FALSE))
+			if (!parse_body_section(client, item+1, FALSE,
+						fetch_data, bodies))
 				return FALSE;
 		} else if (strncmp(item, ".PEEK[", 6) == 0) {
 			/* BODY.PEEK[..] */
-			if (!parse_body_section(client, item+6, data, TRUE))
+			if (!parse_body_section(client, item+6, TRUE,
+						fetch_data, bodies))
 				return FALSE;
 		} else if (strcmp(item, "STRUCTURE") == 0) {
 			/* BODYSTRUCTURE */
-			data->bodystructure = TRUE;
+			*fetch_data |= MAIL_FETCH_IMAP_BODYSTRUCTURE;
 		} else
 			item = NULL;
 		break;
 	case 'E':
 		if (strcmp(item, "ENVELOPE") == 0)
-			data->envelope = TRUE;
+			*fetch_data |= MAIL_FETCH_IMAP_ENVELOPE;
 		else
 			item = NULL;
 		break;
 	case 'F':
 		if (strcmp(item, "FLAGS") == 0)
-			data->flags = TRUE;
+			*fetch_data |= MAIL_FETCH_FLAGS;
 		else if (strcmp(item, "FAST") == 0) {
-			data->flags = TRUE;
-			data->internaldate = TRUE;
-			data->rfc822_size = TRUE;
+			*fetch_data |= MAIL_FETCH_FLAGS |
+				MAIL_FETCH_RECEIVED_DATE |
+				MAIL_FETCH_SIZE;
 		} else if (strcmp(item, "FULL") == 0) {
-			data->flags = TRUE;
-			data->internaldate = TRUE;
-			data->rfc822_size = TRUE;
-			data->envelope = TRUE;
-			data->body = TRUE;
+			*fetch_data |= MAIL_FETCH_FLAGS |
+				MAIL_FETCH_RECEIVED_DATE |
+				MAIL_FETCH_SIZE |
+				MAIL_FETCH_IMAP_ENVELOPE |
+				MAIL_FETCH_IMAP_BODY;
 		} else
 			item = NULL;
 		break;
 	case 'I':
 		if (strcmp(item, "INTERNALDATE") == 0)
-			data->internaldate = TRUE;
+			*fetch_data |= MAIL_FETCH_RECEIVED_DATE;
 		else
 			item = NULL;
 		break;
@@ -181,7 +261,9 @@
 
 		if (*item == '\0') {
 			/* RFC822 */
-			data->rfc822 = TRUE;
+			*fetch_data |= MAIL_FETCH_STREAM_HEADER |
+				MAIL_FETCH_STREAM_BODY;
+			*imap_data |= IMAP_FETCH_RFC822;
 			break;
 		}
 
@@ -192,18 +274,20 @@
 		}
 		item++;
 
-		if (strcmp(item, "HEADER") == 0)
-			data->rfc822_header = TRUE;
-		else if (strcmp(item, "SIZE") == 0)
-			data->rfc822_size = TRUE;
-		else if (strcmp(item, "TEXT") == 0)
-			data->rfc822_text = TRUE;
+		if (strcmp(item, "HEADER") == 0) {
+			*fetch_data |= MAIL_FETCH_STREAM_HEADER;
+			*imap_data |= IMAP_FETCH_RFC822_HEADER;
+		} else if (strcmp(item, "TEXT") == 0) {
+			*fetch_data |= MAIL_FETCH_STREAM_BODY;
+			*imap_data |= IMAP_FETCH_RFC822_TEXT;
+		} else if (strcmp(item, "SIZE") == 0)
+			*fetch_data |= MAIL_FETCH_SIZE;
 		else
 			item = NULL;
 		break;
 	case 'U':
 		if (strcmp(item, "UID") == 0)
-			data->uid = TRUE;
+			*imap_data |= IMAP_FETCH_UID;
 		else
 			item = NULL;
 		break;
@@ -225,9 +309,11 @@
 int cmd_fetch(struct client *client)
 {
 	struct imap_arg *args, *listargs;
-	struct mail_fetch_data data;
+	enum mail_fetch_field fetch_data;
+	enum imap_fetch_field imap_data;
+	struct imap_fetch_body_data *bodies, **bodies_p;
 	const char *messageset;
-	int all_found;
+	int ret;
 
 	if (!client_read_args(client, 2, 0, &args))
 		return FALSE;
@@ -243,33 +329,29 @@
 	}
 
 	/* parse items argument */
-	memset(&data, 0, sizeof(struct mail_fetch_data));
+	fetch_data = 0; imap_data = 0; bodies = NULL; bodies_p = &bodies;
 	if (args[1].type == IMAP_ARG_ATOM) {
-		if (!parse_arg(client, &args[1], &data))
+		if (!parse_arg(client, &args[1], &fetch_data,
+			       &imap_data, &bodies_p))
 			return TRUE;
 	} else {
 		listargs = IMAP_ARG_LIST(&args[1])->args;
 		while (listargs->type != IMAP_ARG_EOL) {
-			if (!parse_arg(client, listargs, &data))
+			if (!parse_arg(client, listargs, &fetch_data,
+				       &imap_data, &bodies_p))
 				return TRUE;
 
 			listargs++;
 		}
 	}
 
-	data.messageset = messageset;
-	data.uidset = client->cmd_uid;
-	if (data.uidset)
-                data.uid = TRUE;
-
-	/* fetch it */
-	if (client->mailbox->fetch(client->mailbox, &data,
-				   client->output, &all_found)) {
+	ret = imap_fetch(client, fetch_data, imap_data,
+			 bodies, messageset, client->cmd_uid);
+	if (ret >= 0) {
 		/* NOTE: syncing isn't allowed here */
                 client_sync_without_expunges(client);
-		client_send_tagline(client, all_found ? "OK Fetch completed." :
-				    "NO Some of the requested messages "
-				    "no longer exist.");
+		client_send_tagline(client, ret > 0 ? "OK Fetch completed." :
+			"NO Some of the requested messages no longer exist.");
 	} else {
 		client_send_storage_error(client);
 	}

Index: cmd-search.c
===================================================================
RCS file: /home/cvs/dovecot/src/imap/cmd-search.c,v
retrieving revision 1.13
retrieving revision 1.14
diff -u -d -r1.13 -r1.14
--- cmd-search.c	8 Jan 2003 20:49:52 -0000	1.13
+++ cmd-search.c	20 Jan 2003 14:52:51 -0000	1.14
@@ -1,8 +1,50 @@
 /* Copyright (C) 2002 Timo Sirainen */
 
 #include "common.h"
+#include "ostream.h"
+#include "str.h"
 #include "commands.h"
-#include "mail-search.h"
+#include "imap-search.h"
+
+#define STRBUF_SIZE 1024
+
+static int imap_search(struct client *client, const char *charset,
+		       struct mail_search_arg *sargs)
+{
+        struct mail_search_context *ctx;
+	const struct mail *mail;
+	string_t *str;
+	int ret, uid, first = TRUE;
+
+	str = t_str_new(STRBUF_SIZE);
+	uid = client->cmd_uid;
+
+	ctx = client->mailbox->search_init(client->mailbox, charset, sargs,
+					   NULL, 0, NULL);
+	if (ctx == NULL)
+		return FALSE;
+
+	str_append(str, "* SEARCH");
+	while ((mail = client->mailbox->search_next(ctx)) != NULL) {
+		if (str_len(str) >= STRBUF_SIZE-MAX_INT_STRLEN) {
+			/* flush */
+			o_stream_send(client->output,
+				      str_data(str), str_len(str));
+			str_truncate(str, 0);
+			first = FALSE;
+		}
+
+		str_printfa(str, " %u", uid ? mail->uid : mail->seq);
+	}
+
+	ret = client->mailbox->search_deinit(ctx);
+
+	if (!first || ret) {
+		str_append(str, "\r\n");
+		o_stream_send(client->output, str_data(str), str_len(str));
+	}
+	return ret;
+}
 
 int cmd_search(struct client *client)
 {
@@ -44,23 +86,19 @@
 
 	pool = pool_alloconly_create("mail_search_args", 2048);
 
-	sargs = mail_search_args_build(pool, args, &error);
+	sargs = imap_search_args_build(pool, args, &error);
 	if (sargs == NULL) {
 		/* error in search arguments */
 		client_send_tagline(client, t_strconcat("NO ", error, NULL));
+	} else if (imap_search(client, charset, sargs)) {
+		/* NOTE: syncing is allowed when returning UIDs */
+		if (client->cmd_uid)
+			client_sync_full(client);
+		else
+			client_sync_without_expunges(client);
+		client_send_tagline(client, "OK Search completed.");
 	} else {
-		if (client->mailbox->search(client->mailbox, charset,
-					    sargs, NULL, MAIL_THREAD_NONE,
-					    client->output, client->cmd_uid)) {
-			/* NOTE: syncing is allowed when returning UIDs */
-			if (client->cmd_uid)
-				client_sync_full(client);
-			else
-				client_sync_without_expunges(client);
-			client_send_tagline(client, "OK Search completed.");
-		} else {
-			client_send_storage_error(client);
-		}
+		client_send_storage_error(client);
 	}
 
 	pool_unref(pool);

Index: cmd-sort.c
===================================================================
RCS file: /home/cvs/dovecot/src/imap/cmd-sort.c,v
retrieving revision 1.8
retrieving revision 1.9
diff -u -d -r1.8 -r1.9
--- cmd-sort.c	8 Jan 2003 20:49:52 -0000	1.8
+++ cmd-sort.c	20 Jan 2003 14:52:51 -0000	1.9
@@ -3,8 +3,8 @@
 #include "common.h"
 #include "buffer.h"
 #include "commands.h"
-#include "mail-search.h"
-#include "mail-sort.h"
+#include "imap-search.h"
+#include "imap-sort.h"
 
 struct sort_name {
 	enum mail_sort_type type;
@@ -110,23 +110,19 @@
 
 	pool = pool_alloconly_create("mail_search_args", 2048);
 
-	sargs = mail_search_args_build(pool, args, &error);
+	sargs = imap_search_args_build(pool, args, &error);
 	if (sargs == NULL) {
 		/* error in search arguments */
 		client_send_tagline(client, t_strconcat("NO ", error, NULL));
+	} else if (imap_sort(client, charset, sargs, sorting)) {
+		/* NOTE: syncing is allowed when returning UIDs */
+		if (client->cmd_uid)
+			client_sync_full(client);
+		else
+			client_sync_without_expunges(client);
+		client_send_tagline(client, "OK Sort completed.");
 	} else {
-		if (client->mailbox->search(client->mailbox, charset,
-					    sargs, sorting, MAIL_THREAD_NONE,
-					    client->output, client->cmd_uid)) {
-			/* NOTE: syncing is allowed when returning UIDs */
-			if (client->cmd_uid)
-				client_sync_full(client);
-			else
-				client_sync_without_expunges(client);
-			client_send_tagline(client, "OK Search completed.");
-		} else {
-			client_send_storage_error(client);
-		}
+		client_send_storage_error(client);
 	}
 
 	pool_unref(pool);

Index: cmd-store.c
===================================================================
RCS file: /home/cvs/dovecot/src/imap/cmd-store.c,v
retrieving revision 1.8
retrieving revision 1.9
diff -u -d -r1.8 -r1.9
--- cmd-store.c	5 Jan 2003 13:09:51 -0000	1.8
+++ cmd-store.c	20 Jan 2003 14:52:51 -0000	1.9
@@ -35,9 +35,8 @@
 int cmd_store(struct client *client)
 {
 	struct imap_arg *args;
-	enum mail_flags flags;
+	struct mail_full_flags flags;
 	enum modify_type modify_type;
-	const char *custflags[MAIL_CUSTOM_FLAGS_COUNT];
 	const char *messageset, *item;
 	int silent, all_found;
 
@@ -62,19 +61,17 @@
 	if (args[2].type == IMAP_ARG_LIST) {
 		if (!client_parse_mail_flags(client,
 					     IMAP_ARG_LIST(&args[2])->args,
-					     IMAP_ARG_LIST(&args[2])->size,
-					     &flags, custflags))
+					     &flags))
 			return TRUE;
 	} else {
-		if (!client_parse_mail_flags(client, &args[2], 1,
-					     &flags, custflags))
+		if (!client_parse_mail_flags(client, args+2, &flags))
 			return TRUE;
 	}
 
 	/* and update the flags */
 	client->sync_flags_send_uid = client->cmd_uid;
 	if (client->mailbox->update_flags(client->mailbox, messageset,
-					  client->cmd_uid, flags, custflags,
+					  client->cmd_uid, &flags,
 					  modify_type, !silent, &all_found)) {
 		/* NOTE: syncing isn't allowed here */
 		client_sync_without_expunges(client);

Index: cmd-thread.c
===================================================================
RCS file: /home/cvs/dovecot/src/imap/cmd-thread.c,v
retrieving revision 1.2
retrieving revision 1.3
diff -u -d -r1.2 -r1.3
--- cmd-thread.c	11 Jan 2003 19:01:56 -0000	1.2
+++ cmd-thread.c	20 Jan 2003 14:52:51 -0000	1.3
@@ -3,8 +3,8 @@
 #include "common.h"
 #include "buffer.h"
 #include "commands.h"
-#include "mail-search.h"
-#include "mail-sort.h"
+#include "imap-search.h"
+#include "imap-thread.h"
 
 int cmd_thread(struct client *client)
 {
@@ -58,23 +58,19 @@
 
 	pool = pool_alloconly_create("mail_search_args", 2048);
 
-	sargs = mail_search_args_build(pool, args, &error);
+	sargs = imap_search_args_build(pool, args, &error);
 	if (sargs == NULL) {
 		/* error in search arguments */
 		client_send_tagline(client, t_strconcat("NO ", error, NULL));
+	} else if (imap_thread(client, charset, sargs, threading)) {
+		/* NOTE: syncing is allowed when returning UIDs */
+		if (client->cmd_uid)
+			client_sync_full(client);
+		else
+			client_sync_without_expunges(client);
+		client_send_tagline(client, "OK Search completed.");
 	} else {
-		if (client->mailbox->search(client->mailbox, charset,
-					    sargs, NULL, threading,
-					    client->output, client->cmd_uid)) {
-			/* NOTE: syncing is allowed when returning UIDs */
-			if (client->cmd_uid)
-				client_sync_full(client);
-			else
-				client_sync_without_expunges(client);
-			client_send_tagline(client, "OK Search completed.");
-		} else {
-			client_send_storage_error(client);
-		}
+		client_send_storage_error(client);
 	}
 
 	pool_unref(pool);

Index: commands-util.c
===================================================================
RCS file: /home/cvs/dovecot/src/imap/commands-util.c,v
retrieving revision 1.15
retrieving revision 1.16
diff -u -d -r1.15 -r1.16
--- commands-util.c	5 Jan 2003 13:09:51 -0000	1.15
+++ commands-util.c	20 Jan 2003 14:52:51 -0000	1.16
@@ -122,37 +122,39 @@
 }
 
 int client_parse_mail_flags(struct client *client, struct imap_arg *args,
-			    size_t args_count, enum mail_flags *flags,
-			    const char *custflags[MAIL_CUSTOM_FLAGS_COUNT])
+			    struct mail_full_flags *flags)
 {
+	/* @UNSAFE */
 	char *atom;
-	size_t pos;
-	int i, custpos;
+	size_t max_flags, flag_pos, i;
 
-	memset(custflags, 0, sizeof(const char *) * MAIL_CUSTOM_FLAGS_COUNT);
+	max_flags = MAIL_CUSTOM_FLAGS_COUNT;
 
-	*flags = 0; custpos = 0;
-	for (pos = 0; pos < args_count; pos++) {
-		if (args[pos].type != IMAP_ARG_ATOM) {
+	memset(flags, 0, sizeof(*flags));
+	flags->custom_flags = t_new(const char *, flags->custom_flags_count);
+
+	flag_pos = 0;
+	while (args->type != IMAP_ARG_EOL) {
+		if (args->type != IMAP_ARG_ATOM) {
 			client_send_command_error(client,
 				"Flags list contains non-atoms.");
 			return FALSE;
 		}
 
-		atom = IMAP_ARG_STR(&args[pos]);
+		atom = IMAP_ARG_STR(args);
 		if (*atom == '\\') {
 			/* system flag */
 			str_ucase(atom);
 			if (strcmp(atom, "\\ANSWERED") == 0)
-				*flags |= MAIL_ANSWERED;
+				flags->flags |= MAIL_ANSWERED;
 			else if (strcmp(atom, "\\FLAGGED") == 0)
-				*flags |= MAIL_FLAGGED;
+				flags->flags |= MAIL_FLAGGED;
 			else if (strcmp(atom, "\\DELETED") == 0)
-				*flags |= MAIL_DELETED;
+				flags->flags |= MAIL_DELETED;
 			else if (strcmp(atom, "\\SEEN") == 0)
-				*flags |= MAIL_SEEN;
+				flags->flags |= MAIL_SEEN;
 			else if (strcmp(atom, "\\DRAFT") == 0)
-				*flags |= MAIL_DRAFT;
+				flags->flags |= MAIL_DRAFT;
 			else {
 				client_send_tagline(client, t_strconcat(
 					"BAD Invalid system flag ",
@@ -161,26 +163,30 @@
 			}
 		} else {
 			/* custom flag - first make sure it's not a duplicate */
-			for (i = 0; i < custpos; i++) {
-				if (strcasecmp(custflags[i], atom) == 0)
+			for (i = 0; i < flag_pos; i++) {
+				if (strcasecmp(flags->custom_flags[i],
+					       atom) == 0)
 					break;
 			}
 
-			if (i == MAIL_CUSTOM_FLAGS_COUNT) {
+			if (i == max_flags) {
 				client_send_tagline(client,
 					"Maximum number of different custom "
 					"flags exceeded");
 				return FALSE;
 			}
 
-			if (i == custpos) {
-				*flags |= 1 << (custpos +
-						MAIL_CUSTOM_FLAG_1_BIT);
-				custflags[custpos++] = atom;
+			if (i == flags->custom_flags_count) {
+				flags->flags |= 1 << (flag_pos +
+						      MAIL_CUSTOM_FLAG_1_BIT);
+				flags->custom_flags[flag_pos++] = atom;
 			}
 		}
+
+		args++;
 	}
 
+	flags->custom_flags_count = flag_pos;
 	return TRUE;
 }
 

Index: commands-util.h
===================================================================
RCS file: /home/cvs/dovecot/src/imap/commands-util.h,v
retrieving revision 1.6
retrieving revision 1.7
diff -u -d -r1.6 -r1.7
--- commands-util.h	5 Jan 2003 13:09:51 -0000	1.6
+++ commands-util.h	20 Jan 2003 14:52:51 -0000	1.7
@@ -24,12 +24,10 @@
 /* Send last mail storage error message to client. */
 void client_send_storage_error(struct client *client);
 
-/* Parse flags, stores custom flag names into custflags[]. The names point to
-   strings in ImapArgList. Returns TRUE if successful, if not sends an error
-   message to client. */
+/* Parse flags. Returns TRUE if successful, if not sends an error message to
+   client. */
 int client_parse_mail_flags(struct client *client, struct imap_arg *args,
-			    size_t args_count, enum mail_flags *flags,
-			    const char *custflags[MAIL_CUSTOM_FLAGS_COUNT]);
+			    struct mail_full_flags *flags);
 
 /* Send FLAGS + PERMANENTFLAGS to client. */
 void client_send_mailbox_flags(struct client *client, struct mailbox *box,




More information about the dovecot-cvs mailing list