dovecot-1.2: fts: Added support for handling multiple namespaces.
dovecot at dovecot.org
dovecot at dovecot.org
Sun Nov 30 01:27:26 EET 2008
details: http://hg.dovecot.org/dovecot-1.2/rev/252b29ac5f43
changeset: 8499:252b29ac5f43
user: Timo Sirainen <tss at iki.fi>
date: Sun Nov 30 01:27:19 2008 +0200
description:
fts: Added support for handling multiple namespaces.
diffstat:
13 files changed, 282 insertions(+), 119 deletions(-)
doc/solr-schema.xml | 1
src/lib-storage/mail-storage-private.h | 4
src/lib-storage/mail-storage.c | 13 -
src/lib-storage/mail-storage.h | 10 -
src/plugins/fts-solr/fts-backend-solr.c | 297 ++++++++++++++++++++++---------
src/plugins/fts-solr/fts-solr-plugin.c | 3
src/plugins/fts-solr/fts-solr-plugin.h | 2
src/plugins/fts-solr/solr-connection.c | 24 +-
src/plugins/fts-solr/solr-connection.h | 7
src/plugins/fts/fts-storage.c | 8
src/plugins/virtual/virtual-config.c | 24 +-
src/plugins/virtual/virtual-storage.c | 4
src/plugins/virtual/virtual-storage.h | 4
diffs (truncated from 858 to 300 lines):
diff -r fb5fedcf4deb -r 252b29ac5f43 doc/solr-schema.xml
--- a/doc/solr-schema.xml Sun Nov 30 01:26:36 2008 +0200
+++ b/doc/solr-schema.xml Sun Nov 30 01:27:19 2008 +0200
@@ -43,6 +43,7 @@ want to modify the tokenizers and filter
<field name="uidv" type="long" indexed="true" stored="true" required="true" />
<field name="box" type="string" indexed="true" stored="true" required="true" />
<field name="user" type="string" indexed="true" stored="true" required="true" />
+ <field name="ns" type="string" indexed="true" stored="true" required="false" />
<field name="last_uid" type="boolean" indexed="true" stored="false" />
<field name="hdr" type="text" indexed="true" stored="false" />
<field name="body" type="text" indexed="true" stored="false" />
diff -r fb5fedcf4deb -r 252b29ac5f43 src/lib-storage/mail-storage-private.h
--- a/src/lib-storage/mail-storage-private.h Sun Nov 30 01:26:36 2008 +0200
+++ b/src/lib-storage/mail-storage-private.h Sun Nov 30 01:27:19 2008 +0200
@@ -146,8 +146,8 @@ struct mailbox_vfuncs {
ARRAY_TYPE(mailboxes) *mailboxes,
bool only_with_msgs);
void (*get_virtual_box_patterns)(struct mailbox *box,
- ARRAY_TYPE(const_string) *includes,
- ARRAY_TYPE(const_string) *excludes);
+ ARRAY_TYPE(mailbox_virtual_patterns) *includes,
+ ARRAY_TYPE(mailbox_virtual_patterns) *excludes);
struct mail *
(*mail_alloc)(struct mailbox_transaction_context *t,
diff -r fb5fedcf4deb -r 252b29ac5f43 src/lib-storage/mail-storage.c
--- a/src/lib-storage/mail-storage.c Sun Nov 30 01:26:36 2008 +0200
+++ b/src/lib-storage/mail-storage.c Sun Nov 30 01:27:19 2008 +0200
@@ -651,13 +651,16 @@ void mailbox_get_virtual_backend_boxes(s
}
void mailbox_get_virtual_box_patterns(struct mailbox *box,
- ARRAY_TYPE(const_string) *includes,
- ARRAY_TYPE(const_string) *excludes)
+ ARRAY_TYPE(mailbox_virtual_patterns) *includes,
+ ARRAY_TYPE(mailbox_virtual_patterns) *excludes)
{
if (box->v.get_virtual_box_patterns == NULL) {
- const char *name = box->name;
-
- array_append(includes, &name, 1);
+ struct mailbox_virtual_pattern pat;
+
+ memset(&pat, 0, sizeof(pat));
+ pat.ns = box->storage->ns;
+ pat.pattern = box->name;
+ array_append(includes, &pat, 1);
} else {
box->v.get_virtual_box_patterns(box, includes, excludes);
}
diff -r fb5fedcf4deb -r 252b29ac5f43 src/lib-storage/mail-storage.h
--- a/src/lib-storage/mail-storage.h Sun Nov 30 01:26:36 2008 +0200
+++ b/src/lib-storage/mail-storage.h Sun Nov 30 01:27:19 2008 +0200
@@ -234,6 +234,12 @@ struct mail_storage_callbacks {
};
+struct mailbox_virtual_pattern {
+ struct mail_namespace *ns;
+ const char *pattern;
+};
+ARRAY_DEFINE_TYPE(mailbox_virtual_patterns, struct mailbox_virtual_pattern);
+
ARRAY_DEFINE_TYPE(mailboxes, struct mailbox *);
typedef void mailbox_notify_callback_t(struct mailbox *box, void *context);
@@ -432,8 +438,8 @@ void mailbox_get_virtual_backend_boxes(s
/* If mailbox is a virtual mailbox, return all mailbox list patterns that
are used to figure out which mailboxes belong to the virtual mailbox. */
void mailbox_get_virtual_box_patterns(struct mailbox *box,
- ARRAY_TYPE(const_string) *includes,
- ARRAY_TYPE(const_string) *excludes);
+ ARRAY_TYPE(mailbox_virtual_patterns) *includes,
+ ARRAY_TYPE(mailbox_virtual_patterns) *excludes);
/* Initialize header lookup for given headers. */
struct mailbox_header_lookup_ctx *
diff -r fb5fedcf4deb -r 252b29ac5f43 src/plugins/fts-solr/fts-backend-solr.c
--- a/src/plugins/fts-solr/fts-backend-solr.c Sun Nov 30 01:26:36 2008 +0200
+++ b/src/plugins/fts-solr/fts-backend-solr.c Sun Nov 30 01:27:19 2008 +0200
@@ -13,10 +13,13 @@
#define SOLR_CMDBUF_SIZE (1024*64)
#define SOLR_MAX_ROWS 100000
+#define FTS_SOLR_MAX_BOX_INC_PATTERNS 5
+#define FTS_SOLR_MAX_BOX_EXC_PATTERNS 5
struct solr_fts_backend {
struct fts_backend backend;
- char *id_username;
+ char *id_username, *id_namespace;
+ struct mail_namespace *default_ns;
};
struct solr_fts_backend_build_context {
@@ -28,17 +31,22 @@ struct solr_fts_backend_build_context {
bool headers;
};
+struct solr_virtual_uid_map_context {
+ struct fts_backend *backend;
+ struct mailbox *box;
+ string_t *vname;
+};
+
struct fts_backend_solr_get_last_uids_context {
+ struct fts_backend *backend;
pool_t pool;
ARRAY_TYPE(fts_backend_uid_map) *last_uids;
+
+ struct mailbox *box;
+ string_t *vname;
};
static struct solr_connection *solr_conn = NULL;
-
-static void solr_quote_str(string_t *dest, const char *str)
-{
- solr_connection_quote_str(solr_conn, dest, str);
-}
static void
xml_encode_data(string_t *dest, const unsigned char *data, unsigned int len)
@@ -97,18 +105,50 @@ static const char *solr_escape_id_str(co
return str_c(tmp);
}
+static void solr_quote(string_t *dest, const char *str)
+{
+ str_append_c(dest, '"');
+ str_append(dest, str_escape(str));
+ str_append_c(dest, '"');
+}
+
+static void solr_quote_http(string_t *dest, const char *str)
+{
+ str_append(dest, "%22");
+ solr_connection_http_escape(solr_conn, dest, str);
+ str_append(dest, "%22");
+}
+
static struct fts_backend *
fts_backend_solr_init(struct mailbox *box)
{
const struct fts_solr_settings *set = &fts_solr_settings;
struct solr_fts_backend *backend;
- const char *username = box->storage->ns->user->username;
+ struct mail_namespace *ns = box->storage->ns;
+ const char *str;
if (solr_conn == NULL)
solr_conn = solr_connection_init(set->url, set->debug);
backend = i_new(struct solr_fts_backend, 1);
- backend->id_username = i_strdup(solr_escape_id_str(username));
+ str = fts_solr_settings.default_ns_prefix;
+ if (str != NULL) {
+ backend->default_ns =
+ mail_namespace_find_prefix(ns->user->namespaces, str);
+ if (backend->default_ns == NULL) {
+ i_fatal("fts_solr: default_ns setting points to "
+ "nonexisting namespace");
+ }
+ } else {
+ backend->default_ns =
+ mail_namespace_find_inbox(ns->user->namespaces);
+ }
+ str = solr_escape_id_str(ns->user->username);
+ backend->id_username = i_strdup(str);
+ if (box->storage->ns != backend->default_ns) {
+ str = solr_escape_id_str(ns->prefix);
+ backend->id_namespace = i_strdup(str);
+ }
backend->backend = fts_backend_solr;
if (set->substring_search)
@@ -120,20 +160,40 @@ static void fts_backend_solr_deinit(stru
{
struct solr_fts_backend *backend = (struct solr_fts_backend *)_backend;
+ i_free(backend->id_namespace);
i_free(backend->id_username);
i_free(backend);
}
-static const char *fts_backend_solr_username(struct fts_backend *_backend)
+static void
+solr_add_ns_query(string_t *str, struct fts_backend *_backend,
+ struct mail_namespace *ns)
{
struct solr_fts_backend *backend = (struct solr_fts_backend *)_backend;
- return backend->id_username;
+ if (ns == backend->default_ns || *ns->prefix == '\0')
+ str_append(str, " -ns:[* TO *]");
+ else {
+ str_append(str, " ns:");
+ solr_quote(str, ns->prefix);
+ }
+}
+
+static void
+solr_add_ns_query_http(string_t *str, struct fts_backend *backend,
+ struct mail_namespace *ns)
+{
+ string_t *tmp;
+
+ tmp = t_str_new(64);
+ solr_add_ns_query(tmp, backend, ns);
+ solr_connection_http_escape(solr_conn, str, str_c(tmp));
}
static int fts_backend_solr_get_last_uid_fallback(struct fts_backend *backend,
uint32_t *last_uid_r)
{
+ struct mailbox *box = backend->box;
struct mailbox_status status;
ARRAY_TYPE(seq_range) uids;
const struct seq_range *uidvals;
@@ -141,13 +201,14 @@ static int fts_backend_solr_get_last_uid
string_t *str;
str = t_str_new(256);
- str_append(str, "fl=uid&rows=1&sort=uid%20desc&q=");
-
- mailbox_get_status(backend->box, STATUS_UIDVALIDITY, &status);
- str_printfa(str, "uidv:%u%%20box:", status.uidvalidity);
- solr_quote_str(str, backend->box->name);
- str_append(str, "%20user:");
- solr_quote_str(str, backend->box->storage->ns->user->username);
+ str_append(str, "fl=uid&rows=1&sort=uid+desc&q=");
+
+ mailbox_get_status(box, STATUS_UIDVALIDITY, &status);
+ str_printfa(str, "uidv:%u+box:", status.uidvalidity);
+ solr_quote_http(str, box->name);
+ solr_add_ns_query_http(str, backend, box->storage->ns);
+ str_append(str, "+user:");
+ solr_quote_http(str, box->storage->ns->user->username);
t_array_init(&uids, 1);
if (solr_connection_select(solr_conn, str_c(str),
@@ -170,6 +231,7 @@ static int fts_backend_solr_get_last_uid
static int fts_backend_solr_get_last_uid(struct fts_backend *backend,
uint32_t *last_uid_r)
{
+ struct mailbox *box = backend->box;
struct mailbox_status status;
ARRAY_TYPE(seq_range) uids;
const struct seq_range *uidvals;
@@ -177,13 +239,14 @@ static int fts_backend_solr_get_last_uid
string_t *str;
str = t_str_new(256);
- str_append(str, "fl=uid&rows=1&q=last_uid:TRUE%20");
-
- mailbox_get_status(backend->box, STATUS_UIDVALIDITY, &status);
- str_printfa(str, "uidv:%u%%20box:", status.uidvalidity);
- solr_quote_str(str, backend->box->name);
- str_append(str, "%20user:");
- solr_quote_str(str, backend->box->storage->ns->user->username);
+ str_append(str, "fl=uid&rows=1&q=last_uid:TRUE+");
+
+ mailbox_get_status(box, STATUS_UIDVALIDITY, &status);
+ str_printfa(str, "uidv:%u+box:", status.uidvalidity);
+ solr_quote_http(str, box->name);
+ solr_add_ns_query_http(str, backend, box->storage->ns);
+ str_append(str, "+user:");
+ solr_quote_http(str, box->storage->ns->user->username);
t_array_init(&uids, 1);
if (solr_connection_select(solr_conn, str_c(str),
@@ -205,41 +268,67 @@ static int fts_backend_solr_get_last_uid
return 0;
}
+static const char *
+solr_get_vmailbox(struct fts_backend *_backend,
+ struct mailbox *box, const char *ns_prefix,
+ const char *mailbox, string_t *dest)
+{
+ struct solr_fts_backend *backend = (struct solr_fts_backend *)_backend;
+ struct mail_namespace *namespaces = box->storage->ns->user->namespaces;
+ struct mail_namespace *ns;
+
+ if (ns_prefix == NULL)
+ ns = backend->default_ns;
+ else {
+ ns = mail_namespace_find_prefix(namespaces, ns_prefix);
+ if (ns == NULL)
+ return FALSE;
+ }
+ return mail_namespace_get_vname(ns, dest, mailbox);
+}
+
static bool
-solr_virtual_get_last_uids(const char *mailbox, uint32_t uidvalidity,
- uint32_t *uid, void *context)
+solr_virtual_get_last_uids(const char *ns_prefix, const char *mailbox,
+ uint32_t uidvalidity, uint32_t *uid, void *context)
{
More information about the dovecot-cvs
mailing list