--- a
+++ b/src/rcldb/searchdatatox.cpp
@@ -0,0 +1,983 @@
+/* Copyright (C) 2006 J.F.Dockes
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the
+ * Free Software Foundation, Inc.,
+ * 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ */
+
+// Handle translation from rcl's SearchData structures to Xapian Queries
+
+#include "autoconfig.h"
+
+#include <stdio.h>
+
+#include <string>
+#include <vector>
+#include <algorithm>
+#include <sstream>
+using namespace std;
+
+#include "xapian.h"
+
+#include "cstr.h"
+#include "rcldb.h"
+#include "rcldb_p.h"
+#include "searchdata.h"
+#include "debuglog.h"
+#include "smallut.h"
+#include "textsplit.h"
+#include "unacpp.h"
+#include "utf8iter.h"
+#include "stoplist.h"
+#include "rclconfig.h"
+#include "termproc.h"
+#include "synfamily.h"
+#include "stemdb.h"
+#include "expansiondbs.h"
+#include "base64.h"
+#include "daterange.h"
+
+namespace Rcl {
+
+typedef vector<SearchDataClause *>::iterator qlist_it_t;
+
+static const int original_term_wqf_booster = 10;
+
+// Expand categories and mime type wild card exps Categories are
+// expanded against the configuration, mimetypes against the index
+// (for wildcards).
+bool SearchData::expandFileTypes(Db &db, vector<string>& tps)
+{
+ const RclConfig *cfg = db.getConf();
+ if (!cfg) {
+ LOGFATAL(("Db::expandFileTypes: null configuration!!\n"));
+ return false;
+ }
+ vector<string> exptps;
+
+ for (vector<string>::iterator it = tps.begin(); it != tps.end(); it++) {
+ if (cfg->isMimeCategory(*it)) {
+ vector<string>tps;
+ cfg->getMimeCatTypes(*it, tps);
+ exptps.insert(exptps.end(), tps.begin(), tps.end());
+ } else {
+ TermMatchResult res;
+ string mt = stringtolower((const string&)*it);
+ // We set casesens|diacsens to get an equivalent of ixTermMatch()
+ db.termMatch(Db::ET_WILD|Db::ET_CASESENS|Db::ET_DIACSENS, string(),
+ mt, res, -1, "mtype");
+ if (res.entries.empty()) {
+ exptps.push_back(it->c_str());
+ } else {
+ for (vector<TermMatchEntry>::const_iterator rit =
+ res.entries.begin(); rit != res.entries.end(); rit++) {
+ exptps.push_back(strip_prefix(rit->term));
+ }
+ }
+ }
+ }
+ sort(exptps.begin(), exptps.end());
+ exptps.erase(unique(exptps.begin(), exptps.end()), exptps.end());
+
+ tps = exptps;
+ return true;
+}
+
+static const char *maxXapClauseMsg =
+ "Maximum Xapian query size exceeded. Increase maxXapianClauses "
+ "in the configuration. ";
+static const char *maxXapClauseCaseDiacMsg =
+ "Or try to use case (C) or diacritics (D) sensitivity qualifiers, or less "
+ "wildcards ?"
+ ;
+
+bool SearchData::clausesToQuery(Rcl::Db &db, SClType tp,
+ vector<SearchDataClause*>& query,
+ string& reason, void *d)
+{
+ Xapian::Query xq;
+ for (qlist_it_t it = query.begin(); it != query.end(); it++) {
+ Xapian::Query nq;
+ if (!(*it)->toNativeQuery(db, &nq)) {
+ LOGERR(("SearchData::clausesToQuery: toNativeQuery failed: %s\n",
+ (*it)->getReason().c_str()));
+ reason += (*it)->getReason() + " ";
+ return false;
+ }
+ if (nq.empty()) {
+ LOGDEB(("SearchData::clausesToQuery: skipping empty clause\n"));
+ continue;
+ }
+ // If this structure is an AND list, must use AND_NOT for excl clauses.
+ // Else this is an OR list, and there can't be excl clauses (checked by
+ // addClause())
+ Xapian::Query::op op;
+ if (tp == SCLT_AND) {
+ if ((*it)->getexclude()) {
+ op = Xapian::Query::OP_AND_NOT;
+ } else {
+ op = Xapian::Query::OP_AND;
+ }
+ } else {
+ op = Xapian::Query::OP_OR;
+ }
+ if (xq.empty()) {
+ if (op == Xapian::Query::OP_AND_NOT)
+ xq = Xapian::Query(op, Xapian::Query::MatchAll, nq);
+ else
+ xq = nq;
+ } else {
+ xq = Xapian::Query(op, xq, nq);
+ }
+ if (int(xq.get_length()) >= getMaxCl()) {
+ LOGERR(("%s\n", maxXapClauseMsg));
+ m_reason += maxXapClauseMsg;
+ if (!o_index_stripchars)
+ m_reason += maxXapClauseCaseDiacMsg;
+ return false;
+ }
+ }
+
+ LOGDEB0(("SearchData::clausesToQuery: got %d clauses\n", xq.get_length()));
+
+ if (xq.empty())
+ xq = Xapian::Query::MatchAll;
+
+ *((Xapian::Query *)d) = xq;
+ return true;
+}
+
+bool SearchData::toNativeQuery(Rcl::Db &db, void *d)
+{
+ LOGDEB(("SearchData::toNativeQuery: stemlang [%s]\n", m_stemlang.c_str()));
+ m_reason.erase();
+
+ db.getConf()->getConfParam("maxTermExpand", &m_maxexp);
+ db.getConf()->getConfParam("maxXapianClauses", &m_maxcl);
+
+ // Walk the clause list translating each in turn and building the
+ // Xapian query tree
+ Xapian::Query xq;
+ if (!clausesToQuery(db, m_tp, m_query, m_reason, &xq)) {
+ LOGERR(("SearchData::toNativeQuery: clausesToQuery failed. reason: %s\n",
+ m_reason.c_str()));
+ return false;
+ }
+
+ if (m_haveDates) {
+ // If one of the extremities is unset, compute db extremas
+ if (m_dates.y1 == 0 || m_dates.y2 == 0) {
+ int minyear = 1970, maxyear = 2100;
+ if (!db.maxYearSpan(&minyear, &maxyear)) {
+ LOGERR(("Can't retrieve index min/max dates\n"));
+ //whatever, go on.
+ }
+
+ if (m_dates.y1 == 0) {
+ m_dates.y1 = minyear;
+ m_dates.m1 = 1;
+ m_dates.d1 = 1;
+ }
+ if (m_dates.y2 == 0) {
+ m_dates.y2 = maxyear;
+ m_dates.m2 = 12;
+ m_dates.d2 = 31;
+ }
+ }
+ LOGDEB(("Db::toNativeQuery: date interval: %d-%d-%d/%d-%d-%d\n",
+ m_dates.y1, m_dates.m1, m_dates.d1,
+ m_dates.y2, m_dates.m2, m_dates.d2));
+ Xapian::Query dq = date_range_filter(m_dates.y1, m_dates.m1, m_dates.d1,
+ m_dates.y2, m_dates.m2, m_dates.d2);
+ if (dq.empty()) {
+ LOGINFO(("Db::toNativeQuery: date filter is empty\n"));
+ }
+ // If no probabilistic query is provided then promote the daterange
+ // filter to be THE query instead of filtering an empty query.
+ if (xq.empty()) {
+ LOGINFO(("Db::toNativeQuery: proba query is empty\n"));
+ xq = dq;
+ } else {
+ xq = Xapian::Query(Xapian::Query::OP_FILTER, xq, dq);
+ }
+ }
+
+
+ if (m_minSize != size_t(-1) || m_maxSize != size_t(-1)) {
+ Xapian::Query sq;
+ char min[50], max[50];
+ sprintf(min, "%lld", (long long)m_minSize);
+ sprintf(max, "%lld", (long long)m_maxSize);
+ if (m_minSize == size_t(-1)) {
+ string value(max);
+ leftzeropad(value, 12);
+ sq = Xapian::Query(Xapian::Query::OP_VALUE_LE, VALUE_SIZE, value);
+ } else if (m_maxSize == size_t(-1)) {
+ string value(min);
+ leftzeropad(value, 12);
+ sq = Xapian::Query(Xapian::Query::OP_VALUE_GE, VALUE_SIZE, value);
+ } else {
+ string minvalue(min);
+ leftzeropad(minvalue, 12);
+ string maxvalue(max);
+ leftzeropad(maxvalue, 12);
+ sq = Xapian::Query(Xapian::Query::OP_VALUE_RANGE, VALUE_SIZE,
+ minvalue, maxvalue);
+ }
+
+ // If no probabilistic query is provided then promote the
+ // filter to be THE query instead of filtering an empty query.
+ if (xq.empty()) {
+ LOGINFO(("Db::toNativeQuery: proba query is empty\n"));
+ xq = sq;
+ } else {
+ xq = Xapian::Query(Xapian::Query::OP_FILTER, xq, sq);
+ }
+ }
+
+ // Add the autophrase if any
+ if (m_autophrase.isNotNull()) {
+ Xapian::Query apq;
+ if (m_autophrase->toNativeQuery(db, &apq)) {
+ xq = xq.empty() ? apq :
+ Xapian::Query(Xapian::Query::OP_AND_MAYBE, xq, apq);
+ }
+ }
+
+ // Add the file type filtering clause if any
+ if (!m_filetypes.empty()) {
+ expandFileTypes(db, m_filetypes);
+
+ Xapian::Query tq;
+ for (vector<string>::iterator it = m_filetypes.begin();
+ it != m_filetypes.end(); it++) {
+ string term = wrap_prefix(mimetype_prefix) + *it;
+ LOGDEB0(("Adding file type term: [%s]\n", term.c_str()));
+ tq = tq.empty() ? Xapian::Query(term) :
+ Xapian::Query(Xapian::Query::OP_OR, tq, Xapian::Query(term));
+ }
+ xq = xq.empty() ? tq : Xapian::Query(Xapian::Query::OP_FILTER, xq, tq);
+ }
+
+ // Add the neg file type filtering clause if any
+ if (!m_nfiletypes.empty()) {
+ expandFileTypes(db, m_nfiletypes);
+
+ Xapian::Query tq;
+ for (vector<string>::iterator it = m_nfiletypes.begin();
+ it != m_nfiletypes.end(); it++) {
+ string term = wrap_prefix(mimetype_prefix) + *it;
+ LOGDEB0(("Adding negative file type term: [%s]\n", term.c_str()));
+ tq = tq.empty() ? Xapian::Query(term) :
+ Xapian::Query(Xapian::Query::OP_OR, tq, Xapian::Query(term));
+ }
+ xq = xq.empty() ? tq : Xapian::Query(Xapian::Query::OP_AND_NOT, xq, tq);
+ }
+
+ *((Xapian::Query *)d) = xq;
+ return true;
+}
+
+// Splitter callback for breaking a user string into simple terms and
+// phrases. This is for parts of the user entry which would appear as
+// a single word because there is no white space inside, but are
+// actually multiple terms to rcldb (ie term1,term2)
+class TextSplitQ : public TextSplitP {
+ public:
+ TextSplitQ(Flags flags, const StopList &_stops, TermProc *prc)
+ : TextSplitP(prc, flags),
+ curnostemexp(false), stops(_stops), alltermcount(0), lastpos(0)
+ {}
+
+ bool takeword(const std::string &term, int pos, int bs, int be)
+ {
+ // Check if the first letter is a majuscule in which
+ // case we do not want to do stem expansion. Need to do this
+ // before unac of course...
+ curnostemexp = unaciscapital(term);
+
+ return TextSplitP::takeword(term, pos, bs, be);
+ }
+
+ bool curnostemexp;
+ vector<string> terms;
+ vector<bool> nostemexps;
+ const StopList &stops;
+ // Count of terms including stopwords: this is for adjusting
+ // phrase/near slack
+ int alltermcount;
+ int lastpos;
+};
+
+class TermProcQ : public TermProc {
+public:
+ TermProcQ() : TermProc(0), m_ts(0) {}
+ void setTSQ(TextSplitQ *ts) {m_ts = ts;}
+
+ bool takeword(const std::string &term, int pos, int bs, int be)
+ {
+ m_ts->alltermcount++;
+ if (m_ts->lastpos < pos)
+ m_ts->lastpos = pos;
+ bool noexpand = be ? m_ts->curnostemexp : true;
+ LOGDEB1(("TermProcQ::takeword: pushing [%s] pos %d noexp %d\n",
+ term.c_str(), pos, noexpand));
+ if (m_terms[pos].size() < term.size()) {
+ m_terms[pos] = term;
+ m_nste[pos] = noexpand;
+ }
+ return true;
+ }
+ bool flush()
+ {
+ for (map<int, string>::const_iterator it = m_terms.begin();
+ it != m_terms.end(); it++) {
+ m_ts->terms.push_back(it->second);
+ m_ts->nostemexps.push_back(m_nste[it->first]);
+ }
+ return true;
+ }
+private:
+ TextSplitQ *m_ts;
+ map<int, string> m_terms;
+ map<int, bool> m_nste;
+};
+
+
+#if 1
+static void listVector(const string& what, const vector<string>&l)
+{
+ string a;
+ for (vector<string>::const_iterator it = l.begin(); it != l.end(); it++) {
+ a = a + *it + " ";
+ }
+ LOGDEB0(("%s: %s\n", what.c_str(), a.c_str()));
+}
+#endif
+
+/** Expand term into term list, using appropriate mode: stem, wildcards,
+ * diacritics...
+ *
+ * @param mods stem expansion, case and diacritics sensitivity control.
+ * @param term input single word
+ * @param oexp output expansion list
+ * @param sterm output original input term if there were no wildcards
+ * @param prefix field prefix in index. We could recompute it, but the caller
+ * has it already. Used in the simple case where there is nothing to expand,
+ * and we just return the prefixed term (else Db::termMatch deals with it).
+ */
+bool SearchDataClauseSimple::expandTerm(Rcl::Db &db,
+ string& ermsg, int mods,
+ const string& term,
+ vector<string>& oexp, string &sterm,
+ const string& prefix)
+{
+ LOGDEB0(("expandTerm: mods 0x%x fld [%s] trm [%s] lang [%s]\n",
+ mods, m_field.c_str(), term.c_str(), getStemLang().c_str()));
+ sterm.clear();
+ oexp.clear();
+ if (term.empty())
+ return true;
+
+ bool maxexpissoft = false;
+ int maxexpand = getSoftMaxExp();
+ if (maxexpand != -1) {
+ maxexpissoft = true;
+ } else {
+ maxexpand = getMaxExp();
+ }
+
+ bool haswild = term.find_first_of(cstr_minwilds) != string::npos;
+
+ // If there are no wildcards, add term to the list of user-entered terms
+ if (!haswild) {
+ m_hldata.uterms.insert(term);
+ sterm = term;
+ }
+ // No stem expansion if there are wildcards or if prevented by caller
+ bool nostemexp = (mods & SDCM_NOSTEMMING) != 0;
+ if (haswild || getStemLang().empty()) {
+ LOGDEB2(("expandTerm: found wildcards or stemlang empty: no exp\n"));
+ nostemexp = true;
+ }
+
+ // noexpansion can be modified further down by possible case/diac expansion
+ bool noexpansion = nostemexp && !haswild;
+
+ int termmatchsens = 0;
+
+ bool diac_sensitive = (mods & SDCM_DIACSENS) != 0;
+ bool case_sensitive = (mods & SDCM_CASESENS) != 0;
+
+ if (o_index_stripchars) {
+ diac_sensitive = case_sensitive = false;
+ } else {
+ // If we are working with a raw index, apply the rules for case and
+ // diacritics sensitivity.
+
+ // If any character has a diacritic, we become
+ // diacritic-sensitive. Note that the way that the test is
+ // performed (conversion+comparison) will automatically ignore
+ // accented characters which are actually a separate letter
+ if (getAutoDiac() && unachasaccents(term)) {
+ LOGDEB0(("expandTerm: term has accents -> diac-sensitive\n"));
+ diac_sensitive = true;
+ }
+
+ // If any character apart the first is uppercase, we become
+ // case-sensitive. The first character is reserved for
+ // turning off stemming. You need to use a query language
+ // modifier to search for Floor in a case-sensitive way.
+ Utf8Iter it(term);
+ it++;
+ if (getAutoCase() && unachasuppercase(term.substr(it.getBpos()))) {
+ LOGDEB0(("expandTerm: term has uppercase -> case-sensitive\n"));
+ case_sensitive = true;
+ }
+
+ // If we are sensitive to case or diacritics turn stemming off
+ if (diac_sensitive || case_sensitive) {
+ LOGDEB0(("expandTerm: diac or case sens set -> stemexpand off\n"));
+ nostemexp = true;
+ }
+
+ if (!case_sensitive || !diac_sensitive)
+ noexpansion = false;
+ }
+
+ if (case_sensitive)
+ termmatchsens |= Db::ET_CASESENS;
+ if (diac_sensitive)
+ termmatchsens |= Db::ET_DIACSENS;
+
+ if (noexpansion) {
+ oexp.push_back(prefix + term);
+ m_hldata.terms[term] = term;
+ LOGDEB(("ExpandTerm: noexpansion: final: %s\n", stringsToString(oexp).c_str()));
+ return true;
+ }
+
+ Db::MatchType mtyp = haswild ? Db::ET_WILD :
+ nostemexp ? Db::ET_NONE : Db::ET_STEM;
+ TermMatchResult res;
+ if (!db.termMatch(mtyp | termmatchsens, getStemLang(), term, res, maxexpand,
+ m_field)) {
+ // Let it go through
+ }
+
+ // Term match entries to vector of terms
+ if (int(res.entries.size()) >= maxexpand && !maxexpissoft) {
+ ermsg = "Maximum term expansion size exceeded."
+ " Maybe use case/diacritics sensitivity or increase maxTermExpand.";
+ return false;
+ }
+ for (vector<TermMatchEntry>::const_iterator it = res.entries.begin();
+ it != res.entries.end(); it++) {
+ oexp.push_back(it->term);
+ }
+ // If the term does not exist at all in the db, the return from
+ // termMatch() is going to be empty, which is not what we want (we
+ // would then compute an empty Xapian query)
+ if (oexp.empty())
+ oexp.push_back(prefix + term);
+
+ // Remember the uterm-to-expansion links
+ for (vector<string>::const_iterator it = oexp.begin();
+ it != oexp.end(); it++) {
+ m_hldata.terms[strip_prefix(*it)] = term;
+ }
+ LOGDEB(("ExpandTerm: final: %s\n", stringsToString(oexp).c_str()));
+ return true;
+}
+
+// Do distribution of string vectors: a,b c,d -> a,c a,d b,c b,d
+void multiply_groups(vector<vector<string> >::const_iterator vvit,
+ vector<vector<string> >::const_iterator vvend,
+ vector<string>& comb,
+ vector<vector<string> >&allcombs)
+{
+ // Remember my string vector and compute next, for recursive calls.
+ vector<vector<string> >::const_iterator myvit = vvit++;
+
+ // Walk the string vector I'm called upon and, for each string,
+ // add it to current result, an call myself recursively on the
+ // next string vector. The last call (last element of the vector of
+ // vectors), adds the elementary result to the output
+
+ // Walk my string vector
+ for (vector<string>::const_iterator strit = (*myvit).begin();
+ strit != (*myvit).end(); strit++) {
+
+ // Add my current value to the string vector we're building
+ comb.push_back(*strit);
+
+ if (vvit == vvend) {
+ // Last call: store current result
+ allcombs.push_back(comb);
+ } else {
+ // Call recursively on next string vector
+ multiply_groups(vvit, vvend, comb, allcombs);
+ }
+ // Pop the value I just added (make room for the next element in my
+ // vector)
+ comb.pop_back();
+ }
+}
+
+void SearchDataClauseSimple::processSimpleSpan(Rcl::Db &db, string& ermsg,
+ const string& span,
+ int mods, void * pq)
+{
+ vector<Xapian::Query>& pqueries(*(vector<Xapian::Query>*)pq);
+ LOGDEB0(("StringToXapianQ::processSimpleSpan: [%s] mods 0x%x\n",
+ span.c_str(), (unsigned int)mods));
+ vector<string> exp;
+ string sterm; // dumb version of user term
+
+ string prefix;
+ const FieldTraits *ftp;
+ if (!m_field.empty() && db.fieldToTraits(m_field, &ftp, true)) {
+ prefix = wrap_prefix(ftp->pfx);
+ }
+
+ if (!expandTerm(db, ermsg, mods, span, exp, sterm, prefix))
+ return;
+
+ // Set up the highlight data. No prefix should go in there
+ for (vector<string>::const_iterator it = exp.begin();
+ it != exp.end(); it++) {
+ m_hldata.groups.push_back(vector<string>(1, it->substr(prefix.size())));
+ m_hldata.slacks.push_back(0);
+ m_hldata.grpsugidx.push_back(m_hldata.ugroups.size() - 1);
+ }
+
+ // Push either term or OR of stem-expanded set
+ Xapian::Query xq(Xapian::Query::OP_OR, exp.begin(), exp.end());
+ m_curcl += exp.size();
+
+ // If sterm (simplified original user term) is not null, give it a
+ // relevance boost. We do this even if no expansion occurred (else
+ // the non-expanded terms in a term list would end-up with even
+ // less wqf). This does not happen if there are wildcards anywhere
+ // in the search.
+ // We normally boost the original term in the stem expansion list. Don't
+ // do it if there are wildcards anywhere, this would skew the results.
+ bool doBoostUserTerm =
+ (m_parentSearch && !m_parentSearch->haveWildCards()) ||
+ (m_parentSearch == 0 && !m_haveWildCards);
+ if (doBoostUserTerm && !sterm.empty()) {
+ xq = Xapian::Query(Xapian::Query::OP_OR, xq,
+ Xapian::Query(prefix+sterm,
+ original_term_wqf_booster));
+ }
+ pqueries.push_back(xq);
+}
+
+// User entry element had several terms: transform into a PHRASE or
+// NEAR xapian query, the elements of which can themselves be OR
+// queries if the terms get expanded by stemming or wildcards (we
+// don't do stemming for PHRASE though)
+void SearchDataClauseSimple::processPhraseOrNear(Rcl::Db &db, string& ermsg,
+ TextSplitQ *splitData,
+ int mods, void *pq,
+ bool useNear, int slack)
+{
+ vector<Xapian::Query> &pqueries(*(vector<Xapian::Query>*)pq);
+ Xapian::Query::op op = useNear ? Xapian::Query::OP_NEAR :
+ Xapian::Query::OP_PHRASE;
+ vector<Xapian::Query> orqueries;
+#ifdef XAPIAN_NEAR_EXPAND_SINGLE_BUF
+ bool hadmultiple = false;
+#endif
+ vector<vector<string> >groups;
+
+ string prefix;
+ const FieldTraits *ftp;
+ if (!m_field.empty() && db.fieldToTraits(m_field, &ftp, true)) {
+ prefix = wrap_prefix(ftp->pfx);
+ }
+
+ if (mods & Rcl::SearchDataClause::SDCM_ANCHORSTART) {
+ orqueries.push_back(Xapian::Query(prefix + start_of_field_term));
+ slack++;
+ }
+
+ // Go through the list and perform stem/wildcard expansion for each element
+ vector<bool>::iterator nxit = splitData->nostemexps.begin();
+ for (vector<string>::iterator it = splitData->terms.begin();
+ it != splitData->terms.end(); it++, nxit++) {
+ LOGDEB0(("ProcessPhrase: processing [%s]\n", it->c_str()));
+ // Adjust when we do stem expansion. Not if disabled by
+ // caller, not inside phrases, and some versions of xapian
+ // will accept only one OR clause inside NEAR.
+ bool nostemexp = *nxit || (op == Xapian::Query::OP_PHRASE)
+#ifdef XAPIAN_NEAR_EXPAND_SINGLE_BUF
+ || hadmultiple
+#endif // single OR inside NEAR
+ ;
+ int lmods = mods;
+ if (nostemexp)
+ lmods |= SearchDataClause::SDCM_NOSTEMMING;
+ string sterm;
+ vector<string> exp;
+ if (!expandTerm(db, ermsg, lmods, *it, exp, sterm, prefix))
+ return;
+ LOGDEB0(("ProcessPhraseOrNear: exp size %d\n", exp.size()));
+ listVector("", exp);
+ // groups is used for highlighting, we don't want prefixes in there.
+ vector<string> noprefs;
+ for (vector<string>::const_iterator it = exp.begin();
+ it != exp.end(); it++) {
+ noprefs.push_back(it->substr(prefix.size()));
+ }
+ groups.push_back(noprefs);
+ orqueries.push_back(Xapian::Query(Xapian::Query::OP_OR,
+ exp.begin(), exp.end()));
+ m_curcl += exp.size();
+ if (m_curcl >= getMaxCl())
+ return;
+#ifdef XAPIAN_NEAR_EXPAND_SINGLE_BUF
+ if (exp.size() > 1)
+ hadmultiple = true;
+#endif
+ }
+
+ if (mods & Rcl::SearchDataClause::SDCM_ANCHOREND) {
+ orqueries.push_back(Xapian::Query(prefix + end_of_field_term));
+ slack++;
+ }
+
+ // Generate an appropriate PHRASE/NEAR query with adjusted slack
+ // For phrases, give a relevance boost like we do for original terms
+ LOGDEB2(("PHRASE/NEAR: alltermcount %d lastpos %d\n",
+ splitData->alltermcount, splitData->lastpos));
+ Xapian::Query xq(op, orqueries.begin(), orqueries.end(),
+ splitData->lastpos + 1 + slack);
+ if (op == Xapian::Query::OP_PHRASE)
+ xq = Xapian::Query(Xapian::Query::OP_SCALE_WEIGHT, xq,
+ original_term_wqf_booster);
+ pqueries.push_back(xq);
+
+ // Add all combinations of NEAR/PHRASE groups to the highlighting data.
+ vector<vector<string> > allcombs;
+ vector<string> comb;
+ multiply_groups(groups.begin(), groups.end(), comb, allcombs);
+
+ // Insert the search groups and slacks in the highlight data, with
+ // a reference to the user entry that generated them:
+ m_hldata.groups.insert(m_hldata.groups.end(),
+ allcombs.begin(), allcombs.end());
+ m_hldata.slacks.insert(m_hldata.slacks.end(), allcombs.size(), slack);
+ m_hldata.grpsugidx.insert(m_hldata.grpsugidx.end(), allcombs.size(),
+ m_hldata.ugroups.size() - 1);
+}
+
+// Trim string beginning with ^ or ending with $ and convert to flags
+static int stringToMods(string& s)
+{
+ int mods = 0;
+ // Check for an anchored search
+ trimstring(s);
+ if (s.length() > 0 && s[0] == '^') {
+ mods |= Rcl::SearchDataClause::SDCM_ANCHORSTART;
+ s.erase(0, 1);
+ }
+ if (s.length() > 0 && s[s.length()-1] == '$') {
+ mods |= Rcl::SearchDataClause::SDCM_ANCHOREND;
+ s.erase(s.length()-1);
+ }
+ return mods;
+}
+
+/**
+ * Turn user entry string (NOT query language) into a list of xapian queries.
+ * We just separate words and phrases, and do wildcard and stem expansion,
+ *
+ * This is used to process data entered into an OR/AND/NEAR/PHRASE field of
+ * the GUI (in the case of NEAR/PHRASE, clausedist adds dquotes to the user
+ * entry).
+ *
+ * This appears awful, and it would seem that the split into
+ * terms/phrases should be performed in the upper layer so that we
+ * only receive pure term or near/phrase pure elements here, but in
+ * fact there are things that would appear like terms to naive code,
+ * and which will actually may be turned into phrases (ie: tom:jerry),
+ * in a manner which intimately depends on the index implementation,
+ * so that it makes sense to process this here.
+ *
+ * The final list contains one query for each term or phrase
+ * - Elements corresponding to a stem-expanded part are an OP_OR
+ * composition of the stem-expanded terms (or a single term query).
+ * - Elements corresponding to phrase/near are an OP_PHRASE/NEAR
+ * composition of the phrase terms (no stem expansion in this case)
+ * @return the subquery count (either or'd stem-expanded terms or phrase word
+ * count)
+ */
+bool SearchDataClauseSimple::processUserString(Rcl::Db &db, const string &iq,
+ string &ermsg, void *pq,
+ int slack, bool useNear)
+{
+ vector<Xapian::Query> &pqueries(*(vector<Xapian::Query>*)pq);
+ int mods = m_modifiers;
+
+ LOGDEB(("StringToXapianQ:pUS:: qstr [%s] fld [%s] mods 0x%x "
+ "slack %d near %d\n",
+ iq.c_str(), m_field.c_str(), mods, slack, useNear));
+ ermsg.erase();
+ m_curcl = 0;
+ const StopList stops = db.getStopList();
+
+ // Simple whitespace-split input into user-level words and
+ // double-quoted phrases: word1 word2 "this is a phrase".
+ //
+ // The text splitter may further still decide that the resulting
+ // "words" are really phrases, this depends on separators:
+ // [paul@dom.net] would still be a word (span), but [about:me]
+ // will probably be handled as a phrase.
+ vector<string> phrases;
+ TextSplit::stringToStrings(iq, phrases);
+
+ // Process each element: textsplit into terms, handle stem/wildcard
+ // expansion and transform into an appropriate Xapian::Query
+ try {
+ for (vector<string>::iterator it = phrases.begin();
+ it != phrases.end(); it++) {
+ LOGDEB0(("strToXapianQ: phrase/word: [%s]\n", it->c_str()));
+ // Anchoring modifiers
+ int amods = stringToMods(*it);
+ int terminc = amods != 0 ? 1 : 0;
+ mods |= amods;
+ // If there are multiple spans in this element, including
+ // at least one composite, we have to increase the slack
+ // else a phrase query including a span would fail.
+ // Ex: "term0@term1 term2" is onlyspans-split as:
+ // 0 term0@term1 0 12
+ // 2 term2 13 18
+ // The position of term2 is 2, not 1, so a phrase search
+ // would fail.
+ // We used to do word split, searching for
+ // "term0 term1 term2" instead, which may have worse
+ // performance, but will succeed.
+ // We now adjust the phrase/near slack by comparing the term count
+ // and the last position
+
+ // The term processing pipeline:
+ TermProcQ tpq;
+ TermProc *nxt = &tpq;
+ TermProcStop tpstop(nxt, stops); nxt = &tpstop;
+ //TermProcCommongrams tpcommon(nxt, stops); nxt = &tpcommon;
+ //tpcommon.onlygrams(true);
+ TermProcPrep tpprep(nxt);
+ if (o_index_stripchars)
+ nxt = &tpprep;
+
+ TextSplitQ splitter(TextSplit::Flags(TextSplit::TXTS_ONLYSPANS |
+ TextSplit::TXTS_KEEPWILD),
+ stops, nxt);
+ tpq.setTSQ(&splitter);
+ splitter.text_to_words(*it);
+
+ slack += splitter.lastpos - splitter.terms.size() + 1;
+
+ LOGDEB0(("strToXapianQ: termcount: %d\n", splitter.terms.size()));
+ switch (splitter.terms.size() + terminc) {
+ case 0:
+ continue;// ??
+ case 1: {
+ int lmods = mods;
+ if (splitter.nostemexps.front())
+ lmods |= SearchDataClause::SDCM_NOSTEMMING;
+ m_hldata.ugroups.push_back(splitter.terms);
+ processSimpleSpan(db, ermsg, splitter.terms.front(),
+ lmods, &pqueries);
+ }
+ break;
+ default:
+ m_hldata.ugroups.push_back(splitter.terms);
+ processPhraseOrNear(db, ermsg, &splitter, mods, &pqueries,
+ useNear, slack);
+ }
+ if (m_curcl >= getMaxCl()) {
+ ermsg = maxXapClauseMsg;
+ if (!o_index_stripchars)
+ ermsg += maxXapClauseCaseDiacMsg;
+ break;
+ }
+ }
+ } catch (const Xapian::Error &e) {
+ ermsg = e.get_msg();
+ } catch (const string &s) {
+ ermsg = s;
+ } catch (const char *s) {
+ ermsg = s;
+ } catch (...) {
+ ermsg = "Caught unknown exception";
+ }
+ if (!ermsg.empty()) {
+ LOGERR(("stringToXapianQueries: %s\n", ermsg.c_str()));
+ return false;
+ }
+ return true;
+}
+
+// Translate a simple OR or AND search clause.
+bool SearchDataClauseSimple::toNativeQuery(Rcl::Db &db, void *p)
+{
+ LOGDEB(("SearchDataClauseSimple::toNativeQuery: fld [%s] val [%s] "
+ "stemlang [%s]\n", m_field.c_str(), m_text.c_str(),
+ getStemLang().c_str()));
+
+ Xapian::Query *qp = (Xapian::Query *)p;
+ *qp = Xapian::Query();
+
+ Xapian::Query::op op;
+ switch (m_tp) {
+ case SCLT_AND: op = Xapian::Query::OP_AND; break;
+ case SCLT_OR: op = Xapian::Query::OP_OR; break;
+ default:
+ LOGERR(("SearchDataClauseSimple: bad m_tp %d\n", m_tp));
+ return false;
+ }
+
+ vector<Xapian::Query> pqueries;
+ if (!processUserString(db, m_text, m_reason, &pqueries))
+ return false;
+ if (pqueries.empty()) {
+ LOGERR(("SearchDataClauseSimple: resolved to null query\n"));
+ return true;
+ }
+
+ *qp = Xapian::Query(op, pqueries.begin(), pqueries.end());
+ if (m_weight != 1.0) {
+ *qp = Xapian::Query(Xapian::Query::OP_SCALE_WEIGHT, *qp, m_weight);
+ }
+ return true;
+}
+
+// Translate a FILENAME search clause. This always comes
+// from a "filename" search from the gui or recollq. A query language
+// "filename:"-prefixed field will not go through here, but through
+// the generic field-processing code.
+//
+// We do not split the entry any more (used to do some crazy thing
+// about expanding multiple fragments in the past). We just take the
+// value blanks and all and expand this against the indexed unsplit
+// file names
+bool SearchDataClauseFilename::toNativeQuery(Rcl::Db &db, void *p)
+{
+ Xapian::Query *qp = (Xapian::Query *)p;
+ *qp = Xapian::Query();
+
+ int maxexp = getSoftMaxExp();
+ if (maxexp == -1)
+ maxexp = getMaxExp();
+
+ vector<string> names;
+ db.filenameWildExp(m_text, names, maxexp);
+ *qp = Xapian::Query(Xapian::Query::OP_OR, names.begin(), names.end());
+
+ if (m_weight != 1.0) {
+ *qp = Xapian::Query(Xapian::Query::OP_SCALE_WEIGHT, *qp, m_weight);
+ }
+ return true;
+}
+
+// Translate a dir: path filtering clause. See comments in .h
+bool SearchDataClausePath::toNativeQuery(Rcl::Db &db, void *p)
+{
+ LOGDEB(("SearchDataClausePath::toNativeQuery: [%s]\n", m_text.c_str()));
+ Xapian::Query *qp = (Xapian::Query *)p;
+ *qp = Xapian::Query();
+
+ if (m_text.empty()) {
+ LOGERR(("SearchDataClausePath: empty path??\n"));
+ m_reason = "Empty path ?";
+ return false;
+ }
+
+ vector<Xapian::Query> orqueries;
+
+ if (m_text[0] == '/')
+ orqueries.push_back(Xapian::Query(wrap_prefix(pathelt_prefix)));
+ else
+ m_text = path_tildexpand(m_text);
+
+ vector<string> vpath;
+ stringToTokens(m_text, vpath, "/");
+
+ for (vector<string>::const_iterator pit = vpath.begin();
+ pit != vpath.end(); pit++){
+
+ string sterm;
+ vector<string> exp;
+ if (!expandTerm(db, m_reason,
+ SDCM_NOSTEMMING|SDCM_CASESENS|SDCM_DIACSENS,
+ *pit, exp, sterm, wrap_prefix(pathelt_prefix))) {
+ return false;
+ }
+ LOGDEB0(("SDataPath::toNative: exp size %d\n", exp.size()));
+ listVector("", exp);
+ if (exp.size() == 1)
+ orqueries.push_back(Xapian::Query(exp[0]));
+ else
+ orqueries.push_back(Xapian::Query(Xapian::Query::OP_OR,
+ exp.begin(), exp.end()));
+ m_curcl += exp.size();
+ if (m_curcl >= getMaxCl())
+ return false;
+ }
+
+ *qp = Xapian::Query(Xapian::Query::OP_PHRASE,
+ orqueries.begin(), orqueries.end());
+
+ if (m_weight != 1.0) {
+ *qp = Xapian::Query(Xapian::Query::OP_SCALE_WEIGHT, *qp, m_weight);
+ }
+ return true;
+}
+
+// Translate NEAR or PHRASE clause.
+bool SearchDataClauseDist::toNativeQuery(Rcl::Db &db, void *p)
+{
+ LOGDEB(("SearchDataClauseDist::toNativeQuery\n"));
+
+ Xapian::Query *qp = (Xapian::Query *)p;
+ *qp = Xapian::Query();
+
+ vector<Xapian::Query> pqueries;
+ Xapian::Query nq;
+
+ // We produce a single phrase out of the user entry then use
+ // stringToXapianQueries() to lowercase and simplify the phrase
+ // terms etc. This will result into a single (complex)
+ // Xapian::Query.
+ if (m_text.find('\"') != string::npos) {
+ m_text = neutchars(m_text, "\"");
+ }
+ string s = cstr_dquote + m_text + cstr_dquote;
+ bool useNear = (m_tp == SCLT_NEAR);
+ if (!processUserString(db, s, m_reason, &pqueries, m_slack, useNear))
+ return false;
+ if (pqueries.empty()) {
+ LOGERR(("SearchDataClauseDist: resolved to null query\n"));
+ return true;
+ }
+
+ *qp = *pqueries.begin();
+ if (m_weight != 1.0) {
+ *qp = Xapian::Query(Xapian::Query::OP_SCALE_WEIGHT, *qp, m_weight);
+ }
+ return true;
+}
+
+} // Namespace Rcl