#include "rclquery.h"

#include "rclquery.h"

#include "rclquery_p.h"

#include "rclquery_p.h"

#include "md5.h"

#include "md5.h"

#include "rclversion.h"

#include "rclversion.h"

#include "cancelcheck.h"

#include "cancelcheck.h"

#ifndef MAX

#ifndef MAX

#define MAX(A,B) (A>B?A:B)

#define MAX(A,B) (A>B?A:B)

#endif

#endif

#ifndef MIN

#ifndef MIN

#define MIN(A,B) (A<B?A:B)

#define MIN(A,B) (A<B?A:B)

#endif

#endif

// Recoll index format version is stored in user metadata. When this change,

// Recoll index format version is stored in user metadata. When this change,

// we can't open the db and will have to reindex.

// we can't open the db and will have to reindex.

static const string RCL_IDX_VERSION_KEY("RCL_IDX_VERSION_KEY");

static const string RCL_IDX_VERSION("1");

// This is the word position offset at which we index the body text

// This is the word position offset at which we index the body text

// (abstract, keywords, etc.. are stored before this)

// (abstract, keywords, etc.. are stored before this)

static const unsigned int baseTextPosition = 100000;

static const unsigned int baseTextPosition = 100000;

const string end_of_field_term = "XXND";

const string end_of_field_term = "XXND";

// This is used as a marker inside the abstract frag lists, but

// This is used as a marker inside the abstract frag lists, but

// normally doesn't remain in final output (which is built with a

// normally doesn't remain in final output (which is built with a

// custom sep. by our caller).

// custom sep. by our caller).

static const string ellipsis("...");

string version_string(){

string version_string(){

    return string("Recoll ") + string(rclversionstr) + string(" + Xapian ") +

    return string("Recoll ") + string(rclversionstr) + string(" + Xapian ") +

        string(Xapian::version_string());

        string(Xapian::version_string());

}

}

// Synthetic abstract marker (to discriminate from abstract actually

// Synthetic abstract marker (to discriminate from abstract actually

// found in document)

// found in document)

// Only ONE field name inside the index data record differs from the

// Only ONE field name inside the index data record differs from the

// Rcl::Doc ones: caption<->title, for a remnant of compatibility with

// Rcl::Doc ones: caption<->title, for a remnant of compatibility with

// omega

// omega

static const string keycap("caption");

// Static/Default table for field->prefix/weight translation. 

// Static/Default table for field->prefix/weight translation. 

// This is logically const after initialization. Can't use a

// This is logically const after initialization. Can't use a

// static object to init this as the static std::string objects may

// static object to init this as the static std::string objects may

// not be ready.

// not be ready.

// "fields" configuration (cf: Db::fieldToTraits()), meaning that the

// "fields" configuration (cf: Db::fieldToTraits()), meaning that the

// entries can be overriden in the configuration, but not

// entries can be overriden in the configuration, but not

// suppressed. 

// suppressed. 

static map<string, FieldTraits> fldToTraits;

static map<string, FieldTraits> fldToTraits;

static void initFldToTraits() 

static void initFldToTraits() 

{

{

    // Can't remember why "abstract" is indexed without a prefix

    // Can't remember why "abstract" is indexed without a prefix

    // (result: it's indexed twice actually). Maybe I'll dare change

    // (result: it's indexed twice actually). Maybe I'll dare change

    // this one day

    // this one day

    fldToTraits[Doc::keyabs] = FieldTraits();

    fldToTraits[Doc::keyabs] = FieldTraits();

    fldToTraits["ext"] = FieldTraits("XE");

    fldToTraits["ext"] = FieldTraits("XE");

    fldToTraits[Doc::keyfn] = FieldTraits("XSFN");

    fldToTraits[Doc::keyfn] = FieldTraits("XSFN");

    fldToTraits[keycap] = FieldTraits("S");

    fldToTraits[Doc::keytt] = FieldTraits("S");

    fldToTraits[Doc::keytt] = FieldTraits("S");

    fldToTraits["subject"] = FieldTraits("S");

    fldToTraits["subject"] = FieldTraits("S");

    fldToTraits[Doc::keyau] = FieldTraits("A");

    fldToTraits[Doc::keyau] = FieldTraits("A");

    fldToTraits["creator"] = FieldTraits("A");

    fldToTraits["creator"] = FieldTraits("A");

    parms.get(Doc::keyurl, doc.url);

    parms.get(Doc::keyurl, doc.url);

    parms.get(Doc::keytp, doc.mimetype);

    parms.get(Doc::keytp, doc.mimetype);

    parms.get(Doc::keyfmt, doc.fmtime);

    parms.get(Doc::keyfmt, doc.fmtime);

    parms.get(Doc::keydmt, doc.dmtime);

    parms.get(Doc::keydmt, doc.dmtime);

    parms.get(Doc::keyoc, doc.origcharset);

    parms.get(Doc::keyoc, doc.origcharset);

    parms.get(keycap, doc.meta[Doc::keytt]);

    parms.get(Doc::keykw, doc.meta[Doc::keykw]);

    parms.get(Doc::keykw, doc.meta[Doc::keykw]);

    parms.get(Doc::keyabs, doc.meta[Doc::keyabs]);

    parms.get(Doc::keyabs, doc.meta[Doc::keyabs]);

    // Possibly remove synthetic abstract indicator (if it's there, we

    // Possibly remove synthetic abstract indicator (if it's there, we

    // used to index the beginning of the text as abstract).

    // used to index the beginning of the text as abstract).

    doc.syntabs = false;

    doc.syntabs = false;

    doc.syntabs = true;

    doc.syntabs = true;

    }

    }

    parms.get(Doc::keyipt, doc.ipath);

    parms.get(Doc::keyipt, doc.ipath);

    parms.get(Doc::keyfs, doc.fbytes);

    parms.get(Doc::keyfs, doc.fbytes);

    parms.get(Doc::keyds, doc.dbytes);

    parms.get(Doc::keyds, doc.dbytes);

            if (ii == (unsigned int)ipos) {

            if (ii == (unsigned int)ipos) {

            sparseDoc[ii] = qterm;

            sparseDoc[ii] = qterm;

            } else if (ii > (unsigned int)ipos && 

            } else if (ii > (unsigned int)ipos && 

                   ii < (unsigned int)ipos + qtrmwrdcnt) {

                   ii < (unsigned int)ipos + qtrmwrdcnt) {

            sparseDoc[ii] = occupiedmarker;

            sparseDoc[ii] = occupiedmarker;

            } else if (!sparseDoc[ii].compare(ellipsis)) {

            // For an empty slot, the test has a side

            // For an empty slot, the test has a side

            // effect of inserting an empty string which

            // effect of inserting an empty string which

            // is what we want

            // is what we want

            sparseDoc[ii] = emptys;

            sparseDoc[ii] = emptys;

            }

            }

        // Add ellipsis at the end. This may be replaced later by

        // Add ellipsis at the end. This may be replaced later by

        // an overlapping extract. Take care not to replace an

        // an overlapping extract. Take care not to replace an

        // empty string here, we really want an empty slot,

        // empty string here, we really want an empty slot,

        // use find()

        // use find()

        if (sparseDoc.find(sto+1) == sparseDoc.end()) {

        if (sparseDoc.find(sto+1) == sparseDoc.end()) {

            sparseDoc[sto+1] = ellipsis;

        }

        }

        // Limit to allocated occurences and total size

        // Limit to allocated occurences and total size

        if (++occurrences >= maxoccs || 

        if (++occurrences >= maxoccs || 

            qtermposs.size() >= maxtotaloccs)

            qtermposs.size() >= maxtotaloccs)

    if (TextSplit::isCJK(*uit))

    if (TextSplit::isCJK(*uit))

        newcjk = true;

        newcjk = true;

    if (!incjk || (incjk && !newcjk))

    if (!incjk || (incjk && !newcjk))

        chunk += " ";

        chunk += " ";

    incjk = newcjk;

    incjk = newcjk;

    if (it->second == ellipsis) {

        vabs.push_back(chunk);

        vabs.push_back(chunk);

        chunk.clear();

        chunk.clear();

    } else {

    } else {

        chunk += it->second;

        chunk += it->second;

    }

    }

            Xapian::DB_CREATE_OR_OVERWRITE;

            Xapian::DB_CREATE_OR_OVERWRITE;

        m_ndb->xwdb = Xapian::WritableDatabase(dir, action);

        m_ndb->xwdb = Xapian::WritableDatabase(dir, action);

                // If db is empty, write the data format version at once

                // If db is empty, write the data format version at once

                // to avoid stupid error messages:

                // to avoid stupid error messages:

                if (m_ndb->xwdb.get_doccount() == 0)

                if (m_ndb->xwdb.get_doccount() == 0)

                    m_ndb->xwdb.set_metadata(RCL_IDX_VERSION_KEY, 

                                             RCL_IDX_VERSION);

        m_ndb->m_iswritable = true;

        m_ndb->m_iswritable = true;

        // We open a readonly object in all cases (possibly in

        // We open a readonly object in all cases (possibly in

        // addition to the r/w one) because some operations

        // addition to the r/w one) because some operations

        // are faster when performed through a Database: no

        // are faster when performed through a Database: no

        // forced flushes on allterms_begin(), ie, used in

        // forced flushes on allterms_begin(), ie, used in

        *error = DbOpenMainDb;

        *error = DbOpenMainDb;

    // Check index format version. Must not try to check a just created or

    // Check index format version. Must not try to check a just created or

    // truncated db

    // truncated db

    if (mode != DbTrunc && m_ndb->xdb().get_doccount() > 0) {

    if (mode != DbTrunc && m_ndb->xdb().get_doccount() > 0) {

        string version = m_ndb->xdb().get_metadata(RCL_IDX_VERSION_KEY);

        if (version.compare(RCL_IDX_VERSION)) {

        m_ndb->m_noversionwrite = true;

        m_ndb->m_noversionwrite = true;

        LOGERR(("Rcl::Db::open: file index [%s], software [%s]\n",

        LOGERR(("Rcl::Db::open: file index [%s], software [%s]\n",

            version.c_str(), RCL_IDX_VERSION.c_str()));

        throw Xapian::DatabaseError("Recoll index version mismatch",

        throw Xapian::DatabaseError("Recoll index version mismatch",

                        "", "");

                        "", "");

        }

        }

    }

    }

    m_mode = mode;

    m_mode = mode;

    string ermsg;

    string ermsg;

    try {

    try {

    bool w = m_ndb->m_iswritable;

    bool w = m_ndb->m_iswritable;

    if (w) {

    if (w) {

        if (!m_ndb->m_noversionwrite)

        if (!m_ndb->m_noversionwrite)

        m_ndb->xwdb.set_metadata(RCL_IDX_VERSION_KEY, RCL_IDX_VERSION);

        LOGDEB(("Rcl::Db:close: xapian will close. May take some time\n"));

        LOGDEB(("Rcl::Db:close: xapian will close. May take some time\n"));

    }

    }

    // Used to do a flush here. Cant see why it should be necessary.

    // Used to do a flush here. Cant see why it should be necessary.

    deleteZ(m_ndb);

    deleteZ(m_ndb);

    if (w)

    if (w)

    if (syntctxlen > 0)

    if (syntctxlen > 0)

    m_synthAbsWordCtxLen = syntctxlen;

    m_synthAbsWordCtxLen = syntctxlen;

}

}

static const int MB = 1024 * 1024;

static const int MB = 1024 * 1024;

static const string nc("\n\r\x0c");

#define RECORD_APPEND(R, NM, VAL) {R += NM + "=" + VAL + "\n";}

#define RECORD_APPEND(R, NM, VAL) {R += NM + "=" + VAL + "\n";}

// Add document in internal form to the database: index the terms in

// Add document in internal form to the database: index the terms in

// the title abstract and body and add special terms for file name,

// the title abstract and body and add special terms for file name,

    RECORD_APPEND(record, Doc::keyipt, doc.ipath);

    RECORD_APPEND(record, Doc::keyipt, doc.ipath);

    if (doc.meta[Doc::keytt].empty())

    if (doc.meta[Doc::keytt].empty())

    doc.meta[Doc::keytt] = doc.utf8fn;

    doc.meta[Doc::keytt] = doc.utf8fn;

    doc.meta[Doc::keytt] = 

    doc.meta[Doc::keytt] = 

    neutchars(truncate_to_word(doc.meta[Doc::keytt], 150), nc);

    if (!doc.meta[Doc::keytt].empty())

    if (!doc.meta[Doc::keytt].empty())

    RECORD_APPEND(record, keycap, doc.meta[Doc::keytt]);

    trimstring(doc.meta[Doc::keykw], " \t\r\n");

    trimstring(doc.meta[Doc::keykw], " \t\r\n");

    doc.meta[Doc::keykw] = 

    doc.meta[Doc::keykw] = 

    neutchars(truncate_to_word(doc.meta[Doc::keykw], 300), nc);

    if (!doc.meta[Doc::keykw].empty())

    if (!doc.meta[Doc::keykw].empty())

    RECORD_APPEND(record, Doc::keykw, doc.meta[Doc::keykw]);

    RECORD_APPEND(record, Doc::keykw, doc.meta[Doc::keykw]);

    // If abstract is empty, we make up one with the beginning of the

    // If abstract is empty, we make up one with the beginning of the

    // document. This is then not indexed, but part of the doc data so

    // document. This is then not indexed, but part of the doc data so

    // don't exist yet.

    // don't exist yet.

    trimstring(doc.meta[Doc::keyabs], " \t\r\n");

    trimstring(doc.meta[Doc::keyabs], " \t\r\n");

    if (doc.meta[Doc::keyabs].empty()) {

    if (doc.meta[Doc::keyabs].empty()) {

    syntabs = true;

    syntabs = true;

    if (!doc.text.empty())

    if (!doc.text.empty())

        neutchars(truncate_to_word(doc.text, m_idxAbsTruncLen), nc);

    } else {

    } else {

    doc.meta[Doc::keyabs] = 

    doc.meta[Doc::keyabs] = 

        neutchars(truncate_to_word(doc.meta[Doc::keyabs], m_idxAbsTruncLen),

        neutchars(truncate_to_word(doc.meta[Doc::keyabs], m_idxAbsTruncLen),

              nc);

    }

    }

    if (!doc.meta[Doc::keyabs].empty())

    if (!doc.meta[Doc::keyabs].empty())

    RECORD_APPEND(record, Doc::keyabs, doc.meta[Doc::keyabs]);

    RECORD_APPEND(record, Doc::keyabs, doc.meta[Doc::keyabs]);

    const set<string>& stored = m_config->getStoredFields();

    const set<string>& stored = m_config->getStoredFields();

    for (set<string>::const_iterator it = stored.begin();

    for (set<string>::const_iterator it = stored.begin();

     it != stored.end(); it++) {

     it != stored.end(); it++) {

    string nm = m_config->fieldCanon(*it);

    string nm = m_config->fieldCanon(*it);

    if (!doc.meta[*it].empty()) {

    if (!doc.meta[*it].empty()) {

        string value = 

        string value = 

        neutchars(truncate_to_word(doc.meta[*it], 150), nc);

        RECORD_APPEND(record, nm, value);

        RECORD_APPEND(record, nm, value);

    }

    }

    }

    }

    // If the file's md5 was computed, add value. This is optionally

    // If the file's md5 was computed, add value. This is optionally

}

}

// Characters that can begin a wildcard or regexp expression. We use skipto

// Characters that can begin a wildcard or regexp expression. We use skipto

// to begin the allterms search with terms that begin with the portion of

// to begin the allterms search with terms that begin with the portion of

// the input string prior to these chars.

// the input string prior to these chars.

const string wildSpecChars = "*?[";

const string regSpecChars = "(.[{";

// Find all index terms that match a wildcard or regular expression

// Find all index terms that match a wildcard or regular expression

bool Db::termMatch(MatchType typ, const string &lang,

bool Db::termMatch(MatchType typ, const string &lang,

           const string &root, 

           const string &root, 

           TermMatchResult& res,

           TermMatchResult& res,

    string droot;

    string droot;

    if (!unacmaybefold(root, droot, "UTF-8", true)) {

    if (!unacmaybefold(root, droot, "UTF-8", true)) {

    LOGERR(("Db::termMatch: unac failed for [%s]\n", root.c_str()));

    LOGERR(("Db::termMatch: unac failed for [%s]\n", root.c_str()));

    return false;

    return false;

    }

    }

    string nochars = typ == ET_WILD ? wildSpecChars : regSpecChars;

    string prefix;

    string prefix;

    if (!field.empty()) {

    if (!field.empty()) {

    const FieldTraits *ftp = 0;

    const FieldTraits *ftp = 0;

    if (!fieldToTraits(field, &ftp) || ftp->pfx.empty()) {

    if (!fieldToTraits(field, &ftp) || ftp->pfx.empty()) {

    XAPTRY(vab = m_ndb->makeAbstract(doc.xdocid, query),

    XAPTRY(vab = m_ndb->makeAbstract(doc.xdocid, query),

           m_ndb->xrdb, m_reason);

           m_ndb->xrdb, m_reason);

    for (vector<string>::const_iterator it = vab.begin(); 

    for (vector<string>::const_iterator it = vab.begin(); 

     it != vab.end(); it++) {

     it != vab.end(); it++) {

    abstract.append(*it);

    abstract.append(*it);

    abstract.append(ellipsis);

    }

    }

    return m_reason.empty() ? true : false;

    return m_reason.empty() ? true : false;

}

}

// Retrieve document defined by Unique doc identifier. This is mainly used

// Retrieve document defined by Unique doc identifier. This is mainly used