Open Source Projects Europe

#ifndef lint

#ifndef lint

#endif

#endif

/*

/*

 *   This program is free software; you can redistribute it and/or modify

 *   This program is free software; you can redistribute it and/or modify

 *   it under the terms of the GNU General Public License as published by

 *   it under the terms of the GNU General Public License as published by

 *   the Free Software Foundation; either version 2 of the License, or

 *   the Free Software Foundation; either version 2 of the License, or

#define MA_ABSTRACT_SIZE 250

#define MA_ABSTRACT_SIZE 250

// This is how many words (context size) we keep around query terms

// This is how many words (context size) we keep around query terms

// when building the abstract

// when building the abstract

#define MA_EXTRACT_WIDTH 4

#define MA_EXTRACT_WIDTH 4

// Data for a xapian database. There could actually be 2 different

// Data for a xapian database. There could actually be 2 different

// ones for indexing or query as there is not much in common.

// ones for indexing or query as there is not much in common.

class Native {

class Native {

 public:

 public:

    bool isopen;

    bool iswritable;

    string basedir;

    // Indexing

    // Indexing

    Xapian::WritableDatabase wdb;

    Xapian::WritableDatabase wdb;

    vector<bool> updated;

    vector<bool> updated;

    bool dbDataToRclDoc(std::string &data, Doc &doc, 

    bool dbDataToRclDoc(std::string &data, Doc &doc, 

            int qopts,

            int qopts,

            Xapian::docid docid,

            Xapian::docid docid,

            const list<string>& terms);

            const list<string>& terms);

    ~Native() {

    ~Native() {

    delete enquire;

    delete enquire;

    }

    }

    bool filterMatch(Db *rdb, Xapian::Document &xdoc) {

    bool filterMatch(Db *rdb, Xapian::Document &xdoc) {

    // Parse xapian document's data and populate doc fields

    // Parse xapian document's data and populate doc fields

    return false;

    return false;

    }

    }

};

};

Db::Db() 

Db::Db() 

{

{

    ndb = new Native;

}

}

Db::~Db()

Db::~Db()

{

{

    LOGDEB1(("Db::~Db\n"));

    LOGDEB1(("Db::~Db\n"));

    if (ndb == 0)

    return;

    return;

    LOGDEB(("Db::~Db: isopen %d iswritable %d\n", ndb->isopen, 

        ndb->iswritable));

    if (ndb->isopen == false)

    return;

    return;

    const char *ermsg = "Unknown error";

    const char *ermsg = "Unknown error";

    try {

    try {

    LOGDEB(("Db::~Db: closing native database\n"));

    LOGDEB(("Db::~Db: closing native database\n"));

    if (ndb->iswritable == true) {

        ndb->wdb.flush();

    }

    }

    delete ndb;

    return;

    return;

    } catch (const Xapian::Error &e) {

    } catch (const Xapian::Error &e) {

    ermsg = e.get_msg().c_str();

    ermsg = e.get_msg().c_str();

    } catch (const string &s) {

    } catch (const string &s) {

    ermsg = s.c_str();

    ermsg = s.c_str();

    LOGERR(("Db::~Db: got exception: %s\n", ermsg));

    LOGERR(("Db::~Db: got exception: %s\n", ermsg));

}

}

bool Db::open(const string& dir, OpenMode mode, int qops)

bool Db::open(const string& dir, OpenMode mode, int qops)

{

{

    if (ndb == 0)

    return false;

    return false;

    LOGDEB(("Db::open: isopen %d iswritable %d\n", ndb->isopen, 

        ndb->iswritable));

    if (ndb->isopen) {

    return false;

    }

    }

    const char *ermsg = "Unknown";

    const char *ermsg = "Unknown";

    try {

    try {

    switch (mode) {

    switch (mode) {

    case DbUpd:

    case DbUpd:

    case DbTrunc: 

    case DbTrunc: 

        {

        {

        int action = (mode == DbUpd) ? Xapian::DB_CREATE_OR_OPEN :

        int action = (mode == DbUpd) ? Xapian::DB_CREATE_OR_OPEN :

            Xapian::DB_CREATE_OR_OVERWRITE;

            Xapian::DB_CREATE_OR_OVERWRITE;

        ndb->wdb = Xapian::WritableDatabase(dir, action);

        LOGDEB(("Db::open: lastdocid: %d\n", 

        LOGDEB(("Db::open: lastdocid: %d\n", 

            ndb->wdb.get_lastdocid()));

        ndb->updated.resize(ndb->wdb.get_lastdocid() + 1);

        for (unsigned int i = 0; i < ndb->updated.size(); i++)

            ndb->updated[i] = false;

        ndb->iswritable = true;

        }

        }

        break;

        break;

    case DbRO:

    case DbRO:

    default:

    default:

        ndb->iswritable = false;

        ndb->db = Xapian::Database(dir);

        break;

        break;

    }

    }

    ndb->isopen = true;

    ndb->basedir = dir;

    return true;

    return true;

    } catch (const Xapian::Error &e) {

    } catch (const Xapian::Error &e) {

    ermsg = e.get_msg().c_str();

    ermsg = e.get_msg().c_str();

    } catch (const string &s) {

    } catch (const string &s) {

    ermsg = s.c_str();

    ermsg = s.c_str();

}

}

// Note: xapian has no close call, we delete and recreate the db

// Note: xapian has no close call, we delete and recreate the db

bool Db::close()

bool Db::close()

{

{

    if (ndb == 0)

    return false;

    return false;

    LOGDEB(("Db::close(): isopen %d iswritable %d\n", ndb->isopen, 

        ndb->iswritable));

    if (ndb->isopen == false)

    return true;

    return true;

    const char *ermsg = "Unknown";

    const char *ermsg = "Unknown";

    try {

    try {

    if (ndb->iswritable == true) {

        ndb->wdb.flush();

        LOGDEB(("Rcl:Db: Called xapian flush\n"));

        LOGDEB(("Rcl:Db: Called xapian flush\n"));

    }

    }

    delete ndb;

    ndb = new Native;

    if (ndb)

        return true;

        return true;

    } catch (const Xapian::Error &e) {

    } catch (const Xapian::Error &e) {

    ermsg = e.get_msg().c_str();

    ermsg = e.get_msg().c_str();

    } catch (const string &s) {

    } catch (const string &s) {

    ermsg = s.c_str();

    ermsg = s.c_str();

    ermsg = "Caught unknown exception";

    ermsg = "Caught unknown exception";

    }

    }

    LOGERR(("Db:close: exception while deleting db: %s\n", ermsg));

    LOGERR(("Db:close: exception while deleting db: %s\n", ermsg));

    return false;

    return false;

}

}

bool Db::isopen()

bool Db::isopen()

{

{

    if (ndb == 0)

    return false;

    return false;

    return ndb->isopen;

}

}

// A small class to hold state while splitting text

// A small class to hold state while splitting text

class mySplitterCB : public TextSplitCB {

class mySplitterCB : public TextSplitCB {

 public:

 public:

    output += " ...";

    output += " ...";

    }

    }

    return output;

    return output;

}

}

static string stripchars(const string &str, string delims)

static string stripchars(const string &str, string delims)

{

{

    string out;

    string out;

    string::size_type startPos, pos;

    string::size_type startPos, pos;

    }

    }

    }

    }

    return out;

    return out;

}

}

// Add document in internal form to the database: index the terms in

// Add document in internal form to the database: index the terms in

// the title abstract and body and add special terms for file name,

// the title abstract and body and add special terms for file name,

// date, mime type ... , create the document data record (more

// date, mime type ... , create the document data record (more

// metadata), and update database

// metadata), and update database

bool Db::add(const string &fn, const Doc &idoc, 

bool Db::add(const string &fn, const Doc &idoc, 

          const struct stat *stp)

          const struct stat *stp)

{

{

    LOGDEB1(("Db::add: fn %s\n", fn.c_str()));

    LOGDEB1(("Db::add: fn %s\n", fn.c_str()));

    if (ndb == 0)

    return false;

    return false;

    Doc doc = idoc;

    Doc doc = idoc;

    // Truncate abstract, title and keywords to reasonable lengths. If

    // Truncate abstract, title and keywords to reasonable lengths. If

    const char *fnc = fn.c_str();

    const char *fnc = fn.c_str();

    // Add db entry or update existing entry:

    // Add db entry or update existing entry:

    try {

    try {

    Xapian::docid did = 

    Xapian::docid did = 

        ndb->wdb.replace_document(uniterm.empty() ? pathterm : uniterm, 

                      newdocument);

                      newdocument);

    if (did < ndb->updated.size()) {

        ndb->updated[did] = true;

        LOGDEB(("Db::add: docid %d updated [%s , %s]\n", did, fnc,

        LOGDEB(("Db::add: docid %d updated [%s , %s]\n", did, fnc,

            doc.ipath.c_str()));

            doc.ipath.c_str()));

    } else {

    } else {

        LOGDEB(("Db::add: docid %d added [%s , %s]\n", did, fnc, 

        LOGDEB(("Db::add: docid %d added [%s , %s]\n", did, fnc, 

            doc.ipath.c_str()));

            doc.ipath.c_str()));

    }

    }

    } catch (...) {

    } catch (...) {

    // FIXME: is this ever actually needed?

    // FIXME: is this ever actually needed?

    try {

    try {

        ndb->wdb.add_document(newdocument);

        LOGDEB(("Db::add: %s added (failed re-seek for duplicate)\n", 

        LOGDEB(("Db::add: %s added (failed re-seek for duplicate)\n", 

            fnc));

            fnc));

    } catch (...) {

    } catch (...) {

        LOGERR(("Db::add: failed again after replace_document\n"));

        LOGERR(("Db::add: failed again after replace_document\n"));

        return false;

        return false;

}

}

// Test if given filename has changed since last indexed:

// Test if given filename has changed since last indexed:

bool Db::needUpdate(const string &filename, const struct stat *stp)

bool Db::needUpdate(const string &filename, const struct stat *stp)

{

{

    if (ndb == 0)

    return false;

    return false;

    // If no document exist with this path, we do need update

    // If no document exist with this path, we do need update

    string hash;

    string hash;

    pathHash(filename, hash, PATHHASHLEN);

    pathHash(filename, hash, PATHHASHLEN);

    // fmtime field which will be identical for all docs inside a

    // fmtime field which will be identical for all docs inside a

    // multi-document file (we currently always reindex all if the

    // multi-document file (we currently always reindex all if the

    // file changed)

    // file changed)

    Xapian::PostingIterator doc;

    Xapian::PostingIterator doc;

    try {

    try {

    if (!ndb->wdb.term_exists(pathterm)) {

        LOGDEB1(("Db::needUpdate: no such path: %s\n", pathterm.c_str()));

        LOGDEB1(("Db::needUpdate: no such path: %s\n", pathterm.c_str()));

        return true;

        return true;

    }

    }

    Xapian::PostingIterator docid0 = ndb->wdb.postlist_begin(pathterm);

    for (Xapian::PostingIterator docid = docid0;

    for (Xapian::PostingIterator docid = docid0;

         docid != ndb->wdb.postlist_end(pathterm); docid++) {

        Xapian::Document doc = ndb->wdb.get_document(*docid);

        // Check the date once. no need to look at the others if

        // Check the date once. no need to look at the others if

        // the db needs updating. Note that the fmtime used to be

        // the db needs updating. Note that the fmtime used to be

        // called mtime, and we're keeping compat

        // called mtime, and we're keeping compat

        if (docid == docid0) {

        if (docid == docid0) {

            return true;

            return true;

        } 

        } 

        }

        }

        // Db is up to date. Make a note that this document exists.

        // Db is up to date. Make a note that this document exists.

        if (*docid < ndb->updated.size())

        ndb->updated[*docid] = true;

    }

    }

    return false;

    return false;

    } catch (const Xapian::Error &e) {

    } catch (const Xapian::Error &e) {

    ermsg = e.get_msg().c_str();

    ermsg = e.get_msg().c_str();

    } catch (...) {

    } catch (...) {

 * Delete stem db for given language

 * Delete stem db for given language

 */

 */

bool Db::deleteStemDb(const string& lang)

bool Db::deleteStemDb(const string& lang)

{

{

    LOGDEB(("Db::deleteStemDb(%s)\n", lang.c_str()));

    LOGDEB(("Db::deleteStemDb(%s)\n", lang.c_str()));

    if (ndb == 0)

    return false;

    return false;

    if (ndb->isopen == false)

    return false;

    return false;

    string dir = stemdbname(ndb->basedir, lang);

    if (wipedir(dir) == 0 && rmdir(dir.c_str()) == 0)

    if (wipedir(dir) == 0 && rmdir(dir.c_str()) == 0)

    return true;

    return true;

    return false;

    return false;

}

}

 * parent terms in the document data.

 * parent terms in the document data.

 */

 */

bool Db::createStemDb(const string& lang)

bool Db::createStemDb(const string& lang)

{

{

    LOGDEB(("Db::createStemDb(%s)\n", lang.c_str()));

    LOGDEB(("Db::createStemDb(%s)\n", lang.c_str()));

    if (ndb == 0)

    return false;

    return false;

    if (ndb->isopen == false)

    return false;

    return false;

    // First build the in-memory stem database:

    // First build the in-memory stem database:

    // We walk the list of all terms, and stem each. 

    // We walk the list of all terms, and stem each. 

    //   If the stem is identical to the term, no need to create an entry

    //   If the stem is identical to the term, no need to create an entry

    int stemdiff=0;  // Count of all different stems

    int stemdiff=0;  // Count of all different stems

    int stemmultiple = 0; // Count of stems with multiple derivatives

    int stemmultiple = 0; // Count of stems with multiple derivatives

    try {

    try {

    Xapian::Stem stemmer(lang);

    Xapian::Stem stemmer(lang);

    Xapian::TermIterator it;

    Xapian::TermIterator it;

    for (it = ndb->wdb.allterms_begin(); 

         it != ndb->wdb.allterms_end(); it++) {

        // If it has any non-lowercase 7bit char, cant be stemmable

        // If it has any non-lowercase 7bit char, cant be stemmable

        string::iterator sit = (*it).begin(), eit = sit + (*it).length();

        string::iterator sit = (*it).begin(), eit = sit + (*it).length();

        if ((sit = find_if(sit, eit, p_notlowerorutf)) != eit) {

        if ((sit = find_if(sit, eit, p_notlowerorutf)) != eit) {

        ++nostem;

        ++nostem;

        // LOGDEB(("stemskipped: [%s], because of 0x%x\n", 

        // LOGDEB(("stemskipped: [%s], because of 0x%x\n", 

        rmdir(dir.c_str());

        rmdir(dir.c_str());

        }

        }

    }

    }

    };

    };

    // Create xapian database for stem relations

    // Create xapian database for stem relations

    string stemdbdir = stemdbname(ndb->basedir, lang);

    // We want to get rid of the db dir in case of error. This gets disarmed

    // We want to get rid of the db dir in case of error. This gets disarmed

    // just before success return.

    // just before success return.

    DirWiper wiper(stemdbdir);

    DirWiper wiper(stemdbdir);

    const char *ermsg = "NOERROR";

    const char *ermsg = "NOERROR";

    Xapian::WritableDatabase sdb;

    Xapian::WritableDatabase sdb;

list<string> Db::getStemLangs()

list<string> Db::getStemLangs()

{

{

    list<string> dirs;

    list<string> dirs;

    LOGDEB(("Db::getStemLang\n"));

    LOGDEB(("Db::getStemLang\n"));

    if (ndb == 0)

    return dirs;

    return dirs;

    string pattern = stemdirstem + "*";

    string pattern = stemdirstem + "*";

    dirs = path_dirglob(ndb->basedir, pattern);

    for (list<string>::iterator it = dirs.begin(); it != dirs.end(); it++) {

    for (list<string>::iterator it = dirs.begin(); it != dirs.end(); it++) {

    *it = path_basename(*it);

    *it = path_basename(*it);

    *it = it->substr(stemdirstem.length(), string::npos);

    *it = it->substr(stemdirstem.length(), string::npos);

    }

    }

    return dirs;

    return dirs;

 *  stem database while we are at it.

 *  stem database while we are at it.

 */

 */

bool Db::purge()

bool Db::purge()

{

{

    LOGDEB(("Db::purge\n"));

    LOGDEB(("Db::purge\n"));

    if (ndb == 0)

    return false;

    return false;

    LOGDEB(("Db::purge: isopen %d iswritable %d\n", ndb->isopen, 

        ndb->iswritable));

    if (ndb->isopen == false || ndb->iswritable == false)

    return false;

    return false;

    // There seems to be problems with the document delete code, when

    // There seems to be problems with the document delete code, when

    // we do this, the database is not actually updated. Especially,

    // we do this, the database is not actually updated. Especially,

    // if we delete a bunch of docs, so that there is a hole in the

    // if we delete a bunch of docs, so that there is a hole in the

    // docids at the beginning, we can't add anything (appears to work

    // docids at the beginning, we can't add anything (appears to work

    // and does nothing). Maybe related to the exceptions below when

    // and does nothing). Maybe related to the exceptions below when

    // trying to delete an unexistant document ?

    // trying to delete an unexistant document ?

    // Flushing before trying the deletes seeems to work around the problem

    // Flushing before trying the deletes seeems to work around the problem

    try {

    try {

    ndb->wdb.flush();

    } catch (...) {

    } catch (...) {

    LOGDEB(("Db::purge: 1st flush failed\n"));

    LOGDEB(("Db::purge: 1st flush failed\n"));

    }

    }

    for (Xapian::docid docid = 1; docid < ndb->updated.size(); ++docid) {

    if (!ndb->updated[docid]) {

        try {

        try {

        ndb->wdb.delete_document(docid);

        LOGDEB(("Db::purge: deleted document #%d\n", docid));

        LOGDEB(("Db::purge: deleted document #%d\n", docid));

        } catch (const Xapian::DocNotFoundError &) {

        } catch (const Xapian::DocNotFoundError &) {

        LOGDEB(("Db::purge: document #%d not found\n", docid));

        LOGDEB(("Db::purge: document #%d not found\n", docid));

        }

        }

    }

    }

    }

    }

    try {

    try {

    ndb->wdb.flush();

    } catch (...) {

    } catch (...) {

    LOGDEB(("Db::purge: 2nd flush failed\n"));

    LOGDEB(("Db::purge: 2nd flush failed\n"));

    }

    }

    return true;

    return true;

}

}

/**

/**

 * Expand term to list of all terms which stem to the same term.

 * Expand term to list of all terms which stem to the same term.

 */

 */

static list<string> stemexpand(Native *ndb, string term, const string& lang)

{

{

    list<string> explist;

    list<string> explist;

    try {

    try {

    Xapian::Stem stemmer(lang);

    Xapian::Stem stemmer(lang);

    string stem = stemmer.stem_word(term);

    string stem = stemmer.stem_word(term);

    LOGDEB(("stemexpand: [%s] stem-> [%s]\n", term.c_str(), stem.c_str()));

    LOGDEB(("stemexpand: [%s] stem-> [%s]\n", term.c_str(), stem.c_str()));

    // Try to fetch the doc from the stem db

    // Try to fetch the doc from the stem db

    string stemdbdir = stemdbname(ndb->basedir, lang);

    Xapian::Database sdb(stemdbdir);

    Xapian::Database sdb(stemdbdir);

    LOGDEB1(("stemexpand: %s lastdocid: %d\n", 

    LOGDEB1(("stemexpand: %s lastdocid: %d\n", 

        stemdbdir.c_str(), sdb.get_lastdocid()));

        stemdbdir.c_str(), sdb.get_lastdocid()));

    if (!sdb.term_exists(stem)) {

    if (!sdb.term_exists(stem)) {

        LOGDEB1(("Db::stemexpand: no term for %s\n", stem.c_str()));

        LOGDEB1(("Db::stemexpand: no term for %s\n", stem.c_str()));

//     composition of the stem-expanded terms (or a single term query).

//     composition of the stem-expanded terms (or a single term query).

//   - Elements corresponding to a phrase are an OP_PHRASE composition of the

//   - Elements corresponding to a phrase are an OP_PHRASE composition of the

//     phrase terms (no stem expansion in this case)

//     phrase terms (no stem expansion in this case)

static void stringToXapianQueries(const string &iq,

static void stringToXapianQueries(const string &iq,

                  const string& stemlang,

                  const string& stemlang,

                  Native *ndb,

                  list<Xapian::Query> &pqueries,

                  list<Xapian::Query> &pqueries,

                  Db::QueryOpts opts = Db::QO_NONE)

                  Db::QueryOpts opts = Db::QO_NONE)

{

{

    string qstring = iq;

    string qstring = iq;

        list<string> exp;  

        list<string> exp;  

        string term1;

        string term1;

        dumb_string(term, term1);

        dumb_string(term, term1);

        // Possibly perform stem compression/expansion

        // Possibly perform stem compression/expansion

        if (!nostemexp && (opts & Db::QO_STEM)) {

        if (!nostemexp && (opts & Db::QO_STEM)) {

            exp = stemexpand(ndb, term1, stemlang);

        } else {

        } else {

            exp.push_back(term1);

            exp.push_back(term1);

        }

        }

        // Push either term or OR of stem-expanded set

        // Push either term or OR of stem-expanded set

bool Db::setQuery(const std::string &iqstring, QueryOpts opts, 

bool Db::setQuery(const std::string &iqstring, QueryOpts opts, 

               const string& stemlang)

               const string& stemlang)

{

{

    LOGDEB(("Db::setQuery: q: [%s], opts 0x%x, stemlang %s\n", 

    LOGDEB(("Db::setQuery: q: [%s], opts 0x%x, stemlang %s\n", 

        iqstring.c_str(), (unsigned int)opts, stemlang.c_str()));

        iqstring.c_str(), (unsigned int)opts, stemlang.c_str()));

    if (!ndb)

    return false;

    return false;

    m_asdata.erase();

    m_asdata.erase();

    dbindices.clear();

    list<Xapian::Query> pqueries;

    list<Xapian::Query> pqueries;

    stringToXapianQueries(iqstring, stemlang, ndb, pqueries, opts);

    ndb->query = Xapian::Query(Xapian::Query::OP_OR, pqueries.begin(), 

                   pqueries.end());

                   pqueries.end());

    delete ndb->enquire;

    ndb->enquire = new Xapian::Enquire(ndb->db);

    ndb->enquire->set_query(ndb->query);

    ndb->mset = Xapian::MSet();

    return true;

    return true;

}

}

// Prepare query out of "advanced search" data

// Prepare query out of "advanced search" data

bool Db::setQuery(AdvSearchData &sdata, QueryOpts opts, 

bool Db::setQuery(AdvSearchData &sdata, QueryOpts opts, 

    LOGDEB((" searched file types: %s\n", ft.c_str()));

    LOGDEB((" searched file types: %s\n", ft.c_str()));

    if (!sdata.topdir.empty())

    if (!sdata.topdir.empty())

    LOGDEB((" restricted to: %s\n", sdata.topdir.c_str()));

    LOGDEB((" restricted to: %s\n", sdata.topdir.c_str()));

    m_asdata = sdata;

    m_asdata = sdata;

    dbindices.clear();

    if (!ndb)

    return false;

    return false;

    list<Xapian::Query> pqueries;

    list<Xapian::Query> pqueries;

    Xapian::Query xq;

    Xapian::Query xq;

    if (!sdata.filename.empty()) {

    if (!sdata.filename.empty()) {

    } // else let it be

    } // else let it be

    LOGDEB((" pattern: [%s]\n", pattern.c_str()));

    LOGDEB((" pattern: [%s]\n", pattern.c_str()));

    // Match pattern against all file names in the db

    // Match pattern against all file names in the db

    Xapian::TermIterator it = ndb->db.allterms_begin(); 

    it.skip_to("XSFN");

    it.skip_to("XSFN");

    list<string> names;

    list<string> names;

    for (;it != ndb->db.allterms_end(); it++) {

        if ((*it).find("XSFN") != 0)

        if ((*it).find("XSFN") != 0)

        break;

        break;

        string fn = (*it).substr(4);

        string fn = (*it).substr(4);

        LOGDEB2(("Matching [%s] and [%s]\n", pattern.c_str(), fn.c_str()));

        LOGDEB2(("Matching [%s] and [%s]\n", pattern.c_str(), fn.c_str()));

        if (fnmatch(pattern.c_str(), fn.c_str(), 0) != FNM_NOMATCH) {

        if (fnmatch(pattern.c_str(), fn.c_str(), 0) != FNM_NOMATCH) {

    // Build a query out of the matching file name terms.

    // Build a query out of the matching file name terms.

    xq = Xapian::Query(Xapian::Query::OP_OR, names.begin(), names.end());

    xq = Xapian::Query(Xapian::Query::OP_OR, names.begin(), names.end());

    }

    }

    if (!sdata.allwords.empty()) {

    if (!sdata.allwords.empty()) {

    stringToXapianQueries(sdata.allwords, stemlang, ndb, pqueries, opts);

    if (!pqueries.empty()) {

    if (!pqueries.empty()) {

        Xapian::Query nq = 

        Xapian::Query nq = 

        Xapian::Query(Xapian::Query::OP_AND, pqueries.begin(),

        Xapian::Query(Xapian::Query::OP_AND, pqueries.begin(),

                  pqueries.end());

                  pqueries.end());

        xq = xq.empty() ? nq :

        xq = xq.empty() ? nq :

        pqueries.clear();

        pqueries.clear();

    }

    }

    }

    }

    if (!sdata.orwords.empty()) {

    if (!sdata.orwords.empty()) {

    stringToXapianQueries(sdata.orwords, stemlang, ndb, pqueries, opts);

    if (!pqueries.empty()) {

    if (!pqueries.empty()) {

        Xapian::Query nq = 

        Xapian::Query nq = 

        Xapian::Query(Xapian::Query::OP_OR, pqueries.begin(),

        Xapian::Query(Xapian::Query::OP_OR, pqueries.begin(),

                   pqueries.end());

                   pqueries.end());

        xq = xq.empty() ? nq :

        xq = xq.empty() ? nq :

    }

    }

    }

    }

    // We do no stem expansion on 'No' words. Should we ?

    // We do no stem expansion on 'No' words. Should we ?

    if (!sdata.nowords.empty()) {

    if (!sdata.nowords.empty()) {

    stringToXapianQueries(sdata.nowords, stemlang, ndb, pqueries);

    if (!pqueries.empty()) {

    if (!pqueries.empty()) {

        Xapian::Query nq;

        Xapian::Query nq;

        nq = Xapian::Query(Xapian::Query::OP_OR, pqueries.begin(),

        nq = Xapian::Query(Xapian::Query::OP_OR, pqueries.begin(),

                   pqueries.end());

                   pqueries.end());

        xq = xq.empty() ? nq :

        xq = xq.empty() ? nq :

    }

    }

    if (!sdata.phrase.empty()) {

    if (!sdata.phrase.empty()) {

    Xapian::Query nq;

    Xapian::Query nq;

    string s = string("\"") + sdata.phrase + string("\"");

    string s = string("\"") + sdata.phrase + string("\"");

    stringToXapianQueries(s, stemlang, ndb, pqueries);

    if (!pqueries.empty()) {

    if (!pqueries.empty()) {

        // There should be a single list element phrase query.

        // There should be a single list element phrase query.

        xq = xq.empty() ? *pqueries.begin() : 

        xq = xq.empty() ? *pqueries.begin() : 

        Xapian::Query(Xapian::Query::OP_AND, xq, *pqueries.begin());

        Xapian::Query(Xapian::Query::OP_AND, xq, *pqueries.begin());

        pqueries.clear();

        pqueries.clear();

        Xapian::Query(Xapian::Query::OP_OR, tq, Xapian::Query(term));

        Xapian::Query(Xapian::Query::OP_OR, tq, Xapian::Query(term));

    }

    }

    xq = xq.empty() ? tq : Xapian::Query(Xapian::Query::OP_FILTER, xq, tq);

    xq = xq.empty() ? tq : Xapian::Query(Xapian::Query::OP_FILTER, xq, tq);

    }

    }

    ndb->query = xq;

    delete ndb->enquire;

    ndb->enquire = new Xapian::Enquire(ndb->db);

    ndb->enquire->set_query(ndb->query);

    ndb->mset = Xapian::MSet();

    // Get the query description and trim the "Xapian::Query"

    // Get the query description and trim the "Xapian::Query"

    sdata.description = ndb->query.get_description();

    if (sdata.description.find("Xapian::Query") == 0)

    if (sdata.description.find("Xapian::Query") == 0)

    sdata.description = sdata.description.substr(strlen("Xapian::Query"));

    sdata.description = sdata.description.substr(strlen("Xapian::Query"));

    LOGDEB(("Db::SetQuery: Q: %s\n", sdata.description.c_str()));

    LOGDEB(("Db::SetQuery: Q: %s\n", sdata.description.c_str()));

    return true;

    return true;

}

}

bool Db::getQueryTerms(list<string>& terms)

bool Db::getQueryTerms(list<string>& terms)

{

{

    if (!ndb)

    return false;

    return false;

    terms.clear();

    terms.clear();

    Xapian::TermIterator it;

    Xapian::TermIterator it;

    for (it = ndb->query.get_terms_begin(); it != ndb->query.get_terms_end();

     it++) {

     it++) {

    terms.push_back(*it);

    terms.push_back(*it);

    }

    }

    return true;

    return true;

}

}

static const int qquantum = 30;

static const int qquantum = 30;

int Db::getResCnt()

int Db::getResCnt()

{

{

    if (!ndb || !ndb->enquire) {

    LOGERR(("Db::getResCnt: no query opened\n"));

    LOGERR(("Db::getResCnt: no query opened\n"));

    return -1;

    return -1;

    }

    }

    if (ndb->mset.size() <= 0) {

    try {

    try {

        ndb->mset = ndb->enquire->get_mset(0, qquantum);

    } catch (const Xapian::DatabaseModifiedError &error) {

    } catch (const Xapian::DatabaseModifiedError &error) {

        ndb->db.reopen();

        ndb->mset = ndb->enquire->get_mset(0, qquantum);

    } catch (const Xapian::Error & error) {

    } catch (const Xapian::Error & error) {

        LOGERR(("enquire->get_mset: exception: %s\n", 

        LOGERR(("enquire->get_mset: exception: %s\n", 

            error.get_msg().c_str()));

            error.get_msg().c_str()));

        return -1;

        return -1;

    }

    }

    }

    }

    return ndb->mset.get_matches_lower_bound();

}

bool Native::dbDataToRclDoc(std::string &data, Doc &doc, 

bool Native::dbDataToRclDoc(std::string &data, Doc &doc, 

                int qopts,

                int qopts,

                Xapian::docid docid, const list<string>& terms)

                Xapian::docid docid, const list<string>& terms)

{

{

// sequence to the internal Xapian hole-y one (the holes being the documents 

// sequence to the internal Xapian hole-y one (the holes being the documents 

// that dont match the filter).

// that dont match the filter).

bool Db::getDoc(int exti, Doc &doc, int *percent)

bool Db::getDoc(int exti, Doc &doc, int *percent)

{

{

    LOGDEB1(("Db::getDoc: exti %d\n", exti));

    LOGDEB1(("Db::getDoc: exti %d\n", exti));

    if (!ndb || !ndb->enquire) {

    LOGERR(("Db::getDoc: no query opened\n"));

    LOGERR(("Db::getDoc: no query opened\n"));

    return false;

    return false;

    }

    }

    // For now the only post-query filter is on dir subtree

    // For now the only post-query filter is on dir subtree

    LOGDEB1(("Topdir %s postqflt %d\n", m_asdata.topdir.c_str(), postqfilter));

    LOGDEB1(("Topdir %s postqflt %d\n", m_asdata.topdir.c_str(), postqfilter));

    int xapi;

    int xapi;

    if (postqfilter) {

    if (postqfilter) {

    // There is a postquery filter, does this fall in already known area ?

    // There is a postquery filter, does this fall in already known area ?

    if (exti >= (int)dbindices.size()) {

        // Have to fetch xapian docs and filter until we get

        // Have to fetch xapian docs and filter until we get

        // enough or fail

        // enough or fail

        dbindices.reserve(exti+1);

        // First xapian doc we fetch is the one after last stored 

        // First xapian doc we fetch is the one after last stored 

        int first = dbindices.size() > 0 ? dbindices.back() + 1 : 0;

        // Loop until we get enough docs

        // Loop until we get enough docs

        while (exti >= (int)dbindices.size()) {

        LOGDEB(("Db::getDoc: fetching %d starting at %d\n",

        LOGDEB(("Db::getDoc: fetching %d starting at %d\n",

            qquantum, first));

            qquantum, first));

        try {

        try {

            ndb->mset = ndb->enquire->get_mset(first, qquantum);

        } catch (const Xapian::DatabaseModifiedError &error) {

        } catch (const Xapian::DatabaseModifiedError &error) {

            ndb->db.reopen();

            ndb->mset = ndb->enquire->get_mset(first, qquantum);

        } catch (const Xapian::Error & error) {

        } catch (const Xapian::Error & error) {

          LOGERR(("enquire->get_mset: exception: %s\n", 

          LOGERR(("enquire->get_mset: exception: %s\n", 

              error.get_msg().c_str()));

              error.get_msg().c_str()));

          abort();

          abort();

        }

        }

        if (ndb->mset.empty()) {

            LOGDEB(("Db::getDoc: got empty mset\n"));

            LOGDEB(("Db::getDoc: got empty mset\n"));

            return false;

            return false;

        }

        }

        first = ndb->mset.get_firstitem();

        for (unsigned int i = 0; i < ndb->mset.size() ; i++) {

            LOGDEB(("Db::getDoc: [%d]\n", i));

            LOGDEB(("Db::getDoc: [%d]\n", i));

            Xapian::Document xdoc = ndb->mset[i].get_document();

            if (ndb->filterMatch(this, xdoc)) {

            dbindices.push_back(first + i);

            }

            }

        }

        }

        first = first + ndb->mset.size();

        }

        }

    }

    }

    xapi = dbindices[exti];

    } else {

    } else {

    xapi = exti;

    xapi = exti;

    }

    }

    // From there on, we work with a xapian enquire item number. Fetch it

    // From there on, we work with a xapian enquire item number. Fetch it

    int first = ndb->mset.get_firstitem();

    int last = first + ndb->mset.size() -1;

    if (!(xapi >= first && xapi <= last)) {

    if (!(xapi >= first && xapi <= last)) {

    LOGDEB(("Fetching for first %d, count %d\n", xapi, qquantum));

    LOGDEB(("Fetching for first %d, count %d\n", xapi, qquantum));

    try {

    try {

      ndb->mset = ndb->enquire->get_mset(xapi, qquantum);

    } catch (const Xapian::DatabaseModifiedError &error) {

    } catch (const Xapian::DatabaseModifiedError &error) {

        ndb->db.reopen();

        ndb->mset = ndb->enquire->get_mset(xapi, qquantum);

    } catch (const Xapian::Error & error) {

    } catch (const Xapian::Error & error) {

      LOGERR(("enquire->get_mset: exception: %s\n", 

      LOGERR(("enquire->get_mset: exception: %s\n", 

          error.get_msg().c_str()));

          error.get_msg().c_str()));

      abort();

      abort();

    }

    }

    if (ndb->mset.empty())

        return false;

        return false;

    first = ndb->mset.get_firstitem();

    last = first + ndb->mset.size() -1;

    }

    }

    LOGDEB1(("Db::getDoc: Qry [%s] win [%d-%d] Estimated results: %d",

    LOGDEB1(("Db::getDoc: Qry [%s] win [%d-%d] Estimated results: %d",

         ndb->query.get_description().c_str(), 

         first, last,

         first, last,

         ndb->mset.get_matches_lower_bound()));

    Xapian::Document xdoc = ndb->mset[xapi-first].get_document();

    Xapian::docid docid = *(ndb->mset[xapi-first]);

    if (percent)

    if (percent)

    *percent = ndb->mset.convert_to_percent(ndb->mset[xapi-first]);

    // Parse xapian document's data and populate doc fields

    // Parse xapian document's data and populate doc fields

    string data = xdoc.get_data();

    string data = xdoc.get_data();

    list<string> terms;

    list<string> terms;

    getQueryTerms(terms);

    getQueryTerms(terms);

    return ndb->dbDataToRclDoc(data, doc, m_qOpts, docid, terms);

}

}

// Retrieve document defined by file name and internal path. Very inefficient,

// Retrieve document defined by file name and internal path. Very inefficient,

// used only for history display. We'd need to enter path+ipath terms in the

// used only for history display. We'd need to enter path+ipath terms in the

// db if we wanted to make this more efficient.

// db if we wanted to make this more efficient.

bool Db::getDoc(const string &fn, const string &ipath, Doc &doc, int *pc)

bool Db::getDoc(const string &fn, const string &ipath, Doc &doc, int *pc)

{

{

    LOGDEB(("Db:getDoc: [%s] (%d) [%s]\n", fn.c_str(), fn.length(),

    LOGDEB(("Db:getDoc: [%s] (%d) [%s]\n", fn.c_str(), fn.length(),

        ipath.c_str()));

        ipath.c_str()));

    if (ndb == 0)

    return false;

    return false;

    // Initialize what we can in any case. If this is history, caller

    // Initialize what we can in any case. If this is history, caller

    // will make partial display in case of error

    // will make partial display in case of error

    doc.ipath = ipath;

    doc.ipath = ipath;

    string pathterm  = "P" + hash;

    string pathterm  = "P" + hash;

    // Look for all documents with this path, searching for the one

    // Look for all documents with this path, searching for the one

    // with the appropriate ipath. This is very inefficient.

    // with the appropriate ipath. This is very inefficient.

    const char *ermsg = "";

    const char *ermsg = "";

    try {

    try {

    if (!ndb->db.term_exists(pathterm)) {

        // Document found in history no longer in the database.

        // Document found in history no longer in the database.

        // We return true (because their might be other ok docs further)

        // We return true (because their might be other ok docs further)

        // but indicate the error with pc = -1

        // but indicate the error with pc = -1

        if (*pc) 

        if (*pc) 

        *pc = -1;

        *pc = -1;

        LOGINFO(("Db:getDoc: path inexistant: [%s] length %d\n",

        LOGINFO(("Db:getDoc: path inexistant: [%s] length %d\n",

             pathterm.c_str(), pathterm.length()));

             pathterm.c_str(), pathterm.length()));

        return true;

        return true;

    }

    }

    for (Xapian::PostingIterator docid = 

    for (Xapian::PostingIterator docid = 

         ndb->db.postlist_begin(pathterm);

         docid != ndb->db.postlist_end(pathterm); docid++) {

        Xapian::Document xdoc = ndb->db.get_document(*docid);

        string data = xdoc.get_data();

        string data = xdoc.get_data();

        list<string> terms;

        list<string> terms;

        if (ndb->dbDataToRclDoc(data, doc, QO_NONE, *docid, terms) 

        && doc.ipath == ipath)

        && doc.ipath == ipath)

        return true;

        return true;

    }

    }

    } catch (const Xapian::Error &e) {

    } catch (const Xapian::Error &e) {

    ermsg = e.get_msg().c_str();

    ermsg = e.get_msg().c_str();