recoll / Code / Diff of /src/common/rclconfig.cpp

Diff of /src/common/rclconfig.cpp [ac09a9] .. [cecd1b]

Switch to unified view

-a/src/common/rclconfig.cpp
+b/src/common/rclconfig.cpp
 ...
 #include "cpuconf.h"
 #include "execmd.h"
 using namespace std;
-// Static, logically const, RclConfig members are initialized once from the
+// Static, logically const, RclConfig members or module static
-// first object build during process initialization.
+// variables are initialized once from the first object build during
+// process initialization.
 // We default to a case- and diacritics-less index for now
 bool o_index_stripchars = true;
+// Default to storing the text contents for generating snippets. This
+// is only an approximate 10% bigger index and produces nicer
+// snippets.
+bool o_index_storedoctext = true;
 bool o_uptodate_test_use_mtime = false;
 string RclConfig::o_localecharset;
 string RclConfig::o_origcwd;
 ...
+    }
     static int m_index_stripchars_init = 0;
     if (!m_index_stripchars_init) {
     getConfParam("indexStripChars", &o_index_stripchars);
+        getConfParam("indexStoreDocText", &o_index_storedoctext);
         getConfParam("testmodifusemtime", &o_uptodate_test_use_mtime);
     m_index_stripchars_init = 1;
+    }
     if (getConfParam("cachedir", m_cachedir)) {
 ...
+    }
     // Build a direct map avoiding all indirections for field to
     // prefix translation
     // Add direct prefixes from the [prefixes] section
-    vector<string>tps = m_fields->getNames("prefixes");
+    vector<string> tps = m_fields->getNames("prefixes");
-    for (vector<string>::const_iterator it = tps.begin();
+    for (const auto& fieldname : tps) {
-   it != tps.end(); it++) {
     string val;
-    m_fields->get(*it, val, "prefixes");
+    m_fields->get(fieldname, val, "prefixes");
     ConfSimple attrs;
     FieldTraits ft;
+        // fieldname = prefix ; attr1=val;attr2=val...
     if (!valueSplitAttributes(val, ft.pfx, attrs)) {
-        LOGERR("readFieldsConfig: bad config line for ["  << *it <<
+        LOGERR("readFieldsConfig: bad config line for ["  << fieldname <<
                    "]: [" << val << "]\n");
         return 0;
+    }
     string tval;
     if (attrs.get("wdfinc", tval))
 ...
         ft.boost = atof(tval.c_str());
     if (attrs.get("pfxonly", tval))
         ft.pfxonly = stringToBool(tval);
     if (attrs.get("noterms", tval))
         ft.noterms = stringToBool(tval);
-    m_fldtotraits[stringtolower(*it)] = ft;
+    m_fldtotraits[stringtolower(fieldname)] = ft;
-    LOGDEB2("readFieldsConfig: ["  << *it << "] -> ["  << ft.pfx <<
+    LOGDEB2("readFieldsConfig: ["  << fieldname << "] -> ["  << ft.pfx <<
                 "] " << ft.wdfinc << " " << ft.boost << "\n");
+    }
+    // Values section
+    tps = m_fields->getNames("values");
+    for (const auto& fieldname : tps) {
+  string canonic = stringtolower(fieldname); // canonic name
+  string val;
+  m_fields->get(fieldname, val, "values");
+  ConfSimple attrs;
+        string svslot;
+        // fieldname = valueslot ; attr1=val;attr2=val...
+  if (!valueSplitAttributes(val, svslot, attrs)) {
+      LOGERR("readFieldsConfig: bad value line for ["  << fieldname <<
+                   "]: [" << val << "]\n");
+      return 0;
+  }
+        uint32_t valueslot = uint32_t(atoi(svslot.c_str()));
+        if (valueslot == 0) {
+            LOGERR("readFieldsConfig: found 0 value slot for [" << fieldname <<
+                   "]: [" << val << "]\n");
+            continue;
+        }
+        string tval;
+        FieldTraits::ValueType valuetype{FieldTraits::STR};
+        if (attrs.get("type", tval)) {
+            if (tval == "string") {
+                valuetype = FieldTraits::STR;
+            } else if (tval == "int") {
+                valuetype = FieldTraits::INT;
+            } else {
+                LOGERR("readFieldsConfig: bad type for value for " <<
+                       fieldname << " : " << tval << endl);
+                return 0;
+            }
+        }
+        int valuelen{0};
+        if (attrs.get("len", tval)) {
+            valuelen = atoi(tval.c_str());
+        }
+        // Find or insert traits entry
+  const auto pit =
+      m_fldtotraits.insert(
+                pair<string, FieldTraits>(canonic, FieldTraits())).first;
+        pit->second.valueslot = valueslot;
+        pit->second.valuetype = valuetype;
+        pit->second.valuelen = valuelen;
+    }
     // Add prefixes for aliases and build alias-to-canonic map while
     // we're at it. Having the aliases in the prefix map avoids an
     // additional indirection at index time.
     tps = m_fields->getNames("aliases");
-    for (vector<string>::const_iterator it = tps.begin();
+    for (const auto& fieldname : tps) {
-         it != tps.end(); it++){
-    string canonic = stringtolower(*it); // canonic name
+    string canonic = stringtolower(fieldname); // canonic name
     FieldTraits ft;
-  map<string, FieldTraits>::const_iterator pit =
-        m_fldtotraits.find(canonic);
+    const auto pit = m_fldtotraits.find(canonic);
     if (pit != m_fldtotraits.end()) {
         ft = pit->second;
+    }
     string aliases;
     m_fields->get(canonic, aliases, "aliases");
     vector<string> l;
     stringToStrings(aliases, l);
-  for (vector<string>::const_iterator ait = l.begin();
+  for (const auto& alias : l) {
-       ait != l.end(); ait++) {
         if (pit != m_fldtotraits.end())
-        m_fldtotraits[stringtolower(*ait)] = ft;
+        m_fldtotraits[stringtolower(alias)] = ft;
-        m_aliastocanon[stringtolower(*ait)] = canonic;
+        m_aliastocanon[stringtolower(alias)] = canonic;
+    }
+    }
     // Query aliases map
     tps = m_fields->getNames("queryaliases");
-    for (vector<string>::const_iterator it = tps.begin();
+    for (const auto& entry: tps) {
-         it != tps.end(); it++){
-    string canonic = stringtolower(*it); // canonic name
+    string canonic = stringtolower(entry); // canonic name
     string aliases;
     m_fields->get(canonic, aliases, "queryaliases");
     vector<string> l;
     stringToStrings(aliases, l);
-  for (vector<string>::const_iterator ait = l.begin();
+  for (const auto& alias : l) {
-       ait != l.end(); ait++) {
-        m_aliastoqcanon[stringtolower(*ait)] = canonic;
+        m_aliastoqcanon[stringtolower(alias)] = canonic;
+    }
+    }
 #if 0
     for (map<string, FieldTraits>::const_iterator it = m_fldtotraits.begin();
      it != m_fldtotraits.end(); it++) {
-    LOGDEB("readFieldsConfig: ["  << *it << "] -> ["  << it->second.pfx <<
+    LOGDEB("readFieldsConfig: ["  << entry << "] -> ["  << it->second.pfx <<
                "] " << it->second.wdfinc << " " << it->second.boost << "\n");
+    }
 #endif
     vector<string> sl = m_fields->getNames("stored");
-    if (!sl.empty()) {
+    for (const auto& fieldname : sl) {
-  for (vector<string>::const_iterator it = sl.begin();
+        m_storedFields.insert(fieldCanon(stringtolower(fieldname)));
-       it != sl.end(); it++) {
-      string fld = fieldCanon(stringtolower(*it));
-      m_storedFields.insert(fld);
+    }
     // Extended file attribute to field translations
     vector<string>xattrs = m_fields->getNames("xattrtofields");
-    for (vector<string>::const_iterator it = xattrs.begin();
+    for (const auto& xattr : xattrs) {
-   it != xattrs.end(); it++) {
     string val;
-    m_fields->get(*it, val, "xattrtofields");
+    m_fields->get(xattr, val, "xattrtofields");
-    m_xattrtofld[*it] = val;
+    m_xattrtofld[xattr] = val;
+    }
     return true;
+}
 ...
 string RclConfig::getPidfile() const
+{
     return path_cat(getCacheDir(), "index.pid");
+}
+/* Eliminate the common leaf part of file paths p1 and p2. Example:
+ * /mnt1/common/part /mnt2/common/part -> /mnt1 /mnt2. This is used
+ * for computing translations for paths when the dataset has been
+ * moved. Of course this could be done more efficiently than by splitting
+ * into vectors, but we don't care.*/
+static string path_diffstems(const string& p1, const string& p2,
+                            string& r1, string& r2)
+{
+    string reason;
+    r1.clear();
+    r2.clear();
+    vector<string> v1, v2;
+    stringToTokens(p1, v1, "/");
+    stringToTokens(p2, v2, "/");
+    unsigned int l1 = v1.size();
+    unsigned int l2 = v2.size();
+    // Search for common leaf part
+    unsigned int cl = 0;
+    for (; cl < MIN(l1, l2); cl++) {
+        if (v1[l1-cl-1] != v2[l2-cl-1]) {
+            break;
+        }
+    }
+    //cerr << "Common length = " << cl << endl;
+    if (cl == 0) {
+        reason = "Input paths are empty or have no common part";
+        return reason;
+    }
+    for (unsigned i = 0; i < l1 - cl; i++) {
+        r1 += "/" + v1[i];
+    }
+    for (unsigned i = 0; i < l2 - cl; i++) {
+        r2 += "/" + v2[i];
+    }
+    return reason;
+}
 void RclConfig::urlrewrite(const string& dbdir, string& url) const
+{
-    LOGDEB2("RclConfig::urlrewrite: dbdir [" << dbdir << "] url [" << url <<
+    LOGDEB1("RclConfig::urlrewrite: dbdir [" << dbdir << "] url [" << url <<
             "]\n");
+    // If orgidxconfdir is set, we assume that this index is for a
+    // movable dataset, with the configuration directory stored inside
+    // the dataset tree. This allows computing automatic path
+    // translations if the dataset has been moved.
+    string orig_confdir;
+    string cur_confdir;
+    string confstemorg, confstemrep;
+    if (m_conf->get("orgidxconfdir", orig_confdir, "")) {
+        if (!m_conf->get("curidxconfdir", cur_confdir, "")) {
+            cur_confdir = m_confdir;
+        }
+        LOGDEB1("RclConfig::urlrewrite: orgidxconfdir: " << orig_confdir <<
+               " cur_confdir " << cur_confdir << endl);
+        string reason = path_diffstems(orig_confdir, cur_confdir,
+                                       confstemorg, confstemrep);
+        if (!reason.empty()) {
+            LOGERR("urlrewrite: path_diffstems failed: " << reason <<
+                   " : orig_confdir [" << orig_confdir <<
+                   "] cur_confdir [" << cur_confdir << endl);
+            confstemorg = confstemrep = "";
+        }
+    }
     // Do path translations exist for this index ?
+    bool needptrans = true;
     if (m_ptrans == 0 || !m_ptrans->hasSubKey(dbdir)) {
     LOGDEB2("RclConfig::urlrewrite: no paths translations (m_ptrans " <<
                 m_ptrans << ")\n");
-  return;
+        needptrans = false;
+    }
+    if (!needptrans && confstemorg.empty()) {
+        return;
+    }
+    bool computeurl = false;
     string path = fileurltolocalpath(url);
     if (path.empty()) {
     LOGDEB2("RclConfig::urlrewrite: not file url\n");
     return;
+    }
+    // Do the movable volume thing.
+    if (!confstemorg.empty() && confstemorg.size() <= path.size() &&
+        !path.compare(0, confstemorg.size(), confstemorg)) {
+        path = path.replace(0, confstemorg.size(), confstemrep);
+        computeurl = true;
+    }
+    if (needptrans) {
-    // For each translation check if the prefix matches the input path,
+        // For each translation check if the prefix matches the input path,
-    // replace and return the result if it does.
+        // replace and return the result if it does.
-    vector<string> opaths = m_ptrans->getNames(dbdir);
+        vector<string> opaths = m_ptrans->getNames(dbdir);
-    for (vector<string>::const_iterator it = opaths.begin();
+        for (const auto& opath: opaths) {
-   it != opaths.end(); it++) {
+            if (opath.size() <= path.size() &&
-  if (it->size() <= path.size() && !path.compare(0, it->size(), *it)) {
+                !path.compare(0, opath.size(), opath)) {
-      string npath;
+                string npath;
-      // This call always succeeds because the key comes from getNames()
+                // Key comes from getNames()=> call must succeed
-      if (m_ptrans->get(*it, npath, dbdir)) {
+                if (m_ptrans->get(opath, npath, dbdir)) {
-      path = path.replace(0, it->size(), npath);
+                    path = path.replace(0, opath.size(), npath);
-      url = path_pathtofileurl(path);
+                    computeurl = true;
+                }
+                break;
+            }
+        }
+    }
-      break;
+    if (computeurl) {
+        url = path_pathtofileurl(path);
+    }
+}
 bool RclConfig::sourceChanged() const
+{