recoll / Code / Diff of /src/utils/smallut.cpp

Diff of /src/utils/smallut.cpp [ab11ea] .. [f13d3f]

Switch to unified view


/* Copyright (C) 2004-2016 J.F.Dockes
 *   This program is free software; you can redistribute it and/or modify
 *   it under the terms of the GNU General Public License as published by
 *   the Free Software Foundation; either version 2 of the License, or
 *   (at your option) any later version.
 *
...
 *   Free Software Foundation, Inc.,
 *   59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
 */

#ifndef TEST_SMALLUT
#ifdef BUILDING_RECOLL
#include "autoconfig.h"
#else
#include "config.h"
#endif

#include <stdio.h>
#include <stdlib.h>
#include <time.h>
#include <ctype.h>
...
#include <list>
#include UNORDERED_MAP_INCLUDE
#include UNORDERED_SET_INCLUDE

#include "smallut.h"




using namespace std;

int stringicmp(const string& s1, const string& s2)










{
    string::const_iterator it1 = s1.begin();
    string::const_iterator it2 = s2.begin();
    string::size_type size1 = s1.length(), size2 = s2.length();
    char c1, c2;

    if (size1 < size2) {
        while (it1 != s1.end()) {
            c1 = ::toupper(*it1);
            c2 = ::toupper(*it2);
            if (c1 != c2) {
                return c1 > c2 ? 1 : -1;
            }
            ++it1;
            ++it2;
        }
        return size1 == size2 ? 0 : -1;
    } else {
        while (it2 != s2.end()) {
            c1 = ::toupper(*it1);
            c2 = ::toupper(*it2);
            if (c1 != c2) {
                return c1 > c2 ? 1 : -1;
            }
            ++it1;
            ++it2;
        }
        return size1 == size2 ? 0 : 1;
    }
}
void stringtolower(string& io)
{
    string::iterator it = io.begin();
    string::iterator ite = io.end();
    while (it != ite) {
        *it = ::tolower(*it);
        it++;
    }
}
string stringtolower(const string& i)
{
    string o = i;
...
    return o;
}
extern int stringisuffcmp(const string& s1, const string& s2)
{
    string::const_reverse_iterator r1 = s1.rbegin(), re1 = s1.rend(),
                                   r2 = s2.rbegin(), re2 = s2.rend();
    while (r1 != re1 && r2 != re2) {
        char c1 = ::toupper(*r1);
        char c2 = ::toupper(*r2);
        if (c1 != c2) {
            return c1 > c2 ? 1 : -1;
        }
        ++r1;
        ++r2;
    }
    return 0;
}

//  s1 is already lowercase
int stringlowercmp(const string& s1, const string& s2)
{
    string::const_iterator it1 = s1.begin();
    string::const_iterator it2 = s2.begin();
    string::size_type size1 = s1.length(), size2 = s2.length();
    char c2;

    if (size1 < size2) {
        while (it1 != s1.end()) {
            c2 = ::tolower(*it2);
            if (*it1 != c2) {
                return *it1 > c2 ? 1 : -1;
            }
            ++it1;
            ++it2;
        }
        return size1 == size2 ? 0 : -1;
    } else {
        while (it2 != s2.end()) {
            c2 = ::tolower(*it2);
            if (*it1 != c2) {
                return *it1 > c2 ? 1 : -1;
            }
            ++it1;
            ++it2;
        }
        return size1 == size2 ? 0 : 1;
    }
}

//  s1 is already uppercase
int stringuppercmp(const string& s1, const string& s2)
{
    string::const_iterator it1 = s1.begin();
    string::const_iterator it2 = s2.begin();
    string::size_type size1 = s1.length(), size2 = s2.length();
    char c2;

    if (size1 < size2) {
        while (it1 != s1.end()) {
            c2 = ::toupper(*it2);
            if (*it1 != c2) {
                return *it1 > c2 ? 1 : -1;
            }
            ++it1;
            ++it2;
        }
        return size1 == size2 ? 0 : -1;
    } else {
        while (it2 != s2.end()) {
            c2 = ::toupper(*it2);
            if (*it1 != c2) {
                return *it1 > c2 ? 1 : -1;
            }
            ++it1;
            ++it2;
        }
        return size1 == size2 ? 0 : 1;
    }
}

// Compare charset names, removing the more common spelling variations
bool samecharset(const string& cs1, const string& cs2)
{
    string mcs1, mcs2;
    // Remove all - and _, turn to lowecase
    for (unsigned int i = 0; i < cs1.length(); i++) {
        if (cs1[i] != '_' && cs1[i] != '-') {
            mcs1 += ::tolower(cs1[i]);

        }
    }
    for (unsigned int i = 0; i < cs2.length(); i++) {
        if (cs2[i] != '_' && cs2[i] != '-') {
            mcs2 += ::tolower(cs2[i]);
        }
    }
    return mcs1 == mcs2;
}

template <class T> bool stringToStrings(const string& s, T& tokens,
                                        const string& addseps)
{
    string current;
    tokens.clear();
    enum states {SPACE, TOKEN, INQUOTE, ESCAPE};
    states state = SPACE;
    for (unsigned int i = 0; i < s.length(); i++) {
        switch (s[i]) {
        case '"':
            switch (state) {
            case SPACE:
                state = INQUOTE;
                continue;
            case TOKEN:
                current += '"';
                continue;
            case INQUOTE:
                tokens.insert(tokens.end(), current);
                current.clear();
                state = SPACE;
                continue;
            case ESCAPE:
                current += '"';
                state = INQUOTE;
                continue;
            }
            break;
        case '\\':
            switch (state) {
            case SPACE:
            case TOKEN:
                current += '\\';
                state = TOKEN;
                continue;
            case INQUOTE:
                state = ESCAPE;
                continue;
            case ESCAPE:
                current += '\\';
                state = INQUOTE;
                continue;
            }
            break;

        case ' ':
        case '\t':
        case '\n':
        case '\r':
            switch (state) {
            case SPACE:
                continue;
            case TOKEN:
                tokens.insert(tokens.end(), current);
                current.clear();
                state = SPACE;
                continue;
            case INQUOTE:
            case ESCAPE:
                current += s[i];
                continue;
            }
            break;

        default:
            if (!addseps.empty() && addseps.find(s[i]) != string::npos) {
                switch (state) {
                case ESCAPE:
                    state = INQUOTE;
                    break;
                case INQUOTE:
                    break;
                case SPACE:
                    tokens.insert(tokens.end(), string(1, s[i]));
                    continue;
                case TOKEN:
                    tokens.insert(tokens.end(), current);
                    current.erase();
                    tokens.insert(tokens.end(), string(1, s[i]));
                    state = SPACE;
                    continue;
                }
            } else switch (state) {
                case ESCAPE:
                    state = INQUOTE;
                    break;
                case SPACE:
                    state = TOKEN;
                    break;
                case TOKEN:
                case INQUOTE:
                    break;
                }
            current += s[i];

        }
    }
    switch (state) {
    case SPACE:
        break;
    case TOKEN:
        tokens.insert(tokens.end(), current);
        break;
    case INQUOTE:
    case ESCAPE:
        return false;
    }
    return true;
}

template bool stringToStrings<list<string> >(const string&,
        list<string>&, const string&);
template bool stringToStrings<vector<string> >(const string&,
        vector<string>&, const string&);
template bool stringToStrings<set<string> >(const string&,
        set<string>&, const string&);
template bool stringToStrings<STD_UNORDERED_SET<string> >
(const string&, STD_UNORDERED_SET<string>&, const string&);

template <class T> void stringsToString(const T& tokens, string& s)
{
    for (typename T::const_iterator it = tokens.begin();
            it != tokens.end(); it++) {
        bool hasblanks = false;
        if (it->find_first_of(" \t\n") != string::npos) {
            hasblanks = true;
        }
        if (it != tokens.begin()) {
            s.append(1, ' ');
        }
        if (hasblanks) {
            s.append(1, '"');
        }
        for (unsigned int i = 0; i < it->length(); i++) {
            char car = it->at(i);
            if (car == '"') {
                s.append(1, '\\');
                s.append(1, car);
            } else {
                s.append(1, car);
            }
        }
        if (hasblanks) {
            s.append(1, '"');
        }
    }
}




template void stringsToString<list<string> >(const list<string>&, string&);
template void stringsToString<vector<string> >(const vector<string>&, string&);
template void stringsToString<set<string> >(const set<string>&, string&);
template <class T> string stringsToString(const T& tokens)
{
    string out;
    stringsToString<T>(tokens, out);
    return out;
}
template string stringsToString<list<string> >(const list<string>&);
template string stringsToString<vector<string> >(const vector<string>&);
template string stringsToString<set<string> >(const set<string>&);

template <class T> void stringsToCSV(const T& tokens, string& s,
                                     char sep)
{
    s.erase();
    for (typename T::const_iterator it = tokens.begin();
            it != tokens.end(); it++) {
        bool needquotes = false;
        if (it->empty() ||
                it->find_first_of(string(1, sep) + "\"\n") != string::npos) {
            needquotes = true;
        }
        if (it != tokens.begin()) {
            s.append(1, sep);
        }
        if (needquotes) {
            s.append(1, '"');
        }
        for (unsigned int i = 0; i < it->length(); i++) {
            char car = it->at(i);
            if (car == '"') {
                s.append(2, '"');
            } else {
                s.append(1, car);
            }
        }
        if (needquotes) {
            s.append(1, '"');
        }
    }
}




template void stringsToCSV<list<string> >(const list<string>&, string&, char);
template void stringsToCSV<vector<string> >(const vector<string>&, string&,
        char);

void stringToTokens(const string& str, vector<string>& tokens,
                    const string& delims, bool skipinit)
{
    string::size_type startPos = 0, pos;

    // Skip initial delims, return empty if this eats all.
    if (skipinit &&
            (startPos = str.find_first_not_of(delims, 0)) == string::npos) {
        return;
    }
    while (startPos < str.size()) {
        // Find next delimiter or end of string (end of token)
        pos = str.find_first_of(delims, startPos);

        // Add token to the vector and adjust start
        if (pos == string::npos) {
            tokens.push_back(str.substr(startPos));
            break;
        } else if (pos == startPos) {
            // Dont' push empty tokens after first
            if (tokens.empty()) {
                tokens.push_back(string());
            }
            startPos = ++pos;
        } else {
            tokens.push_back(str.substr(startPos, pos - startPos));
            startPos = ++pos;
        }
    }
}



bool stringToBool(const string& s)
{
    if (s.empty()) {
        return false;
    }
    if (isdigit(s[0])) {
        int val = atoi(s.c_str());
        return val ? true : false;
    }
    if (s.find_first_of("yYtT") == 0) {
        return true;
    }
    return false;
}

void trimstring(string& s, const char *ws)
{
    string::size_type pos = s.find_first_not_of(ws);
    if (pos == string::npos) {
        s.clear();
        return;
    }
    s.replace(0, pos, string());

    pos = s.find_last_not_of(ws);
    if (pos != string::npos && pos != s.length() - 1) {
        s.replace(pos + 1, string::npos, string());
    }
}

// Remove some chars and replace them with spaces
string neutchars(const string& str, const string& chars)
{
    string out;
    neutchars(str, out, chars);
    return out;
}
void neutchars(const string& str, string& out, const string& chars)
{
    string::size_type startPos, pos;

    for (pos = 0;;) {
        // Skip initial chars, break if this eats all.
        if ((startPos = str.find_first_not_of(chars, pos)) == string::npos) {
            break;
        }
        // Find next delimiter or end of string (end of token)
        pos = str.find_first_of(chars, startPos);
        // Add token to the output. Note: token cant be empty here
        if (pos == string::npos) {
            out += str.substr(startPos);
        } else {
            out += str.substr(startPos, pos - startPos) + " ";
        }
    }
}


/* Truncate a string to a given maxlength, avoiding cutting off midword
 * if reasonably possible. Note: we could also use textsplit, stopping when
 * we have enough, this would be cleanly utf8-aware but would remove
 * punctuation */
static const string cstr_SEPAR = " \t\n\r-:.;,/[]{}";
string truncate_to_word(const string& input, string::size_type maxlen)
{
    string output;
    if (input.length() <= maxlen) {
        output = input;
    } else {
        output = input.substr(0, maxlen);
        string::size_type space = output.find_last_of(cstr_SEPAR);
        // Original version only truncated at space if space was found after
        // maxlen/2. But we HAVE to truncate at space, else we'd need to do
        // utf8 stuff to avoid truncating at multibyte char. In any case,
        // not finding space means that the text probably has no value.
        // Except probably for Asian languages, so we may want to fix this
        // one day
        if (space == string::npos) {
            output.erase();
        } else {
            output.erase(space);
        }
    }
    return output;
}














// Escape things that would look like markup
string escapeHtml(const string& in)
{
    string out;
    for (string::size_type pos = 0; pos < in.length(); pos++) {
        switch (in.at(pos)) {
        case '<':
            out += "&lt;";
            break;
        case '&':
            out += "&amp;";
            break;
        default:
            out += in.at(pos);
        }
    }
    return out;
}

string escapeShell(const string& in)
{
    string out;
    out += "\"";
    for (string::size_type pos = 0; pos < in.length(); pos++) {
        switch (in.at(pos)) {
        case '$':
            out += "\\$";
            break;
        case '`':
            out += "\\`";
            break;
        case '"':
            out += "\\\"";
            break;
        case '\n':
            out += "\\\n";
            break;
        case '\\':
            out += "\\\\";
            break;
        default:
            out += in.at(pos);
        }
    }
    out += "\"";
    return out;
}


// Substitute printf-like percent cmds inside a string
bool pcSubst(const string& in, string& out, const map<char, string>& subs)
{
    string::const_iterator it;
    for (it = in.begin(); it != in.end(); it++) {





        if (*it == '%') {
            if (++it == in.end()) {
                out += '%';
                break;
            }
            if (*it == '%') {
                out += '%';
                continue;
            }
            map<char, string>::const_iterator tr;
            if ((tr = subs.find(*it)) != subs.end()) {
                out += tr->second;
            } else {
                // We used to do "out += *it;" here but this does not make
                // sense
            }
        } else {
            out += *it;
        }
    }
    return true;
}

bool pcSubst(const string& in, string& out, const map<string, string>& subs)
{
    out.erase();
    string::size_type i;
    for (i = 0; i < in.size(); i++) {





        if (in[i] == '%') {
            if (++i == in.size()) {
                out += '%';
                break;
            }
            if (in[i] == '%') {
                out += '%';
                continue;
            }
            string key = "";
            if (in[i] == '(') {
                if (++i == in.size()) {
                    out += string("%(");
                    break;
                }
                string::size_type j = in.find_first_of(")", i);
                if (j == string::npos) {
                    // ??concatenate remaining part and stop
                    out += in.substr(i - 2);
                    break;
                }
                key = in.substr(i, j - i);
                i = j;
            } else {
                key = in[i];
            }
            map<string, string>::const_iterator tr;
            if ((tr = subs.find(key)) != subs.end()) {
                out += tr->second;
            } else {
                // Substitute to nothing, that's the reasonable thing to do
                // instead of keeping the %(key)
                // out += key.size()==1? key : string("(") + key + string(")");
            }
        } else {
            out += in[i];
        }
    }
    return true;
}
inline static int ulltorbuf(unsigned long long val, char *rbuf)
{
    int idx;
    for (idx = 0; val; idx++) {
        rbuf[idx] = '0' + val % 10;
        val /= 10;
    }
    while (val);
    rbuf[idx] = 0;
    return idx;
}

inline static void ullcopyreverse(const char *rbuf, string& buf, int idx)
{
    buf.reserve(idx + 1);
    for (int i = idx - 1; i >= 0; i--) {
        buf.push_back(rbuf[i]);
    }
}

...
        buf = "0";
        return;
    }

    bool neg = val < 0;
    if (neg) {
        val = -val;
    }

    char rbuf[30];
    int idx = ulltorbuf(val, rbuf);

    if (neg) {
        rbuf[idx++] = '-';
    }
    rbuf[idx] = 0;

    ullcopyreverse(rbuf, buf, idx);
    return;
}
...

// Convert byte count into unit (KB/MB...) appropriate for display
string displayableBytes(off_t size)
{
    const char *unit;

    double roundable = 0;
    if (size < 1000) {
        unit = " B ";
        roundable = double(size);
    } else if (size < 1E6) {
        unit = " KB ";
        roundable = double(size) / 1E3;
    } else if (size < 1E9) {
        unit = " MB ";
        roundable = double(size) / 1E6;
    } else {
        unit = " GB ";
        roundable = double(size) / 1E9;
    }
    size = off_t(round(roundable));
    return lltodecstr(size).append(unit);
}

string breakIntoLines(const string& in, unsigned int ll,
                      unsigned int maxlines)
{
    string query = in;
    string oq;
    unsigned int nlines = 0;
    while (query.length() > 0) {
        string ss = query.substr(0, ll);
        if (ss.length() == ll) {
            string::size_type pos = ss.find_last_of(" ");
            if (pos == string::npos) {
                pos = query.find_first_of(" ");
                if (pos != string::npos) {
                    ss = query.substr(0, pos + 1);
                } else {
                    ss = query;
                }
            } else {
                ss = ss.substr(0, pos + 1);
            }
        }
        // This cant happen, but anyway. Be very sure to avoid an infinite loop
        if (ss.length() == 0) {
            oq = query;
            break;
        }
        oq += ss + "\n";
        if (nlines++ >= maxlines) {
            oq += " ... \n";
            break;
        }
        query = query.substr(ss.length());
    }
    return oq;
}

// Date is Y[-M[-D]]
static bool parsedate(vector<string>::const_iterator& it,
                      vector<string>::const_iterator end, DateInterval *dip)
{
    dip->y1 = dip->m1 = dip->d1 = dip->y2 = dip->m2 = dip->d2 = 0;
    if (it->length() > 4 || !it->length() ||
            it->find_first_not_of("0123456789") != string::npos) {
        return false;
    }
    if (it == end || sscanf(it++->c_str(), "%d", &dip->y1) != 1) {
        return false;
    }
    if (it == end || *it == "/") {
        return true;
    }
    if (*it++ != "-") {
        return false;
    }

    if (it->length() > 2 || !it->length() ||
            it->find_first_not_of("0123456789") != string::npos) {
        return false;
    }
    if (it == end || sscanf(it++->c_str(), "%d", &dip->m1) != 1) {
        return false;
    }
    if (it == end || *it == "/") {
        return true;
    }
    if (*it++ != "-") {
        return false;
    }

    if (it->length() > 2 || !it->length() ||
            it->find_first_not_of("0123456789") != string::npos) {
        return false;
    }
    if (it == end || sscanf(it++->c_str(), "%d", &dip->d1) != 1) {
        return false;
    }
...
}

// Called with the 'P' already processed. Period ends at end of string
// or at '/'. We dont' do a lot effort at validation and will happily
// accept 10Y1Y4Y (the last wins)
static bool parseperiod(vector<string>::const_iterator& it,
                        vector<string>::const_iterator end, DateInterval *dip)
{
    dip->y1 = dip->m1 = dip->d1 = dip->y2 = dip->m2 = dip->d2 = 0;
    while (it != end) {
        int value;
...
            return false;
        }
        if (sscanf(it++->c_str(), "%d", &value) != 1) {
            return false;
        }
        if (it == end || it->empty()) {
            return false;
        }
        switch (it->at(0)) {
        case 'Y':
        case 'y':
            dip->y1 = value;
            break;
        case 'M':
        case 'm':
            dip->m1 = value;
            break;
        case 'D':
        case 'd':
            dip->d1 = value;
            break;
        default:
            return false;
        }
        it++;
        if (it == end) {
            return true;
        }
        if (*it == "/") {
            return true;
        }
    }
    return true;
}

#ifdef _WIN32
int setenv(const char *name, const char *value, int overwrite)
{
    if (!overwrite) {
        const char *cp = getenv(name);
        if (cp) {
            return -1;
        }
    }
    return _putenv_s(name, value);
}
void unsetenv(const char *name)
{
...

    tz = getenv("TZ");
    setenv("TZ", "", 1);
    tzset();
    ret = mktime(tm);
    if (tz) {
        setenv("TZ", tz, 1);
    } else {
        unsetenv("TZ");
    }
    tzset();
    return ret;
}

#if 0
static void cerrdip(const string& s, DateInterval *dip)
{
    cerr << s << dip->y1 << "-" << dip->m1 << "-" << dip->d1 << "/"
         << dip->y2 << "-" << dip->m2 << "-" << dip->d2
         << endl;
}
#endif

// Compute date + period. Won't work out of the unix era.
// or pre-1970 dates. Just convert everything to unixtime and
// seconds (with average durations for months/years), add and convert
// back
static bool addperiod(DateInterval *dp, DateInterval *pp)
{
    struct tm tm;
    // Create a struct tm with possibly non normalized fields and let
    // timegm sort it out
    memset(&tm, 0, sizeof(tm));
    tm.tm_year = dp->y1 - 1900 + pp->y1;
    tm.tm_mon = dp->m1 + pp->m1 - 1;
    tm.tm_mday = dp->d1 + pp->d1;
    time_t tres = mktime(&tm);
    localtime_r(&tres, &tm);
    dp->y1 = tm.tm_year + 1900;
    dp->m1 = tm.tm_mon + 1;
...
    return true;
}
int monthdays(int mon, int year)
{
    switch (mon) {
    // We are returning a few too many 29 days februaries, no problem
    case 2:
        return (year % 4) == 0 ? 29 : 28;
    case 1:
    case 3:
    case 5:
    case 7:
    case 8:
    case 10:
    case 12:
        return 31;
    default:
        return 30;
    }
}
bool parsedateinterval(const string& s, DateInterval *dip)
{
    vector<string> vs;
    dip->y1 = dip->m1 = dip->d1 = dip->y2 = dip->m2 = dip->d2 = 0;
    DateInterval p1, p2, d1, d2;
    p1 = p2 = d1 = d2 = *dip;
    bool hasp1 = false, hasp2 = false, hasd1 = false, hasd2 = false,
         hasslash = false;

    if (!stringToStrings(s, vs, "PYMDpymd-/")) {
        return false;
    }
    if (vs.empty()) {
        return false;
    }

    vector<string>::const_iterator it = vs.begin();
    if (*it == "P" || *it == "p") {
        it++;
        if (!parseperiod(it, vs.end(), &p1)) {
...
        } else if (*it == "P" || *it == "p") {
            it++;
            if (!parseperiod(it, vs.end(), &p2)) {
                return false;
            }
            hasp2 = true;
        } else {
            if (!parsedate(it, vs.end(), &d2)) {
                return false;
            }
            hasd2 = true;
...
    // a period or not (actual or infinite indicated by a / + empty)
    //
    // If there is no explicit period, an incomplete date indicates a
    // period of the size of the uncompleted elements. Ex: 1999
    // actually means 1999/P12M
    //
    // If there is a period, the incomplete date should be extended
    // to the beginning or end of the unspecified portion. Ex: 1999/
    // means 1999-01-01/ and /1999 means /1999-12-31
    if (hasd1) {
        if (!(hasslash || hasp2)) {
...
}


void catstrerror(string *reason, const char *what, int _errno)
{
    if (!reason) {
        return;
    }
    if (what) {
        reason->append(what);
    }

    reason->append(": errno: ");

    char nbuf[20];
    sprintf(nbuf, "%d", _errno);
...

#if defined(sun) || defined(_WIN32)
    // Note: sun strerror is noted mt-safe ??
    reason->append(strerror(_errno));
#else
#define ERRBUFSZ 200
    char errbuf[ERRBUFSZ];
    // There are 2 versions of strerror_r.
    // - The GNU one returns a pointer to the message (maybe
    //   static storage or supplied buffer).
    // - The POSIX one always stores in supplied buffer and
    //   returns 0 on success. As the possibility of error and
    //   error code are not specified, we're basically doomed
    //   cause we can't use a test on the 0 value to know if we
    //   were returned a pointer...
    // Also couldn't find an easy way to disable the gnu version without
    // changing the cxxflags globally, so forget it. Recent gnu lib versions
    // normally default to the posix version.
    // At worse we get no message at all here.
    errbuf[0] = 0;
...
    char *ret = (char *)strerror_r(_errno, errbuf, ERRBUFSZ);
    reason->append(errbuf);
#endif
}























































static const char *vlang_to_code[] = {
    "be", "cp1251",
    "bg", "cp1251",
    "cs", "iso-8859-2",
...
    "th", "iso-8859-11",
    "tr", "iso-8859-9",
    "uk", "koi8-u",
};

static const string cstr_cp1252("CP1252");

string langtocode(const string& lang)
{
    static STD_UNORDERED_MAP<string, string> lang_to_code;
    if (lang_to_code.empty()) {
        for (unsigned int i = 0;
                i < sizeof(vlang_to_code) / sizeof(char *); i += 2) {
            lang_to_code[vlang_to_code[i]] = vlang_to_code[i + 1];

        }
    }
    STD_UNORDERED_MAP<string, string>::const_iterator it =
        lang_to_code.find(lang);

    // Use cp1252 by default...
    if (it == lang_to_code.end()) {
        return cstr_cp1252;
    }

    return it->second;
}

string localelang()
{
    const char *lang = getenv("LANG");

    if (lang == 0 || *lang == 0 || !strcmp(lang, "C") ||
            !strcmp(lang, "POSIX")) {
        return "en";
    }
    string locale(lang);
    string::size_type under = locale.find_first_of("_");
    if (under == string::npos) {
        return locale;
    }
    return locale.substr(0, under);
}

// Initialization for static stuff to be called from main thread before going
// multiple
void smallut_init_mt()
{
    // Init langtocode() static table
    langtocode("");
...


// Periods test strings
const char* periods[] = {
    "2001",    // Year 2001
    "2001/",  // 2001 or later
    "2001/P3Y", // 2001 -> 2004 or 2005, ambiguous
    "2001-01-01/P3Y", // 01-2001 -> 01 2004
    "2001-03-03/2001-05-01", // Explicit one
    "P3M/", // 3 months ago to now
    "P1Y1M/2001-03-01", // 2000-02-01/2001-03-01
...

const char *thisprog;
static void cerrdip(const string& s, DateInterval *dip)
{
    cerr << s << dip->y1 << "-" << dip->m1 << "-" << dip->d1 << "/"
         << dip->y2 << "-" << dip->m2 << "-" << dip->d2
         << endl;
}

int main(int argc, char **argv)
{
    thisprog = *argv++;
    argc--;

#if 1
    if (argc <= 0) {
        cerr << "Usage: smallut <stringtosplit>" << endl;
        exit(1);
    }
    string s = *argv++;
    argc--;
    vector<string> vs;
    stringToTokens(s, vs, "/");
    for (vector<string>::const_iterator it = vs.begin(); it != vs.end(); it++) {
        cerr << "[" << *it << "] ";
    }
    cerr << endl;
    exit(0);
#elif 0
    if (argc <= 0) {
        cerr << "Usage: smallut <stringtosplit>" << endl;
        exit(1);
    }
    string s = *argv++;
    argc--;
    vector<string> vs;
    if (!stringToStrings(s, vs, ":-()")) {
        cerr << "Bad entry" << endl;
        exit(1);
    }
    for (vector<string>::const_iterator it = vs.begin(); it != vs.end(); it++) {
        cerr << "[" << *it << "] ";
    }
    cerr << endl;
    exit(0);
#elif 0
    if (argc <= 0) {
        cerr << "Usage: smallut <dateinterval>" << endl;
        exit(1);
    }
    string s = *argv++;
    argc--;
    DateInterval di;
    if (!parsedateinterval(s, &di)) {
        cerr << "Parse failed" << endl;
        exit(1);
    }
...
        }
    }
    exit(0);
#elif 0
    for (int i = 0; i < npairs; i++) {
        {
            int c = stringicmp(pairs[i].s1, pairs[i].s2);
            printf("'%s' %s '%s' ", pairs[i].s1,
                   c == 0 ? "==" : c < 0 ? "<" : ">", pairs[i].s2);
        }
        {
            int cl = stringlowercmp(pairs[i].s1, pairs[i].s2);
            printf("L '%s' %s '%s' ", pairs[i].s1,
                   cl == 0 ? "==" : cl < 0 ? "<" : ">", pairs[i].s2);
        }
        {
            int cu = stringuppercmp(pairs[i].s1, pairs[i].s2);
            printf("U '%s' %s '%s' ", pairs[i].s1,
                   cu == 0 ? "==" : cu < 0 ? "<" : ">", pairs[i].s2);
        }
        printf("\n");
    }
#elif 0
    for (int i = 0; i < nsuffpairs; i++) {
        int c = stringisuffcmp(suffpairs[i].s1, suffpairs[i].s2);
        printf("[%s] %s [%s] \n", suffpairs[i].s1,
               c == 0 ? "matches" : c < 0 ? "<" : ">", suffpairs[i].s2);
    }
#elif 0
    std::string testit("\303\251l\303\251gant");
    for (int sz = 10; sz >= 0; sz--) {
        utf8truncate(testit, sz);
        cout << testit << endl;
    }
#elif 0
    std::string testit("ligne\ndeuxieme ligne\r3eme ligne\r\n");
    cout << "[" << neutchars(testit, "\r\n") << "]" << endl;
    string i, o;
...
    cout << "CSV line: [" << out << "]" << endl;
#elif 0
    string sshort("ABC");
    string slong("ABCD");
    string sshortsmaller("ABB");

    vector<pair<string, string> > cmps;
    cmps.push_back(pair<string, string>(sshort, sshort));
    cmps.push_back(pair<string, string>(sshort, slong));
    cmps.push_back(pair<string, string>(slong, sshort));
    cmps.push_back(pair<string, string>(sshortsmaller, sshort));
    cmps.push_back(pair<string, string>(sshort, sshortsmaller));

    for (vector<pair<string, string> >::const_iterator it = cmps.begin();
            it != cmps.end(); it++) {
        cout << it->first << " " << it->second << " " <<
             stringicmp(it->first, it->second) << endl;
    }
    cout << endl;
    for (vector<pair<string, string> >::const_iterator it = cmps.begin();
            it != cmps.end(); it++) {
        cout << it->first << " " << it->second << " " <<
             stringlowercmp(stringtolower(it->first), it->second) << endl;
    }
    cout << endl;
    for (vector<pair<string, string> >::const_iterator it = cmps.begin();
            it != cmps.end(); it++) {
        cout << it->first << " " << it->second << " " <<
             stringuppercmp(it->first, it->second) << endl;
    }

#endif
}


	a/src/utils/smallut.cpp		b/src/utils/smallut.cpp
1	/* Copyright (C) 2004 J.F.Dockes	1	/* Copyright (C) 2004-2016 J.F.Dockes
2	* This program is free software; you can redistribute it and/or modify	2	* This program is free software; you can redistribute it and/or modify
3	* it under the terms of the GNU General Public License as published by	3	* it under the terms of the GNU General Public License as published by
4	* the Free Software Foundation; either version 2 of the License, or	4	* the Free Software Foundation; either version 2 of the License, or
5	* (at your option) any later version.	5	* (at your option) any later version.
6	*	6	*
	...		...
14	* Free Software Foundation, Inc.,	14	* Free Software Foundation, Inc.,
15	* 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.	15	* 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
16	*/	16	*/
17		17
18	#ifndef TEST_SMALLUT	18	#ifndef TEST_SMALLUT
		19	#ifdef BUILDING_RECOLL
19	#include "autoconfig.h"	20	#include "autoconfig.h"
		21	#else
		22	#include "config.h"
		23	#endif
20		24
21	#include <stdio.h>	25	#include <stdio.h>
22	#include <stdlib.h>	26	#include <stdlib.h>
23	#include <time.h>	27	#include <time.h>
24	#include <ctype.h>	28	#include <ctype.h>
	...		...
32	#include <list>	36	#include <list>
33	#include UNORDERED_MAP_INCLUDE	37	#include UNORDERED_MAP_INCLUDE
34	#include UNORDERED_SET_INCLUDE	38	#include UNORDERED_SET_INCLUDE
35		39
36	#include "smallut.h"	40	#include "smallut.h"
37	#include "utf8iter.h"
38	#include "hldata.h"
39	#include "cstr.h"
40		41
41	using namespace std;	42	using namespace std;
42		43
43	void map_ss_cp_noshr(const map<string,string> s, map<string,string> *d)	44	int stringicmp(const string& s1, const string& s2)
44	{
45	for (map<string,string>::const_iterator it= s.begin();
46	it != s.end(); it++) {
47	d->insert(
48	pair<string,string>(string(it->first.begin(), it->first.end()),
49	string(it->second.begin(), it->second.end())));
50	}
51	}
52
53	int stringicmp(const string & s1, const string& s2)
54	{	45	{
55	string::const_iterator it1 = s1.begin();	46	string::const_iterator it1 = s1.begin();
56	string::const_iterator it2 = s2.begin();	47	string::const_iterator it2 = s2.begin();
57	string::size_type size1 = s1.length(), size2 = s2.length();	48	string::size_type size1 = s1.length(), size2 = s2.length();
58	char c1, c2;	49	char c1, c2;
59		50
60	if (size1 < size2) {	51	if (size1 < size2) {
61	while (it1 != s1.end()) {	52	while (it1 != s1.end()) {
62	c1 = ::toupper(*it1);	53	c1 = ::toupper(*it1);
63	c2 = ::toupper(*it2);	54	c2 = ::toupper(*it2);
64	if (c1 != c2) {	55	if (c1 != c2) {
65	return c1 > c2 ? 1 : -1;	56	return c1 > c2 ? 1 : -1;
66	}	57	}
67	++it1; ++it2;	58	++it1;
68	}	59	++it2;
		60	}
69	return size1 == size2 ? 0 : -1;	61	return size1 == size2 ? 0 : -1;
70	} else {	62	} else {
71	while (it2 != s2.end()) {	63	while (it2 != s2.end()) {
72	c1 = ::toupper(*it1);	64	c1 = ::toupper(*it1);
73	c2 = ::toupper(*it2);	65	c2 = ::toupper(*it2);
74	if (c1 != c2) {	66	if (c1 != c2) {
75	return c1 > c2 ? 1 : -1;	67	return c1 > c2 ? 1 : -1;
76	}	68	}
77	++it1; ++it2;	69	++it1;
78	}	70	++it2;
		71	}
79	return size1 == size2 ? 0 : 1;	72	return size1 == size2 ? 0 : 1;
80	}	73	}
81	}	74	}
82	void stringtolower(string& io)	75	void stringtolower(string& io)
83	{	76	{
84	string::iterator it = io.begin();	77	string::iterator it = io.begin();
85	string::iterator ite = io.end();	78	string::iterator ite = io.end();
86	while (it != ite) {	79	while (it != ite) {
87	it = ::tolower(it);	80	it = ::tolower(it);
88	it++;	81	it++;
89	}	82	}
90	}	83	}
91	string stringtolower(const string& i)	84	string stringtolower(const string& i)
92	{	85	{
93	string o = i;	86	string o = i;
	...		...
95	return o;	88	return o;
96	}	89	}
97	extern int stringisuffcmp(const string& s1, const string& s2)	90	extern int stringisuffcmp(const string& s1, const string& s2)
98	{	91	{
99	string::const_reverse_iterator r1 = s1.rbegin(), re1 = s1.rend(),	92	string::const_reverse_iterator r1 = s1.rbegin(), re1 = s1.rend(),
100	r2 = s2.rbegin(), re2 = s2.rend();	93	r2 = s2.rbegin(), re2 = s2.rend();
101	while (r1 != re1 && r2 != re2) {	94	while (r1 != re1 && r2 != re2) {
102	char c1 = ::toupper(*r1);	95	char c1 = ::toupper(*r1);
103	char c2 = ::toupper(*r2);	96	char c2 = ::toupper(*r2);
104	if (c1 != c2) {	97	if (c1 != c2) {
105	return c1 > c2 ? 1 : -1;	98	return c1 > c2 ? 1 : -1;
106	}	99	}
107	++r1; ++r2;	100	++r1;
		101	++r2;
108	}	102	}
109	return 0;	103	return 0;
110	}	104	}
111		105
112	// s1 is already lowercase	106	// s1 is already lowercase
113	int stringlowercmp(const string & s1, const string& s2)	107	int stringlowercmp(const string& s1, const string& s2)
114	{	108	{
115	string::const_iterator it1 = s1.begin();	109	string::const_iterator it1 = s1.begin();
116	string::const_iterator it2 = s2.begin();	110	string::const_iterator it2 = s2.begin();
117	string::size_type size1 = s1.length(), size2 = s2.length();	111	string::size_type size1 = s1.length(), size2 = s2.length();
118	char c2;	112	char c2;
119		113
120	if (size1 < size2) {	114	if (size1 < size2) {
121	while (it1 != s1.end()) {	115	while (it1 != s1.end()) {
122	c2 = ::tolower(*it2);	116	c2 = ::tolower(*it2);
123	if (*it1 != c2) {	117	if (*it1 != c2) {
124	return *it1 > c2 ? 1 : -1;	118	return *it1 > c2 ? 1 : -1;
125	}	119	}
126	++it1; ++it2;	120	++it1;
127	}	121	++it2;
		122	}
128	return size1 == size2 ? 0 : -1;	123	return size1 == size2 ? 0 : -1;
129	} else {	124	} else {
130	while (it2 != s2.end()) {	125	while (it2 != s2.end()) {
131	c2 = ::tolower(*it2);	126	c2 = ::tolower(*it2);
132	if (*it1 != c2) {	127	if (*it1 != c2) {
133	return *it1 > c2 ? 1 : -1;	128	return *it1 > c2 ? 1 : -1;
134	}	129	}
135	++it1; ++it2;	130	++it1;
136	}	131	++it2;
		132	}
137	return size1 == size2 ? 0 : 1;	133	return size1 == size2 ? 0 : 1;
138	}	134	}
139	}	135	}
140		136
141	// s1 is already uppercase	137	// s1 is already uppercase
142	int stringuppercmp(const string & s1, const string& s2)	138	int stringuppercmp(const string& s1, const string& s2)
143	{	139	{
144	string::const_iterator it1 = s1.begin();	140	string::const_iterator it1 = s1.begin();
145	string::const_iterator it2 = s2.begin();	141	string::const_iterator it2 = s2.begin();
146	string::size_type size1 = s1.length(), size2 = s2.length();	142	string::size_type size1 = s1.length(), size2 = s2.length();
147	char c2;	143	char c2;
148		144
149	if (size1 < size2) {	145	if (size1 < size2) {
150	while (it1 != s1.end()) {	146	while (it1 != s1.end()) {
151	c2 = ::toupper(*it2);	147	c2 = ::toupper(*it2);
152	if (*it1 != c2) {	148	if (*it1 != c2) {
153	return *it1 > c2 ? 1 : -1;	149	return *it1 > c2 ? 1 : -1;
154	}	150	}
155	++it1; ++it2;	151	++it1;
156	}	152	++it2;
		153	}
157	return size1 == size2 ? 0 : -1;	154	return size1 == size2 ? 0 : -1;
158	} else {	155	} else {
159	while (it2 != s2.end()) {	156	while (it2 != s2.end()) {
160	c2 = ::toupper(*it2);	157	c2 = ::toupper(*it2);
161	if (*it1 != c2) {	158	if (*it1 != c2) {
162	return *it1 > c2 ? 1 : -1;	159	return *it1 > c2 ? 1 : -1;
163	}	160	}
164	++it1; ++it2;	161	++it1;
165	}	162	++it2;
		163	}
166	return size1 == size2 ? 0 : 1;	164	return size1 == size2 ? 0 : 1;
167	}	165	}
168	}	166	}
169		167
170	// Compare charset names, removing the more common spelling variations	168	// Compare charset names, removing the more common spelling variations
171	bool samecharset(const string &cs1, const string &cs2)	169	bool samecharset(const string& cs1, const string& cs2)
172	{	170	{
173	string mcs1, mcs2;	171	string mcs1, mcs2;
174	// Remove all - and _, turn to lowecase	172	// Remove all - and _, turn to lowecase
175	for (unsigned int i = 0; i < cs1.length();i++) {	173	for (unsigned int i = 0; i < cs1.length(); i++) {
176	if (cs1[i] != '_' && cs1[i] != '-') {	174	if (cs1[i] != '_' && cs1[i] != '-') {
177	mcs1 += ::tolower(cs1[i]);	175	mcs1 += ::tolower(cs1[i]);
178	}
179	}	176	}
		177	}
180	for (unsigned int i = 0; i < cs2.length();i++) {	178	for (unsigned int i = 0; i < cs2.length(); i++) {
181	if (cs2[i] != '_' && cs2[i] != '-') {	179	if (cs2[i] != '_' && cs2[i] != '-') {
182	mcs2 += ::tolower(cs2[i]);	180	mcs2 += ::tolower(cs2[i]);
183	}	181	}
184	}	182	}
185	return mcs1 == mcs2;	183	return mcs1 == mcs2;
186	}	184	}
187		185
188	template <class T> bool stringToStrings(const string &s, T &tokens,	186	template <class T> bool stringToStrings(const string& s, T& tokens,
189	const string& addseps)	187	const string& addseps)
190	{	188	{
191	string current;	189	string current;
192	tokens.clear();	190	tokens.clear();
193	enum states {SPACE, TOKEN, INQUOTE, ESCAPE};	191	enum states {SPACE, TOKEN, INQUOTE, ESCAPE};
194	states state = SPACE;	192	states state = SPACE;
195	for (unsigned int i = 0; i < s.length(); i++) {	193	for (unsigned int i = 0; i < s.length(); i++) {
196	switch (s[i]) {	194	switch (s[i]) {
197	case '"':	195	case '"':
198	switch(state) {	196	switch (state) {
199	case SPACE:	197	case SPACE:
200	state=INQUOTE; continue;	198	state = INQUOTE;
		199	continue;
201	case TOKEN:	200	case TOKEN:
202	current += '"';	201	current += '"';
203	continue;	202	continue;
204	case INQUOTE:	203	case INQUOTE:
205	tokens.insert(tokens.end(), current);	204	tokens.insert(tokens.end(), current);
206	current.clear();	205	current.clear();
207	state = SPACE;	206	state = SPACE;
208	continue;	207	continue;
209	case ESCAPE:	208	case ESCAPE:
210	current += '"';	209	current += '"';
211	state = INQUOTE;	210	state = INQUOTE;
212	continue;	211	continue;
213	}	212	}
214	break;	213	break;
215	case '\\':	214	case '\\':
216	switch(state) {	215	switch (state) {
217	case SPACE:	216	case SPACE:
218	case TOKEN:	217	case TOKEN:
219	current += '\\';	218	current += '\\';
220	state=TOKEN;	219	state = TOKEN;
221	continue;	220	continue;
222	case INQUOTE:	221	case INQUOTE:
223	state = ESCAPE;	222	state = ESCAPE;
224	continue;	223	continue;
225	case ESCAPE:	224	case ESCAPE:
226	current += '\\';	225	current += '\\';
227	state = INQUOTE;	226	state = INQUOTE;
228	continue;	227	continue;
229	}	228	}
230	break;	229	break;
231		230
232	case ' ':	231	case ' ':
233	case '\t':	232	case '\t':
234	case '\n':	233	case '\n':
235	case '\r':	234	case '\r':
236	switch(state) {	235	switch (state) {
237	case SPACE:	236	case SPACE:
238	continue;	237	continue;
239	case TOKEN:	238	case TOKEN:
240	tokens.insert(tokens.end(), current);	239	tokens.insert(tokens.end(), current);
241	current.clear();	240	current.clear();
242	state = SPACE;	241	state = SPACE;
243	continue;	242	continue;
244	case INQUOTE:	243	case INQUOTE:
245	case ESCAPE:	244	case ESCAPE:
246	current += s[i];	245	current += s[i];
247	continue;	246	continue;
248	}	247	}
249	break;	248	break;
250		249
251	default:	250	default:
252	if (!addseps.empty() && addseps.find(s[i]) != string::npos) {	251	if (!addseps.empty() && addseps.find(s[i]) != string::npos) {
253	switch(state) {	252	switch (state) {
254	case ESCAPE:	253	case ESCAPE:
255	state = INQUOTE;	254	state = INQUOTE;
256	break;	255	break;
257	case INQUOTE:	256	case INQUOTE:
258	break;	257	break;
259	case SPACE:	258	case SPACE:
260	tokens.insert(tokens.end(), string(1, s[i]));	259	tokens.insert(tokens.end(), string(1, s[i]));
261	continue;	260	continue;
262	case TOKEN:	261	case TOKEN:
263	tokens.insert(tokens.end(), current);	262	tokens.insert(tokens.end(), current);
264	current.erase();	263	current.erase();
265	tokens.insert(tokens.end(), string(1, s[i]));	264	tokens.insert(tokens.end(), string(1, s[i]));
266	state = SPACE;	265	state = SPACE;
267	continue;	266	continue;
268	}	267	}
269	} else switch(state) {	268	} else switch (state) {
270	case ESCAPE:	269	case ESCAPE:
271	state = INQUOTE;	270	state = INQUOTE;
272	break;	271	break;
273	case SPACE:	272	case SPACE:
274	state = TOKEN;	273	state = TOKEN;
275	break;	274	break;
276	case TOKEN:	275	case TOKEN:
277	case INQUOTE:	276	case INQUOTE:
278	break;	277	break;
279	}	278	}
280	current += s[i];	279	current += s[i];
281	}
282	}	280	}
		281	}
283	switch(state) {	282	switch (state) {
284	case SPACE:	283	case SPACE:
285	break;	284	break;
286	case TOKEN:	285	case TOKEN:
287	tokens.insert(tokens.end(), current);	286	tokens.insert(tokens.end(), current);
288	break;	287	break;
289	case INQUOTE:	288	case INQUOTE:
290	case ESCAPE:	289	case ESCAPE:
291	return false;	290	return false;
292	}	291	}
293	return true;	292	return true;
294	}	293	}
295		294
296	template bool stringToStrings<list<string> >(const string &,	295	template bool stringToStrings<list<string> >(const string&,
297	list<string> &, const string&);	296	list<string>&, const string&);
298	template bool stringToStrings<vector<string> >(const string &,	297	template bool stringToStrings<vector<string> >(const string&,
299	vector<string> &,const string&);	298	vector<string>&, const string&);
300	template bool stringToStrings<set<string> >(const string &,	299	template bool stringToStrings<set<string> >(const string&,
301	set<string> &, const string&);	300	set<string>&, const string&);
302	template bool stringToStrings<STD_UNORDERED_SET<string> >	301	template bool stringToStrings<STD_UNORDERED_SET<string> >
303	(const string &, STD_UNORDERED_SET<string> &, const string&);	302	(const string&, STD_UNORDERED_SET<string>&, const string&);
304		303
305	template <class T> void stringsToString(const T &tokens, string &s)	304	template <class T> void stringsToString(const T& tokens, string& s)
306	{	305	{
307	for (typename T::const_iterator it = tokens.begin();	306	for (typename T::const_iterator it = tokens.begin();
308	it != tokens.end(); it++) {	307	it != tokens.end(); it++) {
309	bool hasblanks = false;	308	bool hasblanks = false;
310	if (it->find_first_of(" \t\n") != string::npos)	309	if (it->find_first_of(" \t\n") != string::npos) {
311	hasblanks = true;	310	hasblanks = true;
		311	}
312	if (it != tokens.begin())	312	if (it != tokens.begin()) {
313	s.append(1, ' ');	313	s.append(1, ' ');
314	if (hasblanks)	314	}
		315	if (hasblanks) {
315	s.append(1, '"');	316	s.append(1, '"');
		317	}
316	for (unsigned int i = 0; i < it->length(); i++) {	318	for (unsigned int i = 0; i < it->length(); i++) {
317	char car = it->at(i);	319	char car = it->at(i);
318	if (car == '"') {	320	if (car == '"') {
319	s.append(1, '\\');	321	s.append(1, '\\');
320	s.append(1, car);	322	s.append(1, car);
321	} else {	323	} else {
322	s.append(1, car);	324	s.append(1, car);
		325	}
		326	}
		327	if (hasblanks) {
		328	s.append(1, '"');
		329	}
323	}	330	}
324	}	331	}
325	if (hasblanks)
326	s.append(1, '"');
327	}
328	}
329	template void stringsToString<list<string> >(const list<string> &, string &);	332	template void stringsToString<list<string> >(const list<string>&, string&);
330	template void stringsToString<vector<string> >(const vector<string> &,string &);	333	template void stringsToString<vector<string> >(const vector<string>&, string&);
331	template void stringsToString<set<string> >(const set<string> &, string &);	334	template void stringsToString<set<string> >(const set<string>&, string&);
332	template <class T> string stringsToString(const T &tokens)	335	template <class T> string stringsToString(const T& tokens)
333	{	336	{
334	string out;	337	string out;
335	stringsToString<T>(tokens, out);	338	stringsToString<T>(tokens, out);
336	return out;	339	return out;
337	}	340	}
338	template string stringsToString<list<string> >(const list<string> &);	341	template string stringsToString<list<string> >(const list<string>&);
339	template string stringsToString<vector<string> >(const vector<string> &);	342	template string stringsToString<vector<string> >(const vector<string>&);
340	template string stringsToString<set<string> >(const set<string> &);	343	template string stringsToString<set<string> >(const set<string>&);
341		344
342	template <class T> void stringsToCSV(const T &tokens, string &s,	345	template <class T> void stringsToCSV(const T& tokens, string& s,
343	char sep)	346	char sep)
344	{	347	{
345	s.erase();	348	s.erase();
346	for (typename T::const_iterator it = tokens.begin();	349	for (typename T::const_iterator it = tokens.begin();
347	it != tokens.end(); it++) {	350	it != tokens.end(); it++) {
348	bool needquotes = false;	351	bool needquotes = false;
349	if (it->empty() \|\|	352	if (it->empty() \|\|
350	it->find_first_of(string(1, sep) + "\"\n") != string::npos)	353	it->find_first_of(string(1, sep) + "\"\n") != string::npos) {
351	needquotes = true;	354	needquotes = true;
		355	}
352	if (it != tokens.begin())	356	if (it != tokens.begin()) {
353	s.append(1, sep);	357	s.append(1, sep);
354	if (needquotes)	358	}
		359	if (needquotes) {
355	s.append(1, '"');	360	s.append(1, '"');
		361	}
356	for (unsigned int i = 0; i < it->length(); i++) {	362	for (unsigned int i = 0; i < it->length(); i++) {
357	char car = it->at(i);	363	char car = it->at(i);
358	if (car == '"') {	364	if (car == '"') {
359	s.append(2, '"');	365	s.append(2, '"');
360	} else {	366	} else {
361	s.append(1, car);	367	s.append(1, car);
		368	}
		369	}
		370	if (needquotes) {
		371	s.append(1, '"');
		372	}
362	}	373	}
363	}	374	}
364	if (needquotes)
365	s.append(1, '"');
366	}
367	}
368	template void stringsToCSV<list<string> >(const list<string> &, string &, char);	375	template void stringsToCSV<list<string> >(const list<string>&, string&, char);
369	template void stringsToCSV<vector<string> >(const vector<string> &,string &,	376	template void stringsToCSV<vector<string> >(const vector<string>&, string&,
370	char);	377	char);
371		378
372	void stringToTokens(const string& str, vector<string>& tokens,	379	void stringToTokens(const string& str, vector<string>& tokens,
373	const string& delims, bool skipinit)	380	const string& delims, bool skipinit)
374	{	381	{
375	string::size_type startPos = 0, pos;	382	string::size_type startPos = 0, pos;
376		383
377	// Skip initial delims, return empty if this eats all.	384	// Skip initial delims, return empty if this eats all.
378	if (skipinit &&	385	if (skipinit &&
379	(startPos = str.find_first_not_of(delims, 0)) == string::npos) {	386	(startPos = str.find_first_not_of(delims, 0)) == string::npos) {
380	return;	387	return;
381	}	388	}
382	while (startPos < str.size()) {	389	while (startPos < str.size()) {
383	// Find next delimiter or end of string (end of token)	390	// Find next delimiter or end of string (end of token)
384	pos = str.find_first_of(delims, startPos);	391	pos = str.find_first_of(delims, startPos);
385		392
386	// Add token to the vector and adjust start	393	// Add token to the vector and adjust start
387	if (pos == string::npos) {	394	if (pos == string::npos) {
388	tokens.push_back(str.substr(startPos));	395	tokens.push_back(str.substr(startPos));
389	break;	396	break;
390	} else if (pos == startPos) {	397	} else if (pos == startPos) {
391	// Dont' push empty tokens after first	398	// Dont' push empty tokens after first
392	if (tokens.empty())	399	if (tokens.empty()) {
393	tokens.push_back(string());	400	tokens.push_back(string());
		401	}
394	startPos = ++pos;	402	startPos = ++pos;
395	} else {	403	} else {
396	tokens.push_back(str.substr(startPos, pos - startPos));	404	tokens.push_back(str.substr(startPos, pos - startPos));
397	startPos = ++pos;	405	startPos = ++pos;
		406	}
		407	}
398	}	408	}
399	}
400	}
401		409
402	bool stringToBool(const string &s)	410	bool stringToBool(const string& s)
403	{	411	{
404	if (s.empty())	412	if (s.empty()) {
405	return false;	413	return false;
		414	}
406	if (isdigit(s[0])) {	415	if (isdigit(s[0])) {
407	int val = atoi(s.c_str());	416	int val = atoi(s.c_str());
408	return val ? true : false;	417	return val ? true : false;
409	}	418	}
410	if (s.find_first_of("yYtT") == 0)	419	if (s.find_first_of("yYtT") == 0) {
411	return true;	420	return true;
		421	}
412	return false;	422	return false;
413	}	423	}
414		424
415	void trimstring(string &s, const char *ws)	425	void trimstring(string& s, const char *ws)
416	{	426	{
417	string::size_type pos = s.find_first_not_of(ws);	427	string::size_type pos = s.find_first_not_of(ws);
418	if (pos == string::npos) {	428	if (pos == string::npos) {
419	s.clear();	429	s.clear();
420	return;	430	return;
421	}	431	}
422	s.replace(0, pos, string());	432	s.replace(0, pos, string());
423		433
424	pos = s.find_last_not_of(ws);	434	pos = s.find_last_not_of(ws);
425	if (pos != string::npos && pos != s.length()-1)	435	if (pos != string::npos && pos != s.length() - 1) {
426	s.replace(pos+1, string::npos, string());	436	s.replace(pos + 1, string::npos, string());
		437	}
427	}	438	}
428		439
429	// Remove some chars and replace them with spaces	440	// Remove some chars and replace them with spaces
430	string neutchars(const string &str, const string &chars)	441	string neutchars(const string& str, const string& chars)
431	{	442	{
432	string out;	443	string out;
433	neutchars(str, out, chars);	444	neutchars(str, out, chars);
434	return out;	445	return out;
435	}	446	}
436	void neutchars(const string &str, string &out, const string& chars)	447	void neutchars(const string& str, string& out, const string& chars)
437	{	448	{
438	string::size_type startPos, pos;	449	string::size_type startPos, pos;
439		450
440	for (pos = 0;;) {	451	for (pos = 0;;) {
441	// Skip initial chars, break if this eats all.	452	// Skip initial chars, break if this eats all.
442	if ((startPos = str.find_first_not_of(chars, pos)) == string::npos)	453	if ((startPos = str.find_first_not_of(chars, pos)) == string::npos) {
443	break;	454	break;
		455	}
444	// Find next delimiter or end of string (end of token)	456	// Find next delimiter or end of string (end of token)
445	pos = str.find_first_of(chars, startPos);	457	pos = str.find_first_of(chars, startPos);
446	// Add token to the output. Note: token cant be empty here	458	// Add token to the output. Note: token cant be empty here
447	if (pos == string::npos) {	459	if (pos == string::npos) {
448	out += str.substr(startPos);	460	out += str.substr(startPos);
449	} else {	461	} else {
450	out += str.substr(startPos, pos - startPos) + " ";	462	out += str.substr(startPos, pos - startPos) + " ";
451	}	463	}
452	}	464	}
453	}	465	}
454		466
455		467
456	/* Truncate a string to a given maxlength, avoiding cutting off midword	468	/* Truncate a string to a given maxlength, avoiding cutting off midword
457	* if reasonably possible. Note: we could also use textsplit, stopping when	469	* if reasonably possible. Note: we could also use textsplit, stopping when
458	* we have enough, this would be cleanly utf8-aware but would remove	470	* we have enough, this would be cleanly utf8-aware but would remove
459	* punctuation */	471	* punctuation */
460	static const string cstr_SEPAR = " \t\n\r-:.;,/[]{}";	472	static const string cstr_SEPAR = " \t\n\r-:.;,/[]{}";
461	string truncate_to_word(const string &input, string::size_type maxlen)	473	string truncate_to_word(const string& input, string::size_type maxlen)
462	{	474	{
463	string output;	475	string output;
464	if (input.length() <= maxlen) {	476	if (input.length() <= maxlen) {
465	output = input;	477	output = input;
466	} else {	478	} else {
467	output = input.substr(0, maxlen);	479	output = input.substr(0, maxlen);
468	string::size_type space = output.find_last_of(cstr_SEPAR);	480	string::size_type space = output.find_last_of(cstr_SEPAR);
469	// Original version only truncated at space if space was found after	481	// Original version only truncated at space if space was found after
470	// maxlen/2. But we HAVE to truncate at space, else we'd need to do	482	// maxlen/2. But we HAVE to truncate at space, else we'd need to do
471	// utf8 stuff to avoid truncating at multibyte char. In any case,	483	// utf8 stuff to avoid truncating at multibyte char. In any case,
472	// not finding space means that the text probably has no value.	484	// not finding space means that the text probably has no value.
473	// Except probably for Asian languages, so we may want to fix this	485	// Except probably for Asian languages, so we may want to fix this
474	// one day	486	// one day
475	if (space == string::npos) {	487	if (space == string::npos) {