/* Copyright (C) 2004 J.F.Dockes
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the
* Free Software Foundation, Inc.,
* 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
*/
#ifndef TEST_TEXTSPLIT
#include "autoconfig.h"
#include <assert.h>
#include <stdlib.h>
#include <iostream>
#include <string>
#include <algorithm>
#include <cstring>
#include UNORDERED_SET_INCLUDE
#include "textsplit.h"
#include "debuglog.h"
//#define UTF8ITER_CHECK
#include "utf8iter.h"
#include "uproplist.h"
using namespace std;
/**
* Splitting a text into words. The code in this file works with utf-8
* in a semi-clean way (see uproplist.h). Ascii still gets special
* treatment in the sense that many special characters can only be
* ascii (e.g. @, _,...). However, this compromise works quite well
* while being much more light-weight than a full-blown Unicode
* approach (ICU...)
*/
// Ascii character classes: we have three main groups, and then some chars
// are their own class because they want special handling.
//
// We have an array with 256 slots where we keep the character types.
// The array could be fully static, but we use a small function to fill it
// once.
// The array is actually a remnant of the original version which did no utf8.
// Only the lower 127 slots are now used, but keep it at 256
// because it makes some tests in the code simpler.
const unsigned int charclasses_size = 256;
enum CharClass {LETTER=256, SPACE=257, DIGIT=258, WILD=259,
A_ULETTER=260, A_LLETTER=261, SKIP=262};
static int charclasses[charclasses_size];
// Non-ascii UTF-8 characters are handled with sets holding all
// characters with interesting properties. This is far from full-blown
// management of Unicode properties, but seems to do the job well
// enough in most common cases
static vector<unsigned int> vpuncblocks;
static STD_UNORDERED_SET<unsigned int> spunc;
static STD_UNORDERED_SET<unsigned int> visiblewhite;
static STD_UNORDERED_SET<unsigned int> sskip;
class CharClassInit {
public:
CharClassInit()
{
unsigned int i;
// Set default value for all: SPACE
for (i = 0 ; i < 256 ; i ++)
charclasses[i] = SPACE;
char digits[] = "0123456789";
for (i = 0; i < strlen(digits); i++)
charclasses[int(digits[i])] = DIGIT;
char upper[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZ";
for (i = 0; i < strlen(upper); i++)
charclasses[int(upper[i])] = A_ULETTER;
char lower[] = "abcdefghijklmnopqrstuvwxyz";
for (i = 0; i < strlen(lower); i++)
charclasses[int(lower[i])] = A_LLETTER;
char wild[] = "*?[]";
for (i = 0; i < strlen(wild); i++)
charclasses[int(wild[i])] = WILD;
// Characters with special treatment:
//
// The first ones are mostly span-constructing "glue"
// characters, for example those typically allowing us to
// search for an email address as a whole (bob@isp.org instead
// of as a phrase "bob isp org"
//
// The case of the minus sign is a complicated one. It went
// from glue to non-glue to glue along Recoll versions.
// See minus-hyphen-dash.txt in doc/notes
char special[] = ".@+-#'_\n\r\f";
for (i = 0; i < strlen(special); i++)
charclasses[int(special[i])] = special[i];
for (i = 0; i < sizeof(unipunc) / sizeof(int); i++) {
spunc.insert(unipunc[i]);
}
spunc.insert((unsigned int)-1);
for (i = 0; i < sizeof(unipuncblocks) / sizeof(int); i++) {
vpuncblocks.push_back(unipuncblocks[i]);
}
assert((vpuncblocks.size() % 2) == 0);
for (i = 0; i < sizeof(avsbwht) / sizeof(int); i++) {
visiblewhite.insert(avsbwht[i]);
}
for (i = 0; i < sizeof(uniskip) / sizeof(int); i++) {
sskip.insert(uniskip[i]);
}
}
};
static const CharClassInit charClassInitInstance;
static inline int whatcc(unsigned int c)
{
if (c <= 127) {
return charclasses[c];
} else {
if (c == 0x2010) {
// Special treatment for hyphen: handle as ascii minus. See
// doc/notes/minus-hyphen-dash.txt
return 0x2010;
} else if (sskip.find(c) != sskip.end()) {
return SKIP;
} else if (spunc.find(c) != spunc.end()) {
return SPACE;
} else {
vector<unsigned int>::iterator it =
lower_bound(vpuncblocks.begin(), vpuncblocks.end(), c);
if (it == vpuncblocks.end())
return LETTER;
if (c == *it)
return SPACE;
if ((it - vpuncblocks.begin()) % 2 == 1) {
return SPACE;
} else {
return LETTER;
}
}
}
}
// testing whatcc...
#if 0
unsigned int testvalues[] = {'a', '0', 0x80, 0xbf, 0xc0, 0x05c3, 0x1000,
0x2000, 0x2001, 0x206e, 0x206f, 0x20d0, 0x2399,
0x2400, 0x2401, 0x243f, 0x2440, 0xff65};
int ntest = sizeof(testvalues) / sizeof(int);
for (int i = 0; i < ntest; i++) {
int ret = whatcc(testvalues[i]);
printf("Tested value 0x%x, returned value %d %s\n",
testvalues[i], ret, ret == LETTER ? "LETTER" :
ret == SPACE ? "SPACE" : "OTHER");
}
#endif
// CJK Unicode character detection:
//
// 2E80..2EFF; CJK Radicals Supplement
// 3000..303F; CJK Symbols and Punctuation
// 3040..309F; Hiragana
// 30A0..30FF; Katakana
// 3100..312F; Bopomofo
// 3130..318F; Hangul Compatibility Jamo
// 3190..319F; Kanbun
// 31A0..31BF; Bopomofo Extended
// 31C0..31EF; CJK Strokes
// 31F0..31FF; Katakana Phonetic Extensions
// 3200..32FF; Enclosed CJK Letters and Months
// 3300..33FF; CJK Compatibility
// 3400..4DBF; CJK Unified Ideographs Extension A
// 4DC0..4DFF; Yijing Hexagram Symbols
// 4E00..9FFF; CJK Unified Ideographs
// A700..A71F; Modifier Tone Letters
// AC00..D7AF; Hangul Syllables
// F900..FAFF; CJK Compatibility Ideographs
// FE30..FE4F; CJK Compatibility Forms
// FF00..FFEF; Halfwidth and Fullwidth Forms
// 20000..2A6DF; CJK Unified Ideographs Extension B
// 2F800..2FA1F; CJK Compatibility Ideographs Supplement
// Note: the p > 127 test is not necessary, but optimizes away the ascii case
#define UNICODE_IS_CJK(p) \
((p) > 127 && \
(((p) >= 0x2E80 && (p) <= 0x2EFF) || \
((p) >= 0x3000 && (p) <= 0x9FFF) || \
((p) >= 0xA700 && (p) <= 0xA71F) || \
((p) >= 0xAC00 && (p) <= 0xD7AF) || \
((p) >= 0xF900 && (p) <= 0xFAFF) || \
((p) >= 0xFE30 && (p) <= 0xFE4F) || \
((p) >= 0xFF00 && (p) <= 0xFFEF) || \
((p) >= 0x20000 && (p) <= 0x2A6DF) || \
((p) >= 0x2F800 && (p) <= 0x2FA1F)))
bool TextSplit::isCJK(int c)
{
return UNICODE_IS_CJK(c);
}
bool TextSplit::o_processCJK = true;
unsigned int TextSplit::o_CJKNgramLen = 2;
bool TextSplit::o_noNumbers = false;
bool TextSplit::o_deHyphenate = false;
// Final term checkpoint: do some checking (the kind which is simpler
// to do here than in the main loop), then send term to our client.
inline bool TextSplit::emitterm(bool isspan, string &w, int pos,
size_t btstart, size_t btend)
{
LOGDEB2(("TextSplit::emitterm: [%s] pos %d\n", w.c_str(), pos));
int l = int(w.length());
#ifdef TEXTSPLIT_STATS
// Update word length statistics. Do this before we filter out
// long words because stats are used to detect bad text
if (!isspan || m_wordLen == m_span.length())
m_stats.newsamp(m_wordChars);
#endif
if (l > 0 && l < m_maxWordLength) {
// 1 byte word: we index single ascii letters and digits, but
// nothing else. We might want to turn this into a test for a
// single utf8 character instead ?
if (l == 1) {
unsigned int c = ((unsigned int)w[0]) & 0xff;
if (charclasses[c] != A_ULETTER && charclasses[c] != A_LLETTER &&
charclasses[c] != DIGIT &&
(!(m_flags & TXTS_KEEPWILD) || charclasses[c] != WILD)
) {
//cerr << "ERASING single letter term " << c << endl;
return true;
}
}
if (pos != m_prevpos || l != m_prevlen) {
bool ret = takeword(w, pos, int(btstart), int(btend));
m_prevpos = pos;
m_prevlen = int(w.length());
return ret;
}
LOGDEB2(("TextSplit::emitterm:dup: [%s] pos %d\n", w.c_str(), pos));
}
return true;
}
// Check for an acronym/abbreviation ie I.B.M. This only works with
// ascii (no non-ascii utf-8 acronym are possible)
bool TextSplit::span_is_acronym(string *acronym)
{
bool acron = false;
if (m_wordLen != m_span.length() &&
m_span.length() > 2 && m_span.length() <= 20) {
acron = true;
// Check odd chars are '.'
for (unsigned int i = 1 ; i < m_span.length(); i += 2) {
if (m_span[i] != '.') {
acron = false;
break;
}
}
if (acron) {
// Check that even chars are letters
for (unsigned int i = 0 ; i < m_span.length(); i += 2) {
int c = m_span[i];
if (!((c >= 'a' && c <= 'z')||(c >= 'A' && c <= 'Z'))) {
acron = false;
break;
}
}
}
}
if (acron) {
for (unsigned int i = 0; i < m_span.length(); i += 2) {
*acronym += m_span[i];
}
}
return acron;
}
// Generate terms from span. Have to take into account the
// flags: ONLYSPANS, NOSPANS, noNumbers
bool TextSplit::words_from_span(size_t bp)
{
#if 0
cerr << "Span: [" << m_span << "] " << " w_i_s size: " <<
m_words_in_span.size() << " : ";
for (unsigned int i = 0; i < m_words_in_span.size(); i++) {
cerr << " [" << m_words_in_span[i].first << " " <<
m_words_in_span[i].second << "] ";
}
cerr << endl;
#endif
int spanwords = int(m_words_in_span.size());
int pos = m_spanpos;
// Byte position of the span start
size_t spboffs = bp - m_span.size();
if (o_deHyphenate && spanwords == 2 &&
m_span[m_words_in_span[0].second] == '-') {
unsigned int s0 = m_words_in_span[0].first;
unsigned int l0 = m_words_in_span[0].second - m_words_in_span[0].first;
unsigned int s1 = m_words_in_span[1].first;
unsigned int l1 = m_words_in_span[1].second - m_words_in_span[1].first;
string word = m_span.substr(s0, l0) + m_span.substr(s1, l1);
if (l0 && l1)
emitterm(false, word,
m_spanpos, spboffs, spboffs + m_words_in_span[1].second);
}
for (int i = 0;
i < ((m_flags&TXTS_ONLYSPANS) ? 1 : spanwords);
i++, pos++) {
int deb = m_words_in_span[i].first;
for (int j = ((m_flags&TXTS_ONLYSPANS) ? spanwords-1 : i);
j < ((m_flags&TXTS_NOSPANS) ? i+1 : spanwords);
j++) {
int fin = m_words_in_span[j].second;
//cerr << "i " << i << " j " << j << " deb " << deb <<
// " fin " << fin << endl;
if (fin - deb > int(m_span.size()))
break;
string word(m_span.substr(deb, fin-deb));
if (!emitterm(j != i+1, word, pos, spboffs+deb, spboffs+fin))
return false;
}
}
return true;
}
/**
* A method called at word boundaries (different places in
* text_to_words()), to adjust the current state of the parser, and
* possibly generate term(s). While inside a span (words linked by
* glue characters), we just keep track of the word boundaries. Once
* actual white-space is reached, we get called with spanerase set to
* true, and we process the span, calling the emitterm() routine for
* each generated term.
*
* The object flags can modify our behaviour, deciding if we only emit
* single words (bill, recoll, org), only spans (bill@recoll.org), or
* words and spans (bill@recoll.org, recoll.org, jf, recoll...)
*
* @return true if ok, false for error. Splitting should stop in this case.
* @param spanerase Set if the current span is at its end. Process it.
* @param bp The current BYTE position in the stream
*/
inline bool TextSplit::doemit(bool spanerase, size_t _bp)
{
int bp = int(_bp);
LOGDEB2(("TextSplit::doemit: sper %d bp %d spp %d spanwords %u wS %d wL %d "
"inn %d span [%s]\n",
spanerase, bp, m_spanpos, m_words_in_span.size(),
m_wordStart, m_wordLen, m_inNumber, m_span.c_str()));
if (m_wordLen) {
// We have a current word. Remember it
// Limit max span word count
if (m_words_in_span.size() >= 6) {
spanerase = true;
}
m_words_in_span.push_back(pair<int,int>(m_wordStart,
m_wordStart + m_wordLen));
m_wordpos++;
m_wordLen = m_wordChars = 0;
}
if (spanerase) {
// We encountered a span-terminating character. Produce terms.
string acronym;
if (span_is_acronym(&acronym)) {
if (!emitterm(false, acronym, m_spanpos, bp - m_span.length(), bp))
return false;
}
// Maybe trim at end. These are chars that we might keep
// inside a span, but not at the end.
while (m_span.length() > 0) {
switch (*(m_span.rbegin())) {
case '.':
case '-':
case ',':
case '@':
case '_':
case '\'':
m_span.resize(m_span.length()-1);
if (m_words_in_span.size() &&
m_words_in_span.back().second > int(m_span.size()))
m_words_in_span.back().second = int(m_span.size());
if (--bp < 0)
bp = 0;
break;
default:
goto breaktrimloop;
}
}
breaktrimloop:
if (!words_from_span(bp)) {
return false;
}
discardspan();
} else {
m_wordStart = int(m_span.length());
}
return true;
}
void TextSplit::discardspan()
{
m_words_in_span.clear();
m_span.erase();
m_spanpos = m_wordpos;
m_wordStart = 0;
m_wordLen = m_wordChars = 0;
}
static inline bool isalphanum(int what, unsigned int flgs)
{
return what == A_LLETTER || what == A_ULETTER ||
what == DIGIT || what == LETTER ||
((flgs & TextSplit::TXTS_KEEPWILD) && what == WILD);
}
static inline bool isdigit(int what, unsigned int flgs)
{
return what == DIGIT || ((flgs & TextSplit::TXTS_KEEPWILD) && what == WILD);
}
#ifdef TEXTSPLIT_STATS
#define STATS_INC_WORDCHARS ++m_wordChars
#else
#define STATS_INC_WORDCHARS
#endif
/**
* Splitting a text into terms to be indexed.
* We basically emit a word every time we see a separator, but some chars are
* handled specially so that special cases, ie, c++ and jfd@recoll.com etc,
* are handled properly,
*/
bool TextSplit::text_to_words(const string &in)
{
LOGDEB1(("TextSplit::text_to_words: docjk %d (%d) %s%s%s [%s]\n",
o_processCJK, o_CJKNgramLen,
m_flags & TXTS_NOSPANS ? " nospans" : "",
m_flags & TXTS_ONLYSPANS ? " onlyspans" : "",
m_flags & TXTS_KEEPWILD ? " keepwild" : "",
in.substr(0,50).c_str()));
if (in.empty())
return true;
m_span.erase();
m_inNumber = false;
m_wordStart = m_wordLen = m_wordChars = m_prevpos = m_prevlen = m_wordpos
= m_spanpos = 0;
bool pagepending = false;
bool softhyphenpending = false;
// Running count of non-alphanum chars. Reset when we see one;
int nonalnumcnt = 0;
Utf8Iter it(in);
for (; !it.eof(); it++) {
unsigned int c = *it;
nonalnumcnt++;
if (c == (unsigned int)-1) {
LOGERR(("Textsplit: error occured while scanning UTF-8 string\n"));
return false;
}
if (o_processCJK && UNICODE_IS_CJK(c)) {
// CJK character hit.
// Do like at EOF with the current non-cjk data.
if (m_wordLen || m_span.length()) {
if (!doemit(true, it.getBpos()))
return false;
}
// Hand off situation to the cjk routine.
if (!cjk_to_words(&it, &c)) {
LOGERR(("Textsplit: scan error in cjk handler\n"));
return false;
}
// Check for eof, else c contains the first non-cjk
// character after the cjk sequence, just go on.
if (it.eof())
break;
}
int cc = whatcc(c);
switch (cc) {
case SKIP:
// Special-case soft-hyphen. To work, this depends on the
// fact that only SKIP calls "continue" inside the
// switch. All the others will do the softhyphenpending
// reset after the switch
if (c == 0xad) {
softhyphenpending = true;
} else {
softhyphenpending = false;
}
// Skips the softhyphenpending reset
continue;
case DIGIT:
nonalnumcnt = 0;
if (m_wordLen == 0)
m_inNumber = true;
m_wordLen += it.appendchartostring(m_span);
STATS_INC_WORDCHARS;
break;
case SPACE:
nonalnumcnt = 0;
SPACE:
if (m_wordLen || m_span.length()) {
if (!doemit(true, it.getBpos()))
return false;
m_inNumber = false;
}
if (pagepending) {
pagepending = false;
newpage(m_wordpos);
}
break;
case WILD:
if (m_flags & TXTS_KEEPWILD)
goto NORMALCHAR;
else
goto SPACE;
break;
case '-':
case '+':
if (m_wordLen == 0) {
// + or - don't start a term except if this looks like
// it's going to be to be a number
if (isdigit(whatcc(it[it.getCpos()+1]), m_flags)) {
// -10
m_inNumber = true;
m_wordLen += it.appendchartostring(m_span);
STATS_INC_WORDCHARS;
break;
}
} else if (m_inNumber) {
if ((m_span[m_span.length() - 1] == 'e' ||
m_span[m_span.length() - 1] == 'E')) {
if (isdigit(whatcc(it[it.getCpos()+1]), m_flags)) {
m_wordLen += it.appendchartostring(m_span);
STATS_INC_WORDCHARS;
break;
}
}
} else {
if (cc == '+') {
int nextc = it[it.getCpos()+1];
if (nextc == '+' || nextc == -1 || visiblewhite.find(nextc)
!= visiblewhite.end()) {
// someword++[+...] !
m_wordLen += it.appendchartostring(m_span);
STATS_INC_WORDCHARS;
break;
}
} else {
// Treat '-' inside span as glue char
if (!doemit(false, it.getBpos()))
return false;
m_inNumber = false;
m_wordStart += it.appendchartostring(m_span);
break;
}
}
goto SPACE;
break;
case 0x2010:
// Hyphen is replaced with ascii minus
if (m_wordLen != 0) {
// Treat '-' inside span as glue char
if (!doemit(false, it.getBpos()))
return false;
m_inNumber = false;
m_span += '-';
m_wordStart++;
break;
}
goto SPACE;
case '.':
{
// Need a little lookahead here. At worse this gets the end null
int nextc = it[it.getCpos()+1];
int nextwhat = whatcc(nextc);
if (m_inNumber) {
if (!isdigit(nextwhat, m_flags))
goto SPACE;
m_wordLen += it.appendchartostring(m_span);
STATS_INC_WORDCHARS;
break;
} else {
// Found '.' while not in number
// Only letters and digits make sense after
if (!isalphanum(nextwhat, m_flags))
goto SPACE;
// Keep an initial '.' for catching .net, and .34 (aka
// 0.34) but this adds quite a few spurious terms !
if (m_span.length() == 0) {
// Check for number like .1
if (isdigit(nextwhat, m_flags)) {
m_inNumber = true;
}
m_wordLen += it.appendchartostring(m_span);
STATS_INC_WORDCHARS;
break;
}
// '.' between words: span glue
if (m_wordLen) {
if (!doemit(false, it.getBpos()))
return false;
m_wordStart += it.appendchartostring(m_span);
}
}
}
break;
case '@':
case '_':
case '\'':
// If in word, potential span: o'brien, jf@dockes.org,
// else just ignore
if (m_wordLen) {
if (!doemit(false, it.getBpos()))
return false;
m_inNumber = false;
m_wordStart += it.appendchartostring(m_span);
}
break;
case '#': {
int w = whatcc(it[it.getCpos()+1]);
// Keep it only at the beginning of a word (hashtag),
if (m_wordLen == 0 && isalphanum(w, m_flags)) {
m_wordLen += it.appendchartostring(m_span);
STATS_INC_WORDCHARS;
break;
}
// or at the end (special case for c# ...)
if (m_wordLen > 0) {
if (w == SPACE || w == '\n' || w == '\r') {
m_wordLen += it.appendchartostring(m_span);
STATS_INC_WORDCHARS;
break;
}
}
goto SPACE;
}
break;
case '\n':
case '\r':
if (m_span.length() && *m_span.rbegin() == '-') {
// if '-' is the last char before end of line, we
// strip it. We have no way to know if this is added
// because of the line split or if it was part of an
// actual compound word (would need a dictionary to
// check). As soft-hyphen *should* be used if the '-'
// is not part of the text, it is better to properly
// process a real compound word, and produce wrong
// output from wrong text. The word-emitting routine
// will strip the trailing '-'.
goto SPACE;
} else if (softhyphenpending) {
// Don't reset soft-hyphen
continue;
} else {
// Normal case: EOL is white space
goto SPACE;
}
break;
case '\f':
pagepending = true;
goto SPACE;
break;
#ifdef RCL_SPLIT_CAMELCASE
// Camelcase handling.
// If we get uppercase ascii after lowercase ascii, emit word.
// This emits "camel" when hitting the 'C' of camelCase
// Not enabled by defaults as this makes phrase searches quite
// confusing.
// ie "MySQL manual" is matched by "MySQL manual" and
// "my sql manual" but not "mysql manual"
// A possibility would be to emit both my and sql at the
// same position. All non-phrase searches would work, and
// both "MySQL manual" and "mysql manual" phrases would
// match too. "my sql manual" would not match, but this is
// not an issue.
case A_ULETTER:
if (m_span.length() &&
charclasses[(unsigned char)m_span[m_span.length() - 1]] ==
A_LLETTER) {
if (m_wordLen) {
if (!doemit(false, it.getBpos()))
return false;
}
}
goto NORMALCHAR;
// CamelCase handling.
// If we get lowercase after uppercase and the current
// word length is bigger than one, it means we had a
// string of several upper-case letters: an
// acronym (readHTML) or a single letter article (ALittleHelp).
// Emit the uppercase word before proceeding
case A_LLETTER:
if (m_span.length() &&
charclasses[(unsigned char)m_span[m_span.length() - 1]] ==
A_ULETTER && m_wordLen > 1) {
// Multiple upper-case letters. Single letter word
// or acronym which we want to emit now
m_wordLen--;
if (!doemit(false, it.getBpos()))
return false;
// m_wordstart could be 0 here if the span was reset
// for excessive length
if (m_wordStart)
m_wordStart--;
m_wordLen++;
}
goto NORMALCHAR;
#endif /* CAMELCASE */
default:
NORMALCHAR:
nonalnumcnt = 0;
if (m_inNumber && c != 'e' && c != 'E') {
m_inNumber = false;
}
m_wordLen += it.appendchartostring(m_span);
STATS_INC_WORDCHARS;
break;
}
softhyphenpending = false;
}
if (m_wordLen || m_span.length()) {
if (!doemit(true, it.getBpos()))
return false;
}
return true;
}
// Using an utf8iter pointer just to avoid needing its definition in
// textsplit.h
//
// We output ngrams for exemple for char input a b c and ngramlen== 2,
// we generate: a ab b bc c as words
//
// This is very different from the normal behaviour, so we don't use
// the doemit() and emitterm() routines
//
// The routine is sort of a mess and goes to show that we'd probably
// be better off converting the whole buffer to utf32 on entry...
bool TextSplit::cjk_to_words(Utf8Iter *itp, unsigned int *cp)
{
LOGDEB1(("cjk_to_words: m_wordpos %d\n", m_wordpos));
Utf8Iter &it = *itp;
// We use an offset buffer to remember the starts of the utf-8
// characters which we still need to use.
assert(o_CJKNgramLen < o_CJKMaxNgramLen);
unsigned int boffs[o_CJKMaxNgramLen+1];
// Current number of valid offsets;
unsigned int nchars = 0;
unsigned int c = 0;
for (; !it.eof(); it++) {
c = *it;
if (!UNICODE_IS_CJK(c)) {
// Return to normal handler
break;
}
if (whatcc(c) == SPACE) {
// Flush the ngram buffer and go on
nchars = 0;
continue;
}
if (nchars == o_CJKNgramLen) {
// Offset buffer full, shift it. Might be more efficient
// to have a circular one, but things are complicated
// enough already...
for (unsigned int i = 0; i < nchars-1; i++) {
boffs[i] = boffs[i+1];
}
} else {
nchars++;
}
// Take note of byte offset for this character.
boffs[nchars-1] = int(it.getBpos());
// Output all new ngrams: they begin at each existing position
// and end after the new character. onlyspans->only output
// maximum words, nospans=> single chars
if (!(m_flags & TXTS_ONLYSPANS) || nchars == o_CJKNgramLen) {
int btend = int(it.getBpos() + it.getBlen());
int loopbeg = (m_flags & TXTS_NOSPANS) ? nchars-1 : 0;
int loopend = (m_flags & TXTS_ONLYSPANS) ? 1 : nchars;
for (int i = loopbeg; i < loopend; i++) {
if (!takeword(it.buffer().substr(boffs[i],
btend-boffs[i]),
m_wordpos - (nchars-i-1), boffs[i], btend)) {
return false;
}
}
if ((m_flags & TXTS_ONLYSPANS)) {
// Only spans: don't overlap: flush buffer
nchars = 0;
}
}
// Increase word position by one, other words are at an
// existing position. This could be subject to discussion...
m_wordpos++;
}
// If onlyspans is set, there may be things to flush in the buffer
// first
if ((m_flags & TXTS_ONLYSPANS) && nchars > 0 && nchars != o_CJKNgramLen) {
int btend = int(it.getBpos()); // Current char is out
if (!takeword(it.buffer().substr(boffs[0], btend-boffs[0]),
m_wordpos - nchars,
boffs[0], btend)) {
return false;
}
}
m_span.erase();
m_inNumber = false;
m_wordStart = m_wordLen = m_wordChars = m_prevpos = m_prevlen = 0;
m_spanpos = m_wordpos;
*cp = c;
return true;
}
// Specialization for countWords
class TextSplitCW : public TextSplit {
public:
int wcnt;
TextSplitCW(Flags flags) : TextSplit(flags), wcnt(0) {}
bool takeword(const string &, int, int, int) {
wcnt++;
return true;
}
};
int TextSplit::countWords(const string& s, TextSplit::Flags flgs)
{
TextSplitCW splitter(flgs);
splitter.text_to_words(s);
return splitter.wcnt;
}
bool TextSplit::hasVisibleWhite(const string &in)
{
Utf8Iter it(in);
for (; !it.eof(); it++) {
unsigned int c = (unsigned char)*it;
LOGDEB3(("TextSplit::hasVisibleWhite: testing 0x%04x\n", c));
if (c == (unsigned int)-1) {
LOGERR(("hasVisibleWhite: error while scanning UTF-8 string\n"));
return false;
}
if (visiblewhite.find(c) != visiblewhite.end())
return true;
}
return false;
}
template <class T> bool u8stringToStrings(const string &s, T &tokens)
{
Utf8Iter it(s);
string current;
tokens.clear();
enum states {SPACE, TOKEN, INQUOTE, ESCAPE};
states state = SPACE;
for (; !it.eof(); it++) {
unsigned int c = *it;
if (visiblewhite.find(c) != visiblewhite.end())
c = ' ';
LOGDEB3(("TextSplit::stringToStrings: 0x%04x\n", c));
if (c == (unsigned int)-1) {
LOGERR(("TextSplit::stringToStrings: error while "
"scanning UTF-8 string\n"));
return false;
}
switch (c) {
case '"':
switch(state) {
case SPACE: state = INQUOTE; continue;
case TOKEN: goto push_char;
case ESCAPE: state = INQUOTE; goto push_char;
case INQUOTE: tokens.push_back(current);current.clear();
state = SPACE; continue;
}
break;
case '\\':
switch(state) {
case SPACE:
case TOKEN: state=TOKEN; goto push_char;
case INQUOTE: state = ESCAPE; continue;
case ESCAPE: state = INQUOTE; goto push_char;
}
break;
case ' ':
case '\t':
case '\n':
case '\r':
switch(state) {
case SPACE: continue;
case TOKEN: tokens.push_back(current); current.clear();
state = SPACE; continue;
case INQUOTE:
case ESCAPE: goto push_char;
}
break;
default:
switch(state) {
case ESCAPE: state = INQUOTE; break;
case SPACE: state = TOKEN; break;
case TOKEN:
case INQUOTE: break;
}
push_char:
it.appendchartostring(current);
}
}
// End of string. Process residue, and possible error (unfinished quote)
switch(state) {
case SPACE: break;
case TOKEN: tokens.push_back(current); break;
case INQUOTE:
case ESCAPE: return false;
}
return true;
}
bool TextSplit::stringToStrings(const string &s, vector<string> &tokens)
{
return u8stringToStrings<vector<string> >(s, tokens);
}
#else // TEST driver ->
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <errno.h>
#include <fcntl.h>
#include <string.h>
#include <math.h>
#include <iostream>
#include "textsplit.h"
#include "readfile.h"
#include "debuglog.h"
#include "transcode.h"
#include "unacpp.h"
#include "termproc.h"
using namespace std;
class myTermProc : public Rcl::TermProc {
int first;
bool nooutput;
public:
myTermProc() : TermProc(0), first(1), nooutput(false) {}
void setNoOut(bool val) {nooutput = val;}
virtual bool takeword(const string &term, int pos, int bs, int be)
{
if (nooutput)
return true;
FILE *fp = stdout;
if (first) {
fprintf(fp, "%3s %-20s %4s %4s\n", "pos", "Term", "bs", "be");
first = 0;
}
fprintf(fp, "%3d %-20s %4d %4d\n", pos, term.c_str(), bs, be);
return true;
}
};
#define OPT_s 0x1
#define OPT_w 0x2
#define OPT_q 0x4
#define OPT_c 0x8
#define OPT_k 0x10
#define OPT_C 0x20
#define OPT_n 0x40
#define OPT_S 0x80
#define OPT_u 0x100
#define OPT_p 0x200
bool dosplit(const string& data, TextSplit::Flags flags, int op_flags)
{
myTermProc printproc;
Rcl::TermProc *nxt = &printproc;
// Rcl::TermProcCommongrams commonproc(nxt, stoplist);
// if (op_flags & OPT_S)
// nxt = &commonproc;
Rcl::TermProcPrep preproc(nxt);
if (op_flags & OPT_u)
nxt = &preproc;
Rcl::TextSplitP splitter(nxt, flags);
if (op_flags & OPT_q)
printproc.setNoOut(true);
splitter.text_to_words(data);
#ifdef TEXTSPLIT_STATS
TextSplit::Stats::Values v = splitter.getStats();
cout << "Average length: "
<< v.avglen
<< " Standard deviation: "
<< v.sigma
<< " Coef of variation "
<< v.sigma / v.avglen
<< endl;
#endif
return true;
}
static const char *teststrings[] = {
"Un bout de texte \nnormal. 2eme phrase.3eme;quatrieme.\n",
"\"Jean-Francois Dockes\" <jfd@okyz.com>\n",
"n@d @net .net net@ t@v@c c# c++ o'brien 'o'brien'",
"_network_ some_span",
"data123\n",
"134 +134 -14 0.1 .1 2. -1.5 +1.5 1,2 1.54e10 1,2e30 .1e10 1.e-8\n",
"@^#$(#$(*)\n",
"192.168.4.1 one\n\rtwo\r",
"[olala][ululu] (valeur) (23)\n",
"utf-8 ucs-4�� \\nodef\n",
"A b C 2 . +",
"','this\n",
" ,able,test-domain",
" -wl,--export-dynamic",
" ~/.xsession-errors",
"this_very_long_span_this_very_long_span_this_very_long_span",
"soft\xc2\xadhyphen",
"soft\xc2\xad\nhyphen",
"soft\xc2\xad\n\rhyphen",
"real\xe2\x80\x90hyphen",
"real\xe2\x80\x90\nhyphen",
"hyphen-\nminus",
};
const int teststrings_cnt = sizeof(teststrings)/sizeof(char *);
static string teststring1 = " nouvel-an ";
static string thisprog;
static string usage =
" textsplit [opts] [filename]\n"
" -q : no output\n"
" -s : only spans\n"
" -w : only words\n"
" -n : no numbers\n"
" -k : preserve wildcards (?*)\n"
" -c : just count words\n"
" -u : use unac\n"
" -C [charset] : input charset\n"
" -S [stopfile] : stopfile to use for commongrams\n"
" if filename is 'stdin', will read stdin for data (end with ^D)\n\n"
" textplit -p somephrase : display results from stringToStrings()\n"
" \n"
;
static void
Usage(void)
{
cerr << thisprog << ": usage:\n" << usage;
exit(1);
}
static int op_flags;
int main(int argc, char **argv)
{
string charset, stopfile;
thisprog = argv[0];
argc--; argv++;
while (argc > 0 && **argv == '-') {
(*argv)++;
if (!(**argv))
/* Cas du "adb - core" */
Usage();
while (**argv)
switch (*(*argv)++) {
case 'c': op_flags |= OPT_c; break;
case 'C': op_flags |= OPT_C; if (argc < 2) Usage();
charset = *(++argv); argc--;
goto b1;
case 'k': op_flags |= OPT_k; break;
case 'n': op_flags |= OPT_n; break;
case 'p': op_flags |= OPT_p; break;
case 'q': op_flags |= OPT_q; break;
case 's': op_flags |= OPT_s; break;
case 'S': op_flags |= OPT_S; if (argc < 2) Usage();
stopfile = *(++argv); argc--;
goto b1;
case 'u': op_flags |= OPT_u; break;
case 'w': op_flags |= OPT_w; break;
default: Usage(); break;
}
b1: argc--; argv++;
}
DebugLog::getdbl()->setloglevel(DEBDEB1);
DebugLog::setfilename("stderr");
TextSplit::Flags flags = TextSplit::TXTS_NONE;
if (op_flags&OPT_s)
flags = TextSplit::TXTS_ONLYSPANS;
else if (op_flags&OPT_w)
flags = TextSplit::TXTS_NOSPANS;
if (op_flags & OPT_k)
flags = (TextSplit::Flags)(flags | TextSplit::TXTS_KEEPWILD);
if (op_flags & OPT_n)
TextSplit::noNumbers();
Rcl::StopList stoplist;
if (op_flags & OPT_S) {
if (!stoplist.setFile(stopfile)) {
cerr << "Can't read stopfile: " << stopfile << endl;
exit(1);
}
}
string odata, reason;
if (argc == 1) {
const char *filename = *argv++; argc--;
if (op_flags& OPT_p) {
vector<string> tokens;
TextSplit::stringToStrings(filename, tokens);
for (vector<string>::const_iterator it = tokens.begin();
it != tokens.end(); it++) {
cout << "[" << *it << "] ";
}
cout << endl;
exit(0);
}
if (!strcmp(filename, "stdin")) {
char buf[1024];
int nread;
while ((nread = read(0, buf, 1024)) > 0) {
odata.append(buf, nread);
}
} else if (!file_to_string(filename, odata, &reason)) {
cerr << "Failed: file_to_string(" << filename << ") failed: "
<< reason << endl;
exit(1);
}
} else {
if (op_flags & OPT_p)
Usage();
for (int i = 0; i < teststrings_cnt; i++) {
cout << endl << teststrings[i] << endl;
dosplit(teststrings[i], flags, op_flags);
}
exit(0);
}
string& data = odata;
string ndata;
if ((op_flags & OPT_C)) {
if (!transcode(odata, ndata, charset, "UTF-8")) {
cerr << "Failed: transcode error" << endl;
exit(1);
} else {
data = ndata;
}
}
if (op_flags & OPT_c) {
int n = TextSplit::countWords(data, flags);
cout << n << " words" << endl;
} else {
dosplit(data, flags, op_flags);
}
}
#endif // TEST