|
a/src/common/textsplit.h |
|
b/src/common/textsplit.h |
|
... |
|
... |
| 211 |
Stats m_stats;
|
211 |
Stats m_stats;
|
| 212 |
#endif
|
212 |
#endif
|
| 213 |
// Word length in characters. Declared but not updated if !TEXTSPLIT_STATS
|
213 |
// Word length in characters. Declared but not updated if !TEXTSPLIT_STATS
|
| 214 |
unsigned int m_wordChars;
|
214 |
unsigned int m_wordChars;
|
| 215 |
|
215 |
|
|
|
216 |
void clearsplitstate() {
|
|
|
217 |
m_span.clear();
|
|
|
218 |
m_words_in_span.clear();
|
|
|
219 |
m_inNumber = false;
|
|
|
220 |
m_wordStart = m_wordLen = m_wordpos = m_spanpos = m_prevpos =
|
|
|
221 |
m_prevlen = m_wordChars = 0;
|
|
|
222 |
}
|
|
|
223 |
|
| 216 |
// This processes cjk text:
|
224 |
// This processes cjk text:
|
| 217 |
bool cjk_to_words(Utf8Iter *it, unsigned int *cp);
|
225 |
bool cjk_to_words(Utf8Iter *it, unsigned int *cp);
|
| 218 |
|
226 |
|
| 219 |
bool emitterm(bool isspan, std::string &term, int pos, size_t bs,size_t be);
|
227 |
bool emitterm(bool isspan, std::string &term, int pos, size_t bs,size_t be);
|
| 220 |
bool doemit(bool spanerase, size_t bp);
|
228 |
bool doemit(bool spanerase, size_t bp);
|