|
a/src/common/textsplit.h |
|
b/src/common/textsplit.h |
|
... |
|
... |
| 16 |
*/
|
16 |
*/
|
| 17 |
#ifndef _TEXTSPLIT_H_INCLUDED_
|
17 |
#ifndef _TEXTSPLIT_H_INCLUDED_
|
| 18 |
#define _TEXTSPLIT_H_INCLUDED_
|
18 |
#define _TEXTSPLIT_H_INCLUDED_
|
| 19 |
|
19 |
|
| 20 |
#include <string>
|
20 |
#include <string>
|
| 21 |
#include <list>
|
21 |
#include <vector>
|
| 22 |
|
22 |
|
| 23 |
#ifndef NO_NAMESPACES
|
23 |
#ifndef NO_NAMESPACES
|
| 24 |
using std::string;
|
24 |
using std::string;
|
| 25 |
using std::list;
|
25 |
using std::vector;
|
| 26 |
#endif
|
26 |
#endif
|
| 27 |
|
27 |
|
| 28 |
class Utf8Iter;
|
28 |
class Utf8Iter;
|
| 29 |
|
29 |
|
| 30 |
/**
|
30 |
/**
|
|
... |
|
... |
| 92 |
* This has to be kept separate from smallut.cpp's stringsToStrings, which
|
92 |
* This has to be kept separate from smallut.cpp's stringsToStrings, which
|
| 93 |
* basically works only if whitespace is ascii, and which processes
|
93 |
* basically works only if whitespace is ascii, and which processes
|
| 94 |
* non-utf-8 input (iso-8859 config files work ok). This hopefully
|
94 |
* non-utf-8 input (iso-8859 config files work ok). This hopefully
|
| 95 |
* handles all Unicode whitespace, but needs correct utf-8 input
|
95 |
* handles all Unicode whitespace, but needs correct utf-8 input
|
| 96 |
*/
|
96 |
*/
|
| 97 |
static bool stringToStrings(const string &s, list<string> &tokens);
|
97 |
static bool stringToStrings(const string &s, vector<string> &tokens);
|
| 98 |
|
98 |
|
| 99 |
/** Is char CJK ? */
|
99 |
/** Is char CJK ? */
|
| 100 |
static bool isCJK(int c);
|
100 |
static bool isCJK(int c);
|
| 101 |
|
101 |
|
| 102 |
private:
|
102 |
private:
|