|
a/src/common/textsplit.h |
|
b/src/common/textsplit.h |
|
... |
|
... |
16 |
*/
|
16 |
*/
|
17 |
#ifndef _TEXTSPLIT_H_INCLUDED_
|
17 |
#ifndef _TEXTSPLIT_H_INCLUDED_
|
18 |
#define _TEXTSPLIT_H_INCLUDED_
|
18 |
#define _TEXTSPLIT_H_INCLUDED_
|
19 |
|
19 |
|
20 |
#include <string>
|
20 |
#include <string>
|
21 |
#include <list>
|
21 |
#include <vector>
|
22 |
|
22 |
|
23 |
#ifndef NO_NAMESPACES
|
23 |
#ifndef NO_NAMESPACES
|
24 |
using std::string;
|
24 |
using std::string;
|
25 |
using std::list;
|
25 |
using std::vector;
|
26 |
#endif
|
26 |
#endif
|
27 |
|
27 |
|
28 |
class Utf8Iter;
|
28 |
class Utf8Iter;
|
29 |
|
29 |
|
30 |
/**
|
30 |
/**
|
|
... |
|
... |
92 |
* This has to be kept separate from smallut.cpp's stringsToStrings, which
|
92 |
* This has to be kept separate from smallut.cpp's stringsToStrings, which
|
93 |
* basically works only if whitespace is ascii, and which processes
|
93 |
* basically works only if whitespace is ascii, and which processes
|
94 |
* non-utf-8 input (iso-8859 config files work ok). This hopefully
|
94 |
* non-utf-8 input (iso-8859 config files work ok). This hopefully
|
95 |
* handles all Unicode whitespace, but needs correct utf-8 input
|
95 |
* handles all Unicode whitespace, but needs correct utf-8 input
|
96 |
*/
|
96 |
*/
|
97 |
static bool stringToStrings(const string &s, list<string> &tokens);
|
97 |
static bool stringToStrings(const string &s, vector<string> &tokens);
|
98 |
|
98 |
|
99 |
/** Is char CJK ? */
|
99 |
/** Is char CJK ? */
|
100 |
static bool isCJK(int c);
|
100 |
static bool isCJK(int c);
|
101 |
|
101 |
|
102 |
private:
|
102 |
private:
|