Parent: [52bc9f] (diff)

Child: [533068] (diff)

Download this file

utf8iter.h    284 lines (256 with data), 7.5 kB

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
/* Copyright (C) 2004 J.F.Dockes
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the
* Free Software Foundation, Inc.,
* 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
*/
#ifndef _UTF8ITER_H_INCLUDED_
#define _UTF8ITER_H_INCLUDED_
#ifdef UTF8ITER_CHECK
#include "assert.h"
#endif
#include <string>
/**
* A small helper class to iterate over utf8 strings. This is not an
* STL iterator and does not much error checking. It is designed purely
* for recoll usage, where the utf-8 string comes out of iconv in most cases
* and is assumed legal. We just try to catch cases where there would be
* a risk of crash.
*/
class Utf8Iter {
public:
Utf8Iter(const std::string &in)
: m_s(in), m_cl(0), m_pos(0), m_charpos(0)
{
update_cl();
}
const std::string& buffer() const {return m_s;}
void rewind()
{
m_cl = 0;
m_pos = 0;
m_charpos = 0;
update_cl();
}
/** "Direct" access. Awfully inefficient as we skip from start or current
* position at best. This can only be useful for a lookahead from the
* current position */
unsigned int operator[](unsigned int charpos) const
{
std::string::size_type mypos = 0;
unsigned int mycp = 0;
if (charpos >= m_charpos) {
mypos = m_pos;
mycp = m_charpos;
}
int l;
while (mypos < m_s.length() && mycp != charpos) {
l = get_cl(mypos);
if (l <= 0 || !poslok(mypos, l) || !checkvalidat(mypos, l))
return (unsigned int)-1;
mypos += l;
++mycp;
}
if (mypos < m_s.length() && mycp == charpos) {
l = get_cl(mypos);
if (poslok(mypos, l) && checkvalidat(mypos, l))
return getvalueat(mypos, l);
}
return (unsigned int)-1;
}
/** Increment current position to next utf-8 char */
std::string::size_type operator++(int)
{
// Note: m_cl may be zero at eof if user's test not right
// this shouldn't crash the program until actual data access
#ifdef UTF8ITER_CHECK
assert(m_cl != 0);
#endif
if (m_cl == 0)
return std::string::npos;
m_pos += m_cl;
m_charpos++;
update_cl();
return m_pos;
}
/** operator* returns the ucs4 value as a machine integer*/
unsigned int operator*()
{
#ifdef UTF8ITER_CHECK
assert(m_cl > 0);
#endif
return m_cl == 0 ? (unsigned int)-1 : getvalueat(m_pos, m_cl);
}
/** Append current utf-8 possibly multi-byte character to string param.
This needs to be fast. No error checking. */
unsigned int appendchartostring(std::string &out) const {
#ifdef UTF8ITER_CHECK
assert(m_cl != 0);
#endif
out.append(&m_s[m_pos], m_cl);
return m_cl;
}
/** Return current character as string */
operator std::string() {
#ifdef UTF8ITER_CHECK
assert(m_cl != 0);
#endif
return m_cl > 0 ? m_s.substr(m_pos, m_cl) : std::string();
}
bool eof() const {
return m_pos == m_s.length();
}
bool error() const {
return m_cl == 0;
}
/** Return current byte offset in input string */
std::string::size_type getBpos() const {
return m_pos;
}
/** Return current character length */
std::string::size_type getBlen() const {
return m_cl;
}
/** Return current unicode character offset in input string */
std::string::size_type getCpos() const {
return m_charpos;
}
private:
// String we're working with
const std::string& m_s;
// Character length at current position. A value of zero indicates
// an error.
unsigned int m_cl;
// Current byte offset in string.
std::string::size_type m_pos;
// Current character position
unsigned int m_charpos;
// Check position and cl against string length
bool poslok(std::string::size_type p, int l) const {
#ifdef UTF8ITER_CHECK
assert(p != std::string::npos && l > 0 && p + l <= m_s.length());
#endif
return p != std::string::npos && l > 0 && p + l <= m_s.length();
}
// Update current char length in object state, check
// for errors
inline void update_cl()
{
m_cl = 0;
if (m_pos >= m_s.length())
return;
m_cl = get_cl(m_pos);
if (!poslok(m_pos, m_cl)) {
// Used to set eof here for safety, but this is bad because it
// basically prevents the caller to discriminate error and eof.
// m_pos = m_s.length();
m_cl = 0;
return;
}
if (!checkvalidat(m_pos, m_cl)) {
m_cl = 0;
}
}
inline bool checkvalidat(std::string::size_type p, int l) const
{
switch (l) {
case 1:
return (unsigned char)m_s[p] < 128;
case 2:
return (((unsigned char)m_s[p]) & 224) == 192
&& (((unsigned char)m_s[p+1]) & 192) == 128;
case 3:
return (((unsigned char)m_s[p]) & 240) == 224
&& (((unsigned char)m_s[p+1]) & 192) == 128
&& (((unsigned char)m_s[p+2]) & 192) == 128
;
case 4:
return (((unsigned char)m_s[p]) & 248) == 240
&& (((unsigned char)m_s[p+1]) & 192) == 128
&& (((unsigned char)m_s[p+2]) & 192) == 128
&& (((unsigned char)m_s[p+3]) & 192) == 128
;
default:
return false;
}
}
// Get character byte length at specified position. Returns 0 for error.
inline int get_cl(std::string::size_type p) const
{
unsigned int z = (unsigned char)m_s[p];
if (z <= 127) {
return 1;
} else if ((z & 224) == 192) {
return 2;
} else if ((z & 240) == 224) {
return 3;
} else if ((z & 248) == 240) {
return 4;
}
#ifdef UTF8ITER_CHECK
assert(z <= 127 || (z & 224) == 192 || (z & 240) == 224 ||
(z & 248) == 240);
#endif
return 0;
}
// Compute value at given position. No error checking.
inline unsigned int getvalueat(std::string::size_type p, int l) const
{
switch (l) {
case 1:
#ifdef UTF8ITER_CHECK
assert((unsigned char)m_s[p] < 128);
#endif
return (unsigned char)m_s[p];
case 2:
#ifdef UTF8ITER_CHECK
assert(
((unsigned char)m_s[p] & 224) == 192
&& ((unsigned char)m_s[p+1] & 192) == 128
);
#endif
return ((unsigned char)m_s[p] - 192) * 64 +
(unsigned char)m_s[p+1] - 128 ;
case 3:
#ifdef UTF8ITER_CHECK
assert(
(((unsigned char)m_s[p]) & 240) == 224
&& (((unsigned char)m_s[p+1]) & 192) == 128
&& (((unsigned char)m_s[p+2]) & 192) == 128
);
#endif
return ((unsigned char)m_s[p] - 224) * 4096 +
((unsigned char)m_s[p+1] - 128) * 64 +
(unsigned char)m_s[p+2] - 128;
case 4:
#ifdef UTF8ITER_CHECK
assert(
(((unsigned char)m_s[p]) & 248) == 240
&& (((unsigned char)m_s[p+1]) & 192) == 128
&& (((unsigned char)m_s[p+2]) & 192) == 128
&& (((unsigned char)m_s[p+3]) & 192) == 128
);
#endif
return ((unsigned char)m_s[p]-240)*262144 +
((unsigned char)m_s[p+1]-128)*4096 +
((unsigned char)m_s[p+2]-128)*64 +
(unsigned char)m_s[p+3]-128;
default:
#ifdef UTF8ITER_CHECK
assert(l <= 4);
#endif
return (unsigned int)-1;
}
}
};
#endif /* _UTF8ITER_H_INCLUDED_ */