Parent: [8f77b9] (diff)

Download this file

termproc.h    325 lines (292 with data), 10.0 kB

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
/* Copyright (C) 2011 J.F.Dockes
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the
* Free Software Foundation, Inc.,
* 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
*/
#ifndef _TERMPROC_H_INCLUDED_
#define _TERMPROC_H_INCLUDED_
#include <vector>
#include <string>
#include "textsplit.h"
#include "stoplist.h"
#include "smallut.h"
#include "utf8iter.h"
namespace Rcl {
/**
* Termproc objects take term tokens as input and do something
* with them: transform to lowercase, filter out stop words, generate n-grams,
* finally index or generate search clauses, etc. They are chained and can
* be arranged to form different pipelines depending on the desired processing
* steps: for example, optional stoplist or commongram processing.
*
* Shared processing steps are defined in this file. The first and last steps
* are usually defined in the specific module.
* - The front TermProc is typically chained from a TextSplit object
* which generates the original terms, and calls takeword() from its
* own takeword() method.
* - The last TermProc does something with the finalized terms, e.g. adds
* them to the index.
*/
/**
* The base class takes care of chaining: all derived classes call its
* takeword() and flush() methods to ensure that terms go through the pipe.
*/
class TermProc {
public:
TermProc(TermProc* next) : m_next(next) {}
virtual ~TermProc() {}
virtual bool takeword(const string &term, int pos, int bs, int be)
{
if (m_next)
return m_next->takeword(term, pos, bs, be);
else
return true;
}
// newpage() is like takeword(), but for page breaks.
virtual void newpage(int pos)
{
if (m_next)
m_next->newpage(pos);
}
virtual bool flush()
{
if (m_next)
return m_next->flush();
else
return true;
}
private:
TermProc *m_next;
/* Copyconst and assignment private and forbidden */
TermProc(const TermProc &) {}
TermProc& operator=(const TermProc &) {
return *this;
};
};
/**
* Helper specialized TextSplit class, feeds the pipeline:
* - The takeword() method calls a TermProc->takeword().
* - The text_to_words() method also takes care of flushing.
* Both methods can be further specialized by the user (they should then call
* the base methods when they've done the local processing).
*/
class TextSplitP : public TextSplit {
public:
TextSplitP(TermProc *prc, Flags flags = Flags(TXTS_NONE))
: TextSplit(flags), m_prc(prc) {}
virtual bool text_to_words(const string &in) {
bool ret = TextSplit::text_to_words(in);
if (m_prc && !m_prc->flush())
return false;
return ret;
}
virtual bool takeword(const string& term, int pos, int bs, int be) {
if (m_prc)
return m_prc->takeword(term, pos, bs, be);
else
return true;
}
virtual void newpage(int pos) {
if (m_prc)
return m_prc->newpage(pos);
}
private:
TermProc *m_prc;
};
/** Unaccent and lowercase term. If the index is
* not case/diac-sensitive, this is usually the first step in the pipeline
*/
class TermProcPrep : public TermProc {
public:
TermProcPrep(TermProc *nxt)
: TermProc(nxt), m_totalterms(0), m_unacerrors(0)
{
}
virtual bool takeword(const string& itrm, int pos, int bs, int be)
{
m_totalterms++;
string otrm;
if (!unacmaybefold(itrm, otrm, "UTF-8", UNACOP_UNACFOLD)) {
LOGDEB("splitter::takeword: unac [" << itrm << "] failed\n");
m_unacerrors++;
// We don't generate a fatal error because of a bad term,
// but one has to put the limit somewhere
if (m_unacerrors > 500 &&
(double(m_totalterms) / double(m_unacerrors)) < 2.0) {
// More than 1 error for every other term
LOGERR("splitter::takeword: too many unac errors " <<
m_unacerrors << "/" << m_totalterms << "\n");
return false;
}
return true;
}
if (otrm.empty()) {
// It may happen in some weird cases that the output from
// unac is empty (if the word actually consisted entirely
// of diacritics ...) The consequence is that a phrase
// search won't work without addional slack.
return true;
}
// We should have a Japanese stemmer to handle this, but for
// experimenting, let's do it here: remove 'prolounged sound
// mark' and its halfwidth variant from the end of terms.
if ((unsigned int)otrm[0] > 127) {
Utf8Iter it(otrm);
if (TextSplit::isKATAKANA(*it)) {
Utf8Iter itprev = it;
while (*it != (unsigned int)-1) {
itprev = it;
it++;
}
if (*itprev == 0x30fc || *itprev == 0xff70) {
otrm = otrm.substr(0, itprev.getBpos());
}
}
}
if (otrm.empty()) {
return true;
}
// It may also occur that unac introduces spaces in the string
// (when removing isolated accents, may happen for Greek
// for example). This is a pathological situation. We
// index all the resulting terms at the same pos because
// the surrounding code is not designed to handle a pos
// change in here. This means that phrase searches and
// snippets will be wrong, but at least searching for the
// terms will work.
bool hasspace = false;
for (string::const_iterator it = otrm.begin();it < otrm.end();it++) {
if (*it == ' ') {
hasspace=true;
break;
}
}
if (hasspace) {
std::vector<std::string> terms;
stringToTokens(otrm, terms, " ", true);
for (std::vector<std::string>::const_iterator it = terms.begin();
it < terms.end(); it++) {
if (!TermProc::takeword(*it, pos, bs, be)) {
return false;
}
}
return true;
} else {
return TermProc::takeword(otrm, pos, bs, be);
}
}
virtual bool flush()
{
m_totalterms = m_unacerrors = 0;
return TermProc::flush();
}
private:
int m_totalterms;
int m_unacerrors;
};
/** Compare to stop words list and discard if match found */
class TermProcStop : public TermProc {
public:
TermProcStop(TermProc *nxt, const Rcl::StopList& stops)
: TermProc(nxt), m_stops(stops)
{
}
virtual bool takeword(const string& term, int pos, int bs, int be)
{
if (m_stops.isStop(term)) {
return true;
}
return TermProc::takeword(term, pos, bs, be);
}
private:
const Rcl::StopList& m_stops;
};
/** Handle common-gram generation: combine frequent terms with neighbours to
* shorten the positions lists for phrase searches.
* NOTE: This does not currently work because of bad interaction with the
* spans (ie john@domain.com) generation in textsplit. Not used, kept for
* testing only
*/
class TermProcCommongrams : public TermProc {
public:
TermProcCommongrams(TermProc *nxt, const Rcl::StopList& stops)
: TermProc(nxt), m_stops(stops), m_onlygrams(false)
{
}
virtual bool takeword(const string& term, int pos, int bs, int be)
{
LOGDEB1("TermProcCom::takeword: pos " << (pos) << " " << (bs) << " " << (be) << " [" << (term) << "]\n" );
bool isstop = m_stops.isStop(term);
bool twogramemit = false;
if (!m_prevterm.empty() && (m_prevstop || isstop)) {
// create 2-gram. space unnecessary but improves
// the readability of queries
string twogram;
twogram.swap(m_prevterm);
twogram.append(1, ' ');
twogram += term;
// When emitting a complex term we set the bps to 0. This may
// be used by our clients
if (!TermProc::takeword(twogram, m_prevpos, 0, 0))
return false;
twogramemit = true;
#if 0
if (m_stops.isStop(twogram)) {
firstword = twogram;
isstop = false;
}
#endif
}
m_prevterm = term;
m_prevstop = isstop;
m_prevpos = pos;
m_prevsent = false;
m_prevbs = bs;
m_prevbe = be;
// If flags allow, emit the bare term at the current pos.
if (!m_onlygrams || (!isstop && !twogramemit)) {
if (!TermProc::takeword(term, pos, bs, be))
return false;
m_prevsent = true;
}
return true;
}
virtual bool flush()
{
if (!m_prevsent && !m_prevterm.empty())
if (!TermProc::takeword(m_prevterm, m_prevpos, m_prevbs, m_prevbe))
return false;
m_prevterm.clear();
m_prevsent = true;
return TermProc::flush();
}
void onlygrams(bool on)
{
m_onlygrams = on;
}
private:
// The stoplist we're using
const Rcl::StopList& m_stops;
// Remembered data for the last processed term
string m_prevterm;
bool m_prevstop;
int m_prevpos;
int m_prevbs;
int m_prevbe;
bool m_prevsent;
// If this is set, we only emit longest grams
bool m_onlygrams;
};
} // End namespace Rcl
#endif /* _TERMPROC_H_INCLUDED_ */