Parent: [e4e5ee] (diff)

Child: [835874] (diff)

Download this file

rclabsfromtext.cpp    429 lines (382 with data), 15.1 kB

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
/* Copyright (C) 2004-2017 J.F.Dockes
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the
* Free Software Foundation, Inc.,
* 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
*/
#include "autoconfig.h"
#include <math.h>
#include <map>
#include <unordered_map>
#include <deque>
#include <algorithm>
#include <regex>
#include "log.h"
#include "rcldb.h"
#include "rcldb_p.h"
#include "rclquery.h"
#include "rclquery_p.h"
#include "textsplit.h"
#include "hldata.h"
#include "chrono.h"
#include "unacpp.h"
#include "zlibut.h"
using namespace std;
// We now let plaintorich do the highlight tags insertions which is
// wasteful because we have most of the information (but the perf hit
// is small because it's only called on the output fragments, not on
// the whole text). The highlight zone computation code has been left
// around just in case I change my mind.
#undef COMPUTE_HLZONES
namespace Rcl {
//// Fragment cleanup
// Chars we turn to spaces in the Snippets
static const string cstr_nc("\n\r\x0c\\");
// Things that we don't want to repeat in a displayed snippet.
// e.g. > > > > > >
static const string punctcls("[-<>._+,#*=|]");
static const string punctRE = "(" + punctcls + " *)(" + punctcls + " *)+";
static std::regex fixfrag_re(punctRE);
static string fixfrag(const string& infrag)
{
return std::regex_replace(neutchars(infrag, cstr_nc), fixfrag_re, "$2");
}
// Fragment descriptor. A fragment is a text area with one or several
// matched terms and some context. It is ranked according to the
// matched term weights and the near/phrase matches get a boost.
struct MatchFragment {
// Start/End byte offsets of fragment in the document text
int start;
int stop;
// Weight for this fragment (bigger better)
double coef;
#ifdef COMPUTE_HLZONES
// Highlight areas (each is one or several contiguous match
// terms). Because a fragment extends around a match, there
// can be several contiguous or separate matches in a given
// fragment.
vector<pair<int,int>> hlzones;
#endif
// Position of the first matched term (for page number computations)
unsigned int hitpos;
// "best term" for this match (e.g. for use as ext app search term)
string term;
MatchFragment(int sta, int sto, double c,
#ifdef COMPUTE_HLZONES
vector<pair<int,int>>& hl,
#endif
unsigned int pos, string& trm)
: start(sta), stop(sto), coef(c), hitpos(pos) {
#ifdef COMPUTE_HLZONES
hlzones.swap(hl);
#endif
term.swap(trm);
}
};
// Text splitter for finding the match areas in the document text.
class TextSplitABS : public TextSplit {
public:
TextSplitABS(const vector<string>& matchTerms,
const HighlightData& hdata,
unordered_map<string, double>& wordcoefs,
unsigned int ctxwords,
Flags flags = TXTS_NONE)
: TextSplit(flags), m_terms(matchTerms.begin(), matchTerms.end()),
m_hdata(hdata), m_wordcoefs(wordcoefs), m_ctxwords(ctxwords) {
// Take note of the group (phrase/near) terms because we need
// to compute the position lists for them.
for (const auto& group : hdata.groups) {
if (group.size() > 1) {
for (const auto& term: group) {
m_gterms.insert(term);
}
}
}
}
// Accept a word and its position. If the word is a matched term,
// add/update fragment definition.
virtual bool takeword(const std::string& term, int pos, int bts, int bte) {
LOGDEB2("takeword: " << term << endl);
// Remember recent past
m_prevterms.push_back(pair<int,int>(bts,bte));
if (m_prevterms.size() > m_ctxwords+1) {
m_prevterms.pop_front();
}
string dumb;
if (o_index_stripchars) {
if (!unacmaybefold(term, dumb, "UTF-8", UNACOP_UNACFOLD)) {
LOGINFO("abstract: unac failed for [" << term << "]\n");
return true;
}
} else {
dumb = term;
}
if (m_terms.find(dumb) != m_terms.end()) {
// This word is a search term. Extend or create fragment
LOGDEB2("match: [" << dumb << "] current: " << m_curfrag.first <<
", " << m_curfrag.second << " remain " <<
m_remainingWords << endl);
double coef = m_wordcoefs[dumb];
if (!m_remainingWords) {
// No current fragment. Start one
m_curhitpos = baseTextPosition + pos;
m_curfrag.first = m_prevterms.front().first;
m_curfrag.second = m_prevterms.back().second;
#ifdef COMPUTE_HLZONES
m_curhlzones.push_back(pair<int,int>(bts, bte));
#endif
m_curterm = term;
m_curtermcoef = coef;
} else {
LOGDEB2("Extending current fragment: " << m_remainingWords <<
" -> " << m_ctxwords << endl);
m_extcount++;
#ifdef COMPUTE_HLZONES
if (m_prevwordhit) {
m_curhlzones.back().second = bte;
} else {
m_curhlzones.push_back(pair<int,int>(bts, bte));
}
#endif
if (coef > m_curtermcoef) {
m_curterm = term;
m_curtermcoef = coef;
}
}
#ifdef COMPUTE_HLZONES
m_prevwordhit = true;
#endif
m_curfragcoef += coef;
m_remainingWords = m_ctxwords + 1;
if (m_extcount > 3) {
// Limit expansion of contiguous fragments (this is to
// avoid common terms in search causing long
// heavyweight meaningless fragments. Also, limit length).
m_remainingWords = 1;
m_extcount = 0;
}
// If the term is part of a near/phrase group, update its
// positions list
if (m_gterms.find(dumb) != m_gterms.end()) {
// Term group (phrase/near) handling
m_plists[dumb].push_back(pos);
m_gpostobytes[pos] = pair<int,int>(bts, bte);
LOGDEB2("Recorded bpos for " << pos << ": " << bts << " " <<
bte << "\n");
}
}
#ifdef COMPUTE_HLZONES
else {
// Not a matched term
m_prevwordhit = false;
}
#endif
if (m_remainingWords) {
// Fragment currently open. Time to close ?
m_remainingWords--;
m_curfrag.second = bte;
if (m_remainingWords == 0) {
if (m_totalcoef < 5.0 || m_curfragcoef >= 1.0) {
// Don't push bad fragments if we have a lot already
m_fragments.push_back(MatchFragment(m_curfrag.first,
m_curfrag.second,
m_curfragcoef,
#ifdef COMPUTE_HLZONES
m_curhlzones,
#endif
m_curhitpos,
m_curterm
));
}
m_totalcoef += m_curfragcoef;
m_curfragcoef = 0.0;
m_curtermcoef = 0.0;
}
}
return true;
}
const vector<MatchFragment>& getFragments() {
return m_fragments;
}
// After the text is split: use the group terms positions lists to
// find the group matches. We process everything as NEAR (no
// PHRASE specific processing).
void updgroups() {
vector<GroupMatchEntry> tboffs;
// Look for matches to PHRASE and NEAR term groups and finalize
// the matched regions list (sort it by increasing start then
// decreasing length). We process all groups as NEAR (ignore order).
for (unsigned int i = 0; i < m_hdata.groups.size(); i++) {
if (m_hdata.groups[i].size() > 1) {
matchGroup(m_hdata, i, m_plists, m_gpostobytes, tboffs);
}
}
// Sort the fragments by increasing start and decreasing width
std::sort(m_fragments.begin(), m_fragments.end(),
[](const MatchFragment& a, const MatchFragment& b) -> bool {
if (a.start != b.start)
return a.start < b.start;
return a.stop - a.start > b.stop - a.stop;
}
);
// Sort the group regions by increasing start and decreasing width.
std::sort(tboffs.begin(), tboffs.end(),
[](const GroupMatchEntry& a, const GroupMatchEntry& b)
-> bool {
if (a.offs.first != b.offs.first)
return a.offs.first < b.offs.first;
return a.offs.second > b.offs.second;
}
);
// Give a boost to fragments which contain a group match
// (phrase/near), they are dear to the user's heart. list are
// sorted, so we never go back in the fragment list (can
// always start the search where we previously stopped).
auto fragit = m_fragments.begin();
for (const auto& grpmatch : tboffs) {
LOGDEB2("LOOKING FOR FRAGMENT: group: " << grpmatch.offs.first <<
"-" << grpmatch.offs.second << " curfrag " <<
fragit->start << "-" << fragit->stop << endl);
while (fragit->stop < grpmatch.offs.first) {
fragit++;
if (fragit == m_fragments.end()) {
return;
}
}
if (fragit->start <= grpmatch.offs.first &&
fragit->stop >= grpmatch.offs.second) {
// grp in frag
fragit->coef += 10.0;
}
}
return;
}
private:
// Past terms because we need to go back for context before a hit
deque<pair<int,int>> m_prevterms;
// Data about the fragment we are building
pair<int,int> m_curfrag{0,0};
double m_curfragcoef{0.0};
unsigned int m_remainingWords{0};
unsigned int m_extcount{0};
#ifdef COMPUTE_HLZONES
vector<pair<int,int>> m_curhlzones;
bool m_prevwordhit{false};
#endif
// Current sum of fragment weights
double m_totalcoef{0.0};
// Position of 1st term match (for page number computations)
unsigned int m_curhitpos{0};
// "best" term
string m_curterm;
double m_curtermcoef{0.0};
// Group terms, extracted from m_hdata
unordered_set<string> m_gterms;
// group/near terms word positions.
map<string, vector<int> > m_plists;
map<int, pair<int, int> > m_gpostobytes;
// Input
unordered_set<string> m_terms;
const HighlightData& m_hdata;
unordered_map<string, double>& m_wordcoefs;
unsigned int m_ctxwords;
// Result: begin and end byte positions of query terms/groups in text
vector<MatchFragment> m_fragments;
};
int Query::Native::abstractFromText(
Rcl::Db::Native *ndb,
Xapian::docid docid,
const vector<string>& matchTerms,
const multimap<double, vector<string>> byQ,
double totalweight,
int ctxwords,
unsigned int maxtotaloccs,
vector<Snippet>& vabs,
Chrono&
)
{
Xapian::Database& xrdb(ndb->xrdb);
string rawtext;
if (!ndb->getRawText(docid, rawtext)) {
LOGDEB0("abstractFromText: can't fetch text\n");
return ABSRES_ERROR;
}
#if 0 && ! (XAPIAN_MAJOR_VERSION <= 1 && XAPIAN_MINOR_VERSION <= 2) && \
(defined(RAWTEXT_IN_DATA))
// Tryout the Xapian internal method.
string snippet = xmset.snippet(rawtext);
LOGDEB("SNIPPET: [" << snippet << "] END SNIPPET\n");
#endif
// We need the q coefs for individual terms
unordered_map<string, double> wordcoefs;
for (const auto& mment : byQ) {
for (const auto& word : mment.second) {
wordcoefs[word] = mment.first;
}
}
// Note: getTerms() was already called by qualityTerms, so this is
// a bit wasteful. I guess that the performance impact is
// negligible though. To be checked ? We need the highlightdata for the
// phrase/near groups.
HighlightData hld;
if (m_q->m_sd) {
m_q->m_sd->getTerms(hld);
}
TextSplitABS splitter(matchTerms, hld, wordcoefs, ctxwords,
TextSplit::TXTS_ONLYSPANS);
splitter.text_to_words(rawtext);
splitter.updgroups();
// Sort the fragments by decreasing weight
const vector<MatchFragment>& res1 = splitter.getFragments();
vector<MatchFragment> result(res1.begin(), res1.end());
std::sort(result.begin(), result.end(),
[](const MatchFragment& a,
const MatchFragment& b) -> bool {
return a.coef > b.coef;
}
);
vector<int> vpbreaks;
ndb->getPagePositions(docid, vpbreaks);
// Build the output snippets array by merging the fragments, their
// main term and the page positions.
unsigned int count = 0;
for (const auto& entry : result) {
string frag(
fixfrag(rawtext.substr(entry.start, entry.stop - entry.start)));
#ifdef COMPUTE_HLZONES
// This would need to be modified to take tag parameters
// instead of the const strings
static const string starthit("<span style='color: blue;'>");
static const string endhit("</span>");
size_t inslen = 0;
for (const auto& hlzone: entry.hlzones) {
frag.replace(hlzone.first - entry.start + inslen, 0, starthit);
inslen += starthit.size();
frag.replace(hlzone.second - entry.start + inslen, 0, endhit);
inslen += endhit.size();
}
#endif
LOGDEB("=== FRAGMENT: Coef: " << entry.coef << ": " << frag << endl);
int page = 0;
if (vpbreaks.size() > 1) {
page = ndb->getPageNumberForPosition(vpbreaks, entry.hitpos);
if (page < 0)
page = 0;
}
vabs.push_back(Snippet(page, frag).setTerm(entry.term));
if (count++ >= maxtotaloccs)
break;
}
return ABSRES_OK;
}
}