Child: [5863d2] (diff)

Download this file

mimehandler.cpp    379 lines (345 with data), 13.3 kB

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
/*
* Copyright 2004 J.F.Dockes
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the
* Free Software Foundation, Inc.,
* 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
*/
#include "autoconfig.h"
#include <errno.h>
#include <iostream>
#include <string>
#include <vector>
#include <list>
#include <mutex>
using namespace std;
#include "cstr.h"
#include "mimehandler.h"
#include "log.h"
#include "rclconfig.h"
#include "smallut.h"
#include "md5ut.h"
#include "mh_exec.h"
#include "mh_execm.h"
#include "mh_html.h"
#include "mh_mail.h"
#include "mh_mbox.h"
#include "mh_text.h"
#include "mh_symlink.h"
#include "mh_unknown.h"
#include "mh_null.h"
// Performance help: we use a pool of already known and created
// handlers. There can be several instances for a given mime type
// (think email attachment in email message: 2 rfc822 handlers are
// needed simulteanously)
static multimap<string, RecollFilter*> o_handlers;
static list<multimap<string, RecollFilter*>::iterator> o_hlru;
typedef list<multimap<string, RecollFilter*>::iterator>::iterator hlruit_tp;
static std::mutex o_handlers_mutex;
static const unsigned int max_handlers_cache_size = 100;
/* Look for mime handler in pool */
static RecollFilter *getMimeHandlerFromCache(const string& key)
{
std::unique_lock<std::mutex> locker(o_handlers_mutex);
string xdigest;
MD5HexPrint(key, xdigest);
LOGDEB("getMimeHandlerFromCache: " << xdigest << " cache size " <<
o_handlers.size() << "\n");
multimap<string, RecollFilter *>::iterator it = o_handlers.find(key);
if (it != o_handlers.end()) {
RecollFilter *h = it->second;
hlruit_tp it1 = find(o_hlru.begin(), o_hlru.end(), it);
if (it1 != o_hlru.end()) {
o_hlru.erase(it1);
} else {
LOGERR("getMimeHandlerFromCache: lru position not found\n");
}
o_handlers.erase(it);
LOGDEB("getMimeHandlerFromCache: " << xdigest << " found size " <<
o_handlers.size() << "\n");
return h;
}
LOGDEB("getMimeHandlerFromCache: " << xdigest << " not found\n");
return 0;
}
/* Return mime handler to pool */
void returnMimeHandler(RecollFilter *handler)
{
typedef multimap<string, RecollFilter*>::value_type value_type;
if (handler == 0) {
LOGERR("returnMimeHandler: bad parameter\n");
return;
}
handler->clear();
std::unique_lock<std::mutex> locker(o_handlers_mutex);
LOGDEB("returnMimeHandler: returning filter for " <<
handler->get_mime_type() << " cache size " << o_handlers.size() <<
"\n");
// Limit pool size. The pool can grow quite big because there are
// many filter types, each of which can be used in several copies
// at the same time either because it occurs several times in a
// stack (ie mail attachment to mail), or because several threads
// are processing the same mime type at the same time.
multimap<string, RecollFilter *>::iterator it;
if (o_handlers.size() >= max_handlers_cache_size) {
static int once = 1;
if (once) {
once = 0;
for (it = o_handlers.begin(); it != o_handlers.end(); it++) {
LOGDEB1("Cache full. key: " << it->first << "\n");
}
LOGDEB1("Cache LRU size: " << o_hlru.size() << "\n");
}
if (o_hlru.size() > 0) {
it = o_hlru.back();
o_hlru.pop_back();
delete it->second;
o_handlers.erase(it);
}
}
it = o_handlers.insert(value_type(handler->get_id(), handler));
o_hlru.push_front(it);
}
void clearMimeHandlerCache()
{
LOGDEB("clearMimeHandlerCache()\n");
multimap<string, RecollFilter *>::iterator it;
std::unique_lock<std::mutex> locker(o_handlers_mutex);
for (it = o_handlers.begin(); it != o_handlers.end(); it++) {
delete it->second;
}
o_handlers.clear();
}
/** For mime types set as "internal" in mimeconf:
* create appropriate handler object. */
static RecollFilter *mhFactory(RclConfig *config, const string &mime,
bool nobuild, string& id)
{
LOGDEB2("mhFactory(" << mime << ")\n");
string lmime(mime);
stringtolower(lmime);
if (cstr_textplain == lmime) {
LOGDEB2("mhFactory(" << mime << "): returning MimeHandlerText\n");
MD5String("MimeHandlerText", id);
return nobuild ? 0 : new MimeHandlerText(config, id);
} else if ("text/html" == lmime) {
LOGDEB2("mhFactory(" << mime << "): returning MimeHandlerHtml\n");
MD5String("MimeHandlerHtml", id);
return nobuild ? 0 : new MimeHandlerHtml(config, id);
} else if ("text/x-mail" == lmime) {
LOGDEB2("mhFactory(" << mime << "): returning MimeHandlerMbox\n");
MD5String("MimeHandlerMbox", id);
return nobuild ? 0 : new MimeHandlerMbox(config, id);
} else if ("message/rfc822" == lmime) {
LOGDEB2("mhFactory(" << mime << "): returning MimeHandlerMail\n");
MD5String("MimeHandlerMail", id);
return nobuild ? 0 : new MimeHandlerMail(config, id);
} else if ("inode/symlink" == lmime) {
LOGDEB2("mhFactory(" << mime << "): ret MimeHandlerSymlink\n");
MD5String("MimeHandlerSymlink", id);
return nobuild ? 0 : new MimeHandlerSymlink(config, id);
} else if ("application/x-zerosize" == lmime) {
LOGDEB("mhFactory(" << mime << "): ret MimeHandlerNull\n");
MD5String("MimeHandlerNull", id);
return nobuild ? 0 : new MimeHandlerNull(config, id);
} else if (lmime.find("text/") == 0) {
// Try to handle unknown text/xx as text/plain. This
// only happen if the text/xx was defined as "internal" in
// mimeconf, not at random. For programs, for example this
// allows indexing and previewing as text/plain (no filter
// exec) but still opening with a specific editor.
LOGDEB2("mhFactory(" << mime << "): returning MimeHandlerText(x)\n");
MD5String("MimeHandlerText", id);
return nobuild ? 0 : new MimeHandlerText(config, id);
} else {
// We should not get there. It means that "internal" was set
// as a handler in mimeconf for a mime type we actually can't
// handle.
LOGERR("mhFactory: mime type [" << lmime <<
"] set as internal but unknown\n");
MD5String("MimeHandlerUnknown", id);
return nobuild ? 0 : new MimeHandlerUnknown(config, id);
}
}
static const string cstr_mh_charset("charset");
/**
* Create a filter that executes an external program or script
* A filter def can look like:
* someprog -v -t " h i j";charset= xx; mimetype=yy
* A semi-colon list of attr=value pairs may come after the exec spec.
* This list is treated by replacing semi-colons with newlines and building
* a confsimple. This is done quite brutally and we don't support having
* a ';' inside a quoted string for now. Can't see a use for it.
*/
MimeHandlerExec *mhExecFactory(RclConfig *cfg, const string& mtype, string& hs,
bool multiple, const string& id)
{
ConfSimple attrs;
string cmdstr;
if (!cfg->valueSplitAttributes(hs, cmdstr, attrs)) {
LOGERR("mhExecFactory: bad config line for [" <<
mtype << "]: [" << hs << "]\n");
return 0;
}
// Split command name and args, and build exec object
vector<string> cmdtoks;
stringToStrings(cmdstr, cmdtoks);
if (cmdtoks.empty()) {
LOGERR("mhExecFactory: bad config line for [" << mtype <<
"]: [" << hs << "]\n");
return 0;
}
MimeHandlerExec *h = multiple ?
new MimeHandlerExecMultiple(cfg, id) :
new MimeHandlerExec(cfg, id);
vector<string>::iterator it = cmdtoks.begin();
// Special-case python and perl on windows: we need to also locate the
// first argument which is the script name "python somescript.py".
// On Unix, thanks to #!, we usually just run "somescript.py", but need
// the same change if we ever want to use the same cmdling as windows
if (!stringlowercmp("python", *it) || !stringlowercmp("perl", *it)) {
if (cmdtoks.size() < 2) {
LOGERR("mhExecFactory: python/perl cmd: no script?. [" <<
mtype << "]: [" << hs << "]\n");
}
vector<string>::iterator it1(it);
it1++;
*it1 = cfg->findFilter(*it1);
}
h->params.push_back(cfg->findFilter(*it++));
h->params.insert(h->params.end(), it, cmdtoks.end());
// Handle additional attributes. We substitute the semi-colons
// with newlines and use a ConfSimple
string value;
if (attrs.get(cstr_mh_charset, value))
h->cfgFilterOutputCharset = stringtolower((const string&)value);
if (attrs.get(cstr_dj_keymt, value))
h->cfgFilterOutputMtype = stringtolower((const string&)value);
#if 0
string scmd;
for (it = h->params.begin(); it != h->params.end(); it++) {
scmd += string("[") + *it + "] ";
}
LOGDEB("mhExecFactory:mt [" << mtype << "] cfgmt [" <<
h->cfgFilterOutputMtype << "] cfgcs [" <<
h->cfgFilterOutputCharset << "] cmd: [" << scmd << "]\n");
#endif
return h;
}
/* Get handler/filter object for given mime type: */
RecollFilter *getMimeHandler(const string &mtype, RclConfig *cfg,
bool filtertypes)
{
LOGDEB("getMimeHandler: mtype [" << mtype << "] filtertypes " <<
filtertypes << "\n");
RecollFilter *h = 0;
// Get handler definition for mime type. We do this even if an
// appropriate handler object may be in the cache.
// This is fast, and necessary to conform to the
// configuration, (ie: text/html might be filtered out by
// indexedmimetypes but an html handler could still be in the
// cache because it was needed by some other interning stack).
string hs;
hs = cfg->getMimeHandlerDef(mtype, filtertypes);
string id;
if (!hs.empty()) {
// Got a handler definition line
// Break definition into type (internal/exec/execm)
// and name/command string
string::size_type b1 = hs.find_first_of(" \t");
string handlertype = hs.substr(0, b1);
string cmdstr;
if (b1 != string::npos) {
cmdstr = hs.substr(b1);
trimstring(cmdstr);
}
bool internal = !stringlowercmp("internal", handlertype);
if (internal) {
// For internal types let the factory compute the id
mhFactory(cfg, cmdstr.empty() ? mtype : cmdstr, true, id);
} else {
// exec/execm: use the md5 of the def line
MD5String(hs, id);
}
// Do we already have a handler object in the cache ?
h = getMimeHandlerFromCache(id);
if (h != 0)
goto out;
LOGDEB2("getMimeHandler: " << mtype << " not in cache\n");
// Not in cache.
if (internal) {
// If there is a parameter after "internal" it's the mime
// type to use. This is so that we can have bogus mime
// types like text/x-purple-html-log (for ie: specific
// icon) and still use the html filter on them. This is
// partly redundant with the localfields/rclaptg, but
// better and the latter will probably go away at some
// point in the future.
LOGDEB2("handlertype internal, cmdstr [" << cmdstr << "]\n");
h = mhFactory(cfg, cmdstr.empty() ? mtype : cmdstr, false, id);
goto out;
} else if (!stringlowercmp("dll", handlertype)) {
} else {
if (cmdstr.empty()) {
LOGERR("getMimeHandler: bad line for " << mtype << ": " <<
hs << "\n");
goto out;
}
if (!stringlowercmp("exec", handlertype)) {
h = mhExecFactory(cfg, mtype, cmdstr, false, id);
goto out;
} else if (!stringlowercmp("execm", handlertype)) {
h = mhExecFactory(cfg, mtype, cmdstr, true, id);
goto out;
} else {
LOGERR("getMimeHandler: bad line for " << mtype << ": " <<
hs << "\n");
goto out;
}
}
}
// We get here if there was no specific error, but there is no
// identified mime type, or no handler associated.
// Finally, unhandled files are either ignored or their name and
// generic metadata is indexed, depending on configuration
{
bool indexunknown = false;
cfg->getConfParam("indexallfilenames", &indexunknown);
if (indexunknown) {
MD5String("MimeHandlerUnknown", id);
if ((h = getMimeHandlerFromCache(id)) == 0)
h = new MimeHandlerUnknown(cfg, id);
}
goto out;
}
out:
if (h) {
h->set_property(RecollFilter::DEFAULT_CHARSET, cfg->getDefCharset());
// In multithread context, and in case this handler is out
// from the cache, it may have a config pointer belonging to
// another thread. Fix it.
h->setConfig(cfg);
}
return h;
}
/// Can this mime type be interned (according to config) ?
bool canIntern(const std::string mtype, RclConfig *cfg)
{
if (mtype.empty())
return false;
string hs = cfg->getMimeHandlerDef(mtype);
if (hs.empty())
return false;
return true;
}