recoll / Code / Diff of /src/internfile/internfile.cpp

Diff of /src/internfile/internfile.cpp [643f4d] .. [b2a0e6]

Switch to unified view


...
void FileInterner::init(const string &f, const struct stat *stp, RclConfig *cnf,
                        int flags, const string *imime)
{
    m_fn = f;

    // Compute udi for the input file. This is used by filters which
    // manage some kind of cache.  Indexing by udi makes things easier
    // because they sometimes get a temp as actual input.
    string udi;
    make_udi(f, cstr_null, udi);

    cnf->setKeyDir(path_getfather(m_fn));

...
    cnf->getConfParam("usesystemfilecommand", &usfci);

    // In general, even when the input mime type is set (when
    // previewing), we can't use it: it's the type for the actual
    // document, but this can be part of a compound document, and
    // we're dealing with the top level file here, or this could be a
    // compressed file. The flag tells us we really can use it
    // (e.g. the beagle indexer sets it).
    if (flags & FIF_doUseInputMimetype) {
        if (!imime) {
            LOGERR(("FileInterner:: told to use null imime\n"));
            return;
        }
        l_mime = *imime;
    } else {
        LOGDEB(("FileInterner:: [%s] mime [%s] preview %d\n", 
                f.c_str(), imime?imime->c_str() : "(null)", m_forPreview));

        // Run mime type identification in any case (see comment above).

        l_mime = mimetype(m_fn, stp, m_cfg, usfci);

        // If identification fails, try to use the input parameter. This
        // is then normally not a compressed type (it's the mime type from
        // the db), and is only set when previewing, not for indexing
...
    return;
    }
    df->set_property(Dijon::Filter::OPERATING_MODE, 
                m_forPreview ? "view" : "index");

    bool result = false;
    df->set_docsize(data.length());
    if (df->is_data_input_ok(Dijon::Filter::DOCUMENT_STRING)) {
    result = df->set_document_string(data);
    } else if (df->is_data_input_ok(Dijon::Filter::DOCUMENT_DATA)) {
    result = df->set_document_data(data.c_str(), data.length());
    } else if (df->is_data_input_ok(Dijon::Filter::DOCUMENT_FILE_NAME)) {
  TempFile temp = dataToTempFile(data, m_mimetype);
  if (temp.isNotNull() && 
        (result = df->set_document_file(temp->filename()))) {
      m_tmpflgs[m_handlers.size()] = true;
        m_tempfiles.push_back(temp);
  }
    }


    if (!result) {
    LOGINFO(("FileInterner:: set_doc failed inside for mtype %s\n", 
                 m_mimetype.c_str()));
    delete df;
    return;
    }
...
}

// Create a temporary file for a block of data (ie: attachment) found
// while walking the internal document tree, with a type for which the
// handler needs an actual file (ie : external script).
TempFile FileInterner::dataToTempFile(const string& dt, const string& mt)

{
    // Create temp file with appropriate suffix for mime type
    TempFile temp(new TempFileInternal(m_cfg->getSuffixFromMimeType(mt)));
    if (!temp->ok()) {






    LOGERR(("FileInterner::dataToTempFile: cant create tempfile: %s\n",
        temp->getreason().c_str()));
  return TempFile();
    }

    int fd = open(temp->filename(), O_WRONLY);
    if (fd < 0) {
    LOGERR(("FileInterner::dataToTempFile: open(%s) failed errno %d\n",
        temp->filename(), errno));
  return TempFile();
    }
    if (write(fd, dt.c_str(), dt.length()) != (int)dt.length()) {
    close(fd);
    LOGERR(("FileInterner::dataToTempFile: write to %s failed errno %d\n",
        temp->filename(), errno));
  return TempFile();
    }
    close(fd);

    return temp;
}

// See if the error string is formatted as a missing helper message,
// accumulate helper name if it is
void FileInterner::checkExternalMissing(const string& msg, const string& mt)
...
// Remove handler from stack. Clean up temp file if needed.
void FileInterner::popHandler()
{
    if (m_handlers.empty())
    return;
    int i = m_handlers.size() - 1;
    if (m_tmpflgs[i]) {
    m_tempfiles.pop_back();
    m_tmpflgs[i] = false;
    }
    returnMimeHandler(m_handlers.back());
...
    if (newflt->is_data_input_ok(Dijon::Filter::DOCUMENT_STRING)) {
    setres = newflt->set_document_string(*txt);
    } else if (newflt->is_data_input_ok(Dijon::Filter::DOCUMENT_DATA)) {
    setres = newflt->set_document_data(txt->c_str(), txt->length());
    } else if (newflt->is_data_input_ok(Dijon::Filter::DOCUMENT_FILE_NAME)) {
  TempFile temp = dataToTempFile(*txt, mimetype);
  if (temp.isNotNull() && 
        (setres = newflt->set_document_file(temp->filename()))) {
        m_tmpflgs[m_handlers.size()] = true;
        m_tempfiles.push_back(temp);

        // Hack here, but really helps perfs: if we happen to
        // create a temp file for, ie, an image attachment, keep
      // it around for preview to use it through get_imgtmp()
        if (!mimetype.compare(0, 6, "image/")) {
        m_imgtmp = m_tempfiles.back();

        }
    }
    }
    if (!setres) {
    LOGINFO(("FileInterner::addHandler: set_doc failed inside %s "

	a/src/internfile/internfile.cpp		b/src/internfile/internfile.cpp
	...		...
199	void FileInterner::init(const string &f, const struct stat stp, RclConfig cnf,	199	void FileInterner::init(const string &f, const struct stat stp, RclConfig cnf,
200	int flags, const string *imime)	200	int flags, const string *imime)
201	{	201	{
202	m_fn = f;	202	m_fn = f;
203		203
204	// This is used by filters which manage some kind of cache.	204	// Compute udi for the input file. This is used by filters which
205	// Indexing by udi makes things easier (because they sometimes get a temp	205	// manage some kind of cache. Indexing by udi makes things easier
206	// as input	206	// because they sometimes get a temp as actual input.
207	string udi;	207	string udi;
208	make_udi(f, cstr_null, udi);	208	make_udi(f, cstr_null, udi);
209		209
210	cnf->setKeyDir(path_getfather(m_fn));	210	cnf->setKeyDir(path_getfather(m_fn));
211		211
	...		...
214	cnf->getConfParam("usesystemfilecommand", &usfci);	214	cnf->getConfParam("usesystemfilecommand", &usfci);
215		215
216	// In general, even when the input mime type is set (when	216	// In general, even when the input mime type is set (when
217	// previewing), we can't use it: it's the type for the actual	217	// previewing), we can't use it: it's the type for the actual
218	// document, but this can be part of a compound document, and	218	// document, but this can be part of a compound document, and
219	// we're dealing with the top level file here.	219	// we're dealing with the top level file here, or this could be a
		220	// compressed file. The flag tells us we really can use it
		221	// (e.g. the beagle indexer sets it).
220	if (flags & FIF_doUseInputMimetype) {	222	if (flags & FIF_doUseInputMimetype) {
221	if (!imime) {	223	if (!imime) {
222	LOGERR(("FileInterner:: told to use null imime\n"));	224	LOGERR(("FileInterner:: told to use null imime\n"));
223	return;	225	return;
224	}	226	}
225	l_mime = *imime;	227	l_mime = *imime;
226	} else {	228	} else {
227	LOGDEB(("FileInterner:: [%s] mime [%s] preview %d\n",	229	LOGDEB(("FileInterner:: [%s] mime [%s] preview %d\n",
228	f.c_str(), imime?imime->c_str() : "(null)", m_forPreview));	230	f.c_str(), imime?imime->c_str() : "(null)", m_forPreview));
229		231
230	// We need to run mime type identification in any case to check	232	// Run mime type identification in any case (see comment above).
231	// for a compressed file.
232	l_mime = mimetype(m_fn, stp, m_cfg, usfci);	233	l_mime = mimetype(m_fn, stp, m_cfg, usfci);
233		234
234	// If identification fails, try to use the input parameter. This	235	// If identification fails, try to use the input parameter. This
235	// is then normally not a compressed type (it's the mime type from	236	// is then normally not a compressed type (it's the mime type from
236	// the db), and is only set when previewing, not for indexing	237	// the db), and is only set when previewing, not for indexing
	...		...
342	return;	343	return;
343	}	344	}
344	df->set_property(Dijon::Filter::OPERATING_MODE,	345	df->set_property(Dijon::Filter::OPERATING_MODE,
345	m_forPreview ? "view" : "index");	346	m_forPreview ? "view" : "index");
346		347
347	bool setres = false;	348	bool result = false;
348	df->set_docsize(data.length());	349	df->set_docsize(data.length());
349	if (df->is_data_input_ok(Dijon::Filter::DOCUMENT_STRING)) {	350	if (df->is_data_input_ok(Dijon::Filter::DOCUMENT_STRING)) {
350	setres = df->set_document_string(data);	351	result = df->set_document_string(data);
351	} else if (df->is_data_input_ok(Dijon::Filter::DOCUMENT_DATA)) {	352	} else if (df->is_data_input_ok(Dijon::Filter::DOCUMENT_DATA)) {
352	setres = df->set_document_data(data.c_str(), data.length());	353	result = df->set_document_data(data.c_str(), data.length());
353	} else if (df->is_data_input_ok(Dijon::Filter::DOCUMENT_FILE_NAME)) {	354	} else if (df->is_data_input_ok(Dijon::Filter::DOCUMENT_FILE_NAME)) {
354	string filename;	355	TempFile temp = dataToTempFile(data, m_mimetype);
355	if (dataToTempFile(data, m_mimetype, filename)) {	356	if (temp.isNotNull() &&
356	if (!(setres=df->set_document_file(filename))) {	357	(result = df->set_document_file(temp->filename()))) {
357	m_tmpflgs[0] = false;	358	m_tmpflgs[m_handlers.size()] = true;
358	m_tempfiles.pop_back();	359	m_tempfiles.push_back(temp);
		360	}
359	}	361	}
360	}
361	}
362	if (!setres) {	362	if (!result) {
363	LOGINFO(("FileInterner:: set_doc failed inside for mtype %s\n",	363	LOGINFO(("FileInterner:: set_doc failed inside for mtype %s\n",
364	m_mimetype.c_str()));	364	m_mimetype.c_str()));
365	delete df;	365	delete df;
366	return;	366	return;
367	}	367	}
	...		...
434	}	434	}
435		435
436	// Create a temporary file for a block of data (ie: attachment) found	436	// Create a temporary file for a block of data (ie: attachment) found
437	// while walking the internal document tree, with a type for which the	437	// while walking the internal document tree, with a type for which the
438	// handler needs an actual file (ie : external script).	438	// handler needs an actual file (ie : external script).
439	bool FileInterner::dataToTempFile(const string& dt, const string& mt,	439	TempFile FileInterner::dataToTempFile(const string& dt, const string& mt)
440	string& fn)
441	{	440	{
442	// Find appropriate suffix for mime type	441	// Create temp file with appropriate suffix for mime type
443	TempFile temp(new TempFileInternal(m_cfg->getSuffixFromMimeType(mt)));	442	TempFile temp(new TempFileInternal(m_cfg->getSuffixFromMimeType(mt)));
444	if (temp->ok()) {	443	if (!temp->ok()) {
445	// We are called before the handler is actually on the stack, so the
446	// index is m_handlers.size(). m_tmpflgs is a static array, so this is
447	// no problem
448	m_tmpflgs[m_handlers.size()] = true;
449	m_tempfiles.push_back(temp);
450	} else {
451	LOGERR(("FileInterner::dataToTempFile: cant create tempfile: %s\n",	444	LOGERR(("FileInterner::dataToTempFile: cant create tempfile: %s\n",
452	temp->getreason().c_str()));	445	temp->getreason().c_str()));
453	return false;	446	return TempFile();
454	}	447	}
455		448
456	int fd = open(temp->filename(), O_WRONLY);	449	int fd = open(temp->filename(), O_WRONLY);
457	if (fd < 0) {	450	if (fd < 0) {
458	LOGERR(("FileInterner::dataToTempFile: open(%s) failed errno %d\n",	451	LOGERR(("FileInterner::dataToTempFile: open(%s) failed errno %d\n",
459	temp->filename(), errno));	452	temp->filename(), errno));
460	return false;	453	return TempFile();
461	}	454	}
462	if (write(fd, dt.c_str(), dt.length()) != (int)dt.length()) {	455	if (write(fd, dt.c_str(), dt.length()) != (int)dt.length()) {
463	close(fd);	456	close(fd);
464	LOGERR(("FileInterner::dataToTempFile: write to %s failed errno %d\n",	457	LOGERR(("FileInterner::dataToTempFile: write to %s failed errno %d\n",
465	temp->filename(), errno));	458	temp->filename(), errno));
466	return false;	459	return TempFile();
467	}	460	}
468	close(fd);	461	close(fd);
469	fn = temp->filename();
470	return true;	462	return temp;
471	}	463	}
472		464
473	// See if the error string is formatted as a missing helper message,	465	// See if the error string is formatted as a missing helper message,
474	// accumulate helper name if it is	466	// accumulate helper name if it is
475	void FileInterner::checkExternalMissing(const string& msg, const string& mt)	467	void FileInterner::checkExternalMissing(const string& msg, const string& mt)
	...		...
705	// Remove handler from stack. Clean up temp file if needed.	697	// Remove handler from stack. Clean up temp file if needed.
706	void FileInterner::popHandler()	698	void FileInterner::popHandler()
707	{	699	{
708	if (m_handlers.empty())	700	if (m_handlers.empty())
709	return;	701	return;
710	int i = m_handlers.size()-1;	702	int i = m_handlers.size() - 1;
711	if (m_tmpflgs[i]) {	703	if (m_tmpflgs[i]) {
712	m_tempfiles.pop_back();	704	m_tempfiles.pop_back();
713	m_tmpflgs[i] = false;	705	m_tmpflgs[i] = false;
714	}	706	}
715	returnMimeHandler(m_handlers.back());	707	returnMimeHandler(m_handlers.back());
	...		...
775	if (newflt->is_data_input_ok(Dijon::Filter::DOCUMENT_STRING)) {	767	if (newflt->is_data_input_ok(Dijon::Filter::DOCUMENT_STRING)) {
776	setres = newflt->set_document_string(*txt);	768	setres = newflt->set_document_string(*txt);
777	} else if (newflt->is_data_input_ok(Dijon::Filter::DOCUMENT_DATA)) {	769	} else if (newflt->is_data_input_ok(Dijon::Filter::DOCUMENT_DATA)) {
778	setres = newflt->set_document_data(txt->c_str(), txt->length());	770	setres = newflt->set_document_data(txt->c_str(), txt->length());
779	} else if (newflt->is_data_input_ok(Dijon::Filter::DOCUMENT_FILE_NAME)) {	771	} else if (newflt->is_data_input_ok(Dijon::Filter::DOCUMENT_FILE_NAME)) {
780	string filename;	772	TempFile temp = dataToTempFile(*txt, mimetype);
781	if (dataToTempFile(*txt, mimetype, filename)) {	773	if (temp.isNotNull() &&
782	if (!(setres = newflt->set_document_file(filename))) {	774	(setres = newflt->set_document_file(temp->filename()))) {
783	m_tmpflgs[m_handlers.size()] = false;	775	m_tmpflgs[m_handlers.size()] = true;
784	m_tempfiles.pop_back();	776	m_tempfiles.push_back(temp);
785	} else {
786	// Hack here, but really helps perfs: if we happen to	777	// Hack here, but really helps perfs: if we happen to
787	// create a temp file for, ie, an image attachment,	778	// create a temp file for, ie, an image attachment, keep
788	// keep it around for preview reuse	779	// it around for preview to use it through get_imgtmp()
789	if (!mimetype.compare(0, 6, "image/")) {	780	if (!mimetype.compare(0, 6, "image/")) {
790	m_imgtmp = m_tempfiles.back();	781	m_imgtmp = m_tempfiles.back();
791	}
792	}	782	}
793	}	783	}
794	}	784	}
795	if (!setres) {	785	if (!setres) {
796	LOGINFO(("FileInterner::addHandler: set_doc failed inside %s "	786	LOGINFO(("FileInterner::addHandler: set_doc failed inside %s "