/* Copyright (C) 2005 J.F.Dockes
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the
* Free Software Foundation, Inc.,
* 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
*/
#include "autoconfig.h"
#include <stdio.h>
#include <unistd.h>
#include <sys/stat.h>
#include <errno.h>
#include <iostream>
#include <string>
#ifndef NO_NAMESPACES
using namespace std;
#endif /* NO_NAMESPACES */
#include "cstr.h"
#include "mh_text.h"
#include "csguess.h"
#include "debuglog.h"
#include "readfile.h"
#include "md5.h"
#include "rclconfig.h"
const int MB = 1024*1024;
const int KB = 1024;
// Process a plain text file
bool MimeHandlerText::set_document_file(const string &fn)
{
LOGDEB(("MimeHandlerText::set_document_file: [%s]\n", fn.c_str()));
RecollFilter::set_document_file(fn);
m_fn = fn;
// file size for oversize check
struct stat st;
if (stat(m_fn.c_str(), &st) < 0) {
LOGERR(("MimeHandlerText::set_document_file: stat(%s) errno %d\n",
m_fn.c_str(), errno));
return false;
}
// Max file size parameter: texts over this size are not indexed
int maxmbs = 20;
m_config->getConfParam("textfilemaxmbs", &maxmbs);
if (maxmbs == -1 || st.st_size / MB <= maxmbs) {
// Text file page size: if set, we split text files into
// multiple documents
int ps = 1000;
m_config->getConfParam("textfilepagekbs", &ps);
if (ps != -1) {
ps *= KB;
m_paging = true;
}
m_pagesz = size_t(ps);
string reason;
LOGDEB(("calling file_to_string\n"));
// file_to_string() takes pagesz == size_t(-1) to mean read all.
if (!file_to_string(fn, m_text, 0, m_pagesz, &reason)) {
LOGERR(("MimeHandlerText: can't read file: %s\n", reason.c_str()));
return false;
}
LOGDEB(("file_to_string OK\n"));
m_offs = m_text.length();
}
string md5, xmd5;
MD5String(m_text, md5);
m_metaData[cstr_dj_keymd5] = MD5HexPrint(md5, xmd5);
m_havedoc = true;
return true;
}
bool MimeHandlerText::set_document_string(const string& otext)
{
m_text = otext;
string md5, xmd5;
MD5String(m_text, md5);
m_metaData[cstr_dj_keymd5] = MD5HexPrint(md5, xmd5);
m_havedoc = true;
return true;
}
bool MimeHandlerText::skip_to_document(const string& ipath)
{
long long t;
if (sscanf(ipath.c_str(), "%lld", &t) != 1) {
LOGERR(("MimeHandlerText::skip_to_document: bad ipath offs [%s]\n",
ipath.c_str()));
return false;
}
m_offs = (off_t)t;
readnext();
return true;
}
bool MimeHandlerText::next_document()
{
LOGDEB(("MimeHandlerText::next_document: m_havedoc %d\n", int(m_havedoc)));
if (m_havedoc == false)
return false;
// We transcode even if defcharset is supposedly already utf-8:
// this validates the encoding.
m_metaData[cstr_dj_keyorigcharset] = m_dfltInputCharset;
m_metaData[cstr_dj_keymt] = cstr_textplain;
size_t srclen = m_text.length();
m_metaData[cstr_dj_keycontent].swap(m_text);
// txtdcode() truncates the text if transcoding fails
(void)txtdcode("mh_text");
// If the text length is 0 (the file is empty or oversize), or we are
// not paging, we're done
if (srclen == 0 || !m_paging) {
m_havedoc = false;
return true;
} else {
// Paging: set ipath then read next chunk.
// Don't set ipath for the first chunk to avoid having 2
// records for small files (one for the file, one for the
// first chunk). This is a hack. The right thing to do would
// be to use a different mtype for files over the page size,
// and keep text/plain only for smaller files.
char buf[30];
sprintf(buf, "%lld", (long long)(m_offs - srclen));
if (m_offs - srclen != 0)
m_metaData[cstr_dj_keyipath] = buf;
readnext();
return true;
}
}
bool MimeHandlerText::readnext()
{
string reason;
m_text.clear();
if (!file_to_string(m_fn, m_text, m_offs, m_pagesz, &reason)) {
LOGERR(("MimeHandlerText: can't read file: %s\n", reason.c_str()));
m_havedoc = false;
return false;
}
if (m_text.length() == 0) {
// EOF
m_havedoc = false;
return true;
}
// If possible try to adjust the chunk to end right after a line
// Don't do this for the last chunk
if (m_text.length() == m_pagesz) {
string::size_type pos = m_text.find_last_of("\n\r");
if (pos != string::npos && pos != 0) {
m_text.erase(pos);
}
}
m_offs += m_text.length();
return true;
}