recoll / Code / Diff of /src/index/indexer.cpp

Diff of /src/index/indexer.cpp [cca706] .. [01d963]

Switch to unified view


#ifndef lint
static char rcsid[] = "@(#$Id: indexer.cpp,v 1.38 2006-10-16 15:33:08 dockes Exp $ (C) 2004 J.F.Dockes";
#endif
/*
 *   This program is free software; you can redistribute it and/or modify
 *   it under the terms of the GNU General Public License as published by
 *   the Free Software Foundation; either version 2 of the License, or
...
#include <stdio.h>
#include <sys/stat.h>
#include <unistd.h>
#include <errno.h>
#include <strings.h>
#include <fnmatch.h>

#include <iostream>
#include <list>
#include <map>
#include <algorithm>
...
#endif
    return true;
}

/** 
 * Index individual files, out of a full tree run. No database purging
 */
bool DbIndexer::indexFiles(const list<string> &filenames)
{
    if (!init())
    return false;

    list<string>::const_iterator it;
    for (it = filenames.begin(); it != filenames.end(); it++) {
  string dir = path_getfather(*it);
  m_config->setKeyDir(dir);
    int abslen;
    if (m_config->getConfParam("idxabsmlen", &abslen))
        m_db.setAbstractParams(abslen, -1, -1);
    struct stat stb;
    if (stat(it->c_str(), &stb) != 0) {
...
    if (!S_ISREG(stb.st_mode)) {
        LOGERR(("DbIndexer::indexFiles: %s: not a regular file\n", 
            it->c_str()));
        continue;
    }

  static string lstdir;
  static list<string> skpl;
  if (lstdir.compare(dir)) {
      LOGDEB(("Recomputing list of skipped names\n"));
      string skipped; 
      if (m_config->getConfParam("skippedNames", skipped)) {
      stringToStrings(skipped, skpl);
      lstdir = dir;
      }
  }
  if (!skpl.empty()) {
      list<string>::const_iterator skit;
      string fn = path_getsimple(*it);
      for (skit = skpl.begin(); skit != skpl.end(); skit++) {
      if (fnmatch(skit->c_str(), fn.c_str(), 0) == 0) {
          LOGDEB(("Skipping [%s] :matches skip list\n", fn.c_str()));
          goto skipped;
      }
      }
  }

    if (processone(*it, &stb, FsTreeWalker::FtwRegular) != 
        FsTreeWalker::FtwOk) {
        LOGERR(("DbIndexer::indexFiles: Database error\n"));
        return false;
    }
    skipped: 
  false; // Need a statement here to make compiler happy ??
    }

    // The close would be done in our destructor, but we want status here
    if (!m_db.close()) {
    LOGERR(("DbIndexer::indexfiles: error closing database in %s\n", 
        m_dbdir.c_str()));
    return false;
...
ConfIndexer::~ConfIndexer()
{
     deleteZ(m_dbindexer);
}



















bool ConfIndexer::index(bool resetbefore)
{
    list<string> tdl = m_config->getTopdirs();
    if (tdl.empty()) {
    m_reason = "Top directory list (topdirs param.) not found in config"
        "or Directory list parse error";
    return false;
    }

	a/src/index/indexer.cpp		b/src/index/indexer.cpp
1	#ifndef lint	1	#ifndef lint
2	static char rcsid[] = "@(#$Id: indexer.cpp,v 1.37 2006-10-12 14:46:02 dockes Exp $ (C) 2004 J.F.Dockes";	2	static char rcsid[] = "@(#$Id: indexer.cpp,v 1.38 2006-10-16 15:33:08 dockes Exp $ (C) 2004 J.F.Dockes";
3	#endif	3	#endif
4	/*	4	/*
5	* This program is free software; you can redistribute it and/or modify	5	* This program is free software; you can redistribute it and/or modify
6	* it under the terms of the GNU General Public License as published by	6	* it under the terms of the GNU General Public License as published by
7	* the Free Software Foundation; either version 2 of the License, or	7	* the Free Software Foundation; either version 2 of the License, or
	...		...
24	#include <stdio.h>	24	#include <stdio.h>
25	#include <sys/stat.h>	25	#include <sys/stat.h>
26	#include <unistd.h>	26	#include <unistd.h>
27	#include <errno.h>	27	#include <errno.h>
28	#include <strings.h>	28	#include <strings.h>
		29	#include <fnmatch.h>
29		30
30	#include <iostream>	31	#include <iostream>
31	#include <list>	32	#include <list>
32	#include <map>	33	#include <map>
33	#include <algorithm>	34	#include <algorithm>
	...		...
205	#endif	206	#endif
206	return true;	207	return true;
207	}	208	}
208		209
209	/**	210	/**
210	Index individual files, out of a full tree run. No database purging	211	* Index individual files, out of a full tree run. No database purging
211	*/	212	*/
212	bool DbIndexer::indexFiles(const list<string> &filenames)	213	bool DbIndexer::indexFiles(const list<string> &filenames)
213	{	214	{
214	if (!init())	215	if (!init())
215	return false;	216	return false;
216		217
217	list<string>::const_iterator it;	218	list<string>::const_iterator it;
218	for (it = filenames.begin(); it != filenames.end();it++) {	219	for (it = filenames.begin(); it != filenames.end(); it++) {
219	m_config->setKeyDir(path_getfather(*it));	220	string dir = path_getfather(*it);
		221	m_config->setKeyDir(dir);
220	int abslen;	222	int abslen;
221	if (m_config->getConfParam("idxabsmlen", &abslen))	223	if (m_config->getConfParam("idxabsmlen", &abslen))
222	m_db.setAbstractParams(abslen, -1, -1);	224	m_db.setAbstractParams(abslen, -1, -1);
223	struct stat stb;	225	struct stat stb;
224	if (stat(it->c_str(), &stb) != 0) {	226	if (stat(it->c_str(), &stb) != 0) {
	...		...
229	if (!S_ISREG(stb.st_mode)) {	231	if (!S_ISREG(stb.st_mode)) {
230	LOGERR(("DbIndexer::indexFiles: %s: not a regular file\n",	232	LOGERR(("DbIndexer::indexFiles: %s: not a regular file\n",
231	it->c_str()));	233	it->c_str()));
232	continue;	234	continue;
233	}	235	}
		236
		237	static string lstdir;
		238	static list<string> skpl;
		239	if (lstdir.compare(dir)) {
		240	LOGDEB(("Recomputing list of skipped names\n"));
		241	string skipped;
		242	if (m_config->getConfParam("skippedNames", skipped)) {
		243	stringToStrings(skipped, skpl);
		244	lstdir = dir;
		245	}
		246	}
		247	if (!skpl.empty()) {
		248	list<string>::const_iterator skit;
		249	string fn = path_getsimple(*it);
		250	for (skit = skpl.begin(); skit != skpl.end(); skit++) {
		251	if (fnmatch(skit->c_str(), fn.c_str(), 0) == 0) {
		252	LOGDEB(("Skipping [%s] :matches skip list\n", fn.c_str()));
		253	goto skipped;
		254	}
		255	}
		256	}
		257
234	if (processone(*it, &stb, FsTreeWalker::FtwRegular) !=	258	if (processone(*it, &stb, FsTreeWalker::FtwRegular) !=
235	FsTreeWalker::FtwOk) {	259	FsTreeWalker::FtwOk) {
236	LOGERR(("DbIndexer::indexFiles: Database error\n"));	260	LOGERR(("DbIndexer::indexFiles: Database error\n"));
237	return false;	261	return false;
238	}	262	}
		263	skipped:
		264	false; // Need a statement here to make compiler happy ??
239	}	265	}
		266
240	// The close would be done in our destructor, but we want status here	267	// The close would be done in our destructor, but we want status here
241	if (!m_db.close()) {	268	if (!m_db.close()) {
242	LOGERR(("DbIndexer::indexfiles: error closing database in %s\n",	269	LOGERR(("DbIndexer::indexfiles: error closing database in %s\n",
243	m_dbdir.c_str()));	270	m_dbdir.c_str()));
244	return false;	271	return false;
	...		...
369	ConfIndexer::~ConfIndexer()	396	ConfIndexer::~ConfIndexer()
370	{	397	{
371	deleteZ(m_dbindexer);	398	deleteZ(m_dbindexer);
372	}	399	}
373		400
374	list<string> topdirsToList(RclConfig *conf)
375	{
376	list<string> tdl;
377	// Retrieve the list of directories to be indexed.
378	string topdirs;
379	if (!conf->getConfParam("topdirs", topdirs)) {
380	LOGERR(("ConfIndexer::index: no top directories in configuration\n"));
381	return tdl;
382	}
383	if (!stringToStrings(topdirs, tdl)) {
384	LOGERR(("ConfIndexer::index: parse error for directory list\n"));
385	}
386	for (list<string>::iterator it = tdl.begin(); it != tdl.end(); it++) {
387	it = path_tildexpand(it);
388	}
389	return tdl;
390	}
391
392	bool ConfIndexer::index(bool resetbefore)	401	bool ConfIndexer::index(bool resetbefore)
393	{	402	{
394	list<string> tdl = topdirsToList(m_config);	403	list<string> tdl = m_config->getTopdirs();
395	if (tdl.empty()) {	404	if (tdl.empty()) {
396	m_reason = "Top directory list (topdirs param.) not found in config"	405	m_reason = "Top directory list (topdirs param.) not found in config"
397	"or Directory list parse error";	406	"or Directory list parse error";
398	return false;	407	return false;
399	}	408	}