recoll / Code / Diff of /src/index/recollindex.cpp

Diff of /src/index/recollindex.cpp [6a60ac] .. [c7c9c4]

Switch to unified view


...
#define OPT_x     0x800
#define OPT_l     0x1000
#define OPT_b     0x2000
#define OPT_f     0x4000
#define OPT_C     0x8000
#define OPT_Z     0x10000

ReExec *o_reexec;

// Globals for atexit cleanup
static ConfIndexer *confindexer;
...
    LOGDEB(("Got signal, registering stop request\n"));
    CancelCheck::instance().setCancel();
    stopindexing = 1;
}

static void makeIndexerOrExit(RclConfig *config, bool inPlaceReset)
{
    if (!confindexer) {
    confindexer = new ConfIndexer(config, updater);
  if (inPlaceReset)
      confindexer->setInPlaceReset();
    }
    if (!confindexer) {
        cerr << "Cannot create indexer" << endl;
        exit(1);
    }

}

void rclIxIonice(RclConfig *config)
{
    string clss, classdata;
...
// Index a list of files. We just check that they belong to one of the
// topdirs subtrees, and call the indexer method. 
//
// This is called either from the command line or from the monitor. In
// this case we're called repeatedly in the same process, and the
// confindexer is only created once by makeIndexerOrExit (but the db closed and
// flushed every time)
bool indexfiles(RclConfig *config, list<string> &filenames)
{
    if (filenames.empty())
    return true;
    makeIndexerOrExit(config, (op_flags & OPT_Z) != 0);

    return confindexer->indexFiles(filenames, (op_flags&OPT_f) ? 
                   ConfIndexer::IxFIgnoreSkip : 
                   ConfIndexer::IxFNone);
}

// Delete a list of files. Same comments about call contexts as indexfiles.
bool purgefiles(RclConfig *config, list<string> &filenames)
{
    if (filenames.empty())
    return true;
    makeIndexerOrExit(config, (op_flags & OPT_Z) != 0);

    return confindexer->purgeFiles(filenames);
}

// Create stemming and spelling databases
bool createAuxDbs(RclConfig *config)
{
    makeIndexerOrExit(config, false);


    if (!confindexer->createStemmingDatabases())
    return false;

    if (!confindexer->createAspellDict())
...
}

// Create additional stem database 
static bool createstemdb(RclConfig *config, const string &lang)
{
    makeIndexerOrExit(config, false);

    return confindexer->createStemDb(lang);
}

static const char *thisprog;

static const char usage [] =
"\n"
"recollindex [-h] \n"
"    Print help\n"
"recollindex [-z|-Z] \n"
"    Index everything according to configuration file\n"
"    -z : reset database before starting indexing\n"
"    -Z : in place reset: consider all documents as changed. Can also\n"
"         be combined with -i but not -m\n"
#ifdef RCL_MONITOR
"recollindex -m [-w <secs>] -x [-D] [-C]\n"
"    Perform real time indexing. Don't become a daemon if -D is set.\n"
"    -w sets number of seconds to wait before starting.\n"
"    -C disables monitoring config for changes/reexecuting.\n"
...
"    -f : ignore skippedPaths and skippedNames while doing this\n"
"recollindex -l\n"
"    List available stemming languages\n"
"recollindex -s <lang>\n"
"    Build stem database for additional language <lang>\n"
#ifdef FUTURE_IMPROVEMENT
"recollindex -b\n"
"    Process the Beagle queue\n"
#endif
#ifdef RCL_USE_ASPELL
"recollindex -S\n"
"    Build aspell spelling dictionary.>\n"
#endif
"Common options:\n"
...
{
    pid_t pid;
    if ((pid = pidfile->open()) != 0) {
    cerr << "Can't become exclusive indexer: " << pidfile->getreason() << 
        ". Return (other pid?): " << pid << endl;
  exit(1);
    }
    if (pidfile->write_pid() != 0) {
  cerr << "Can't become exclusive indexer: " << pidfile->getreason() <<
      endl;
    exit(1);
    }
}

int main(int argc, char **argv)
...
        case 'w':   op_flags |= OPT_w; if (argc < 2)  Usage();
        if ((sscanf(*(++argv), "%d", &sleepsecs)) != 1) 
            Usage(); 
        argc--; goto b1;
        case 'x': op_flags |= OPT_x; break;
      case 'Z': op_flags |= OPT_Z; break;
        case 'z': op_flags |= OPT_z; break;
        default: Usage(); break;
        }
    b1: argc--; argv++;
    }
...
    }
#endif

    if ((op_flags & OPT_z) && (op_flags & (OPT_i|OPT_e)))
    Usage();
    if ((op_flags & OPT_Z) && (op_flags & (OPT_m)))
  Usage();

    string reason;
    RclInitFlags flags = (op_flags & OPT_m) && !(op_flags&OPT_D) ? 
    RCLINIT_DAEMON : RCLINIT_NONE;
    config = recollinit(flags, cleanup, sigcleanup, reason, &a_config);
...
    exit(1);
    }
    o_reexec->atexit(cleanup);

    bool rezero(op_flags & OPT_z);
    bool inPlaceReset(op_flags & OPT_Z);
    Pidfile pidfile(config->getPidfile());
    updater = new MyUpdater(config);

    if (setpriority(PRIO_PROCESS, 0, 20) != 0) {
        LOGINFO(("recollindex: can't setpriority(), errno %d\n", errno));
...
    // Try to ionice. This does not work on all platforms
    rclIxIonice(config);

    if (op_flags & (OPT_i|OPT_e)) {
    lockorexit(&pidfile);


    list<string> filenames;

    if (argc == 0) {
        // Read from stdin
...
    } else if (op_flags & OPT_s) {
    if (argc != 1) 
        Usage();
    string lang = *argv++; argc--;
    exit(!createstemdb(config, lang));
#ifdef RCL_USE_ASPELL
    } else if (op_flags & OPT_S) {
  makeIndexerOrExit(config, inPlaceReset);
        exit(!confindexer->createAspellDict());
#endif // ASPELL

#ifdef RCL_MONITOR
    } else if (op_flags & OPT_m) {
    if (argc != 0) 
        Usage();
...
          fprintf(stderr, "daemon() failed, errno %d\n", errno);
          LOGERR(("daemon() failed, errno %d\n", errno));
          exit(1);
        }
    }
  // Need to rewrite pid, it changed
    pidfile.write_pid();

        // Not too sure if I have to redo the nice thing after daemon(),
        // can't hurt anyway (easier than testing on all platforms...)
        if (setpriority(PRIO_PROCESS, 0, 20) != 0) {
...
        LOGDEB(("X11 session went away during initial sleep period\n"));
        exit(0);
          }
        }
    }
  makeIndexerOrExit(config, inPlaceReset);
    if (!confindexer->index(rezero, ConfIndexer::IxTAll) || stopindexing) {
      LOGERR(("recollindex, initial indexing pass failed, not going into monitor mode\n"));
      exit(1);
    }
        if (updater) {
...
    bool monret = startMonitor(config, opts);
    MONDEB(("Monitor returned %d, exiting\n", monret));
    exit(monret == false);
#endif // MONITOR







    } else if (op_flags & OPT_b) {
        cerr << "Not yet" << endl;
        return 1;
    } else {
    lockorexit(&pidfile);
  makeIndexerOrExit(config, inPlaceReset);

    bool status = confindexer->index(rezero, ConfIndexer::IxTAll);
    if (!status) 
        cerr << "Indexing failed" << endl;
        if (!confindexer->getReason().empty())
            cerr << confindexer->getReason() << endl;

	a/src/index/recollindex.cpp		b/src/index/recollindex.cpp
	...		...
63	#define OPT_x 0x800	63	#define OPT_x 0x800
64	#define OPT_l 0x1000	64	#define OPT_l 0x1000
65	#define OPT_b 0x2000	65	#define OPT_b 0x2000
66	#define OPT_f 0x4000	66	#define OPT_f 0x4000
67	#define OPT_C 0x8000	67	#define OPT_C 0x8000
		68	#define OPT_Z 0x10000
68		69
69	ReExec *o_reexec;	70	ReExec *o_reexec;
70		71
71	// Globals for atexit cleanup	72	// Globals for atexit cleanup
72	static ConfIndexer *confindexer;	73	static ConfIndexer *confindexer;
	...		...
145	LOGDEB(("Got signal, registering stop request\n"));	146	LOGDEB(("Got signal, registering stop request\n"));
146	CancelCheck::instance().setCancel();	147	CancelCheck::instance().setCancel();
147	stopindexing = 1;	148	stopindexing = 1;
148	}	149	}
149		150
150	static bool makeIndexer(RclConfig *config)	151	static void makeIndexerOrExit(RclConfig *config, bool inPlaceReset)
151	{	152	{
152	if (!confindexer)	153	if (!confindexer) {
153	confindexer = new ConfIndexer(config, updater);	154	confindexer = new ConfIndexer(config, updater);
		155	if (inPlaceReset)
		156	confindexer->setInPlaceReset();
		157	}
154	if (!confindexer) {	158	if (!confindexer) {
155	cerr << "Cannot create indexer" << endl;	159	cerr << "Cannot create indexer" << endl;
156	exit(1);	160	exit(1);
157	}	161	}
158	return true;
159	}	162	}
160		163
161	void rclIxIonice(RclConfig *config)	164	void rclIxIonice(RclConfig *config)
162	{	165	{
163	string clss, classdata;	166	string clss, classdata;
	...		...
170	// Index a list of files. We just check that they belong to one of the	173	// Index a list of files. We just check that they belong to one of the
171	// topdirs subtrees, and call the indexer method.	174	// topdirs subtrees, and call the indexer method.
172	//	175	//
173	// This is called either from the command line or from the monitor. In	176	// This is called either from the command line or from the monitor. In
174	// this case we're called repeatedly in the same process, and the	177	// this case we're called repeatedly in the same process, and the
175	// confindexer is only created once by makeIndexer (but the db closed and	178	// confindexer is only created once by makeIndexerOrExit (but the db closed and
176	// flushed every time)	179	// flushed every time)
177	bool indexfiles(RclConfig *config, list<string> &filenames)	180	bool indexfiles(RclConfig *config, list<string> &filenames)
178	{	181	{
179	if (filenames.empty())	182	if (filenames.empty())
180	return true;	183	return true;
181	if (!makeIndexer(config))	184	makeIndexerOrExit(config, (op_flags & OPT_Z) != 0);
182	return false;
183	return confindexer->indexFiles(filenames, (op_flags&OPT_f) ?	185	return confindexer->indexFiles(filenames, (op_flags&OPT_f) ?
184	ConfIndexer::IxFIgnoreSkip :	186	ConfIndexer::IxFIgnoreSkip :
185	ConfIndexer::IxFNone);	187	ConfIndexer::IxFNone);
186	}	188	}
187		189
188	// Delete a list of files. Same comments about call contexts as indexfiles.	190	// Delete a list of files. Same comments about call contexts as indexfiles.
189	bool purgefiles(RclConfig *config, list<string> &filenames)	191	bool purgefiles(RclConfig *config, list<string> &filenames)
190	{	192	{
191	if (filenames.empty())	193	if (filenames.empty())
192	return true;	194	return true;
193	if (!makeIndexer(config))	195	makeIndexerOrExit(config, (op_flags & OPT_Z) != 0);
194	return false;
195	return confindexer->purgeFiles(filenames);	196	return confindexer->purgeFiles(filenames);
196	}	197	}
197		198
198	// Create stemming and spelling databases	199	// Create stemming and spelling databases
199	bool createAuxDbs(RclConfig *config)	200	bool createAuxDbs(RclConfig *config)
200	{	201	{
201	if (!makeIndexer(config))	202	makeIndexerOrExit(config, false);
202	return false;
203		203
204	if (!confindexer->createStemmingDatabases())	204	if (!confindexer->createStemmingDatabases())
205	return false;	205	return false;
206		206
207	if (!confindexer->createAspellDict())	207	if (!confindexer->createAspellDict())
	...		...
211	}	211	}
212		212
213	// Create additional stem database	213	// Create additional stem database
214	static bool createstemdb(RclConfig *config, const string &lang)	214	static bool createstemdb(RclConfig *config, const string &lang)
215	{	215	{
216	if (!makeIndexer(config))	216	makeIndexerOrExit(config, false);
217	return false;
218	return confindexer->createStemDb(lang);	217	return confindexer->createStemDb(lang);
219	}	218	}
220		219
221	static const char *thisprog;	220	static const char *thisprog;
222		221
223	static const char usage [] =	222	static const char usage [] =
224	"\n"	223	"\n"
225	"recollindex [-h] \n"	224	"recollindex [-h] \n"
226	" Print help\n"	225	" Print help\n"
227	"recollindex [-z] \n"	226	"recollindex [-z\|-Z] \n"
228	" Index everything according to configuration file\n"	227	" Index everything according to configuration file\n"
229	" -z : reset database before starting indexing\n"	228	" -z : reset database before starting indexing\n"
		229	" -Z : in place reset: consider all documents as changed. Can also\n"
		230	" be combined with -i but not -m\n"
230	#ifdef RCL_MONITOR	231	#ifdef RCL_MONITOR
231	"recollindex -m [-w <secs>] -x [-D] [-C]\n"	232	"recollindex -m [-w <secs>] -x [-D] [-C]\n"
232	" Perform real time indexing. Don't become a daemon if -D is set.\n"	233	" Perform real time indexing. Don't become a daemon if -D is set.\n"
233	" -w sets number of seconds to wait before starting.\n"	234	" -w sets number of seconds to wait before starting.\n"
234	" -C disables monitoring config for changes/reexecuting.\n"	235	" -C disables monitoring config for changes/reexecuting.\n"
	...		...
243	" -f : ignore skippedPaths and skippedNames while doing this\n"	244	" -f : ignore skippedPaths and skippedNames while doing this\n"
244	"recollindex -l\n"	245	"recollindex -l\n"
245	" List available stemming languages\n"	246	" List available stemming languages\n"
246	"recollindex -s <lang>\n"	247	"recollindex -s <lang>\n"
247	" Build stem database for additional language <lang>\n"	248	" Build stem database for additional language <lang>\n"
		249	#ifdef FUTURE_IMPROVEMENT
248	"recollindex -b\n"	250	"recollindex -b\n"
249	" Process the Beagle queue\n"	251	" Process the Beagle queue\n"
		252	#endif
250	#ifdef RCL_USE_ASPELL	253	#ifdef RCL_USE_ASPELL
251	"recollindex -S\n"	254	"recollindex -S\n"
252	" Build aspell spelling dictionary.>\n"	255	" Build aspell spelling dictionary.>\n"
253	#endif	256	#endif
254	"Common options:\n"	257	"Common options:\n"
	...		...
270	{	273	{
271	pid_t pid;	274	pid_t pid;
272	if ((pid = pidfile->open()) != 0) {	275	if ((pid = pidfile->open()) != 0) {
273	cerr << "Can't become exclusive indexer: " << pidfile->getreason() <<	276	cerr << "Can't become exclusive indexer: " << pidfile->getreason() <<
274	". Return (other pid?): " << pid << endl;	277	". Return (other pid?): " << pid << endl;
		278	exit(1);
		279	}
		280	if (pidfile->write_pid() != 0) {
		281	cerr << "Can't become exclusive indexer: " << pidfile->getreason() <<
		282	endl;
275	exit(1);	283	exit(1);
276	}	284	}
277	}	285	}
278		286
279	int main(int argc, char **argv)	287	int main(int argc, char **argv)
	...		...
313	case 'w': op_flags \|= OPT_w; if (argc < 2) Usage();	321	case 'w': op_flags \|= OPT_w; if (argc < 2) Usage();
314	if ((sscanf(*(++argv), "%d", &sleepsecs)) != 1)	322	if ((sscanf(*(++argv), "%d", &sleepsecs)) != 1)
315	Usage();	323	Usage();
316	argc--; goto b1;	324	argc--; goto b1;
317	case 'x': op_flags \|= OPT_x; break;	325	case 'x': op_flags \|= OPT_x; break;
		326	case 'Z': op_flags \|= OPT_Z; break;
318	case 'z': op_flags \|= OPT_z; break;	327	case 'z': op_flags \|= OPT_z; break;
319	default: Usage(); break;	328	default: Usage(); break;
320	}	329	}
321	b1: argc--; argv++;	330	b1: argc--; argv++;
322	}	331	}
	...		...
330	}	339	}
331	#endif	340	#endif
332		341
333	if ((op_flags & OPT_z) && (op_flags & (OPT_i\|OPT_e)))	342	if ((op_flags & OPT_z) && (op_flags & (OPT_i\|OPT_e)))
334	Usage();	343	Usage();
		344	if ((op_flags & OPT_Z) && (op_flags & (OPT_m)))
		345	Usage();
335		346
336	string reason;	347	string reason;
337	RclInitFlags flags = (op_flags & OPT_m) && !(op_flags&OPT_D) ?	348	RclInitFlags flags = (op_flags & OPT_m) && !(op_flags&OPT_D) ?
338	RCLINIT_DAEMON : RCLINIT_NONE;	349	RCLINIT_DAEMON : RCLINIT_NONE;
339	config = recollinit(flags, cleanup, sigcleanup, reason, &a_config);	350	config = recollinit(flags, cleanup, sigcleanup, reason, &a_config);
	...		...
342	exit(1);	353	exit(1);
343	}	354	}
344	o_reexec->atexit(cleanup);	355	o_reexec->atexit(cleanup);
345		356
346	bool rezero(op_flags & OPT_z);	357	bool rezero(op_flags & OPT_z);
		358	bool inPlaceReset(op_flags & OPT_Z);
347	Pidfile pidfile(config->getPidfile());	359	Pidfile pidfile(config->getPidfile());
348	updater = new MyUpdater(config);	360	updater = new MyUpdater(config);
349		361
350	if (setpriority(PRIO_PROCESS, 0, 20) != 0) {	362	if (setpriority(PRIO_PROCESS, 0, 20) != 0) {
351	LOGINFO(("recollindex: can't setpriority(), errno %d\n", errno));	363	LOGINFO(("recollindex: can't setpriority(), errno %d\n", errno));
	...		...
353	// Try to ionice. This does not work on all platforms	365	// Try to ionice. This does not work on all platforms
354	rclIxIonice(config);	366	rclIxIonice(config);
355		367
356	if (op_flags & (OPT_i\|OPT_e)) {	368	if (op_flags & (OPT_i\|OPT_e)) {
357	lockorexit(&pidfile);	369	lockorexit(&pidfile);
358	pidfile.write_pid();
359		370
360	list<string> filenames;	371	list<string> filenames;
361		372
362	if (argc == 0) {	373	if (argc == 0) {
363	// Read from stdin	374	// Read from stdin
	...		...
392	} else if (op_flags & OPT_s) {	403	} else if (op_flags & OPT_s) {
393	if (argc != 1)	404	if (argc != 1)
394	Usage();	405	Usage();
395	string lang = *argv++; argc--;	406	string lang = *argv++; argc--;
396	exit(!createstemdb(config, lang));	407	exit(!createstemdb(config, lang));
		408	#ifdef RCL_USE_ASPELL
		409	} else if (op_flags & OPT_S) {
		410	makeIndexerOrExit(config, inPlaceReset);
		411	exit(!confindexer->createAspellDict());
		412	#endif // ASPELL
397		413
398	#ifdef RCL_MONITOR	414	#ifdef RCL_MONITOR
399	} else if (op_flags & OPT_m) {	415	} else if (op_flags & OPT_m) {
400	if (argc != 0)	416	if (argc != 0)
401	Usage();	417	Usage();
	...		...
406	fprintf(stderr, "daemon() failed, errno %d\n", errno);	422	fprintf(stderr, "daemon() failed, errno %d\n", errno);
407	LOGERR(("daemon() failed, errno %d\n", errno));	423	LOGERR(("daemon() failed, errno %d\n", errno));
408	exit(1);	424	exit(1);
409	}	425	}
410	}	426	}
		427	// Need to rewrite pid, it changed
411	pidfile.write_pid();	428	pidfile.write_pid();
412		429
413	// Not too sure if I have to redo the nice thing after daemon(),	430	// Not too sure if I have to redo the nice thing after daemon(),
414	// can't hurt anyway (easier than testing on all platforms...)	431	// can't hurt anyway (easier than testing on all platforms...)
415	if (setpriority(PRIO_PROCESS, 0, 20) != 0) {	432	if (setpriority(PRIO_PROCESS, 0, 20) != 0) {
	...		...
427	LOGDEB(("X11 session went away during initial sleep period\n"));	444	LOGDEB(("X11 session went away during initial sleep period\n"));
428	exit(0);	445	exit(0);
429	}	446	}
430	}	447	}
431	}	448	}
432	confindexer = new ConfIndexer(config, updater);	449	makeIndexerOrExit(config, inPlaceReset);
433	if (!confindexer->index(rezero, ConfIndexer::IxTAll) \|\| stopindexing) {	450	if (!confindexer->index(rezero, ConfIndexer::IxTAll) \|\| stopindexing) {
434	LOGERR(("recollindex, initial indexing pass failed, not going into monitor mode\n"));	451	LOGERR(("recollindex, initial indexing pass failed, not going into monitor mode\n"));
435	exit(1);	452	exit(1);
436	}	453	}
437	if (updater) {	454	if (updater) {
	...		...
450	bool monret = startMonitor(config, opts);	467	bool monret = startMonitor(config, opts);
451	MONDEB(("Monitor returned %d, exiting\n", monret));	468	MONDEB(("Monitor returned %d, exiting\n", monret));
452	exit(monret == false);	469	exit(monret == false);
453	#endif // MONITOR	470	#endif // MONITOR
454		471
455	#ifdef RCL_USE_ASPELL
456	} else if (op_flags & OPT_S) {
457	if (!makeIndexer(config))
458	exit(1);
459	exit(!confindexer->createAspellDict());
460	#endif // ASPELL
461	} else if (op_flags & OPT_b) {	472	} else if (op_flags & OPT_b) {
462	cerr << "Not yet" << endl;	473	cerr << "Not yet" << endl;
463	return 1;	474	return 1;
464	} else {	475	} else {
465	lockorexit(&pidfile);	476	lockorexit(&pidfile);
466	pidfile.write_pid();	477	makeIndexerOrExit(config, inPlaceReset);
467	confindexer = new ConfIndexer(config, updater);
468	bool status = confindexer->index(rezero, ConfIndexer::IxTAll);	478	bool status = confindexer->index(rezero, ConfIndexer::IxTAll);
469	if (!status)	479	if (!status)
470	cerr << "Indexing failed" << endl;	480	cerr << "Indexing failed" << endl;
471	if (!confindexer->getReason().empty())	481	if (!confindexer->getReason().empty())
472	cerr << confindexer->getReason() << endl;	482	cerr << confindexer->getReason() << endl;