recoll / Code / Diff of /src/utils/fstreewalk.cpp

Diff of /src/utils/fstreewalk.cpp [caf5d1] .. [7876fb]

Switch to unified view


...
#include "autoconfig.h"
#endif

#ifndef TEST_FSTREEWALK

#include <stdio.h>
#include <dirent.h>
#include <sys/stat.h>
#include <errno.h>
#include <fnmatch.h>
#include <cstring>
...
    return dev < r.dev || (dev == r.dev && ino < r.ino);
    }
};

class FsTreeWalker::Internal {
public:
    Internal(int opts)
    : options(opts), depthswitch(4), maxdepth(-1), errors(0)
    {
    }
    int options;
    int depthswitch;
    int maxdepth;
    int basedepth;
    stringstream reason;
    vector<string> skippedNames;
    vector<string> skippedPaths;
    // When doing Breadth or FilesThenDirs traversal, we keep a list
    // of directory paths to be processed, and we do not recurse.
...
    {
    errors++;
    reason << call << "(" << param << ") : " << errno << " : " << 
        strerror(errno) << endl;
    }

};

FsTreeWalker::FsTreeWalker(int opts)
{
    data = new Internal(opts);
}

FsTreeWalker::~FsTreeWalker()
{
    delete data;
}

void FsTreeWalker::setOpts(int opts)
{
    if (data) {
    data->options = opts;


    }
}
int FsTreeWalker::getOpts()






{
    if (data) {
  return data->options;
    } else {
  return 0;
    }
}
void FsTreeWalker::setDepthSwitch(int ds)
{
    if (data) {
        data->depthswitch = ds;
    }
}
void FsTreeWalker::setMaxDepth(int md)
{
    if (data) {
        data->maxdepth = md;
    }
}

string FsTreeWalker::getReason()
{
...

    if ((data->options & FtwTravMask) == 0) {
        data->options |= FtwTravNatural;
    }

    data->basedepth = slashcount(top); // Only used for breadthxx

    struct stat st;
    // We always follow symlinks at this point. Makes more sense.
    if (stat(top.c_str(), &st) == -1) {
    // Note that we do not return an error if the stat call
    // fails. A temp file may have gone away.
...
                data->dirs.pop_front();
                nfather = path_getfather(dir);
                if (data->options & FtwTravBreadthThenDepth) {
                    // Check if new depth warrants switch to depth first
                    // traversal (will happen on next loop iteration).
                    int curdepth = slashcount(dir) - data->basedepth;
                    if (curdepth >= data->depthswitch) {
                        //fprintf(stderr, "SWITCHING TO DEPTH FIRST\n");
                        data->options &= ~FtwTravMask;
                        data->options |= FtwTravFilesThenDirs;
                    }
...
    return cb.processone(top, stp, FtwRegular);
    } else {
    return status;
    }


    int curdepth = slashcount(top) - data->basedepth;
    if (data->maxdepth >= 0 && curdepth >= data->maxdepth) {
  LOGDEB1(("FsTreeWalker::iwalk: Maxdepth reached: [%s]\n", top.c_str()));
  return status;
    }

    // This is a directory, read it and process entries:

    // Detect if directory already seen. This could just be several
    // symlinks pointing to the same place (if FtwFollow is set), it
    // could also be some other kind of cycle. In any case, there is
...

    struct dirent *ent;
    while ((ent = readdir(d)) != 0) {
        string fn;
        struct stat st;
  // Maybe skip dotfiles
  if ((data->options & FtwSkipDotFiles) && ent->d_name[0] == '.')
      continue;
    // Skip . and ..
    if (!strcmp(ent->d_name, ".") || !strcmp(ent->d_name, "..")) 
        continue;

    // Skipped file names match ?
...
#define OPT_b     0x20
#define OPT_d     0x40
#define OPT_m     0x80
#define OPT_L     0x100
#define OPT_w     0x200
#define OPT_M     0x400
#define OPT_D     0x800

class myCB : public FsTreeWalkerCB {
 public:
    FsTreeWalker::Status processone(const string &path, 
                                    const struct stat *st,
...
" -L : follow symbolic links\n"
" -b : use breadth first walk\n"
" -d : use almost depth first (dir files, then subdirs)\n"
" -m : use breadth up to 4 deep then switch to -d\n"
" -w : unset default FNM_PATHNAME when using fnmatch() to match skipped paths\n"
" -M <depth>: limit depth (works with -b/m/d)\n"
" -D : skip dotfiles\n"
;
static void
Usage(void)
{
    fprintf(stderr, "%s: usage:\n%s", thisprog, usage);
...

int main(int argc, const char **argv)
{
    vector<string> patterns;
    vector<string> paths;
    int maxdepth = -1;

    thisprog = argv[0];
    argc--; argv++;

    while (argc > 0 && **argv == '-') {
  (*argv)++;
  if (!(**argv))
      /* Cas du "adb - core" */
      Usage();
  while (**argv)
      switch (*(*argv)++) {
      case 'b':  op_flags |= OPT_b; break;
      case 'c':  op_flags |= OPT_c; break;
      case 'd':  op_flags |= OPT_d; break;
      case 'D':   op_flags |= OPT_D; break;
      case 'L':  op_flags |= OPT_L; break;
      case 'm':  op_flags |= OPT_m; break;
      case 'M':   op_flags |= OPT_M; if (argc < 2)  Usage();
      maxdepth = atoi(*(++argv));
      argc--; 
      goto b1;
      case 'p':  op_flags |= OPT_p; if (argc < 2)  Usage();
      patterns.push_back(*(++argv));
      argc--; 
      goto b1;
      case 'P':  op_flags |= OPT_P; if (argc < 2)  Usage();
      paths.push_back(*(++argv));
     argc--; 
     goto b1;
      case 'r':  op_flags |= OPT_r; break;
      case 'w':  op_flags |= OPT_w; break;
      default: Usage();  break;
      }
    b1: argc--; argv++;
    }

    if (argc != 1)
  Usage();
    string topdir = *argv++;argc--;

    int opt = 0;
    if (op_flags & OPT_r)
  opt |= FsTreeWalker::FtwNoRecurse;
    if (op_flags & OPT_c)
  opt |= FsTreeWalker::FtwNoCanon;
    if (op_flags & OPT_L)
  opt |= FsTreeWalker::FtwFollow;
    if (op_flags & OPT_D)
  opt |= FsTreeWalker::FtwSkipDotFiles;

    if (op_flags & OPT_b)
  opt |= FsTreeWalker::FtwTravBreadth;
    else if (op_flags & OPT_d)
  opt |= FsTreeWalker::FtwTravFilesThenDirs;
    else if (op_flags & OPT_m)
  opt |= FsTreeWalker::FtwTravBreadthThenDepth;

    string reason;
    if (!recollinit(0, 0, reason)) {
  fprintf(stderr, "Init failed: %s\n", reason.c_str());
  exit(1);
    }
    if (op_flags & OPT_w) {
  FsTreeWalker::setNoFnmPathname();
    }
    FsTreeWalker walker;
    walker.setOpts(opt); 
    walker.setMaxDepth(maxdepth);
    walker.setSkippedNames(patterns);
    walker.setSkippedPaths(paths);
    myCB cb;
    walker.walk(topdir, cb);
    if (walker.getErrCnt() > 0)
  cout << walker.getReason();
}

#endif // TEST_FSTREEWALK

	a/src/utils/fstreewalk.cpp		b/src/utils/fstreewalk.cpp
	...		...
18	#include "autoconfig.h"	18	#include "autoconfig.h"
19	#endif	19	#endif
20		20
21	#ifndef TEST_FSTREEWALK	21	#ifndef TEST_FSTREEWALK
22		22
		23	#include <stdio.h>
23	#include <dirent.h>	24	#include <dirent.h>
24	#include <sys/stat.h>	25	#include <sys/stat.h>
25	#include <errno.h>	26	#include <errno.h>
26	#include <fnmatch.h>	27	#include <fnmatch.h>
27	#include <cstring>	28	#include <cstring>
	...		...
56	return dev < r.dev \|\| (dev == r.dev && ino < r.ino);	57	return dev < r.dev \|\| (dev == r.dev && ino < r.ino);
57	}	58	}
58	};	59	};
59		60
60	class FsTreeWalker::Internal {	61	class FsTreeWalker::Internal {
		62	public:
		63	Internal(int opts)
		64	: options(opts), depthswitch(4), maxdepth(-1), errors(0)
		65	{
		66	}
61	int options;	67	int options;
62	int depthswitch;	68	int depthswitch;
		69	int maxdepth;
		70	int basedepth;
63	stringstream reason;	71	stringstream reason;
64	vector<string> skippedNames;	72	vector<string> skippedNames;
65	vector<string> skippedPaths;	73	vector<string> skippedPaths;
66	// When doing Breadth or FilesThenDirs traversal, we keep a list	74	// When doing Breadth or FilesThenDirs traversal, we keep a list
67	// of directory paths to be processed, and we do not recurse.	75	// of directory paths to be processed, and we do not recurse.
	...		...
72	{	80	{
73	errors++;	81	errors++;
74	reason << call << "(" << param << ") : " << errno << " : " <<	82	reason << call << "(" << param << ") : " << errno << " : " <<
75	strerror(errno) << endl;	83	strerror(errno) << endl;
76	}	84	}
77	friend class FsTreeWalker;
78	};	85	};
79		86
80	FsTreeWalker::FsTreeWalker(int opts)	87	FsTreeWalker::FsTreeWalker(int opts)
81	{	88	{
82	data = new Internal;	89	data = new Internal(opts);
		90	}
		91
		92	FsTreeWalker::~FsTreeWalker()
		93	{
		94	delete data;
		95	}
		96
		97	void FsTreeWalker::setOpts(int opts)
		98	{
83	if (data) {	99	if (data) {
84	data->options = opts;	100	data->options = opts;
85	data->depthswitch = 4;
86	data->errors = 0;
87	}	101	}
88	}	102	}
89		103	int FsTreeWalker::getOpts()
90	FsTreeWalker::~FsTreeWalker()
91	{
92	delete data;
93	}
94
95	void FsTreeWalker::setOpts(Options opts, int depthswitch)
96	{	104	{
97	if (data) {	105	if (data) {
98	data->options = opts;	106	return data->options;
		107	} else {
		108	return 0;
		109	}
		110	}
		111	void FsTreeWalker::setDepthSwitch(int ds)
		112	{
		113	if (data) {
99	data->depthswitch = depthswitch;	114	data->depthswitch = ds;
		115	}
		116	}
		117	void FsTreeWalker::setMaxDepth(int md)
		118	{
		119	if (data) {
		120	data->maxdepth = md;
100	}	121	}
101	}	122	}
102		123
103	string FsTreeWalker::getReason()	124	string FsTreeWalker::getReason()
104	{	125	{
	...		...
196		217
197	if ((data->options & FtwTravMask) == 0) {	218	if ((data->options & FtwTravMask) == 0) {
198	data->options \|= FtwTravNatural;	219	data->options \|= FtwTravNatural;
199	}	220	}
200		221
201	int basedepth = slashcount(top); // Only used for breadthThenDepth	222	data->basedepth = slashcount(top); // Only used for breadthxx
202
203	struct stat st;	223	struct stat st;
204	// We always follow symlinks at this point. Makes more sense.	224	// We always follow symlinks at this point. Makes more sense.
205	if (stat(top.c_str(), &st) == -1) {	225	if (stat(top.c_str(), &st) == -1) {
206	// Note that we do not return an error if the stat call	226	// Note that we do not return an error if the stat call
207	// fails. A temp file may have gone away.	227	// fails. A temp file may have gone away.
	...		...
238	data->dirs.pop_front();	258	data->dirs.pop_front();
239	nfather = path_getfather(dir);	259	nfather = path_getfather(dir);
240	if (data->options & FtwTravBreadthThenDepth) {	260	if (data->options & FtwTravBreadthThenDepth) {
241	// Check if new depth warrants switch to depth first	261	// Check if new depth warrants switch to depth first
242	// traversal (will happen on next loop iteration).	262	// traversal (will happen on next loop iteration).
243	int curdepth = slashcount(dir) - basedepth;	263	int curdepth = slashcount(dir) - data->basedepth;
244	if (curdepth >= data->depthswitch) {	264	if (curdepth >= data->depthswitch) {
245	//fprintf(stderr, "SWITCHING TO DEPTH FIRST\n");	265	//fprintf(stderr, "SWITCHING TO DEPTH FIRST\n");
246	data->options &= ~FtwTravMask;	266	data->options &= ~FtwTravMask;
247	data->options \|= FtwTravFilesThenDirs;	267	data->options \|= FtwTravFilesThenDirs;
248	}	268	}
	...		...
307	return cb.processone(top, stp, FtwRegular);	327	return cb.processone(top, stp, FtwRegular);
308	} else {	328	} else {
309	return status;	329	return status;
310	}	330	}
311		331
		332
		333	int curdepth = slashcount(top) - data->basedepth;
		334	if (data->maxdepth >= 0 && curdepth >= data->maxdepth) {
		335	LOGDEB1(("FsTreeWalker::iwalk: Maxdepth reached: [%s]\n", top.c_str()));
		336	return status;
		337	}
		338
312	// This is a directory, read it and process entries:	339	// This is a directory, read it and process entries:
313		340
314	// Detect if directory already seen. This could just be several	341	// Detect if directory already seen. This could just be several
315	// symlinks pointing to the same place (if FtwFollow is set), it	342	// symlinks pointing to the same place (if FtwFollow is set), it
316	// could also be some other kind of cycle. In any case, there is	343	// could also be some other kind of cycle. In any case, there is
	...		...
343		370
344	struct dirent *ent;	371	struct dirent *ent;
345	while ((ent = readdir(d)) != 0) {	372	while ((ent = readdir(d)) != 0) {
346	string fn;	373	string fn;
347	struct stat st;	374	struct stat st;
		375	// Maybe skip dotfiles
		376	if ((data->options & FtwSkipDotFiles) && ent->d_name[0] == '.')
		377	continue;
348	// Skip . and ..	378	// Skip . and ..
349	if (!strcmp(ent->d_name, ".") \|\| !strcmp(ent->d_name, ".."))	379	if (!strcmp(ent->d_name, ".") \|\| !strcmp(ent->d_name, ".."))
350	continue;	380	continue;
351		381
352	// Skipped file names match ?	382	// Skipped file names match ?
	...		...
437	#define OPT_b 0x20	467	#define OPT_b 0x20
438	#define OPT_d 0x40	468	#define OPT_d 0x40
439	#define OPT_m 0x80	469	#define OPT_m 0x80
440	#define OPT_L 0x100	470	#define OPT_L 0x100
441	#define OPT_w 0x200	471	#define OPT_w 0x200
		472	#define OPT_M 0x400
		473	#define OPT_D 0x800
442		474
443	class myCB : public FsTreeWalkerCB {	475	class myCB : public FsTreeWalkerCB {
444	public:	476	public:
445	FsTreeWalker::Status processone(const string &path,	477	FsTreeWalker::Status processone(const string &path,
446	const struct stat *st,	478	const struct stat *st,
	...		...
487	" -L : follow symbolic links\n"	519	" -L : follow symbolic links\n"
488	" -b : use breadth first walk\n"	520	" -b : use breadth first walk\n"
489	" -d : use almost depth first (dir files, then subdirs)\n"	521	" -d : use almost depth first (dir files, then subdirs)\n"
490	" -m : use breadth up to 4 deep then switch to -d\n"	522	" -m : use breadth up to 4 deep then switch to -d\n"
491	" -w : unset default FNM_PATHNAME when using fnmatch() to match skipped paths\n"	523	" -w : unset default FNM_PATHNAME when using fnmatch() to match skipped paths\n"
		524	" -M <depth>: limit depth (works with -b/m/d)\n"
		525	" -D : skip dotfiles\n"
492	;	526	;
493	static void	527	static void
494	Usage(void)	528	Usage(void)
495	{	529	{
496	fprintf(stderr, "%s: usage:\n%s", thisprog, usage);	530	fprintf(stderr, "%s: usage:\n%s", thisprog, usage);
	...		...
499		533
500	int main(int argc, const char **argv)	534	int main(int argc, const char **argv)
501	{	535	{
502	vector<string> patterns;	536	vector<string> patterns;
503	vector<string> paths;	537	vector<string> paths;
		538	int maxdepth = -1;
		539
504	thisprog = argv[0];	540	thisprog = argv[0];
505	argc--; argv++;	541	argc--; argv++;
506
507	while (argc > 0 && **argv == '-') {	542	while (argc > 0 && **argv == '-') {
508	(*argv)++;	543	(*argv)++;
509	if (!(**argv))	544	if (!(**argv))
510	/* Cas du "adb - core" */	545	/* Cas du "adb - core" */
511	Usage();	546	Usage();
512	while (**argv)	547	while (**argv)
513	switch ((argv)++) {	548	switch ((argv)++) {
514	case 'b': op_flags \|= OPT_b; break;	549	case 'b': op_flags \|= OPT_b; break;
515	case 'c': op_flags \|= OPT_c; break;	550	case 'c': op_flags \|= OPT_c; break;
516	case 'd': op_flags \|= OPT_d; break;	551	case 'd': op_flags \|= OPT_d; break;
		552	case 'D': op_flags \|= OPT_D; break;
517	case 'L': op_flags \|= OPT_L; break;	553	case 'L': op_flags \|= OPT_L; break;
518	case 'm': op_flags \|= OPT_m; break;	554	case 'm': op_flags \|= OPT_m; break;
		555	case 'M': op_flags \|= OPT_M; if (argc < 2) Usage();
		556	maxdepth = atoi(*(++argv));
		557	argc--;
		558	goto b1;
519	case 'p': op_flags \|= OPT_p; if (argc < 2) Usage();	559	case 'p': op_flags \|= OPT_p; if (argc < 2) Usage();
520	patterns.push_back(*(++argv));	560	patterns.push_back(*(++argv));
521	argc--;	561	argc--;
522	goto b1;	562	goto b1;
523	case 'P': op_flags \|= OPT_P; if (argc < 2) Usage();	563	case 'P': op_flags \|= OPT_P; if (argc < 2) Usage();
524	paths.push_back(*(++argv));	564	paths.push_back(*(++argv));
525	argc--;	565	argc--;
526	goto b1;	566	goto b1;
527	case 'r': op_flags \|= OPT_r; break;	567	case 'r': op_flags \|= OPT_r; break;
528	case 'w': op_flags \|= OPT_w; break;	568	case 'w': op_flags \|= OPT_w; break;
529	default: Usage(); break;	569	default: Usage(); break;
530	}	570	}
531	b1: argc--; argv++;	571	b1: argc--; argv++;
532	}	572	}
533		573
534	if (argc != 1)	574	if (argc != 1)
535	Usage();	575	Usage();
536	string topdir = *argv++;argc--;	576	string topdir = *argv++;argc--;
537		577
538	int opt = 0;	578	int opt = 0;
539	if (op_flags & OPT_r)	579	if (op_flags & OPT_r)
540	opt \|= FsTreeWalker::FtwNoRecurse;	580	opt \|= FsTreeWalker::FtwNoRecurse;
541	if (op_flags & OPT_c)	581	if (op_flags & OPT_c)
542	opt \|= FsTreeWalker::FtwNoCanon;	582	opt \|= FsTreeWalker::FtwNoCanon;
543	if (op_flags & OPT_L)	583	if (op_flags & OPT_L)
544	opt \|= FsTreeWalker::FtwFollow;	584	opt \|= FsTreeWalker::FtwFollow;
		585	if (op_flags & OPT_D)
		586	opt \|= FsTreeWalker::FtwSkipDotFiles;
545		587
546	if (op_flags & OPT_b)	588	if (op_flags & OPT_b)
547	opt \|= FsTreeWalker::FtwTravBreadth;	589	opt \|= FsTreeWalker::FtwTravBreadth;
548	else if (op_flags & OPT_d)	590	else if (op_flags & OPT_d)
549	opt \|= FsTreeWalker::FtwTravFilesThenDirs;	591	opt \|= FsTreeWalker::FtwTravFilesThenDirs;
550	else if (op_flags & OPT_m)	592	else if (op_flags & OPT_m)
551	opt \|= FsTreeWalker::FtwTravBreadthThenDepth;	593	opt \|= FsTreeWalker::FtwTravBreadthThenDepth;
552		594
553	string reason;	595	string reason;
554	if (!recollinit(0, 0, reason)) {	596	if (!recollinit(0, 0, reason)) {
555	fprintf(stderr, "Init failed: %s\n", reason.c_str());	597	fprintf(stderr, "Init failed: %s\n", reason.c_str());
556	exit(1);	598	exit(1);
557	}	599	}
558	if (op_flags & OPT_w) {	600	if (op_flags & OPT_w) {
559	FsTreeWalker::setNoFnmPathname();	601	FsTreeWalker::setNoFnmPathname();
560	}	602	}
561	FsTreeWalker walker(opt);	603	FsTreeWalker walker;
		604	walker.setOpts(opt);
		605	walker.setMaxDepth(maxdepth);
562	walker.setSkippedNames(patterns);	606	walker.setSkippedNames(patterns);
563	walker.setSkippedPaths(paths);	607	walker.setSkippedPaths(paths);
564	myCB cb;	608	myCB cb;
565	walker.walk(topdir, cb);	609	walker.walk(topdir, cb);
566	if (walker.getErrCnt() > 0)	610	if (walker.getErrCnt() > 0)
567	cout << walker.getReason();	611	cout << walker.getReason();
568	}	612	}
569		613
570	#endif // TEST_FSTREEWALK	614	#endif // TEST_FSTREEWALK