recoll / Code / Diff of /src/utils/fstreewalk.cpp

Diff of /src/utils/fstreewalk.cpp [e5f41a] .. [5fb98c]

Switch to unified view


...
#include <cstring>
#include <algorithm>

#include <sstream>
#include <list>
#include <set>

#include "debuglog.h"
#include "pathut.h"
#include "fstreewalk.h"

...
using namespace std;
#endif /* NO_NAMESPACES */

const int FsTreeWalker::FtwTravMask = FtwTravNatural|
    FtwTravBreadth|FtwTravFilesThenDirs|FtwTravBreadthThenDepth;

class DirId {
public:
    dev_t dev;
    ino_t ino;
    DirId(dev_t d, ino_t i) : dev(d), ino(i) {}
    bool operator<(const DirId& r) const 
    {
  return dev < r.dev || (dev == r.dev && ino < r.ino);
    }
};

class FsTreeWalker::Internal {
    int options;
    int depthswitch;
    stringstream reason;
...
    list<string> skippedPaths;
    // When doing Breadth or FilesThenDirs traversal, we keep a list
    // of directory paths to be processed, and we do not recurse.
    list<string> dirs;
    int errors;
    set<DirId> donedirs;
    void logsyserr(const char *call, const string &param) 
    {
    errors++;
    reason << call << "(" << param << ") : " << errno << " : " << 
        strerror(errno) << endl;
...
    } else {
    return status;
    }

    // This is a directory, read it and process entries:

    // Detect if directory already seen. This could just be several
    // symlinks pointing to the same place (if FtwFollow is set), it
    // could also be some other kind of cycle. In any case, there is
    // no point in entering again.
    // For now, we'll ignore the "other kind of cycle" part and only monitor
    // this is FtwFollow is set
    if (data->options & FtwFollow) {
  DirId dirid(stp->st_dev, stp->st_ino);
  if (data->donedirs.find(dirid) != data->donedirs.end()) {
      LOGINFO(("Not processing [%s] (already seen as other path)\n", 
           top.c_str()));
      return status;
  }
  data->donedirs.insert(dirid);
    }

    DIR *d = opendir(top.c_str());
    if (d == 0) {
    data->logsyserr("opendir", top);
    switch (errno) {
    case EPERM:
...
#define OPT_r     0x8
#define OPT_c     0x10
#define OPT_b     0x20
#define OPT_d     0x40
#define OPT_m     0x80
#define OPT_L     0x100

class myCB : public FsTreeWalkerCB {
 public:
    FsTreeWalker::Status processone(const string &path, 
                                    const struct stat *st,
...
// real    14m53.245s user    0m4.244s sys     0m34.494s
// time trfstreewalk -b / > /data/tmp/breadth;
// real    17m10.585s user    0m4.532s sys     0m35.033s

static char usage [] =
"trfstreewalk [-p pattern] [-P ignpath] [-r] [-c] [-L] topdir\n"
" -r : norecurse\n"
" -c : no path canonification\n"
" -L : follow symbolic links\n"
" -b : use breadth first walk\n"
" -d : use almost depth first (dir files, then subdirs)\n"
" -m : use breadth up to 4 deep then switch to -d\n"
;
static void
...
    while (**argv)
      switch (*(*argv)++) {
      case 'b': op_flags |= OPT_b; break;
      case 'c': op_flags |= OPT_c; break;
      case 'd': op_flags |= OPT_d; break;
      case 'L':   op_flags |= OPT_L; break;
      case 'm': op_flags |= OPT_m; break;
      case 'r': op_flags |= OPT_r; break;
      case 'p': op_flags |= OPT_p; if (argc < 2)  Usage();
      patterns.push_back(*(++argv));
      argc--; 
...
  int opt = 0;
  if (op_flags & OPT_r)
      opt |= FsTreeWalker::FtwNoRecurse;
  if (op_flags & OPT_c)
      opt |= FsTreeWalker::FtwNoCanon;
  if (op_flags & OPT_L)
      opt |= FsTreeWalker::FtwFollow;

  if (op_flags & OPT_b)
      opt |= FsTreeWalker::FtwTravBreadth;
  else if (op_flags & OPT_d)
      opt |= FsTreeWalker::FtwTravFilesThenDirs;

	a/src/utils/fstreewalk.cpp		b/src/utils/fstreewalk.cpp
	...		...
30	#include <cstring>	30	#include <cstring>
31	#include <algorithm>	31	#include <algorithm>
32		32
33	#include <sstream>	33	#include <sstream>
34	#include <list>	34	#include <list>
		35	#include <set>
35		36
36	#include "debuglog.h"	37	#include "debuglog.h"
37	#include "pathut.h"	38	#include "pathut.h"
38	#include "fstreewalk.h"	39	#include "fstreewalk.h"
39		40
	...		...
41	using namespace std;	42	using namespace std;
42	#endif /* NO_NAMESPACES */	43	#endif /* NO_NAMESPACES */
43		44
44	const int FsTreeWalker::FtwTravMask = FtwTravNatural\|	45	const int FsTreeWalker::FtwTravMask = FtwTravNatural\|
45	FtwTravBreadth\|FtwTravFilesThenDirs\|FtwTravBreadthThenDepth;	46	FtwTravBreadth\|FtwTravFilesThenDirs\|FtwTravBreadthThenDepth;
		47
		48	class DirId {
		49	public:
		50	dev_t dev;
		51	ino_t ino;
		52	DirId(dev_t d, ino_t i) : dev(d), ino(i) {}
		53	bool operator<(const DirId& r) const
		54	{
		55	return dev < r.dev \|\| (dev == r.dev && ino < r.ino);
		56	}
		57	};
46		58
47	class FsTreeWalker::Internal {	59	class FsTreeWalker::Internal {
48	int options;	60	int options;
49	int depthswitch;	61	int depthswitch;
50	stringstream reason;	62	stringstream reason;
	...		...
52	list<string> skippedPaths;	64	list<string> skippedPaths;
53	// When doing Breadth or FilesThenDirs traversal, we keep a list	65	// When doing Breadth or FilesThenDirs traversal, we keep a list
54	// of directory paths to be processed, and we do not recurse.	66	// of directory paths to be processed, and we do not recurse.
55	list<string> dirs;	67	list<string> dirs;
56	int errors;	68	int errors;
		69	set<DirId> donedirs;
57	void logsyserr(const char *call, const string &param)	70	void logsyserr(const char *call, const string &param)
58	{	71	{
59	errors++;	72	errors++;
60	reason << call << "(" << param << ") : " << errno << " : " <<	73	reason << call << "(" << param << ") : " << errno << " : " <<
61	strerror(errno) << endl;	74	strerror(errno) << endl;
	...		...
290	} else {	303	} else {
291	return status;	304	return status;
292	}	305	}
293		306
294	// This is a directory, read it and process entries:	307	// This is a directory, read it and process entries:
		308
		309	// Detect if directory already seen. This could just be several
		310	// symlinks pointing to the same place (if FtwFollow is set), it
		311	// could also be some other kind of cycle. In any case, there is
		312	// no point in entering again.
		313	// For now, we'll ignore the "other kind of cycle" part and only monitor
		314	// this is FtwFollow is set
		315	if (data->options & FtwFollow) {
		316	DirId dirid(stp->st_dev, stp->st_ino);
		317	if (data->donedirs.find(dirid) != data->donedirs.end()) {
		318	LOGINFO(("Not processing [%s] (already seen as other path)\n",
		319	top.c_str()));
		320	return status;
		321	}
		322	data->donedirs.insert(dirid);
		323	}
		324
295	DIR *d = opendir(top.c_str());	325	DIR *d = opendir(top.c_str());
296	if (d == 0) {	326	if (d == 0) {
297	data->logsyserr("opendir", top);	327	data->logsyserr("opendir", top);
298	switch (errno) {	328	switch (errno) {
299	case EPERM:	329	case EPERM:
	...		...
393	#define OPT_r 0x8	423	#define OPT_r 0x8
394	#define OPT_c 0x10	424	#define OPT_c 0x10
395	#define OPT_b 0x20	425	#define OPT_b 0x20
396	#define OPT_d 0x40	426	#define OPT_d 0x40
397	#define OPT_m 0x80	427	#define OPT_m 0x80
		428	#define OPT_L 0x100
398		429
399	class myCB : public FsTreeWalkerCB {	430	class myCB : public FsTreeWalkerCB {
400	public:	431	public:
401	FsTreeWalker::Status processone(const string &path,	432	FsTreeWalker::Status processone(const string &path,
402	const struct stat *st,	433	const struct stat *st,
	...		...
435	// real 14m53.245s user 0m4.244s sys 0m34.494s	466	// real 14m53.245s user 0m4.244s sys 0m34.494s
436	// time trfstreewalk -b / > /data/tmp/breadth;	467	// time trfstreewalk -b / > /data/tmp/breadth;
437	// real 17m10.585s user 0m4.532s sys 0m35.033s	468	// real 17m10.585s user 0m4.532s sys 0m35.033s
438		469
439	static char usage [] =	470	static char usage [] =
440	"trfstreewalk [-p pattern] [-P ignpath] [-r] [-c] topdir\n"	471	"trfstreewalk [-p pattern] [-P ignpath] [-r] [-c] [-L] topdir\n"
441	" -r : norecurse\n"	472	" -r : norecurse\n"
442	" -c : no path canonification\n"	473	" -c : no path canonification\n"
		474	" -L : follow symbolic links\n"
443	" -b : use breadth first walk\n"	475	" -b : use breadth first walk\n"
444	" -d : use almost depth first (dir files, then subdirs)\n"	476	" -d : use almost depth first (dir files, then subdirs)\n"
445	" -m : use breadth up to 4 deep then switch to -d\n"	477	" -m : use breadth up to 4 deep then switch to -d\n"
446	;	478	;
447	static void	479	static void
	...		...
466	while (**argv)	498	while (**argv)
467	switch ((argv)++) {	499	switch ((argv)++) {
468	case 'b': op_flags \|= OPT_b; break;	500	case 'b': op_flags \|= OPT_b; break;
469	case 'c': op_flags \|= OPT_c; break;	501	case 'c': op_flags \|= OPT_c; break;
470	case 'd': op_flags \|= OPT_d; break;	502	case 'd': op_flags \|= OPT_d; break;
		503	case 'L': op_flags \|= OPT_L; break;
471	case 'm': op_flags \|= OPT_m; break;	504	case 'm': op_flags \|= OPT_m; break;
472	case 'r': op_flags \|= OPT_r; break;	505	case 'r': op_flags \|= OPT_r; break;
473	case 'p': op_flags \|= OPT_p; if (argc < 2) Usage();	506	case 'p': op_flags \|= OPT_p; if (argc < 2) Usage();
474	patterns.push_back(*(++argv));	507	patterns.push_back(*(++argv));
475	argc--;	508	argc--;
	...		...
490	int opt = 0;	523	int opt = 0;
491	if (op_flags & OPT_r)	524	if (op_flags & OPT_r)
492	opt \|= FsTreeWalker::FtwNoRecurse;	525	opt \|= FsTreeWalker::FtwNoRecurse;
493	if (op_flags & OPT_c)	526	if (op_flags & OPT_c)
494	opt \|= FsTreeWalker::FtwNoCanon;	527	opt \|= FsTreeWalker::FtwNoCanon;
		528	if (op_flags & OPT_L)
		529	opt \|= FsTreeWalker::FtwFollow;
495		530
496	if (op_flags & OPT_b)	531	if (op_flags & OPT_b)
497	opt \|= FsTreeWalker::FtwTravBreadth;	532	opt \|= FsTreeWalker::FtwTravBreadth;
498	else if (op_flags & OPT_d)	533	else if (op_flags & OPT_d)
499	opt \|= FsTreeWalker::FtwTravFilesThenDirs;	534	opt \|= FsTreeWalker::FtwTravFilesThenDirs;