recoll / Code / Diff of /src/utils/circache.cpp

Diff of /src/utils/circache.cpp [4b8166] .. [d91d47]

Switch to unified view


/*
 *   This program is free software; you can redistribute it and/or modify
 *   it under the terms of the GNU General Public License as published by
 *   the Free Software Foundation; either version 2 of the License, or
 *   (at your option) any later version.
 *
 *   This program is distributed in the hope that it will be useful,
 *   but WITHOUT ANY WARRANTY; without even the implied warranty of
 *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *   GNU General Public License for more details.
 *
 *   You should have received a copy of the GNU General Public License
 *   along with this program; if not, write to the
 *   Free Software Foundation, Inc.,
 *   59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
 */
#ifndef lint
static char rcsid[] = "@(#$Id: $ (C) 2009 J.F.Dockes";
#endif

#ifndef TEST_CIRCACHE
...
#include <sstream>
#include <iostream>

#include "circache.h"
#include "conftree.h"
#include "debuglog.h"

using namespace std;

/*
 * File structure:
 * - Starts with a 1-KB header block, with a param dictionary.

 * - Stored items follow. Each item has a header and 2 segments for
 *   the metadata and the data. 
 *   The segment sizes are stored in the ascii header/marker:
 *     circacheSizes = xxx yyy zzz
 *     xxx bytes of metadata
 *     yyy bytes of data
 *     zzz bytes of padding up to next object (only one entry has non zero)
 *
 * There is a write position, which can be at eof while 
 * the file is growing, or inside the file if we are recycling. This is stored
 * in the header, together with the maximum size
 *
 * If we are recycling, we have to take care to compute the size of the 
 * possible remaining area from the last object invalidated by the write, 
 * pad it with neutral data and store the size in the new header.
 */

// First block size
#define CIRCACHE_FIRSTBLOCK_SIZE 1024

// Entry header.
// The 32 bits size are stored as hex integers so the maximum size is
// 13 + 3x8 + 6 = 43
...
    }

    // Name for the cache file
    string datafn(const string& d)
    {
        return  path_cat(d, "circache.crch");
    }

    bool writefirstblock()
    {
        assert(m_fd >= 0);
...
            return false;
        }
        return true;
    }

    CCScanHook::status scan(off_t startoffset, CCScanHook *user, 
                            bool fold = false)
    {
        assert(m_fd >= 0);

        off_t so0 = startoffset;
        bool already_folded = false;
        
        while (true) {
            if (already_folded && startoffset == so0)
                return CCScanHook::Eof;

            EntryHeaderData d;
            CCScanHook::status st;
            switch ((st = readentryheader(startoffset, d))) {
            case CCScanHook::Continue: break;
            case CCScanHook::Eof:
                if (fold && !already_folded) {
                    already_folded = true;
                    startoffset = CIRCACHE_FIRSTBLOCK_SIZE;
                    continue;
                }
                /* FALLTHROUGH */
            default:
                return st;
            }

            char *bf;
            if ((bf = buf(d.dicsize+1)) == 0) {
                return CCScanHook::Error;
            }
            bf[d.dicsize] = 0;
...
{
    assert(m_d != 0);
    struct stat st;
    if (stat(m_dir.c_str(), &st) < 0) {
        if (mkdir(m_dir.c_str(), 0777) < 0) {
            m_d->m_reason << "CirCache::create: mkdir(" << m_dir << 
                ") failed" << " errno " << errno;
            return false;
        }
    }

    if ((m_d->m_fd = ::open(m_d->datafn(m_dir).c_str(), 
...

    char buf[CIRCACHE_FIRSTBLOCK_SIZE];
    memset(buf, 0, CIRCACHE_FIRSTBLOCK_SIZE);
    if (::write(m_d->m_fd, buf, CIRCACHE_FIRSTBLOCK_SIZE) != 
        CIRCACHE_FIRSTBLOCK_SIZE) {
        m_d->m_reason << "CirCache::create: write header failed, errno " 
                      << errno;
        return false;
    }
    return m_d->writefirstblock();
}

bool CirCache::open(OpMode mode)
{
    assert(m_d != 0);
    if (m_d->m_fd >= 0)
        ::close(m_d->m_fd);

    if ((m_d->m_fd = ::open(m_d->datafn(m_dir).c_str(), 
                          mode == CC_OPREAD ? O_RDONLY : O_RDWR)) < 0) {
        m_d->m_reason << "CirCache::open: open(" << m_d->datafn(m_dir) << 
            ") failed " << "errno " << errno;
        return false;
    }
    return m_d->readfirstblock();
}

class CCScanHookDump : public  CCScanHook {
public:
    virtual status takeone(off_t offs, const string& udi, unsigned int dicsize, 
                           unsigned int datasize, unsigned int padsize) 
    {
        cout << "Scan: offs " << offs << " dicsize " << dicsize 
             << " datasize " << datasize << " padsize " << padsize << 
            " udi [" << udi << "]" << endl;
        return Continue;
    }
};

bool CirCache::dump()
{
    CCScanHookDump dumper;
    off_t start = m_d->m_nheadoffs > CIRCACHE_FIRSTBLOCK_SIZE ? 
        m_d->m_nheadoffs : CIRCACHE_FIRSTBLOCK_SIZE;
    switch (m_d->scan(start, &dumper, true)) {
    case CCScanHook::Stop: 
        cout << "Scan returns Stop??" << endl;
        return false;
    case CCScanHook::Continue: 
        cout << "Scan returns Continue ?? " << CCScanHook::Continue << " " <<
            getReason() << endl;
        return false;
    case CCScanHook::Error: 
        cout << "Scan returns Error: " << getReason() << endl;
        return false;
    case CCScanHook::Eof:
        cout << "Scan returns Eof" << endl;
        return true;
    default:
        cout << "Scan returns Unknown ??" << endl;
        return false;
    }
}

class CCScanHookGetter : public  CCScanHook {
public:
    string  m_udi;
    int     m_targinstance;
    int     m_instance;
    off_t   m_offs;
    EntryHeaderData m_hd;

    CCScanHookGetter(const string &udi, int ti)
        : m_udi(udi), m_targinstance(ti), m_instance(0), m_offs(0){}

    virtual status takeone(off_t offs, const string& udi, unsigned int dicsize, 
                           unsigned int datasize, unsigned int padsize) 
    {
        cerr << "offs " << offs << " udi [" << udi << "] dicsize " << dicsize 
             << " datasize " << datasize << " padsize " << padsize << endl;
        if (!m_udi.compare(udi)) {
            m_instance++;
            m_offs = offs;
            m_hd.dicsize = dicsize;
            m_hd.datasize = datasize;
            m_hd.padsize = padsize;
            if (m_instance == m_targinstance)
                return Stop;
        }
        return Continue;
    }
};

// instance == -1 means get latest. Otherwise specify from 1+
bool CirCache::get(const string& udi, string& dict, string& data, int instance)
{
    assert(m_d != 0);
    if (m_d->m_fd < 0) {
        m_d->m_reason << "CirCache::get: not open";
        return false;
    }

    LOGDEB(("CirCache::get: udi [%s], instance\n", udi.c_str(), instance));

    CCScanHookGetter getter(udi, instance);
    off_t start = m_d->m_nheadoffs > CIRCACHE_FIRSTBLOCK_SIZE ? 
        m_d->m_nheadoffs : CIRCACHE_FIRSTBLOCK_SIZE;

    CCScanHook::status ret = m_d->scan(start, &getter, true);
    if (ret == CCScanHook::Eof) {
        if (getter.m_instance == 0)
            return false;
    } else if (ret != CCScanHook::Stop) {
        return false;
    }
    off_t offs = getter.m_offs + CIRCACHE_HEADER_SIZE;
    if (lseek(m_d->m_fd, offs, 0) != offs) {
        m_d->m_reason << "CirCache::get: lseek(" << offs << ") failed: " << 




            errno;
        return false;









    }
    char *bf = m_d->buf(getter.m_hd.dicsize);
    if (bf == 0)
        return false;
    if (read(m_d->m_fd, bf, getter.m_hd.dicsize) != int(getter.m_hd.dicsize)) {
        m_d->m_reason << "CirCache::get: read() failed: errno " << errno;
        return false;
    }
    dict.assign(bf, getter.m_hd.dicsize);


    bf = m_d->buf(getter.m_hd.datasize);
    if (bf == 0)
        return false;
    if (read(m_d->m_fd, bf, getter.m_hd.datasize) != int(getter.m_hd.datasize)){
        m_d->m_reason << "CirCache::get: read() failed: errno " << errno;
        return false;
    }
    data.assign(bf, getter.m_hd.datasize);

    return true;
}

class CCScanHookSpacer : public  CCScanHook {
public:
...
        : sizewanted(sz), sizeseen(0) {assert(sz > 0);}

    virtual status takeone(off_t offs, const string& udi, unsigned int dicsize, 
                           unsigned int datasize, unsigned int padsize) 
    {
        LOGDEB(("ScanSpacer: offs %u dicsz %u datasz %u padsz %u udi[%s]\n",
                (unsigned int)offs, dicsize, datasize, padsize, udi.c_str()));
        sizeseen += CIRCACHE_HEADER_SIZE + dicsize + datasize + padsize;
        if (sizeseen >= sizewanted)
            return Stop;
        return Continue;
    }
...
    int nsize = CIRCACHE_HEADER_SIZE + dic.size() + data.size();
    int nwriteoffs = 0;
    int npadsize = 0;
    bool extending = false;

    LOGDEB2(("CirCache::put: nsize %d oheadoffs %d\n", 
             nsize, m_d->m_oheadoffs));

    if (st.st_size < m_d->m_maxsize) {
        // If we are still growing the file, things are simple
        nwriteoffs = lseek(m_d->m_fd, 0, SEEK_END);
        npadsize = 0;
...
            if (m_d->readentryheader(m_d->m_nheadoffs, pd) != 
                CCScanHook::Continue) {
                return false;
            }
            assert(int(pd.padsize) == m_d->m_npadsize);
            LOGDEB2(("CirCache::put: recovering previous padsize %d\n",
                     pd.padsize));
            pd.padsize = 0;
            if (!m_d->writeentryheader(m_d->m_nheadoffs, pd)) {
                return false;
            }
            nwriteoffs = m_d->m_oheadoffs - recovpadsize;
...
        }

        if (nsize <= recovpadsize) {
            // If the new entry fits entirely in the pad area from the
            // latest one, no need to recycle the oldest entries.
            LOGDEB2(("CirCache::put: new fits in old padsize %d\n,"
                     recovpadsize));
            npadsize = recovpadsize - nsize;
        } else {
            // Scan the file until we have enough space for the new entry,
            // and determine the pad size up to the 1st preserved entry
            int scansize = nsize - recovpadsize;
            LOGDEB2(("CirCache::put: scanning for size %d from offs %u\n",
                    scansize, (unsigned int)m_d->m_oheadoffs));
            CCScanHookSpacer spacer(scansize);
            switch (m_d->scan(m_d->m_oheadoffs, &spacer)) {
            case CCScanHook::Stop: 
                LOGDEB2(("CirCache::put: Scan ok, sizeseen %d\n", 
                         spacer.sizeseen));
                npadsize = spacer.sizeseen - scansize;
                break;
            case CCScanHook::Eof:
                // npadsize is 0
                extending = true;
...
                return false;
            }
        }
    }
    
    LOGDEB2(("CirCache::put: writing %d at %d padsize %d\n", 
             nsize, nwriteoffs, npadsize));
    if (lseek(m_d->m_fd, nwriteoffs, 0) != nwriteoffs) {
        m_d->m_reason << "CirCache::put: lseek failed: " << errno;
        return false;
    }
    char *bf = m_d->buf(CIRCACHE_HEADER_SIZE);
...
static char *thisprog;

static char usage [] =
" -c <dirname> : create\n"
" -p <dirname> <apath> [apath ...] : put files\n"
" -d <dirname> : dump\n"
" -g [-i instance] <dirname> <udi>: get\n"
;
static void
Usage(FILE *fp = stderr)
{
    fprintf(fp, "%s: usage:\n%s", thisprog, usage);
...
}

static int     op_flags;
#define OPT_MOINS 0x1
#define OPT_c     0x2 

#define OPT_p     0x8
#define OPT_g     0x10
#define OPT_d     0x20
#define OPT_i     0x40

int main(int argc, char **argv)
{
  int instance = -1;

  thisprog = argv[0];
  argc--; argv++;

  while (argc > 0 && **argv == '-') {
    (*argv)++;
...
    while (**argv)
      switch (*(*argv)++) {
      case 'c': op_flags |= OPT_c; break;
      case 'p': op_flags |= OPT_p; break;
      case 'g': op_flags |= OPT_g; break;
      case 'd':   op_flags |= OPT_d; break;
      case 'i':  op_flags |= OPT_i; if (argc < 2)  Usage();
    if ((sscanf(*(++argv), "%d", &instance)) != 1) 
      Usage(); 
    argc--; 
    goto b1;
      default: Usage(); break;
      }
...
              exit(1);
          }
      }
      cc.open(CirCache::CC_OPREAD);
  } else if (op_flags & OPT_g) {
      string udi = *argv++;argc--;
      if (!cc.open(CirCache::CC_OPREAD)) {
          cerr << "Open failed: " << cc.getReason() << endl;
          exit(1);
      }
      string dic, data;
      if (!cc.get(udi, dic, data, instance)) {
          cerr << "Get failed: " << cc.getReason() << endl;
          exit(1);
      }
      cout << "Dict: [" << dic << "]" << endl;
  } else if (op_flags & OPT_d) {
      if (!cc.open(CirCache::CC_OPREAD)) {
          cerr << "Open failed: " << cc.getReason() << endl;
          exit(1);
      }
      cc.dump();
  } else
      Usage();

  exit(0);
}

	a/src/utils/circache.cpp		b/src/utils/circache.cpp
		1	/*
		2	* This program is free software; you can redistribute it and/or modify
		3	* it under the terms of the GNU General Public License as published by
		4	* the Free Software Foundation; either version 2 of the License, or
		5	* (at your option) any later version.
		6	*
		7	* This program is distributed in the hope that it will be useful,
		8	* but WITHOUT ANY WARRANTY; without even the implied warranty of
		9	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
		10	* GNU General Public License for more details.
		11	*
		12	* You should have received a copy of the GNU General Public License
		13	* along with this program; if not, write to the
		14	* Free Software Foundation, Inc.,
		15	* 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
		16	*/
1	#ifndef lint	17	#ifndef lint
2	static char rcsid[] = "@(#$Id: $ (C) 2009 J.F.Dockes";	18	static char rcsid[] = "@(#$Id: $ (C) 2009 J.F.Dockes";
3	#endif	19	#endif
4		20
5	#ifndef TEST_CIRCACHE	21	#ifndef TEST_CIRCACHE
	...		...
15	#include <sstream>	31	#include <sstream>
16	#include <iostream>	32	#include <iostream>
17		33
18	#include "circache.h"	34	#include "circache.h"
19	#include "conftree.h"	35	#include "conftree.h"
		36	#include "debuglog.h"
20		37
21	using namespace std;	38	using namespace std;
22		39
23	/*	40	/*
24	* File structure:	41	* File structure:
25	* - Starts with a 1-KB header block, with a param dictionary, ascii-space	42	* - Starts with a 1-KB header block, with a param dictionary.
26	* filled.
27	* - Stored items follow. Each item has 2 segments for the metadata and the	43	* - Stored items follow. Each item has a header and 2 segments for
		44	* the metadata and the data.
28	* data. The segment sizes are stored in an ascii header/marker.	45	* The segment sizes are stored in the ascii header/marker:
29	* circacheSizes = xxx yyy zzz	46	* circacheSizes = xxx yyy zzz
30	* xxx bytes of metadata	47	* xxx bytes of metadata
31	* yyy bytes of data	48	* yyy bytes of data
32	* zzz bytes of padding up to next object	49	* zzz bytes of padding up to next object (only one entry has non zero)
33	*	50	*
34	* There is a write position, which can be at eof while	51	* There is a write position, which can be at eof while
35	* the file is growing, or inside the file if we are recycling. This is stored	52	* the file is growing, or inside the file if we are recycling. This is stored
36	* in the header, together with the maximum size	53	* in the header, together with the maximum size
37	*	54	*
38	* If we are recycling, we have to take care to compute the size of the	55	* If we are recycling, we have to take care to compute the size of the
39	* possible remaining area from the last object invalidated by the write,	56	* possible remaining area from the last object invalidated by the write,
40	* pad it with neutral data and store the size in the new header.	57	* pad it with neutral data and store the size in the new header.
41	*/	58	*/
42		59
43	// First block in file.	60	// First block size
44	#define CIRCACHE_FIRSTBLOCK_SIZE 1024	61	#define CIRCACHE_FIRSTBLOCK_SIZE 1024
45		62
46	// Entry header.	63	// Entry header.
47	// The 32 bits size are stored as hex integers so the maximum size is	64	// The 32 bits size are stored as hex integers so the maximum size is
48	// 13 + 3x8 + 6 = 43	65	// 13 + 3x8 + 6 = 43
	...		...
109	}	126	}
110		127
111	// Name for the cache file	128	// Name for the cache file
112	string datafn(const string& d)	129	string datafn(const string& d)
113	{	130	{
114	return path_cat(d, "circache");	131	return path_cat(d, "circache.crch");
115	}	132	}
116		133
117	bool writefirstblock()	134	bool writefirstblock()
118	{	135	{
119	assert(m_fd >= 0);	136	assert(m_fd >= 0);
	...		...
226	return false;	243	return false;
227	}	244	}
228	return true;	245	return true;
229	}	246	}
230		247
231	CCScanHook::status scan(off_t startoffset, CCScanHook *user)	248	CCScanHook::status scan(off_t startoffset, CCScanHook *user,
		249	bool fold = false)
232	{	250	{
233	assert(m_fd >= 0);	251	assert(m_fd >= 0);
234		252
		253	off_t so0 = startoffset;
		254	bool already_folded = false;
		255
235	while (true) {	256	while (true) {
		257	if (already_folded && startoffset == so0)
		258	return CCScanHook::Eof;
		259
236	EntryHeaderData d;	260	EntryHeaderData d;
237	CCScanHook::status st;	261	CCScanHook::status st;
238	if ((st = readentryheader(startoffset, d)) !=	262	switch ((st = readentryheader(startoffset, d))) {
239	CCScanHook::Continue) {	263	case CCScanHook::Continue: break;
		264	case CCScanHook::Eof:
		265	if (fold && !already_folded) {
		266	already_folded = true;
		267	startoffset = CIRCACHE_FIRSTBLOCK_SIZE;
		268	continue;
		269	}
		270	/* FALLTHROUGH */
		271	default:
240	return st;	272	return st;
241	}	273	}
		274
242	char *bf;	275	char *bf;
243	if ((bf = buf(d.dicsize+1)) == 0) {	276	if ((bf = buf(d.dicsize+1)) == 0) {
244	return CCScanHook::Error;	277	return CCScanHook::Error;
245	}	278	}
246	bf[d.dicsize] = 0;	279	bf[d.dicsize] = 0;
	...		...
294	{	327	{
295	assert(m_d != 0);	328	assert(m_d != 0);
296	struct stat st;	329	struct stat st;
297	if (stat(m_dir.c_str(), &st) < 0) {	330	if (stat(m_dir.c_str(), &st) < 0) {
298	if (mkdir(m_dir.c_str(), 0777) < 0) {	331	if (mkdir(m_dir.c_str(), 0777) < 0) {
299	m_d->m_reason << "CirCache::create: mkdir(" << m_dir << ") failed" <<	332	m_d->m_reason << "CirCache::create: mkdir(" << m_dir <<
300	" errno " << errno;	333	") failed" << " errno " << errno;
301	return false;	334	return false;
302	}	335	}
303	}	336	}
304		337
305	if ((m_d->m_fd = ::open(m_d->datafn(m_dir).c_str(),	338	if ((m_d->m_fd = ::open(m_d->datafn(m_dir).c_str(),
	...		...
315		348
316	char buf[CIRCACHE_FIRSTBLOCK_SIZE];	349	char buf[CIRCACHE_FIRSTBLOCK_SIZE];
317	memset(buf, 0, CIRCACHE_FIRSTBLOCK_SIZE);	350	memset(buf, 0, CIRCACHE_FIRSTBLOCK_SIZE);
318	if (::write(m_d->m_fd, buf, CIRCACHE_FIRSTBLOCK_SIZE) !=	351	if (::write(m_d->m_fd, buf, CIRCACHE_FIRSTBLOCK_SIZE) !=
319	CIRCACHE_FIRSTBLOCK_SIZE) {	352	CIRCACHE_FIRSTBLOCK_SIZE) {
320	m_d->m_reason << "CirCache::create: write header failed, errno " << errno;	353	m_d->m_reason << "CirCache::create: write header failed, errno "
		354	<< errno;
321	return false;	355	return false;
322	}	356	}
323	return m_d->writefirstblock();	357	return m_d->writefirstblock();
		358	}
		359
		360	bool CirCache::open(OpMode mode)
		361	{
		362	assert(m_d != 0);
		363	if (m_d->m_fd >= 0)
		364	::close(m_d->m_fd);
		365
		366	if ((m_d->m_fd = ::open(m_d->datafn(m_dir).c_str(),
		367	mode == CC_OPREAD ? O_RDONLY : O_RDWR)) < 0) {
		368	m_d->m_reason << "CirCache::open: open(" << m_d->datafn(m_dir) <<
		369	") failed " << "errno " << errno;
		370	return false;
		371	}
		372	return m_d->readfirstblock();
324	}	373	}
325		374
326	class CCScanHookDump : public CCScanHook {	375	class CCScanHookDump : public CCScanHook {
327	public:	376	public:
328	virtual status takeone(off_t offs, const string& udi, unsigned int dicsize,	377	virtual status takeone(off_t offs, const string& udi, unsigned int dicsize,
329	unsigned int datasize, unsigned int padsize)	378	unsigned int datasize, unsigned int padsize)
330	{	379	{
331	cout << "udi [" << udi << "] dicsize " << dicsize << " datasize "	380	cout << "Scan: offs " << offs << " dicsize " << dicsize
332	<< datasize << " padsize " << padsize << endl;	381	<< " datasize " << datasize << " padsize " << padsize <<
		382	" udi [" << udi << "]" << endl;
333	return Continue;	383	return Continue;
334	}	384	}
335	};	385	};
336		386
337	bool CirCache::open(OpMode mode)	387	bool CirCache::dump()
		388	{
		389	CCScanHookDump dumper;
		390	off_t start = m_d->m_nheadoffs > CIRCACHE_FIRSTBLOCK_SIZE ?
		391	m_d->m_nheadoffs : CIRCACHE_FIRSTBLOCK_SIZE;
		392	switch (m_d->scan(start, &dumper, true)) {
		393	case CCScanHook::Stop:
		394	cout << "Scan returns Stop??" << endl;
		395	return false;
		396	case CCScanHook::Continue:
		397	cout << "Scan returns Continue ?? " << CCScanHook::Continue << " " <<
		398	getReason() << endl;
		399	return false;
		400	case CCScanHook::Error:
		401	cout << "Scan returns Error: " << getReason() << endl;
		402	return false;
		403	case CCScanHook::Eof:
		404	cout << "Scan returns Eof" << endl;
		405	return true;
		406	default:
		407	cout << "Scan returns Unknown ??" << endl;
		408	return false;
		409	}
		410	}
		411
		412	class CCScanHookGetter : public CCScanHook {
		413	public:
		414	string m_udi;
		415	int m_targinstance;
		416	int m_instance;
		417	off_t m_offs;
		418	EntryHeaderData m_hd;
		419
		420	CCScanHookGetter(const string &udi, int ti)
		421	: m_udi(udi), m_targinstance(ti), m_instance(0), m_offs(0){}
		422
		423	virtual status takeone(off_t offs, const string& udi, unsigned int dicsize,
		424	unsigned int datasize, unsigned int padsize)
		425	{
		426	cerr << "offs " << offs << " udi [" << udi << "] dicsize " << dicsize
		427	<< " datasize " << datasize << " padsize " << padsize << endl;
		428	if (!m_udi.compare(udi)) {
		429	m_instance++;
		430	m_offs = offs;
		431	m_hd.dicsize = dicsize;
		432	m_hd.datasize = datasize;
		433	m_hd.padsize = padsize;
		434	if (m_instance == m_targinstance)
		435	return Stop;
		436	}
		437	return Continue;
		438	}
		439	};
		440
		441	// instance == -1 means get latest. Otherwise specify from 1+
		442	bool CirCache::get(const string& udi, string& dict, string& data, int instance)
338	{	443	{
339	assert(m_d != 0);	444	assert(m_d != 0);
340	if (m_d->m_fd >= 0)	445	if (m_d->m_fd < 0) {
341	::close(m_d->m_fd);	446	m_d->m_reason << "CirCache::get: not open";
		447	return false;
		448	}
342		449
343	if ((m_d->m_fd = ::open(m_d->datafn(m_dir).c_str(),	450	LOGDEB(("CirCache::get: udi [%s], instance\n", udi.c_str(), instance));
344	mode == CC_OPREAD ? O_RDONLY : O_RDWR)) < 0) {	451
345	m_d->m_reason << "CirCache::open: open(" << m_d->datafn(m_dir) <<	452	CCScanHookGetter getter(udi, instance);
346	") failed " << "errno " << errno;	453	off_t start = m_d->m_nheadoffs > CIRCACHE_FIRSTBLOCK_SIZE ?
		454	m_d->m_nheadoffs : CIRCACHE_FIRSTBLOCK_SIZE;
		455
		456	CCScanHook::status ret = m_d->scan(start, &getter, true);
		457	if (ret == CCScanHook::Eof) {
		458	if (getter.m_instance == 0)
347	return false;	459	return false;
		460	} else if (ret != CCScanHook::Stop) {
		461	return false;
348	}	462	}
349	bool ret = m_d->readfirstblock();	463	off_t offs = getter.m_offs + CIRCACHE_HEADER_SIZE;
350		464	if (lseek(m_d->m_fd, offs, 0) != offs) {
351	if (mode == CC_OPREAD) {	465	m_d->m_reason << "CirCache::get: lseek(" << offs << ") failed: " <<
352	CCScanHookDump dumper;
353	switch (m_d->scan(CIRCACHE_FIRSTBLOCK_SIZE, &dumper)) {
354	case CCScanHook::Stop:
355	cerr << "Scan returns Stop" << endl;
356	break;	466	errno;
357	case CCScanHook::Continue:	467	return false;
358	cerr << "Scan returns Continue ?? " << CCScanHook::Continue << " " <<
359	getReason() << endl;
360	break;
361	case CCScanHook::Error:
362	cerr << "Scan returns Error: " << getReason() << endl;
363	break;
364	case CCScanHook::Eof:
365	cerr << "Scan returns Eof" << endl;
366	break;
367	}	468	}
		469	char *bf = m_d->buf(getter.m_hd.dicsize);
		470	if (bf == 0)
		471	return false;
		472	if (read(m_d->m_fd, bf, getter.m_hd.dicsize) != int(getter.m_hd.dicsize)) {
		473	m_d->m_reason << "CirCache::get: read() failed: errno " << errno;
		474	return false;
368	}	475	}
369	return ret;	476	dict.assign(bf, getter.m_hd.dicsize);
370	}
371		477
372	bool CirCache::get(const string& udi, string dic, string data)	478	bf = m_d->buf(getter.m_hd.datasize);
373	{	479	if (bf == 0)
		480	return false;
		481	if (read(m_d->m_fd, bf, getter.m_hd.datasize) != int(getter.m_hd.datasize)){
		482	m_d->m_reason << "CirCache::get: read() failed: errno " << errno;
		483	return false;
		484	}
		485	data.assign(bf, getter.m_hd.datasize);
		486
374	return true;	487	return true;
375	}	488	}
376		489
377	class CCScanHookSpacer : public CCScanHook {	490	class CCScanHookSpacer : public CCScanHook {
378	public:	491	public:
	...		...
383	: sizewanted(sz), sizeseen(0) {assert(sz > 0);}	496	: sizewanted(sz), sizeseen(0) {assert(sz > 0);}
384		497
385	virtual status takeone(off_t offs, const string& udi, unsigned int dicsize,	498	virtual status takeone(off_t offs, const string& udi, unsigned int dicsize,
386	unsigned int datasize, unsigned int padsize)	499	unsigned int datasize, unsigned int padsize)
387	{	500	{
388	cout << "udi [" << udi << "] dicsize " << dicsize << " datasize "	501	LOGDEB(("ScanSpacer: offs %u dicsz %u datasz %u padsz %u udi[%s]\n",
389	<< datasize << " padsize " << padsize << endl;	502	(unsigned int)offs, dicsize, datasize, padsize, udi.c_str()));
390	sizeseen += CIRCACHE_HEADER_SIZE + dicsize + datasize + padsize;	503	sizeseen += CIRCACHE_HEADER_SIZE + dicsize + datasize + padsize;
391	if (sizeseen >= sizewanted)	504	if (sizeseen >= sizewanted)
392	return Stop;	505	return Stop;
393	return Continue;	506	return Continue;
394	}	507	}
	...		...
423	int nsize = CIRCACHE_HEADER_SIZE + dic.size() + data.size();	536	int nsize = CIRCACHE_HEADER_SIZE + dic.size() + data.size();
424	int nwriteoffs = 0;	537	int nwriteoffs = 0;
425	int npadsize = 0;	538	int npadsize = 0;
426	bool extending = false;	539	bool extending = false;
427		540
428	cerr << "CirCache::PUT: nsize " << nsize <<	541	LOGDEB2(("CirCache::put: nsize %d oheadoffs %d\n",
429	" oheadoffs " << m_d->m_oheadoffs << endl;	542	nsize, m_d->m_oheadoffs));
430		543
431	if (st.st_size < m_d->m_maxsize) {	544	if (st.st_size < m_d->m_maxsize) {
432	// If we are still growing the file, things are simple	545	// If we are still growing the file, things are simple
433	nwriteoffs = lseek(m_d->m_fd, 0, SEEK_END);	546	nwriteoffs = lseek(m_d->m_fd, 0, SEEK_END);
434	npadsize = 0;	547	npadsize = 0;
	...		...
448	if (m_d->readentryheader(m_d->m_nheadoffs, pd) !=	561	if (m_d->readentryheader(m_d->m_nheadoffs, pd) !=
449	CCScanHook::Continue) {	562	CCScanHook::Continue) {
450	return false;	563	return false;
451	}	564	}
452	assert(int(pd.padsize) == m_d->m_npadsize);	565	assert(int(pd.padsize) == m_d->m_npadsize);
453	cerr << "CirCache::put: recovering previous padsize " <<	566	LOGDEB2(("CirCache::put: recovering previous padsize %d\n",
454	pd.padsize << endl;	567	pd.padsize));
455	pd.padsize = 0;	568	pd.padsize = 0;
456	if (!m_d->writeentryheader(m_d->m_nheadoffs, pd)) {	569	if (!m_d->writeentryheader(m_d->m_nheadoffs, pd)) {
457	return false;	570	return false;
458	}	571	}
459	nwriteoffs = m_d->m_oheadoffs - recovpadsize;	572	nwriteoffs = m_d->m_oheadoffs - recovpadsize;
	...		...
461	}	574	}
462		575
463	if (nsize <= recovpadsize) {	576	if (nsize <= recovpadsize) {
464	// If the new entry fits entirely in the pad area from the	577	// If the new entry fits entirely in the pad area from the
465	// latest one, no need to recycle the oldest entries.	578	// latest one, no need to recycle the oldest entries.
466	cerr << "CirCache::put: new fits in old padsize " <<	579	LOGDEB2(("CirCache::put: new fits in old padsize %d\n,"
467	recovpadsize << endl;	580	recovpadsize));
468	npadsize = recovpadsize - nsize;	581	npadsize = recovpadsize - nsize;
469	} else {	582	} else {
470	// Scan the file until we have enough space for the new entry,	583	// Scan the file until we have enough space for the new entry,
471	// and determine the pad size up to the 1st preserved entry	584	// and determine the pad size up to the 1st preserved entry
472	int scansize = nsize - recovpadsize;	585	int scansize = nsize - recovpadsize;
473	cerr << "CirCache::put: scanning for size " << scansize <<	586	LOGDEB2(("CirCache::put: scanning for size %d from offs %u\n",
474	" from offset " << m_d->m_oheadoffs << endl;	587	scansize, (unsigned int)m_d->m_oheadoffs));
475	CCScanHookSpacer spacer(scansize);	588	CCScanHookSpacer spacer(scansize);
476	switch (m_d->scan(m_d->m_oheadoffs, &spacer)) {	589	switch (m_d->scan(m_d->m_oheadoffs, &spacer)) {
477	case CCScanHook::Stop:	590	case CCScanHook::Stop:
478	cerr << "put: Scan ok, sizeseen " << spacer.sizeseen << endl;	591	LOGDEB2(("CirCache::put: Scan ok, sizeseen %d\n",
		592	spacer.sizeseen));
479	npadsize = spacer.sizeseen - scansize;	593	npadsize = spacer.sizeseen - scansize;
480	break;	594	break;
481	case CCScanHook::Eof:	595	case CCScanHook::Eof:
482	// npadsize is 0	596	// npadsize is 0
483	extending = true;	597	extending = true;
	...		...
487	return false;	601	return false;
488	}	602	}
489	}	603	}
490	}	604	}
491		605
492	cerr << "CirCache::put: writing " << nsize << " at " << nwriteoffs <<	606	LOGDEB2(("CirCache::put: writing %d at %d padsize %d\n",
493	" padsize " << npadsize << endl;	607	nsize, nwriteoffs, npadsize));
494	if (lseek(m_d->m_fd, nwriteoffs, 0) != nwriteoffs) {	608	if (lseek(m_d->m_fd, nwriteoffs, 0) != nwriteoffs) {
495	m_d->m_reason << "CirCache::put: lseek failed: " << errno;	609	m_d->m_reason << "CirCache::put: lseek failed: " << errno;
496	return false;	610	return false;
497	}	611	}
498	char *bf = m_d->buf(CIRCACHE_HEADER_SIZE);	612	char *bf = m_d->buf(CIRCACHE_HEADER_SIZE);
	...		...
549	static char *thisprog;	663	static char *thisprog;
550		664
551	static char usage [] =	665	static char usage [] =
552	" -c <dirname> : create\n"	666	" -c <dirname> : create\n"
553	" -p <dirname> <apath> [apath ...] : put files\n"	667	" -p <dirname> <apath> [apath ...] : put files\n"
		668	" -d <dirname> : dump\n"
		669	" -g [-i instance] <dirname> <udi>: get\n"
554	;	670	;
555	static void	671	static void
556	Usage(FILE *fp = stderr)	672	Usage(FILE *fp = stderr)
557	{	673	{
558	fprintf(fp, "%s: usage:\n%s", thisprog, usage);	674	fprintf(fp, "%s: usage:\n%s", thisprog, usage);
	...		...
560	}	676	}
561		677
562	static int op_flags;	678	static int op_flags;
563	#define OPT_MOINS 0x1	679	#define OPT_MOINS 0x1
564	#define OPT_c 0x2	680	#define OPT_c 0x2
565	#define OPT_b 0x4
566	#define OPT_p 0x8	681	#define OPT_p 0x8
567	#define OPT_g 0x10	682	#define OPT_g 0x10
		683	#define OPT_d 0x20
		684	#define OPT_i 0x40
568		685
569	int main(int argc, char **argv)	686	int main(int argc, char **argv)
570	{	687	{
571	int count = 10;	688	int instance = -1;
572		689
573	thisprog = argv[0];	690	thisprog = argv[0];
574	argc--; argv++;	691	argc--; argv++;
575		692
576	while (argc > 0 && **argv == '-') {	693	while (argc > 0 && **argv == '-') {
577	(*argv)++;	694	(*argv)++;
	...		...
581	while (**argv)	698	while (**argv)
582	switch ((argv)++) {	699	switch ((argv)++) {
583	case 'c': op_flags \|= OPT_c; break;	700	case 'c': op_flags \|= OPT_c; break;
584	case 'p': op_flags \|= OPT_p; break;	701	case 'p': op_flags \|= OPT_p; break;
585	case 'g': op_flags \|= OPT_g; break;	702	case 'g': op_flags \|= OPT_g; break;
		703	case 'd': op_flags \|= OPT_d; break;
586	case 'b': op_flags \|= OPT_b; if (argc < 2) Usage();	704	case 'i': op_flags \|= OPT_i; if (argc < 2) Usage();
587	if ((sscanf(*(++argv), "%d", &count)) != 1)	705	if ((sscanf(*(++argv), "%d", &instance)) != 1)
588	Usage();	706	Usage();
589	argc--;	707	argc--;
590	goto b1;	708	goto b1;
591	default: Usage(); break;	709	default: Usage(); break;
592	}	710	}
	...		...
631	exit(1);	749	exit(1);
632	}	750	}
633	}	751	}
634	cc.open(CirCache::CC_OPREAD);	752	cc.open(CirCache::CC_OPREAD);
635	} else if (op_flags & OPT_g) {	753	} else if (op_flags & OPT_g) {
		754	string udi = *argv++;argc--;
636	if (!cc.open(CirCache::CC_OPREAD)) {	755	if (!cc.open(CirCache::CC_OPREAD)) {
637	cerr << "Open failed: " << cc.getReason() << endl;	756	cerr << "Open failed: " << cc.getReason() << endl;
638	exit(1);	757	exit(1);
639	}	758	}
		759	string dic, data;
		760	if (!cc.get(udi, dic, data, instance)) {
		761	cerr << "Get failed: " << cc.getReason() << endl;
		762	exit(1);
		763	}
		764	cout << "Dict: [" << dic << "]" << endl;
		765	} else if (op_flags & OPT_d) {
		766	if (!cc.open(CirCache::CC_OPREAD)) {
		767	cerr << "Open failed: " << cc.getReason() << endl;
		768	exit(1);
		769	}
		770	cc.dump();
640	} else	771	} else
641	Usage();	772	Usage();
642		773
643	exit(0);	774	exit(0);
644	}	775	}