--- a
+++ b/src/utils/circache.cpp
@@ -0,0 +1,646 @@
+#ifndef lint
+static char rcsid[] = "@(#$Id: $ (C) 2009 J.F.Dockes";
+#endif
+
+#ifndef TEST_CIRCACHE
+
+#include <assert.h>
+#include <errno.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <sys/types.h>
+#include <sys/uio.h>
+#include <unistd.h>
+
+#include <sstream>
+#include <iostream>
+
+#include "circache.h"
+#include "conftree.h"
+
+using namespace std;
+
+/*
+ * File structure:
+ * - Starts with a 1-KB header block, with a param dictionary, ascii-space 
+ *   filled. 
+ * - Stored items follow. Each item has 2 segments for the metadata and the 
+ *   data. The segment sizes are stored in an ascii header/marker.
+ *     circacheSizes = xxx yyy zzz
+ *     xxx bytes of metadata
+ *     yyy bytes of data
+ *     zzz bytes of padding up to next object
+ *
+ * There is a write position, which can be at eof while 
+ * the file is growing, or inside the file if we are recycling. This is stored
+ * in the header, together with the maximum size
+ *
+ * If we are recycling, we have to take care to compute the size of the 
+ * possible remaining area from the last object invalidated by the write, 
+ * pad it with neutral data and store the size in the new header.
+ */
+
+// First block in file. 
+#define CIRCACHE_FIRSTBLOCK_SIZE 1024
+
+// Entry header.
+// The 32 bits size are stored as hex integers so the maximum size is
+// 13 + 3x8 + 6 = 43
+#define CIRCACHE_HEADER_SIZE 50
+const char *headerformat = "circacheSizes = %x %x %x";
+class EntryHeaderData {
+public:
+    EntryHeaderData() : dicsize(0), datasize(0), padsize(0) {}
+    unsigned int dicsize;
+    unsigned int datasize;
+    unsigned int padsize;
+};
+
+// A callback class for the header-hopping function.
+class CCScanHook {
+public:
+    virtual ~CCScanHook() {}
+    enum status {Stop, Continue, Error, Eof};
+    virtual status takeone(off_t offs, const string& udi, unsigned int dicsize, 
+                           unsigned int datasize, unsigned int padsize) = 0;
+};
+
+class CirCacheInternal {
+public:
+    int m_fd;
+    // Maximum file size, after which we begin reusing old space
+    off_t m_maxsize; 
+    // Offset of the oldest header. 
+    off_t m_oheadoffs;
+    // Offset of last write (newest header)
+    off_t m_nheadoffs;
+    // Pad size for newest entry. 
+    int   m_npadsize;
+    // A place to hold data when reading
+    char *m_buffer;
+    size_t m_bufsiz;
+    // Error messages
+    ostringstream m_reason;
+
+    CirCacheInternal()
+        : m_fd(-1), m_maxsize(-1), m_oheadoffs(-1), 
+          m_nheadoffs(0), m_npadsize(0), m_buffer(0), m_bufsiz(0)
+    {}
+
+    ~CirCacheInternal()
+    {
+        if (m_fd >= 0)
+            close(m_fd);
+        if (m_buffer)
+            free(m_buffer);
+    }
+
+    char *buf(size_t sz)
+    {
+        if (m_bufsiz >= sz)
+            return m_buffer;
+        if ((m_buffer = (char *)realloc(m_buffer, sz))) {
+            m_bufsiz = sz;
+        } else {
+            m_reason << "CirCache:: realloc(" << sz << ") failed";
+            m_bufsiz = 0;
+        }
+        return m_buffer;
+    }
+
+    // Name for the cache file
+    string datafn(const string& d)
+    {
+        return  path_cat(d, "circache");
+    }
+
+    bool writefirstblock()
+    {
+        assert(m_fd >= 0);
+
+        ostringstream s;
+        s << 
+            "maxsize = " << m_maxsize << "\n" <<
+            "oheadoffs = " << m_oheadoffs << "\n" <<
+            "nheadoffs = " << m_nheadoffs << "\n" <<
+            "npadsize = " << m_npadsize << "\n" <<
+            "                                                              " <<
+            "                                                              " <<
+            "\0";
+
+        int sz = int(s.str().size());
+        assert(sz < CIRCACHE_FIRSTBLOCK_SIZE);
+        lseek(m_fd, 0, 0);
+        if (write(m_fd, s.str().c_str(), sz) != sz) {
+            m_reason << "writefirstblock: write() failed: errno " << errno;
+            return false;
+        }
+        return true;
+    }
+
+    bool readfirstblock()
+    {
+        assert(m_fd >= 0);
+
+        char *bf = buf(CIRCACHE_FIRSTBLOCK_SIZE);
+        if (!bf)
+            return false;
+        lseek(m_fd, 0, 0);
+        if (read(m_fd, bf, CIRCACHE_FIRSTBLOCK_SIZE) != 
+            CIRCACHE_FIRSTBLOCK_SIZE) {
+            m_reason << "readfirstblock: read() failed: errno " << errno;
+            return false;
+        }
+        string s(bf, CIRCACHE_FIRSTBLOCK_SIZE);
+        ConfSimple conf(s, 1);
+        string value;
+        if (!conf.get("maxsize", value, "")) {
+            m_reason << "readfirstblock: conf get maxsize failed";
+            return false;
+        }
+        m_maxsize = atol(value.c_str());
+        if (!conf.get("oheadoffs", value, "")) {
+            m_reason << "readfirstblock: conf get oheadoffs failed";
+            return false;
+        }
+        m_oheadoffs = atol(value.c_str());
+        if (!conf.get("nheadoffs", value, "")) {
+            m_reason << "readfirstblock: conf get nheadoffs failed";
+            return false;
+        }
+        m_nheadoffs = atol(value.c_str());
+        if (!conf.get("npadsize", value, "")) {
+            m_reason << "readfirstblock: conf get npadsize failed";
+            return false;
+        }
+        m_npadsize = atol(value.c_str());
+        return true;
+    }            
+
+    CCScanHook::status readentryheader(off_t offset, EntryHeaderData& d)
+    {
+        assert(m_fd >= 0);
+
+        if (lseek(m_fd, offset, 0) != offset) {
+            m_reason << "readentryheader: lseek(" << offset << 
+                ") failed: errno " << errno;
+            return CCScanHook::Error;
+        }
+        char *bf = buf(CIRCACHE_HEADER_SIZE);
+        if (bf == 0) {
+            return CCScanHook::Error;
+        }
+        int ret = read(m_fd, bf, CIRCACHE_HEADER_SIZE);
+        if (ret == 0) {
+            // Eof
+            m_reason << " Eof ";
+            return CCScanHook::Eof;
+        }
+        if (ret != CIRCACHE_HEADER_SIZE) {
+            m_reason << " readheader: read failed errno " << errno;
+            return CCScanHook::Error;
+        }
+        if (sscanf(bf, headerformat, &d.dicsize, &d.datasize, 
+                   &d.padsize) != 3) {
+            m_reason << " readentryheader: bad header at " << 
+                offset << " [" << bf << "]";
+            return CCScanHook::Error;
+        }
+        return CCScanHook::Continue;
+    }
+
+    bool writeentryheader(off_t offset, const EntryHeaderData& d)
+    {
+        char *bf = buf(CIRCACHE_HEADER_SIZE);
+        if (bf == 0)
+            return false;
+        memset(bf, 0, CIRCACHE_HEADER_SIZE);
+        sprintf(bf, headerformat, d.dicsize, d.datasize, d.padsize);
+        if (lseek(m_fd, offset, 0) != offset) {
+            m_reason << "CirCache::weh: lseek(" << offset << 
+                ") failed: errno " << errno;
+            return false;
+        }
+        if (write(m_fd, bf, CIRCACHE_HEADER_SIZE) !=  CIRCACHE_HEADER_SIZE) {
+            m_reason << "CirCache::weh: write failed. errno " << errno;
+            return false;
+        }
+        return true;
+    }
+
+    CCScanHook::status scan(off_t startoffset, CCScanHook *user)
+    {
+        assert(m_fd >= 0);
+
+        while (true) {
+            EntryHeaderData d;
+            CCScanHook::status st;
+            if ((st = readentryheader(startoffset, d)) != 
+                CCScanHook::Continue) {
+                return st;
+            }
+            char *bf;
+            if ((bf = buf(d.dicsize+1)) == 0) {
+                return CCScanHook::Error;
+            }
+            bf[d.dicsize] = 0;
+            if (read(m_fd, bf, d.dicsize) != int(d.dicsize)) {
+                m_reason << "scan: read failed errno " << errno;
+                return CCScanHook::Error;
+            }
+            string b(bf, d.dicsize);
+            ConfSimple conf(b, 1);
+            
+            string udi;
+            if (!conf.get("udi", udi, "")) {
+                m_reason << "scan: no udi in dic";
+                return CCScanHook::Error;
+            }
+                
+            // Call callback
+            CCScanHook::status a = 
+                user->takeone(startoffset, udi, d.dicsize, d.datasize, 
+                              d.padsize);
+            switch (a) {
+            case CCScanHook::Continue: 
+                break;
+            default:
+                return a;
+            }
+            startoffset += CIRCACHE_HEADER_SIZE + d.dicsize + 
+                d.datasize + d.padsize;
+        }
+    }
+};
+
+CirCache::CirCache(const string& dir)
+    : m_dir(dir)
+{
+    m_d = new CirCacheInternal;
+}
+
+CirCache::~CirCache()
+{
+    delete m_d;
+    m_d = 0;
+}
+
+string CirCache::getReason()
+{
+    return m_d ? m_d->m_reason.str() : "Not initialized";
+}
+
+bool CirCache::create(off_t m_maxsize)
+{
+    assert(m_d != 0);
+    struct stat st;
+    if (stat(m_dir.c_str(), &st) < 0) {
+        if (mkdir(m_dir.c_str(), 0777) < 0) {
+            m_d->m_reason << "CirCache::create: mkdir(" << m_dir << ") failed" <<
+                " errno " << errno;
+            return false;
+        }
+    }
+
+    if ((m_d->m_fd = ::open(m_d->datafn(m_dir).c_str(), 
+                          O_CREAT | O_RDWR | O_TRUNC, 
+                          0666)) < 0) {
+        m_d->m_reason << "CirCache::create: open/creat(" << 
+            m_d->datafn(m_dir) << ") failed " << "errno " << errno;
+            return false;
+    }
+
+    m_d->m_maxsize = m_maxsize;
+    m_d->m_oheadoffs = CIRCACHE_FIRSTBLOCK_SIZE;
+
+    char buf[CIRCACHE_FIRSTBLOCK_SIZE];
+    memset(buf, 0, CIRCACHE_FIRSTBLOCK_SIZE);
+    if (::write(m_d->m_fd, buf, CIRCACHE_FIRSTBLOCK_SIZE) != 
+        CIRCACHE_FIRSTBLOCK_SIZE) {
+        m_d->m_reason << "CirCache::create: write header failed, errno " << errno;
+        return false;
+    }
+    return m_d->writefirstblock();
+}
+
+class CCScanHookDump : public  CCScanHook {
+public:
+    virtual status takeone(off_t offs, const string& udi, unsigned int dicsize, 
+                           unsigned int datasize, unsigned int padsize) 
+    {
+        cout << "udi [" << udi << "] dicsize " << dicsize << " datasize "
+             << datasize << " padsize " << padsize << endl;
+        return Continue;
+    }
+};
+
+bool CirCache::open(OpMode mode)
+{
+    assert(m_d != 0);
+    if (m_d->m_fd >= 0)
+        ::close(m_d->m_fd);
+
+    if ((m_d->m_fd = ::open(m_d->datafn(m_dir).c_str(), 
+                          mode == CC_OPREAD ? O_RDONLY : O_RDWR)) < 0) {
+        m_d->m_reason << "CirCache::open: open(" << m_d->datafn(m_dir) << 
+            ") failed " << "errno " << errno;
+        return false;
+    }
+    bool ret = m_d->readfirstblock();
+
+    if (mode == CC_OPREAD) {
+        CCScanHookDump dumper;
+        switch (m_d->scan(CIRCACHE_FIRSTBLOCK_SIZE, &dumper)) {
+        case CCScanHook::Stop: 
+            cerr << "Scan returns Stop" << endl;
+            break;
+        case CCScanHook::Continue: 
+            cerr << "Scan returns Continue ?? " << CCScanHook::Continue << " " <<
+                getReason() << endl;
+            break;
+        case CCScanHook::Error: 
+            cerr << "Scan returns Error: " << getReason() << endl;
+            break;
+        case CCScanHook::Eof:
+            cerr << "Scan returns Eof" << endl;
+            break;
+        }
+    }
+    return ret;
+}
+
+bool CirCache::get(const string& udi, string dic, string data)
+{
+    return true;
+}
+
+class CCScanHookSpacer : public  CCScanHook {
+public:
+    unsigned int sizewanted;
+    unsigned int sizeseen;
+    
+    CCScanHookSpacer(int sz)
+        : sizewanted(sz), sizeseen(0) {assert(sz > 0);}
+
+    virtual status takeone(off_t offs, const string& udi, unsigned int dicsize, 
+                           unsigned int datasize, unsigned int padsize) 
+    {
+        cout << "udi [" << udi << "] dicsize " << dicsize << " datasize "
+             << datasize << " padsize " << padsize << endl;
+        sizeseen += CIRCACHE_HEADER_SIZE + dicsize + datasize + padsize;
+        if (sizeseen >= sizewanted)
+            return Stop;
+        return Continue;
+    }
+};
+
+bool CirCache::put(const string& udi, const string& idic, const string& data)
+{
+    assert(m_d != 0);
+    if (m_d->m_fd < 0) {
+        m_d->m_reason << "CirCache::put: not open";
+        return false;
+    }
+    string dic = idic;
+
+    // If udi is not already in the metadata, need to add it
+    string u;
+    ConfSimple conf(dic, 0);
+    if (!conf.get("udi", u, "")) {
+        ostringstream s;
+        conf.set("udi", udi, "");
+        conf.write(s);
+        dic = s.str();
+    }
+
+    struct stat st;
+    if (fstat(m_d->m_fd, &st) < 0) {
+        m_d->m_reason << "CirCache::put: fstat failed. errno " << errno;
+        return false;
+    }
+
+    // Characteristics for the new entry
+    int nsize = CIRCACHE_HEADER_SIZE + dic.size() + data.size();
+    int nwriteoffs = 0;
+    int npadsize = 0;
+    bool extending = false;
+
+    cerr << "CirCache::PUT: nsize " << nsize << 
+        " oheadoffs " << m_d->m_oheadoffs << endl;
+
+    if (st.st_size < m_d->m_maxsize) {
+        // If we are still growing the file, things are simple
+        nwriteoffs = lseek(m_d->m_fd, 0, SEEK_END);
+        npadsize = 0;
+        extending = true;
+    } else {
+        // We'll write at the oldest header, minus the possible
+        // padsize for the previous (latest) one.
+        int recovpadsize = m_d->m_oheadoffs == CIRCACHE_FIRSTBLOCK_SIZE ?
+            0 : m_d->m_npadsize;
+        if (recovpadsize == 0) {
+            // No padsize to recover
+            nwriteoffs = m_d->m_oheadoffs;
+        } else {
+            // Need to read the latest entry's header, to rewrite it with a 
+            // zero pad size
+            EntryHeaderData pd;
+            if (m_d->readentryheader(m_d->m_nheadoffs, pd) != 
+                CCScanHook::Continue) {
+                return false;
+            }
+            assert(int(pd.padsize) == m_d->m_npadsize);
+            cerr << "CirCache::put: recovering previous padsize " << 
+                pd.padsize << endl;
+            pd.padsize = 0;
+            if (!m_d->writeentryheader(m_d->m_nheadoffs, pd)) {
+                return false;
+            }
+            nwriteoffs = m_d->m_oheadoffs - recovpadsize;
+            // If we fail between here and the end, the file is hosed.
+        }
+
+        if (nsize <= recovpadsize) {
+            // If the new entry fits entirely in the pad area from the
+            // latest one, no need to recycle the oldest entries.
+            cerr << "CirCache::put: new fits in old padsize " << 
+                recovpadsize << endl;
+            npadsize = recovpadsize - nsize;
+        } else {
+            // Scan the file until we have enough space for the new entry,
+            // and determine the pad size up to the 1st preserved entry
+            int scansize = nsize - recovpadsize;
+            cerr << "CirCache::put: scanning for size " << scansize << 
+                " from offset " << m_d->m_oheadoffs << endl;
+            CCScanHookSpacer spacer(scansize);
+            switch (m_d->scan(m_d->m_oheadoffs, &spacer)) {
+            case CCScanHook::Stop: 
+                cerr << "put: Scan ok, sizeseen " << spacer.sizeseen << endl;
+                npadsize = spacer.sizeseen - scansize;
+                break;
+            case CCScanHook::Eof:
+                // npadsize is 0
+                extending = true;
+                break;
+            case CCScanHook::Continue: 
+            case CCScanHook::Error: 
+                return false;
+            }
+        }
+    }
+    
+    cerr << "CirCache::put: writing " << nsize << " at " << nwriteoffs << 
+        " padsize " << npadsize << endl;
+    if (lseek(m_d->m_fd, nwriteoffs, 0) != nwriteoffs) {
+        m_d->m_reason << "CirCache::put: lseek failed: " << errno;
+        return false;
+    }
+    char *bf = m_d->buf(CIRCACHE_HEADER_SIZE);
+    if (bf == 0) 
+        return false;
+    memset(bf, 0, CIRCACHE_HEADER_SIZE);
+    sprintf(bf, headerformat, dic.size(), data.size(), npadsize);
+    struct iovec vecs[3];
+    vecs[0].iov_base = bf;
+    vecs[0].iov_len = CIRCACHE_HEADER_SIZE;
+    vecs[1].iov_base = (void *)dic.c_str();
+    vecs[1].iov_len = dic.size();
+    vecs[2].iov_base = (void *)data.c_str();
+    vecs[2].iov_len = data.size();
+    if (writev(m_d->m_fd, vecs, 3) !=  nsize) {
+        m_d->m_reason << "put: write failed. errno " << errno;
+        if (extending)
+            ftruncate(m_d->m_fd, m_d->m_oheadoffs);
+        return false;
+    }
+
+    // Update first block information
+    m_d->m_nheadoffs = nwriteoffs;
+    m_d->m_npadsize  = npadsize;
+    m_d->m_oheadoffs = nwriteoffs + nsize + npadsize;
+    if (nwriteoffs + nsize >= m_d->m_maxsize) {
+        // If we are at the biggest allowed size or we are currently
+        // growing a young file, the oldest header is at BOT.
+        m_d->m_oheadoffs = CIRCACHE_FIRSTBLOCK_SIZE;
+    }
+    return m_d->writefirstblock();
+    return true;
+}
+
+#else // TEST ->
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <errno.h>
+#include <string.h>
+
+#include <string>
+#include <iostream>
+
+#include "circache.h"
+#include "fileudi.h"
+#include "conftree.h"
+#include "readfile.h"
+#include "debuglog.h"
+
+using namespace std;
+
+static char *thisprog;
+
+static char usage [] =
+" -c <dirname> : create\n"
+" -p <dirname> <apath> [apath ...] : put files\n"
+;
+static void
+Usage(FILE *fp = stderr)
+{
+    fprintf(fp, "%s: usage:\n%s", thisprog, usage);
+    exit(1);
+}
+
+static int     op_flags;
+#define OPT_MOINS 0x1
+#define OPT_c	  0x2 
+#define OPT_b	  0x4 
+#define OPT_p     0x8
+#define OPT_g     0x10
+
+int main(int argc, char **argv)
+{
+  int count = 10;
+    
+  thisprog = argv[0];
+  argc--; argv++;
+
+  while (argc > 0 && **argv == '-') {
+    (*argv)++;
+    if (!(**argv))
+      /* Cas du "adb - core" */
+      Usage();
+    while (**argv)
+      switch (*(*argv)++) {
+      case 'c':	op_flags |= OPT_c; break;
+      case 'p':	op_flags |= OPT_p; break;
+      case 'g':	op_flags |= OPT_g; break;
+      case 'b':	op_flags |= OPT_b; if (argc < 2)  Usage();
+	if ((sscanf(*(++argv), "%d", &count)) != 1) 
+	  Usage(); 
+	argc--; 
+	goto b1;
+      default: Usage();	break;
+      }
+  b1: argc--; argv++;
+  }
+
+  DebugLog::getdbl()->setloglevel(DEBDEB1);
+  DebugLog::setfilename("stderr");
+
+  if (argc < 1)
+    Usage();
+  string dir = *argv++;argc--;
+
+  CirCache cc(dir);
+
+  if (op_flags & OPT_c) {
+      if (!cc.create(100*1024)) {
+          cerr << "Create failed:" << cc.getReason() << endl;
+          exit(1);
+      }
+  } else if (op_flags & OPT_p) {
+      if (argc < 1)
+          Usage();
+      if (!cc.open(CirCache::CC_OPWRITE)) {
+          cerr << "Open failed: " << cc.getReason() << endl;
+          exit(1);
+      }
+      while (argc) {
+          string fn = *argv++;argc--;
+          char dic[1000];
+          sprintf(dic, "#whatever...\nmimetype = text/plain\n");
+          string data, reason;
+          if (!file_to_string(fn, data, &reason)) {
+              cerr << "File_to_string: " << reason << endl;
+              exit(1);
+          }
+          string udi;
+          make_udi(fn, "", udi);
+   
+          if (!cc.put(udi, dic, data)) {
+              cerr << "Put failed: " << cc.getReason() << endl;
+              exit(1);
+          }
+      }
+      cc.open(CirCache::CC_OPREAD);
+  } else if (op_flags & OPT_g) {
+      if (!cc.open(CirCache::CC_OPREAD)) {
+          cerr << "Open failed: " << cc.getReason() << endl;
+          exit(1);
+      }
+  } else
+      Usage();
+
+  exit(0);
+}
+
+#endif