Open Source Projects Europe

// A callback class for the header-hopping function.

// A callback class for the header-hopping function.

class CCScanHook {

class CCScanHook {

public:

public:

    virtual ~CCScanHook() {}

    virtual ~CCScanHook() {}

    enum status {Stop, Continue, Error, Eof};

    enum status {Stop, Continue, Error, Eof};

                           const EntryHeaderData& d) = 0;

                           const EntryHeaderData& d) = 0;

};

};

// We have an auxiliary in-memory multimap of hashed-udi -> offset to

// We have an auxiliary in-memory multimap of hashed-udi -> offset to

// speed things up. This is created the first time the file is scanned

// speed things up. This is created the first time the file is scanned

            }

            }

        }

        }

        return false;

        return false;

    }

    }

};

};

class CirCacheInternal {

class CirCacheInternal {

public:

public:

    int m_fd;

    int m_fd;

    ////// These are cache persistent state and written to the first block:

    ////// These are cache persistent state and written to the first block:

    // Maximum file size, after which we begin reusing old space

    // Maximum file size, after which we begin reusing old space

    // Offset of the oldest header, or max file offset (file size)

    // Offset of the oldest header, or max file offset (file size)

    // while the file is growing. This is the next write position.

    // while the file is growing. This is the next write position.

    // Offset of last write (newest header)

    // Offset of last write (newest header)

    // Pad size for newest entry.

    // Pad size for newest entry.

    // Keep history or only last entry

    // Keep history or only last entry

    bool  m_uniquentries;

    bool  m_uniquentries;

    ///////////////////// End header entries

    ///////////////////// End header entries

    // A place to hold data when reading

    // A place to hold data when reading

    // Error messages

    // Error messages

    ostringstream m_reason;

    ostringstream m_reason;

    // State for rewind/next/getcurrent operation. This could/should

    // State for rewind/next/getcurrent operation. This could/should

    // be moved to a separate iterator.

    // be moved to a separate iterator.

    EntryHeaderData m_ithd;

    EntryHeaderData m_ithd;

    // Offset cache

    // Offset cache

    kh_type m_ofskh;

    kh_type m_ofskh;

    bool    m_ofskhcplt; // Has cache been fully read since open?

    bool    m_ofskhcplt; // Has cache been fully read since open?

    // Add udi->offset translation to map

    // Add udi->offset translation to map

        UdiH h(udi);

        UdiH h(udi);

        LOGDEB2("Circache::khEnter: h "  << (h.asHexString()) << " offs "  << ((ULONG)ofs) << " udi ["  << (udi) << "]\n" );

        LOGDEB2("Circache::khEnter: h "  << (h.asHexString()) << " offs "  << ((ULONG)ofs) << " udi ["  << (udi) << "]\n" );

        pair<kh_type::iterator, kh_type::iterator> p = m_ofskh.equal_range(h);

        pair<kh_type::iterator, kh_type::iterator> p = m_ofskh.equal_range(h);

    }

    }

    // Return vector of candidate offsets for udi (possibly several

    // Return vector of candidate offsets for udi (possibly several

    // because there may be hash collisions, and also multiple

    // because there may be hash collisions, and also multiple

    // instances).

    // instances).

        ofss.clear();

        ofss.clear();

        UdiH h(udi);

        UdiH h(udi);

        LOGDEB2("Circache::khFind: h "  << (h.asHexString()) << " udi ["  << (udi) << "]\n" );

        LOGDEB2("Circache::khFind: h "  << (h.asHexString()) << " udi ["  << (udi) << "]\n" );

            ofss.push_back(it->second);

            ofss.push_back(it->second);

        }

        }

        return true;

        return true;

    }

    }

    // Clear entry for udi/offs

    // Clear entry for udi/offs

        UdiH h(ref.first);

        UdiH h(ref.first);

        pair<kh_type::iterator, kh_type::iterator> p = m_ofskh.equal_range(h);

        pair<kh_type::iterator, kh_type::iterator> p = m_ofskh.equal_range(h);

        if (p.first != m_ofskh.end() && (p.first->first == h)) {

        if (p.first != m_ofskh.end() && (p.first->first == h)) {

            for (kh_type::iterator it = p.first; it != p.second;) {

            for (kh_type::iterator it = p.first; it != p.second;) {

                kh_type::iterator tmp = it++;

                kh_type::iterator tmp = it++;

            }

            }

        }

        }

        return true;

        return true;

    }

    }

    // Clear entries for vector of udi/offs

    // Clear entries for vector of udi/offs

                it != udis.end(); it++) {

                it != udis.end(); it++) {

            khClear(*it);

            khClear(*it);

        }

        }

        return true;

        return true;

    }

    }

            m_uniquentries = stringToBool(value);

            m_uniquentries = stringToBool(value);

        }

        }

        return true;

        return true;

    }

    }

                          bool eraseData = false) {

                          bool eraseData = false) {

        if (m_fd < 0) {

        if (m_fd < 0) {

            m_reason << "writeEntryHeader: not open ";

            m_reason << "writeEntryHeader: not open ";

            return false;

            return false;

        }

        }

            }

            }

        }

        }

        return true;

        return true;

    }

    }

        if (m_fd < 0) {

        if (m_fd < 0) {

            m_reason << "readEntryHeader: not open ";

            m_reason << "readEntryHeader: not open ";

            return CCScanHook::Error;

            return CCScanHook::Error;

        }

        }

        }

        }

        LOGDEB2("Circache:readEntryHeader: dcsz "  << (d.dicsize) << " dtsz "  << (d.datasize) << " pdsz "  << (d.padsize) << " flgs "  << (d.flags) << "\n" );

        LOGDEB2("Circache:readEntryHeader: dcsz "  << (d.dicsize) << " dtsz "  << (d.datasize) << " pdsz "  << (d.padsize) << " flgs "  << (d.flags) << "\n" );

        return CCScanHook::Continue;

        return CCScanHook::Continue;

    }

    }

                            bool fold = false) {

                            bool fold = false) {

        if (m_fd < 0) {

        if (m_fd < 0) {

            m_reason << "scan: not open ";

            m_reason << "scan: not open ";

            return CCScanHook::Error;

            return CCScanHook::Error;

        }

        }

        bool already_folded = false;

        bool already_folded = false;

        while (true) {

        while (true) {

            if (already_folded && startoffset == so0) {

            if (already_folded && startoffset == so0) {

                m_ofskhcplt = true;

                m_ofskhcplt = true;

            startoffset += CIRCACHE_HEADER_SIZE + d.dicsize +

            startoffset += CIRCACHE_HEADER_SIZE + d.dicsize +

                           d.datasize + d.padsize;

                           d.datasize + d.padsize;

        }

        }

    }

    }

        if (readEntryHeader(hoffs, d) != CCScanHook::Continue) {

        if (readEntryHeader(hoffs, d) != CCScanHook::Continue) {

            return false;

            return false;

        }

        }

        string dic;

        string dic;

        if (!readDicData(hoffs, d, dic, 0)) {

        if (!readDicData(hoffs, d, dic, 0)) {

            return false;

            return false;

        }

        }

        return true;

        return true;

    }

    }

                     string* data) {

                     string* data) {

        // This syscall could be avoided in some cases if we saved the offset

        // This syscall could be avoided in some cases if we saved the offset

        // at each seek. In most cases, we just read the header and we are

        // at each seek. In most cases, we just read the header and we are

        // at the right position

        // at the right position

        if (lseek(m_fd, offs, 0) != offs) {

        if (lseek(m_fd, offs, 0) != offs) {

            m_reason << "CirCache::get: lseek(" << offs << ") failed: " <<

            m_reason << "CirCache::get: lseek(" << offs << ") failed: " <<

// A scan callback which just records the last header offset and

// A scan callback which just records the last header offset and

// padsize seen. This is used with a scan(nofold) to find the last

// padsize seen. This is used with a scan(nofold) to find the last

// physical record in the file

// physical record in the file

class CCScanHookRecord : public  CCScanHook {

class CCScanHookRecord : public  CCScanHook {

public:

public:

    CCScanHookRecord()

    CCScanHookRecord()

        : headoffs(0), padsize(0) {

        : headoffs(0), padsize(0) {

    }

    }

                           const EntryHeaderData& d) {

                           const EntryHeaderData& d) {

        headoffs = offs;

        headoffs = offs;

        padsize = d.padsize;

        padsize = d.padsize;

        LOGDEB2("CCScanHookRecord::takeone: offs "  << (lltodecstr(headoffs)) << " padsize "  << (lltodecstr(padsize)) << "\n" );

        LOGDEB2("CCScanHookRecord::takeone: offs "  << (lltodecstr(headoffs)) << " padsize "  << (lltodecstr(padsize)) << "\n" );

        return Continue;

        return Continue;

string CirCache::getpath()

string CirCache::getpath()

{

{

    return m_d->datafn(m_dir);

    return m_d->datafn(m_dir);

}

}

{

{

    LOGDEB("CirCache::create: ["  << (m_dir) << "] maxsz "  << (lltodecstr((long long)maxsize)) << " flags 0x"  << (flags) << "\n" );

    LOGDEB("CirCache::create: ["  << (m_dir) << "] maxsz "  << (lltodecstr((long long)maxsize)) << " flags 0x"  << (flags) << "\n" );

    if (m_d == 0) {

    if (m_d == 0) {

        LOGERR("CirCache::create: null data\n" );

        LOGERR("CirCache::create: null data\n" );

        return false;

        return false;

    return m_d->readfirstblock();

    return m_d->readfirstblock();

}

}

class CCScanHookDump : public  CCScanHook {

class CCScanHookDump : public  CCScanHook {

public:

public:

                           const EntryHeaderData& d) {

                           const EntryHeaderData& d) {

        cout << "Scan: offs " << offs << " dicsize " << d.dicsize

        cout << "Scan: offs " << offs << " dicsize " << d.dicsize

             << " datasize " << d.datasize << " padsize " << d.padsize <<

             << " datasize " << d.datasize << " padsize " << d.padsize <<

             " flags " << d.flags <<

             " flags " << d.flags <<

             " udi [" << udi << "]" << endl;

             " udi [" << udi << "]" << endl;

{

{

    CCScanHookDump dumper;

    CCScanHookDump dumper;

    // Start at oldest header. This is eof while the file is growing, scan will

    // Start at oldest header. This is eof while the file is growing, scan will

    // fold to bot at once.

    // fold to bot at once.

    switch (m_d->scan(start, &dumper, true)) {

    switch (m_d->scan(start, &dumper, true)) {

    case CCScanHook::Stop:

    case CCScanHook::Stop:

        cout << "Scan returns Stop??" << endl;

        cout << "Scan returns Stop??" << endl;

        return false;

        return false;

class CCScanHookGetter : public  CCScanHook {

class CCScanHookGetter : public  CCScanHook {

public:

public:

    string  m_udi;

    string  m_udi;

    int     m_targinstance;

    int     m_targinstance;

    int     m_instance;

    int     m_instance;

    EntryHeaderData m_hd;

    EntryHeaderData m_hd;

    CCScanHookGetter(const string& udi, int ti)

    CCScanHookGetter(const string& udi, int ti)

        : m_udi(udi), m_targinstance(ti), m_instance(0), m_offs(0) {}

        : m_udi(udi), m_targinstance(ti), m_instance(0), m_offs(0) {}

                           const EntryHeaderData& d) {

                           const EntryHeaderData& d) {

        LOGDEB2("Circache:Scan: off "  << (long(offs)) << " udi ["  << (udi) << "] dcsz "  << ((UINT)d.dicsize) << " dtsz "  << ((UINT)d.datasize) << " pdsz "  << ((UINT)d.padsize) << "  flgs "  << (d.flags) << "\n" );

        LOGDEB2("Circache:Scan: off "  << (long(offs)) << " udi ["  << (udi) << "] dcsz "  << ((UINT)d.dicsize) << " dtsz "  << ((UINT)d.datasize) << " pdsz "  << ((UINT)d.padsize) << "  flgs "  << (d.flags) << "\n" );

        if (!m_udi.compare(udi)) {

        if (!m_udi.compare(udi)) {

            m_instance++;

            m_instance++;

            m_offs = offs;

            m_offs = offs;

    // If memory map is up to date, use it:

    // If memory map is up to date, use it:

    if (m_d->m_ofskhcplt) {

    if (m_d->m_ofskhcplt) {

        LOGDEB1("CirCache::get: using ofskh\n" );

        LOGDEB1("CirCache::get: using ofskh\n" );

        //m_d->khDump();

        //m_d->khDump();

        if (m_d->khFind(udi, ofss)) {

        if (m_d->khFind(udi, ofss)) {

            LOGDEB1("Circache::get: h found, colls "  << (ofss.size()) << "\n" );

            LOGDEB1("Circache::get: h found, colls "  << (ofss.size()) << "\n" );

            int finst = 1;

            int finst = 1;

            EntryHeaderData d_good;

            EntryHeaderData d_good;

                    it != ofss.end(); it++) {

                    it != ofss.end(); it++) {

                LOGDEB1("Circache::get: trying offs "  << ((ULONG)*it) << "\n" );

                LOGDEB1("Circache::get: trying offs "  << ((ULONG)*it) << "\n" );

                EntryHeaderData d;

                EntryHeaderData d;

                string fudi;

                string fudi;

                if (!m_d->readHUdi(*it, d, fudi)) {

                if (!m_d->readHUdi(*it, d, fudi)) {

            // Else try to scan anyway.

            // Else try to scan anyway.

        }

        }

    }

    }

    CCScanHookGetter getter(udi, instance);

    CCScanHookGetter getter(udi, instance);

    CCScanHook::status ret = m_d->scan(start, &getter, true);

    CCScanHook::status ret = m_d->scan(start, &getter, true);

    if (ret == CCScanHook::Eof) {

    if (ret == CCScanHook::Eof) {

        if (getter.m_instance == 0) {

        if (getter.m_instance == 0) {

            return false;

            return false;

            LOGERR("CirCache::erase : cache not updated after get\n" );

            LOGERR("CirCache::erase : cache not updated after get\n" );

            return false;

            return false;

        }

        }

    }

    }

    if (!m_d->khFind(udi, ofss)) {

    if (!m_d->khFind(udi, ofss)) {

        // Udi not in there,  erase ok

        // Udi not in there,  erase ok

        LOGDEB("CirCache::erase: khFind returns none\n" );

        LOGDEB("CirCache::erase: khFind returns none\n" );

        return true;

        return true;

    }

    }

        LOGDEB2("CirCache::erase: reading at "  << ((unsigned long)*it) << "\n" );

        LOGDEB2("CirCache::erase: reading at "  << ((unsigned long)*it) << "\n" );

        EntryHeaderData d;

        EntryHeaderData d;

        string fudi;

        string fudi;

        if (!m_d->readHUdi(*it, d, fudi)) {

        if (!m_d->readHUdi(*it, d, fudi)) {

            return false;

            return false;

// Used to scan the file ahead until we accumulated enough space for the new

// Used to scan the file ahead until we accumulated enough space for the new

// entry.

// entry.

class CCScanHookSpacer : public  CCScanHook {

class CCScanHookSpacer : public  CCScanHook {

public:

public:

        : sizewanted(sz), sizeseen(0) {

        : sizewanted(sz), sizeseen(0) {

        assert(sz > 0);

        assert(sz > 0);

    }

    }

                           const EntryHeaderData& d) {

                           const EntryHeaderData& d) {

        LOGDEB2("Circache:ScanSpacer:off "  << ((UINT)offs) << " dcsz "  << (d.dicsize) << " dtsz "  << (d.datasize) << " pdsz "  << (d.padsize) << " udi["  << (udi) << "]\n" );

        LOGDEB2("Circache:ScanSpacer:off "  << ((UINT)offs) << " dcsz "  << (d.dicsize) << " dtsz "  << (d.datasize) << " pdsz "  << (d.padsize) << " udi["  << (udi) << "]\n" );

        sizeseen += CIRCACHE_HEADER_SIZE + d.dicsize + d.datasize + d.padsize;

        sizeseen += CIRCACHE_HEADER_SIZE + d.dicsize + d.datasize + d.padsize;

        squashed_udis.push_back(make_pair(udi, offs));

        squashed_udis.push_back(make_pair(udi, offs));

        if (sizeseen >= sizewanted) {

        if (sizeseen >= sizewanted) {

        m_d->m_reason << "CirCache::put: fstat failed. errno " << errno;

        m_d->m_reason << "CirCache::put: fstat failed. errno " << errno;

        return false;

        return false;

    }

    }

    // Characteristics for the new entry.

    // Characteristics for the new entry.

    bool extending = false;

    bool extending = false;

    LOGDEB("CirCache::put: nsz "  << (nsize) << " oheadoffs "  << (m_d->m_oheadoffs) << "\n" );

    LOGDEB("CirCache::put: nsz "  << (nsize) << " oheadoffs "  << (m_d->m_oheadoffs) << "\n" );

    // Check if we can recover some pad space from the (physically) previous

    // Check if we can recover some pad space from the (physically) previous

    // entry.

    // entry.

                         0 : m_d->m_npadsize;

                         0 : m_d->m_npadsize;

    if (recovpadsize != 0) {

    if (recovpadsize != 0) {

        // Need to read the latest entry's header, to rewrite it with a

        // Need to read the latest entry's header, to rewrite it with a

        // zero pad size

        // zero pad size

        EntryHeaderData pd;

        EntryHeaderData pd;

        npadsize = 0;

        npadsize = 0;

        extending = true;

        extending = true;

    } else {

    } else {

        // Scan the file until we have enough space for the new entry,

        // Scan the file until we have enough space for the new entry,

        // and determine the pad size up to the 1st preserved entry

        // and determine the pad size up to the 1st preserved entry

        LOGDEB("CirCache::put: scanning for size "  << (scansize) << " from offs "  << ((UINT)m_d->m_oheadoffs) << "\n" );

        LOGDEB("CirCache::put: scanning for size "  << (scansize) << " from offs "  << ((UINT)m_d->m_oheadoffs) << "\n" );

        CCScanHookSpacer spacer(scansize);

        CCScanHookSpacer spacer(scansize);

        switch (m_d->scan(m_d->m_oheadoffs, &spacer)) {

        switch (m_d->scan(m_d->m_oheadoffs, &spacer)) {

        case CCScanHook::Stop:

        case CCScanHook::Stop:

            LOGDEB("CirCache::put: Scan ok, sizeseen "  << (spacer.sizeseen) << "\n" );

            LOGDEB("CirCache::put: Scan ok, sizeseen "  << (spacer.sizeseen) << "\n" );

        return false;

        return false;

    }

    }

    eof = false;

    eof = false;

        LOGERR("CirCache::rewind: seek to EOF failed\n" );

        LOGERR("CirCache::rewind: seek to EOF failed\n" );

        return false;

        return false;

    }

    }

    // Read oldest header. This is either at the position pointed to

    // Read oldest header. This is either at the position pointed to

    // by oheadoffs, or after the first block if the file is still

    // by oheadoffs, or after the first block if the file is still

    if (op_flags & OPT_c) {

    if (op_flags & OPT_c) {

        if (argc != 1) {

        if (argc != 1) {

            Usage();

            Usage();

        }

        }

        argc--;

        argc--;

        int flags = 0;

        int flags = 0;

        if (op_flags & OPT_u) {

        if (op_flags & OPT_u) {

            flags |= CirCache::CC_CRUNIQUE;

            flags |= CirCache::CC_CRUNIQUE;

        }

        }