a/src/utils/circache.cpp b/src/utils/circache.cpp
...
...
28
#include <assert.h>
28
#include <assert.h>
29
#include <memory.h>
29
#include <memory.h>
30
#include <zlib.h>
30
#include <zlib.h>
31
31
32
#include "chrono.h"
32
#include "chrono.h"
33
#include MEMORY_INCLUDE
34
33
35
34
#ifndef _WIN32
36
#ifndef _WIN32
35
#include <sys/uio.h>
37
#include <sys/uio.h>
36
#define O_BINARY 0
38
#define O_BINARY 0
37
#else
39
#else
...
...
855
        return Continue;
857
        return Continue;
856
    }
858
    }
857
};
859
};
858
860
859
// instance == -1 means get latest. Otherwise specify from 1+
861
// instance == -1 means get latest. Otherwise specify from 1+
860
bool CirCache::get(const string& udi, string& dic, string& data, int instance)
862
bool CirCache::get(const string& udi, string& dic, string *data, int instance)
861
{
863
{
862
    Chrono chron;
864
    Chrono chron;
863
    if (m_d->m_fd < 0) {
865
    if (m_d->m_fd < 0) {
864
        m_d->m_reason << "CirCache::get: no data or not open";
866
        m_d->m_reason << "CirCache::get: no data or not open";
865
        return false;
867
        return false;
...
...
898
                    }
900
                    }
899
                }
901
                }
900
            }
902
            }
901
            // Did we read an appropriate entry ?
903
            // Did we read an appropriate entry ?
902
            if (o_good != 0 && (instance == -1 || instance == finst)) {
904
            if (o_good != 0 && (instance == -1 || instance == finst)) {
903
                bool ret = m_d->readDicData(o_good, d_good, dic, &data);
905
                bool ret = m_d->readDicData(o_good, d_good, dic, data);
904
                LOGDEB0(("Circache::get: hfound, %d mS\n",
906
                LOGDEB0(("Circache::get: hfound, %d mS\n",
905
                         chron.millis()));
907
                         chron.millis()));
906
                return ret;
908
                return ret;
907
            }
909
            }
908
            // Else try to scan anyway.
910
            // Else try to scan anyway.
...
...
918
            return false;
920
            return false;
919
        }
921
        }
920
    } else if (ret != CCScanHook::Stop) {
922
    } else if (ret != CCScanHook::Stop) {
921
        return false;
923
        return false;
922
    }
924
    }
923
    bool bret =
924
        m_d->readDicData(getter.m_offs, getter.m_hd, dic, &data);
925
    bool bret = m_d->readDicData(getter.m_offs, getter.m_hd, dic, data);
925
    LOGDEB0(("Circache::get: scanfound, %d mS\n", chron.millis()));
926
    LOGDEB0(("Circache::get: scanfound, %d mS\n", chron.millis()));
926
927
927
    return bret;
928
    return bret;
928
}
929
}
929
930
...
...
941
    LOGDEB0(("CirCache::erase: udi [%s]\n", udi.c_str()));
942
    LOGDEB0(("CirCache::erase: udi [%s]\n", udi.c_str()));
942
943
943
    // If the mem cache is not up to date, update it, we're too lazy
944
    // If the mem cache is not up to date, update it, we're too lazy
944
    // to do a scan
945
    // to do a scan
945
    if (!m_d->m_ofskhcplt) {
946
    if (!m_d->m_ofskhcplt) {
946
        string dic, data;
947
        string dic;
947
        get("nosuchudi probably exists", dic, data);
948
        get("nosuchudi probably exists", dic);
948
        if (!m_d->m_ofskhcplt) {
949
        if (!m_d->m_ofskhcplt) {
949
            LOGERR(("CirCache::erase : cache not updated after get\n"));
950
            LOGERR(("CirCache::erase : cache not updated after get\n"));
950
            return false;
951
            return false;
951
        }
952
        }
952
    }
953
    }
...
...
1263
        return false;
1264
        return false;
1264
    }
1265
    }
1265
    return true;
1266
    return true;
1266
}
1267
}
1267
1268
1268
bool CirCache::getCurrent(string& udi, string& dic, string& data)
1269
bool CirCache::getCurrent(string& udi, string& dic, string *data)
1269
{
1270
{
1270
    if (m_d == 0) {
1271
    if (m_d == 0) {
1271
        LOGERR(("CirCache::getCurrent: null data\n"));
1272
        LOGERR(("CirCache::getCurrent: null data\n"));
1272
        return false;
1273
        return false;
1273
    }
1274
    }
1274
    if (!m_d->readDicData(m_d->m_itoffs, m_d->m_ithd, dic, &data)) {
1275
    if (!m_d->readDicData(m_d->m_itoffs, m_d->m_ithd, dic, data)) {
1275
        return false;
1276
        return false;
1276
    }
1277
    }
1277
1278
1278
    ConfSimple conf(dic, 1);
1279
    ConfSimple conf(dic, 1);
1279
    conf.get("udi", udi, cstr_null);
1280
    conf.get("udi", udi, cstr_null);
...
...
1365
    }
1366
    }
1366
    LOGDEB0(("inflateToDynBuf: ok, output size %d\n", d_stream.total_out));
1367
    LOGDEB0(("inflateToDynBuf: ok, output size %d\n", d_stream.total_out));
1367
    return true;
1368
    return true;
1368
}
1369
}
1369
1370
1371
// Copy all entries from occ to ncc. Both are already open.
1372
static bool copyall(STD_SHARED_PTR<CirCache> occ,
1373
                    STD_SHARED_PTR<CirCache> ncc, int& nentries,
1374
    ostringstream& msg)
1375
{
1376
    bool eof = false;
1377
    if (!occ->rewind(eof)) {
1378
        if (!eof) {
1379
            msg << "Initial rewind failed" << endl;
1380
            return false;
1381
        }
1382
    }
1383
    nentries = 0;
1384
    while (!eof) {
1385
        string udi, sdic, data;
1386
        if (!occ->getCurrent(udi, sdic, &data)) {
1387
            msg << "getCurrent failed: " << occ->getReason() << endl;
1388
            return false;
1389
        }
1390
        // Shouldn't getcurrent deal with this ?
1391
        if (sdic.size() == 0) {
1392
            //cerr << "Skip empty entry" << endl;
1393
            occ->next(eof);
1394
            continue;
1395
        }
1396
        ConfSimple dic(sdic);
1397
        if (!dic.ok()) {
1398
            msg << "Could not parse entry attributes dic" << endl;
1399
            return false;
1400
        }
1401
        //cerr << "UDI: " << udi << endl;
1402
        if (!ncc->put(udi, &dic, data)) {
1403
            msg << "put failed: " << ncc->getReason() << " sdic [" << sdic <<
1404
                 "]" << endl;
1405
            return false;
1406
        }
1407
        nentries++;
1408
        occ->next(eof);
1409
    }
1410
    return true;
1411
}
1412
1413
// Append all entries from sdir to ddir
1414
int CirCache::append(const string ddir, const string& sdir, string *reason)
1415
{
1416
    ostringstream msg;
1417
    // Open source file
1418
    STD_SHARED_PTR<CirCache> occ(new CirCache(sdir));
1419
    if (!occ->open(CirCache::CC_OPREAD)) {
1420
        if (reason) {
1421
            msg << "Open failed in " << sdir << " : " <<
1422
                occ->getReason() << endl;
1423
            *reason = msg.str();
1424
        }
1425
        return -1;
1426
    }
1427
    // Open dest file
1428
    STD_SHARED_PTR<CirCache> ncc(new CirCache(ddir));
1429
    if (!ncc->open(CirCache::CC_OPWRITE)) {
1430
        if (reason) {
1431
            msg << "Open failed in " << ddir << " : " <<
1432
                ncc->getReason() << endl;
1433
            *reason = msg.str();
1434
        }
1435
        return -1;
1436
    }
1437
1438
    int nentries;
1439
    if (!copyall(occ, ncc, nentries, msg)) {
1440
        if (reason) {
1441
            *reason = msg.str();
1442
        }
1443
        return -1;
1444
    }
1445
1446
    return nentries;
1447
}
1448
1449
1370
#else // TEST ->
1450
#else // TEST ->
1371
#include "autoconfig.h"
1451
#include "autoconfig.h"
1372
1452
1373
#include <stdio.h>
1453
#include <stdio.h>
1374
#include <stdlib.h>
1454
#include <stdlib.h>
...
...
1378
#include <sys/stat.h>
1458
#include <sys/stat.h>
1379
#include <unistd.h>
1459
#include <unistd.h>
1380
1460
1381
#include <string>
1461
#include <string>
1382
#include <iostream>
1462
#include <iostream>
1463
#include MEMORY_INCLUDE
1383
1464
1384
#include "circache.h"
1465
#include "circache.h"
1385
#include "fileudi.h"
1466
#include "fileudi.h"
1386
#include "conftree.h"
1467
#include "conftree.h"
1387
#include "readfile.h"
1468
#include "readfile.h"
1388
#include "debuglog.h"
1469
#include "debuglog.h"
1470
#include "smallut.h"
1389
1471
1390
using namespace std;
1472
using namespace std;
1391
1392
// Copy all entries from occ to ncc. Both are already open.
1393
bool copyall(STD_SHARED_PTR<CirCache> occ,
1394
             STD_SHARED_PTR<CirCache> ncc, int& nentries)
1395
{
1396
    bool eof = false;
1397
    if (!occ->rewind(eof)) {
1398
        if (!eof) {
1399
            cerr << "Initial rewind failed" << endl;
1400
            return false;
1401
        }
1402
    }
1403
    nentries = 0;
1404
    while (!eof) {
1405
        string udi, sdic, data;
1406
        if (!occ->getCurrent(udi, sdic, data)) {
1407
            cerr << "getCurrent failed: " << occ->getReason() << endl;
1408
            return false;
1409
        }
1410
        // Shouldn't getcurrent deal with this ?
1411
        if (sdic.size() == 0) {
1412
            //cerr << "Skip empty entry" << endl;
1413
            occ->next(eof);
1414
            continue;
1415
        }
1416
        ConfSimple dic(sdic);
1417
        if (!dic.ok()) {
1418
            cerr << "Could not parse entry attributes dic" << endl;
1419
            return false;
1420
        }
1421
        //cerr << "UDI: " << udi << endl;
1422
        if (!ncc->put(udi, &dic, data)) {
1423
            cerr << "put failed: " << ncc->getReason() << " sdic [" << sdic <<
1424
                 "]" << endl;
1425
            return false;
1426
        }
1427
        nentries++;
1428
        occ->next(eof);
1429
    }
1430
    return true;
1431
}
1432
1433
#warning resizecc is not useful, create(newsize) works !
1434
1435
// Wrote the following at a point where I thought that simple resizing
1436
// did not work. Upon further study (or updates?), it appears it does
1437
// ! So the following code is not useful actually
1438
1439
// Resize circache. This can't be done easily if the write point is
1440
// inside the file (we already reached the old max size). We create a
1441
// new file with the new size and copy the old entries into it. The
1442
// old file is then renamed into a backup and the new file renamed in
1443
// place.
1444
bool resizecc(const string& dir, int newmbs)
1445
{
1446
    // Create object for existing file and get the file name
1447
    STD_SHARED_PTR<CirCache> occ(new CirCache(dir));
1448
    string ofn = occ->getpath();
1449
1450
    // Check for previous backup
1451
    string backupfn = ofn + ".orig";
1452
    if (access(backupfn.c_str(), 0) >= 0) {
1453
        cerr << "Backup file " << backupfn <<
1454
             " exists, please move it out of the way" << endl;
1455
        return false;
1456
    }
1457
1458
    if (!occ->open(CirCache::CC_OPREAD)) {
1459
        cerr << "Open failed in " << dir << " : " << occ->getReason() << endl;
1460
        return false;
1461
    }
1462
1463
    // Create the new empty file in a temporary directory
1464
    string tmpdir = path_cat(dir, "tmp");
1465
    if (access(tmpdir.c_str(), 0) < 0) {
1466
        if (mkdir(tmpdir.c_str(), 0700) < 0) {
1467
            cerr << "Cant create temporary directory " << tmpdir << " ";
1468
            perror("mkdir");
1469
            return false;
1470
        }
1471
    }
1472
    STD_SHARED_PTR<CirCache> ncc(new CirCache(tmpdir));
1473
    string nfn = ncc->getpath();
1474
    if (!ncc->create(off_t(newmbs) * 1000 * 1024,
1475
                     CirCache::CC_CRUNIQUE | CirCache::CC_CRTRUNCATE)) {
1476
        cerr << "Cant create new file in " << tmpdir << " : " <<
1477
             ncc->getReason() << endl;
1478
        return false;
1479
    }
1480
1481
    int nentries;
1482
    if (!copyall(occ, ncc, nentries)) {
1483
        cerr << "Copy failed\n";
1484
        return false;
1485
    }
1486
1487
    // Done with our objects here, there is no close() method, so
1488
    // delete them
1489
    occ.reset();
1490
    ncc.reset();
1491
1492
    // Create backup by renaming the old file
1493
    if (rename(ofn.c_str(), backupfn.c_str()) < 0) {
1494
        cerr << "Could not create backup " << backupfn << " : ";
1495
        perror("rename");
1496
        return false;
1497
    }
1498
    cout << "Created backup file " << backupfn << endl;
1499
1500
    // Move the new file in place.
1501
    if (rename(nfn.c_str(), ofn.c_str()) < 0) {
1502
        cerr << "Could not rename new file from " << nfn << " to " <<
1503
             ofn << " : ";
1504
        perror("rename");
1505
        return false;
1506
    }
1507
    cout << "Resize done, copied " << nentries << " entries " << endl;
1508
    return true;
1509
}
1510
1511
// Append all entries from sdir to ddir
1512
bool appendcc(const string ddir, const string& sdir)
1513
{
1514
    // Open source file
1515
    STD_SHARED_PTR<CirCache> occ(new CirCache(sdir));
1516
    if (!occ->open(CirCache::CC_OPREAD)) {
1517
        cerr << "Open failed in " << sdir << " : " << occ->getReason() << endl;
1518
        return false;
1519
    }
1520
    // Open dest file
1521
    STD_SHARED_PTR<CirCache> ncc(new CirCache(ddir));
1522
    if (!ncc->open(CirCache::CC_OPWRITE)) {
1523
        cerr << "Open failed in " << ddir << " : " << ncc->getReason() << endl;
1524
        return false;
1525
    }
1526
1527
    int nentries;
1528
    if (!copyall(occ, ncc, nentries)) {
1529
        cerr << "Copy failed\n";
1530
        return false;
1531
    }
1532
1533
    occ.reset();
1534
    ncc.reset();
1535
1536
    cout << "Copy done, copied " << nentries << " entries " << endl;
1537
    return true;
1538
}
1539
1473
1540
static char *thisprog;
1474
static char *thisprog;
1541
1475
1542
static char usage [] =
1476
static char usage [] =
1543
    " -c [-u] <dirname> <sizekbs>: create\n"
1477
    " -c [-u] <dirname> <sizekbs>: create\n"
1544
    " -p <dirname> <apath> [apath ...] : put files\n"
1478
    " -p <dirname> <apath> [apath ...] : put files\n"
1545
    " -d <dirname> : dump\n"
1479
    " -d <dirname> : dump\n"
1546
    " -g [-i instance] [-D] <dirname> <udi>: get\n"
1480
    " -g [-i instance] [-D] <dirname> <udi>: get\n"
1547
    "   -D: also dump data\n"
1481
    "   -D: also dump data\n"
1548
    " -e <dirname> <udi> : erase\n"
1482
    " -e <dirname> <udi> : erase\n"
1549
    " -s <dirname> <newmbs> : resize\n"
1550
    " -a <targetdir> <dir> [<dir> ...]: append old content to target\n"
1483
    " -a <targetdir> <dir> [<dir> ...]: append old content to target\n"
1551
    "  The target should be first resized to hold all the data, else only\n"
1484
    "  The target should be first resized to hold all the data, else only\n"
1552
    "  as many entries as capacity permit will be retained\n"
1485
    "  as many entries as capacity permit will be retained\n"
1553
    ;
1486
    ;
1554
1487
...
...
1567
#define OPT_d     0x20
1500
#define OPT_d     0x20
1568
#define OPT_i     0x40
1501
#define OPT_i     0x40
1569
#define OPT_D     0x80
1502
#define OPT_D     0x80
1570
#define OPT_u     0x100
1503
#define OPT_u     0x100
1571
#define OPT_e     0x200
1504
#define OPT_e     0x200
1572
#define OPT_s     0x400
1573
#define OPT_a     0x800
1505
#define OPT_a     0x800
1574
1506
1575
int main(int argc, char **argv)
1507
int main(int argc, char **argv)
1576
{
1508
{
1577
    int instance = -1;
1509
    int instance = -1;
...
...
1618
                argc--;
1550
                argc--;
1619
                goto b1;
1551
                goto b1;
1620
            case 'p':
1552
            case 'p':
1621
                op_flags |= OPT_p;
1553
                op_flags |= OPT_p;
1622
                break;
1554
                break;
1623
            case 's':
1624
                op_flags |= OPT_s;
1625
                break;
1626
            case 'u':
1555
            case 'u':
1627
                op_flags |= OPT_u;
1556
                op_flags |= OPT_u;
1628
                break;
1557
                break;
1629
            default:
1558
            default:
1630
                Usage();
1559
                Usage();
...
...
1658
        }
1587
        }
1659
        if (!cc.create(sizekb * 1024, flags)) {
1588
        if (!cc.create(sizekb * 1024, flags)) {
1660
            cerr << "Create failed:" << cc.getReason() << endl;
1589
            cerr << "Create failed:" << cc.getReason() << endl;
1661
            exit(1);
1590
            exit(1);
1662
        }
1591
        }
1663
    } else if (op_flags & OPT_s) {
1664
        if (argc != 1) {
1665
            Usage();
1666
        }
1667
        int newmbs = atoi(*argv++);
1668
        argc--;
1669
        if (!resizecc(dir, newmbs)) {
1670
            exit(1);
1671
        }
1672
    } else if (op_flags & OPT_a) {
1592
    } else if (op_flags & OPT_a) {
1673
        if (argc < 1) {
1593
        if (argc < 1) {
1674
            Usage();
1594
            Usage();
1675
        }
1595
        }
1676
        while (argc) {
1596
        while (argc) {
1597
            string reason;
1677
            if (!appendcc(dir, *argv++)) {
1598
            if (CirCache::append(dir, *argv++, &reason) < 0) {
1599
                cerr << reason << endl;
1678
                return 1;
1600
                return 1;
1679
            }
1601
            }
1680
            argc--;
1602
            argc--;
1681
        }
1603
        }
1682
    } else if (op_flags & OPT_p) {
1604
    } else if (op_flags & OPT_p) {
...
...
1696
                cerr << "File_to_string: " << reason << endl;
1618
                cerr << "File_to_string: " << reason << endl;
1697
                exit(1);
1619
                exit(1);
1698
            }
1620
            }
1699
            string udi;
1621
            string udi;
1700
            make_udi(fn, "", udi);
1622
            make_udi(fn, "", udi);
1701
            sprintf(dic, "#whatever...\nmimetype = text/plain\nudi=%s\n",
1623
            string cmd("xdg-mime query filetype ");
1702
                    udi.c_str());
1624
            // Should do more quoting here...
1625
            cmd += "'" + fn + "'";
1626
            FILE *fp = popen(cmd.c_str(), "r");
1627
            char* buf=0;
1628
            size_t sz = 0;
1629
            ::getline(&buf, &sz, fp);
1630
            pclose(fp);
1631
            string mimetype(buf);
1632
            free(buf);
1633
            trimstring(mimetype, "\n\r");
1634
            cout << "Got [" << mimetype << "]\n";
1635
1703
            string sdic;
1636
            string s;
1704
            sdic.assign(dic, strlen(dic));
1705
            ConfSimple conf(sdic);
1637
            ConfSimple conf(s);
1638
            conf.set("udi", udi);
1639
            conf.set("mimetype", mimetype);
1640
            //ostringstream str; conf.write(str); cout << str.str() << endl;
1706
1641
1707
            if (!cc.put(udi, &conf, data, 0)) {
1642
            if (!cc.put(udi, &conf, data, 0)) {
1708
                cerr << "Put failed: " << cc.getReason() << endl;
1643
                cerr << "Put failed: " << cc.getReason() << endl;
1709
                cerr << "conf: [";
1644
                cerr << "conf: [";
1710
                conf.write(cerr);
1645
                conf.write(cerr);
...
...
1720
        }
1655
        }
1721
        while (argc) {
1656
        while (argc) {
1722
            string udi = *argv++;
1657
            string udi = *argv++;
1723
            argc--;
1658
            argc--;
1724
            string dic, data;
1659
            string dic, data;
1725
            if (!cc.get(udi, dic, data, instance)) {
1660
            if (!cc.get(udi, dic, &data, instance)) {
1726
                cerr << "Get failed: " << cc.getReason() << endl;
1661
                cerr << "Get failed: " << cc.getReason() << endl;
1727
                exit(1);
1662
                exit(1);
1728
            }
1663
            }
1729
            cout << "Dict: [" << dic << "]" << endl;
1664
            cout << "Dict: [" << dic << "]" << endl;
1730
            if (op_flags & OPT_D) {
1665
            if (op_flags & OPT_D) {