|
a/Ming/ming/datastore.py |
|
b/Ming/ming/datastore.py |
|
... |
|
... |
| 9 |
|
9 |
|
| 10 |
from .utils import parse_uri
|
10 |
from .utils import parse_uri
|
| 11 |
|
11 |
|
| 12 |
log = logging.getLogger(__name__)
|
12 |
log = logging.getLogger(__name__)
|
| 13 |
|
13 |
|
| 14 |
class tl_property(object):
|
|
|
| 15 |
|
|
|
| 16 |
def __init__(self, tl_attr='_tl_value'):
|
|
|
| 17 |
self.tl_attr = tl_attr
|
|
|
| 18 |
self.name = self.func = None
|
|
|
| 19 |
|
|
|
| 20 |
def __call__(self, func):
|
|
|
| 21 |
self.func = func
|
|
|
| 22 |
self.name = func.__name__
|
|
|
| 23 |
return self
|
|
|
| 24 |
|
|
|
| 25 |
def __get__(self, obj, type=None):
|
|
|
| 26 |
tl = getattr(obj, self.tl_attr)
|
|
|
| 27 |
try:
|
|
|
| 28 |
value = getattr(tl, self.name)
|
|
|
| 29 |
except AttributeError:
|
|
|
| 30 |
value = self.func(obj)
|
|
|
| 31 |
setattr(tl, self.name, value)
|
|
|
| 32 |
return value
|
|
|
| 33 |
|
|
|
| 34 |
class DataStore(object):
|
14 |
class DataStore(object):
|
| 35 |
"""Manages a connections to Mongo, with seprate connections per thread."""
|
15 |
"""Manages a connections to Mongo, with seprate connections per thread."""
|
| 36 |
|
16 |
|
| 37 |
def __init__(self, master='mongo://localhost:27017/gutenberg', slave=None,
|
17 |
def __init__(self, master='mongo://localhost:27017/gutenberg', slave=None,
|
| 38 |
connect_retry=10):
|
18 |
connect_retry=3):
|
| 39 |
# self._tl_value = ThreadLocal()
|
19 |
# self._tl_value = ThreadLocal()
|
| 40 |
self._conn = None
|
20 |
self._conn = None
|
| 41 |
self._lock = Lock()
|
21 |
self._lock = Lock()
|
| 42 |
self._connect_retry = connect_retry
|
22 |
self._connect_retry = connect_retry
|
| 43 |
self.configure(master, slave)
|
23 |
self.configure(master, slave)
|
|
... |
|
... |
| 48 |
|
28 |
|
| 49 |
def configure(self, master='mongo://localhost:27017/gutenberg', slave=None):
|
29 |
def configure(self, master='mongo://localhost:27017/gutenberg', slave=None):
|
| 50 |
log.disabled = 0 # @%#$@ logging fileconfig disables our logger
|
30 |
log.disabled = 0 # @%#$@ logging fileconfig disables our logger
|
| 51 |
if isinstance(master, basestring):
|
31 |
if isinstance(master, basestring):
|
| 52 |
master = [ master ]
|
32 |
master = [ master ]
|
|
|
33 |
if isinstance(slave, basestring):
|
|
|
34 |
slave = [ slave ]
|
| 53 |
if slave is None: slave = []
|
35 |
if slave is None: slave = []
|
| 54 |
assert master, 'You MUST supply at least one master mongo connection'
|
36 |
assert master, 'You MUST supply at least one master mongo connection'
|
| 55 |
self.master_args = [ parse_uri(s) for s in master if s ]
|
37 |
self.master_args = [ parse_uri(s) for s in master if s ]
|
| 56 |
self.slave_args = [ parse_uri(s) for s in slave if s ]
|
38 |
self.slave_args = [ parse_uri(s) for s in slave if s ]
|
| 57 |
if len(self.master_args) > 2:
|
39 |
if len(self.master_args) > 2:
|
|
... |
|
... |
| 84 |
self._conn = Connection.paired(
|
66 |
self._conn = Connection.paired(
|
| 85 |
(str(self.master_args[0]['host']), int(self.master_args[0]['port'])),
|
67 |
(str(self.master_args[0]['host']), int(self.master_args[0]['port'])),
|
| 86 |
(str(self.master_args[1]['host']), int(self.master_args[1]['port'])),
|
68 |
(str(self.master_args[1]['host']), int(self.master_args[1]['port'])),
|
| 87 |
pool_size=16)
|
69 |
pool_size=16)
|
| 88 |
else:
|
70 |
else:
|
| 89 |
if self.master_args:
|
71 |
assert self.master_args, 'You must specify a master connection'
|
| 90 |
try:
|
72 |
try:
|
| 91 |
master = Connection(str(self.master_args[0]['host']), int(self.master_args[0]['port']),
|
73 |
master = Connection(str(self.master_args[0]['host']), int(self.master_args[0]['port']),
|
| 92 |
pool_size=8)
|
74 |
pool_size=8)
|
| 93 |
except:
|
75 |
except:
|
| 94 |
if self.slave_args:
|
76 |
if self.slave_args:
|
| 95 |
log.exception('Cannot connect to master: %s will use slave: %s' % (self.master_args, self.slave_args))
|
77 |
log.exception('Cannot connect to master: %s will use slave: %s' % (self.master_args, self.slave_args))
|
| 96 |
# and continue... to use the slave only
|
78 |
# and continue... to use the slave only
|
| 97 |
master = None
|
79 |
master = None
|
| 98 |
else:
|
80 |
else:
|
| 99 |
raise
|
81 |
raise
|
| 100 |
else:
|
|
|
| 101 |
log.info('No master connection specified, using slaves only: %s' % self.slave_args)
|
|
|
| 102 |
master = None
|
|
|
| 103 |
|
82 |
|
| 104 |
if self.slave_args:
|
83 |
if self.slave_args:
|
| 105 |
slave = [ Connection(str(a['host']), int(a['port']), pool_size=16, slave_okay=True)
|
84 |
slave = [ Connection(str(a['host']), int(a['port']), pool_size=16, slave_okay=True)
|
| 106 |
for a in self.slave_args ]
|
85 |
for a in self.slave_args ]
|
| 107 |
if master:
|
86 |
if master:
|