|
a/src/mediaserver/cdplugins/uprcl/uprcl-app.py |
|
b/src/mediaserver/cdplugins/uprcl/uprcl-app.py |
|
... |
|
... |
17 |
|
17 |
|
18 |
import sys
|
18 |
import sys
|
19 |
import os
|
19 |
import os
|
20 |
import json
|
20 |
import json
|
21 |
import re
|
21 |
import re
|
22 |
import conftree
|
|
|
23 |
import cmdtalkplugin
|
22 |
import cmdtalkplugin
|
24 |
import threading
|
|
|
25 |
import subprocess
|
|
|
26 |
import time
|
|
|
27 |
from timeit import default_timer as timer
|
|
|
28 |
|
23 |
|
29 |
import uprclfolders
|
24 |
import uprclfolders
|
30 |
import uprcltags
|
25 |
import uprcltags
|
31 |
import uprcluntagged
|
26 |
import uprcluntagged
|
32 |
import uprclsearch
|
27 |
import uprclsearch
|
33 |
import uprclhttp
|
28 |
import uprclhttp
|
34 |
import uprclindex
|
29 |
import uprclindex
|
35 |
|
30 |
|
36 |
from uprclutils import uplog,g_myprefix,rcldirentry,stringToStrings,findmyip
|
31 |
from uprclutils import uplog, g_myprefix, rcldirentry
|
|
|
32 |
from uprclinit import g_pathprefix, g_httphp, g_dblock, g_rclconfdir
|
|
|
33 |
import uprclinit
|
37 |
|
34 |
|
38 |
# The recoll documents
|
35 |
#####
|
39 |
g_rcldocs = []
|
36 |
# Initialize communication with our parent process: pipe and method
|
40 |
|
37 |
# call dispatch
|
41 |
# Func name to method mapper
|
38 |
# Func name to method mapper
|
42 |
dispatcher = cmdtalkplugin.Dispatch()
|
39 |
dispatcher = cmdtalkplugin.Dispatch()
|
43 |
# Pipe message handler
|
40 |
# Pipe message handler
|
44 |
msgproc = cmdtalkplugin.Processor(dispatcher)
|
41 |
msgproc = cmdtalkplugin.Processor(dispatcher)
|
45 |
|
42 |
|
46 |
def _uprcl_init_worker():
|
|
|
47 |
global httphp, pathprefix, rclconfdir, g_rcldocs
|
|
|
48 |
|
|
|
49 |
# pathprefix would typically be something like "/uprcl". It's used
|
|
|
50 |
# for dispatching URLs to the right plugin for processing. We
|
|
|
51 |
# strip it whenever we need a real file path
|
|
|
52 |
if "UPMPD_PATHPREFIX" not in os.environ:
|
|
|
53 |
raise Exception("No UPMPD_PATHPREFIX in environment")
|
|
|
54 |
pathprefix = os.environ["UPMPD_PATHPREFIX"]
|
|
|
55 |
if "UPMPD_CONFIG" not in os.environ:
|
|
|
56 |
raise Exception("No UPMPD_CONFIG in environment")
|
|
|
57 |
upconfig = conftree.ConfSimple(os.environ["UPMPD_CONFIG"])
|
|
|
58 |
|
|
|
59 |
httphp = upconfig.get("uprclhostport")
|
|
|
60 |
if httphp is None:
|
|
|
61 |
ip = findmyip()
|
|
|
62 |
httphp = ip + ":" + "9090"
|
|
|
63 |
uplog("uprclhostport not in config, using %s" % httphp)
|
|
|
64 |
|
|
|
65 |
rclconfdir = upconfig.get("uprclconfdir")
|
|
|
66 |
if rclconfdir is None:
|
|
|
67 |
uplog("uprclconfdir not in config, using /var/cache/upmpdcli/uprcl")
|
|
|
68 |
rclconfdir = "/var/cache/upmpdcli/uprcl"
|
|
|
69 |
rcltopdirs = upconfig.get("uprclmediadirs")
|
|
|
70 |
if rcltopdirs is None:
|
|
|
71 |
raise Exception("uprclmediadirs not in config")
|
|
|
72 |
|
|
|
73 |
pthstr = upconfig.get("uprclpaths")
|
|
|
74 |
if pthstr is None:
|
|
|
75 |
uplog("uprclpaths not in config")
|
|
|
76 |
pthlist = stringToStrings(rcltopdirs)
|
|
|
77 |
pthstr = ""
|
|
|
78 |
for p in pthlist:
|
|
|
79 |
pthstr += p + ":" + p + ","
|
|
|
80 |
pthstr = pthstr.rstrip(",")
|
|
|
81 |
uplog("Path translation: pthstr: %s" % pthstr)
|
|
|
82 |
lpth = pthstr.split(',')
|
|
|
83 |
pathmap = {}
|
|
|
84 |
for ptt in lpth:
|
|
|
85 |
l = ptt.split(':')
|
|
|
86 |
pathmap[l[0]] = l[1]
|
|
|
87 |
|
|
|
88 |
|
|
|
89 |
# Update or create index.
|
|
|
90 |
uplog("Creating updating index in %s for %s"%(rclconfdir, rcltopdirs))
|
|
|
91 |
start = timer()
|
|
|
92 |
uprclindex.runindexer(rclconfdir, rcltopdirs)
|
|
|
93 |
# Wait for indexer
|
|
|
94 |
while not uprclindex.indexerdone():
|
|
|
95 |
time.sleep(.5)
|
|
|
96 |
fin = timer()
|
|
|
97 |
uplog("Indexing took %.2f Seconds" % (fin - start))
|
|
|
98 |
|
|
|
99 |
g_rcldocs = uprclfolders.inittree(rclconfdir, httphp, pathprefix)
|
|
|
100 |
uprcltags.recolltosql(g_rcldocs)
|
|
|
101 |
uprcluntagged.recoll2untagged(g_rcldocs)
|
|
|
102 |
|
|
|
103 |
host,port = httphp.split(':')
|
|
|
104 |
if True:
|
|
|
105 |
# Running the server as a thread. We get into trouble because
|
|
|
106 |
# something somewhere writes to stdout a bunch of --------.
|
|
|
107 |
# Could not find where they come from, happens after a sigpipe
|
|
|
108 |
# when a client closes a stream. The --- seem to happen before
|
|
|
109 |
# and after the exception strack trace, e.g:
|
|
|
110 |
# ----------------------------------------
|
|
|
111 |
# Exception happened during processing of request from ('192...
|
|
|
112 |
# Traceback...
|
|
|
113 |
# [...]
|
|
|
114 |
# error: [Errno 32] Broken pipe
|
|
|
115 |
# ----------------------------------------
|
|
|
116 |
#
|
|
|
117 |
# **Finally**: found it: the handle_error SocketServer method
|
|
|
118 |
# was writing to stdout. Overriding it should have fixed the
|
|
|
119 |
# issue. Otoh the separate process approach works, so we kept
|
|
|
120 |
# it for now
|
|
|
121 |
httpthread = threading.Thread(target=uprclhttp.runHttp,
|
|
|
122 |
kwargs = {'host':host ,
|
|
|
123 |
'port':int(port),
|
|
|
124 |
'pthstr':pthstr,
|
|
|
125 |
'pathprefix':pathprefix})
|
|
|
126 |
httpthread.daemon = True
|
|
|
127 |
httpthread.start()
|
|
|
128 |
else:
|
|
|
129 |
# Running the HTTP server as a separate process
|
|
|
130 |
cmdpath = os.path.join(os.path.dirname(sys.argv[0]), 'uprclhttp.py')
|
|
|
131 |
cmd = subprocess.Popen((cmdpath, host, port, pthstr, pathprefix),
|
|
|
132 |
stdin = open('/dev/null'),
|
|
|
133 |
stdout = sys.stderr,
|
|
|
134 |
stderr = sys.stderr,
|
|
|
135 |
close_fds = True)
|
|
|
136 |
global _initrunning
|
|
|
137 |
_initrunning = False
|
|
|
138 |
msgproc.log("Init done")
|
|
|
139 |
|
|
|
140 |
_initrunning = True
|
|
|
141 |
def _uprcl_init():
|
|
|
142 |
global _initrunning
|
|
|
143 |
_initrunning = True
|
|
|
144 |
initthread = threading.Thread(target=_uprcl_init_worker)
|
|
|
145 |
initthread.daemon = True
|
|
|
146 |
initthread.start()
|
|
|
147 |
|
|
|
148 |
def _ready():
|
|
|
149 |
global _initrunning
|
|
|
150 |
if _initrunning:
|
|
|
151 |
return False
|
|
|
152 |
else:
|
|
|
153 |
return True
|
|
|
154 |
|
|
|
155 |
@dispatcher.record('trackuri')
|
43 |
@dispatcher.record('trackuri')
|
156 |
def trackuri(a):
|
44 |
def trackuri(a):
|
157 |
# This is used for plugins which generate temporary local urls
|
45 |
# This is used for plugins which generate temporary local urls
|
158 |
# pointing to the microhttpd instance. The microhttpd
|
46 |
# pointing to the microhttpd instance. The microhttpd
|
159 |
# answer_to_connection() routine in plgwithslave calls 'trackuri'
|
47 |
# answer_to_connection() routine in plgwithslave calls 'trackuri'
|
|
... |
|
... |
223 |
entries = []
|
111 |
entries = []
|
224 |
nocache = "0"
|
112 |
nocache = "0"
|
225 |
if bflg == 'meta':
|
113 |
if bflg == 'meta':
|
226 |
raise Exception("uprcl-app: browse: can't browse meta for now")
|
114 |
raise Exception("uprcl-app: browse: can't browse meta for now")
|
227 |
else:
|
115 |
else:
|
228 |
if not _ready():
|
116 |
if not uprclinit.ready():
|
229 |
entries = [rcldirentry(objid + 'notready', objid,
|
117 |
entries = [rcldirentry(objid + 'notready', objid,
|
230 |
'Initializing...'),]
|
118 |
'Initializing...'),]
|
231 |
nocache = "1"
|
119 |
nocache = "1"
|
232 |
elif not idpath:
|
120 |
elif not idpath:
|
233 |
entries = _rootentries()
|
121 |
entries = _rootentries()
|
234 |
else:
|
122 |
else:
|
235 |
entries = _browsedispatch(objid, bflg, httphp, pathprefix)
|
123 |
entries = _browsedispatch(objid, bflg, g_httphp, g_pathprefix)
|
|
|
124 |
g_dblock.release_read()
|
236 |
|
125 |
|
237 |
#msgproc.log("%s" % entries)
|
126 |
#msgproc.log("%s" % entries)
|
238 |
encoded = json.dumps(entries)
|
127 |
encoded = json.dumps(entries)
|
239 |
return {"entries" : encoded, "nocache":nocache}
|
128 |
return {"entries" : encoded, "nocache":nocache}
|
240 |
|
129 |
|
|
... |
|
... |
247 |
raise Exception("bad objid [%s]" % objid)
|
136 |
raise Exception("bad objid [%s]" % objid)
|
248 |
|
137 |
|
249 |
upnps = a['origsearch']
|
138 |
upnps = a['origsearch']
|
250 |
nocache = "0"
|
139 |
nocache = "0"
|
251 |
|
140 |
|
252 |
if not _ready():
|
141 |
if not uprclinit.ready():
|
253 |
entries = [rcldirentry(objid + 'notready', objid, 'Initializing...'),]
|
142 |
entries = [rcldirentry(objid + 'notready', objid, 'Initializing...'),]
|
254 |
nocache = "1"
|
143 |
nocache = "1"
|
255 |
else:
|
144 |
else:
|
256 |
entries = uprclsearch.search(rclconfdir, objid, upnps, g_myprefix,
|
145 |
entries = uprclsearch.search(g_rclconfdir, objid, upnps, g_myprefix,
|
257 |
httphp, pathprefix)
|
146 |
g_httphp, g_pathprefix)
|
|
|
147 |
g_dblock.release_read()
|
258 |
|
148 |
|
259 |
encoded = json.dumps(entries)
|
149 |
encoded = json.dumps(entries)
|
260 |
return {"entries" : encoded, "nocache":nocache}
|
150 |
return {"entries" : encoded, "nocache":nocache}
|
261 |
|
151 |
|
262 |
|
152 |
|
263 |
_uprcl_init()
|
153 |
uprclinit.uprcl_init()
|
264 |
|
|
|
265 |
msgproc.log("Uprcl running")
|
154 |
msgproc.log("Uprcl running")
|
266 |
msgproc.mainloop()
|
155 |
msgproc.mainloop()
|