|
a/Allura/allura/scripts/refreshrepo.py |
|
b/Allura/allura/scripts/refreshrepo.py |
|
... |
|
... |
| 66 |
continue
|
66 |
continue
|
| 67 |
|
67 |
|
| 68 |
if options.clean:
|
68 |
if options.clean:
|
| 69 |
ci_ids = list(c.app.repo.all_commit_ids())
|
69 |
ci_ids = list(c.app.repo.all_commit_ids())
|
| 70 |
log.info("Deleting mongo data for %i commits...", len(ci_ids))
|
70 |
log.info("Deleting mongo data for %i commits...", len(ci_ids))
|
|
|
71 |
# like the tree_ids themselves below, we need to process these in
|
|
|
72 |
# chunks to avoid hitting the BSON max size limit
|
| 71 |
tree_ids = [
|
73 |
tree_ids = []
|
|
|
74 |
for ci_ids_chunk in chunked_list(ci_ids, 3000):
|
|
|
75 |
tree_ids.extend([
|
| 72 |
tree_id for doc in
|
76 |
tree_id for doc in
|
| 73 |
M.repo.TreesDoc.m.find({"_id": {"$in": ci_ids}},
|
77 |
M.repo.TreesDoc.m.find({"_id": {"$in": ci_ids_chunk}},
|
| 74 |
{"tree_ids": 1})
|
78 |
{"tree_ids": 1})
|
| 75 |
for tree_id in doc.get("tree_ids", [])]
|
79 |
for tree_id in doc.get("tree_ids", [])])
|
| 76 |
|
80 |
|
| 77 |
i = M.repo.CommitDoc.m.find({"_id": {"$in": ci_ids}}).count()
|
81 |
i = M.repo.CommitDoc.m.find({"_id": {"$in": ci_ids_chunk}}).count()
|
|
|
82 |
if i:
|
| 78 |
log.info("Deleting %i CommitDoc docs...", i)
|
83 |
log.info("Deleting %i CommitDoc docs...", i)
|
| 79 |
M.repo.CommitDoc.m.remove({"_id": {"$in": ci_ids}})
|
84 |
M.repo.CommitDoc.m.remove({"_id": {"$in": ci_ids_chunk}})
|
| 80 |
|
85 |
|
| 81 |
# delete these in chunks, otherwise the query doc can
|
86 |
# delete these in chunks, otherwise the query doc can
|
| 82 |
# exceed the max BSON size limit (16MB at the moment)
|
87 |
# exceed the max BSON size limit (16MB at the moment)
|
| 83 |
for tree_ids_chunk in chunked_list(tree_ids, 300000):
|
88 |
for tree_ids_chunk in chunked_list(tree_ids, 300000):
|
| 84 |
i = M.repo.TreeDoc.m.find({"_id": {"$in": tree_ids_chunk}}).count()
|
89 |
i = M.repo.TreeDoc.m.find({"_id": {"$in": tree_ids_chunk}}).count()
|
|
|
90 |
if i:
|
| 85 |
log.info("Deleting %i TreeDoc docs...", i)
|
91 |
log.info("Deleting %i TreeDoc docs...", i)
|
| 86 |
M.repo.TreeDoc.m.remove({"_id": {"$in": tree_ids_chunk}})
|
92 |
M.repo.TreeDoc.m.remove({"_id": {"$in": tree_ids_chunk}})
|
| 87 |
del tree_ids
|
93 |
del tree_ids
|
| 88 |
|
94 |
|
| 89 |
# delete these after TreeDoc and LastCommitDoc so that if
|
95 |
# delete these after TreeDoc and LastCommitDoc so that if
|
| 90 |
# we crash, we don't lose the ability to delete those
|
96 |
# we crash, we don't lose the ability to delete those
|
|
|
97 |
for ci_ids_chunk in chunked_list(ci_ids, 3000):
|
|
|
98 |
# delete TreesDocs
|
| 91 |
i = M.repo.TreesDoc.m.find({"_id": {"$in": ci_ids}}).count()
|
99 |
i = M.repo.TreesDoc.m.find({"_id": {"$in": ci_ids_chunk}}).count()
|
|
|
100 |
if i:
|
| 92 |
log.info("Deleting %i TreesDoc docs...", i)
|
101 |
log.info("Deleting %i TreesDoc docs...", i)
|
| 93 |
M.repo.TreesDoc.m.remove({"_id": {"$in": ci_ids}})
|
102 |
M.repo.TreesDoc.m.remove({"_id": {"$in": ci_ids_chunk}})
|
| 94 |
|
103 |
|
| 95 |
# delete LastCommitDocs
|
104 |
# delete LastCommitDocs
|
| 96 |
i = M.repo.LastCommitDoc.m.find(dict(commit_ids={'$in': ci_ids})).count()
|
105 |
i = M.repo.LastCommitDoc.m.find(dict(commit_ids={'$in': ci_ids_chunk})).count()
|
|
|
106 |
if i:
|
| 97 |
log.info("Deleting %i remaining LastCommitDoc docs, by repo id...", i)
|
107 |
log.info("Deleting %i remaining LastCommitDoc docs, by repo id...", i)
|
| 98 |
M.repo.LastCommitDoc.m.remove(dict(commit_ids={'$in': ci_ids}))
|
108 |
M.repo.LastCommitDoc.m.remove(dict(commit_ids={'$in': ci_ids_chunk}))
|
| 99 |
|
109 |
|
| 100 |
i = M.repo.DiffInfoDoc.m.find({"_id": {"$in": ci_ids}}).count()
|
110 |
i = M.repo.DiffInfoDoc.m.find({"_id": {"$in": ci_ids_chunk}}).count()
|
|
|
111 |
if i:
|
| 101 |
log.info("Deleting %i DiffInfoDoc docs...", i)
|
112 |
log.info("Deleting %i DiffInfoDoc docs...", i)
|
| 102 |
M.repo.DiffInfoDoc.m.remove({"_id": {"$in": ci_ids}})
|
113 |
M.repo.DiffInfoDoc.m.remove({"_id": {"$in": ci_ids_chunk}})
|
| 103 |
|
114 |
|
| 104 |
i = M.repo.CommitRunDoc.m.find({"commit_ids": {"$in": ci_ids}}).count()
|
115 |
i = M.repo.CommitRunDoc.m.find({"commit_ids": {"$in": ci_ids_chunk}}).count()
|
|
|
116 |
if i:
|
| 105 |
log.info("Deleting %i CommitRunDoc docs...", i)
|
117 |
log.info("Deleting %i CommitRunDoc docs...", i)
|
| 106 |
M.repo.CommitRunDoc.m.remove({"commit_ids": {"$in": ci_ids}})
|
118 |
M.repo.CommitRunDoc.m.remove({"commit_ids": {"$in": ci_ids_chunk}})
|
| 107 |
del ci_ids
|
119 |
del ci_ids
|
| 108 |
|
120 |
|
| 109 |
try:
|
121 |
try:
|
| 110 |
if options.all:
|
122 |
if options.all:
|
| 111 |
log.info('Refreshing ALL commits in %r', c.app.repo)
|
123 |
log.info('Refreshing ALL commits in %r', c.app.repo)
|