|
a/src/main/java/net/timbusproject/dpes/alternative/kb/PackageUniverseBuilder.java |
|
b/src/main/java/net/timbusproject/dpes/alternative/kb/PackageUniverseBuilder.java |
|
... |
|
... |
| 16 |
* See the License for the specific language governing permissions and limitation under the License.
|
16 |
* See the License for the specific language governing permissions and limitation under the License.
|
| 17 |
*/
|
17 |
*/
|
| 18 |
|
18 |
|
| 19 |
package net.timbusproject.dpes.alternative.kb;
|
19 |
package net.timbusproject.dpes.alternative.kb;
|
| 20 |
|
20 |
|
| 21 |
import java.io.File;
|
21 |
import java.io.FileInputStream;
|
| 22 |
import java.io.FileNotFoundException;
|
22 |
import java.io.FileNotFoundException;
|
| 23 |
import java.io.IOException;
|
23 |
import java.io.IOException;
|
| 24 |
import java.io.StringWriter;
|
24 |
import java.io.ObjectInputStream;
|
| 25 |
import java.util.ArrayList;
|
25 |
import java.util.ArrayList;
|
| 26 |
import java.util.List;
|
26 |
import java.util.List;
|
| 27 |
import java.util.SortedMap;
|
27 |
import java.util.SortedMap;
|
| 28 |
import java.util.TreeMap;
|
28 |
import java.util.TreeMap;
|
| 29 |
import java.util.concurrent.Callable;
|
29 |
import java.util.concurrent.Callable;
|
|
... |
|
... |
| 40 |
import org.apache.commons.cli.HelpFormatter;
|
40 |
import org.apache.commons.cli.HelpFormatter;
|
| 41 |
import org.apache.commons.cli.Option;
|
41 |
import org.apache.commons.cli.Option;
|
| 42 |
import org.apache.commons.cli.Options;
|
42 |
import org.apache.commons.cli.Options;
|
| 43 |
import org.apache.commons.cli.ParseException;
|
43 |
import org.apache.commons.cli.ParseException;
|
| 44 |
import org.apache.commons.io.IOUtils;
|
44 |
import org.apache.commons.io.IOUtils;
|
| 45 |
|
45 |
import org.apache.commons.lang3.time.DurationFormatUtils;
|
| 46 |
import com.thoughtworks.xstream.XStream;
|
|
|
| 47 |
import com.thoughtworks.xstream.io.xml.DomDriver;
|
|
|
| 48 |
|
46 |
|
| 49 |
/**
|
47 |
/**
|
| 50 |
* @author Rudolf Mayer
|
48 |
* @author Rudolf Mayer
|
| 51 |
*/
|
49 |
*/
|
| 52 |
public class PackageUniverseBuilder {
|
50 |
public class PackageUniverseBuilder {
|
| 53 |
|
51 |
|
| 54 |
public static final String aptitudeSearchCommand = "aptitude search -F %p \".*\"";
|
52 |
public static final String aptitudeSearchCommand = "aptitude search -F %p \".*\"";
|
| 55 |
|
53 |
|
| 56 |
private static final XStream xStream = new XStream(new DomDriver());
|
|
|
| 57 |
|
|
|
| 58 |
static final Logger logUniverse = Logger.getLogger("PackaUniverseBuilder");
|
54 |
static final Logger logUniverse = Logger.getLogger("PackaUniverseBuilder");
|
| 59 |
|
55 |
|
| 60 |
public static void main(String[] args) throws IOException {
|
56 |
public static void main(String[] args) throws IOException, ClassNotFoundException {
|
| 61 |
|
57 |
|
| 62 |
long startTime = System.currentTimeMillis();
|
|
|
| 63 |
|
|
|
| 64 |
// search all packages
|
58 |
// this holds all the packages
|
| 65 |
ArrayList<String> allPackageNames = VirtualPackageAlternativeIdentifier.searchPackages(aptitudeSearchCommand);
|
|
|
| 66 |
SortedMap<String, Package> allPackages = new TreeMap<String, Package>();
|
59 |
SortedMap<String, Package> allPackages = new TreeMap<String, Package>();
|
| 67 |
System.out.println("Found " + allPackageNames.size() + " packages.\n");
|
|
|
| 68 |
|
60 |
|
| 69 |
Options options = new Options();
|
61 |
Options options = new Options();
|
| 70 |
Option optLoadProviders = new Option("f", "file", true, "Load package information from an XML serialised file");
|
62 |
options.addOption(new Option("l", "loadPackages", true, "Load package information from a serialised file"));
|
| 71 |
options.addOption(optLoadProviders);
|
63 |
options.addOption(new Option("x", "loadPackagesXML", true,
|
|
|
64 |
"Load package information from an XML serialised file"));
|
|
|
65 |
options.addOption(new Option("j", "loadPackagesJSON", true,
|
|
|
66 |
"Load package information from a JSON serialised file"));
|
| 72 |
|
67 |
|
| 73 |
CommandLineParser parser = new BasicParser();
|
68 |
CommandLineParser parser = new BasicParser();
|
| 74 |
try {
|
69 |
try {
|
| 75 |
CommandLine cmd = parser.parse(options, args);
|
70 |
CommandLine cmd = parser.parse(options, args);
|
| 76 |
if (cmd.hasOption("file")) { // load the package information from the file
|
71 |
if (cmd.hasOption("loadPackages")) { // load the package information from the serialised file
|
| 77 |
String providersFileName = cmd.getOptionValue("file");
|
72 |
String providersFileName = cmd.getOptionValue("loadPackages");
|
| 78 |
logUniverse.info("Trying to load package information from XML " + providersFileName);
|
73 |
logUniverse.info("Trying to load package information from XML " + providersFileName);
|
|
|
74 |
allPackages = (SortedMap<String, Package>) new ObjectInputStream(new FileInputStream(providersFileName)).readObject();
|
|
|
75 |
logUniverse.info("\tDone, found " + allPackages.size() + " packages.");
|
|
|
76 |
}
|
|
|
77 |
if (cmd.hasOption("loadPackagesXML")) { // load the package information from the XML file
|
|
|
78 |
String providersFileName = cmd.getOptionValue("loadPackagesXML");
|
|
|
79 |
logUniverse.info("Trying to load package information from XML " + providersFileName);
|
| 79 |
allPackages = (SortedMap<String, Package>) xStream.fromXML(new File(providersFileName));
|
80 |
allPackages = (SortedMap<String, Package>) SerialisationUtils.readFromXMLFile(providersFileName);
|
| 80 |
logUniverse.info("\tDone!");
|
81 |
logUniverse.info("\tDone, found " + allPackages.size() + " packages.");
|
| 81 |
}
|
82 |
}
|
|
|
83 |
if (cmd.hasOption("loadPackagesJSON")) { // load the package information from the XML file
|
|
|
84 |
String providersFileName = cmd.getOptionValue("loadPackagesJSON");
|
|
|
85 |
logUniverse.info("Trying to load package information from JSON " + providersFileName);
|
|
|
86 |
allPackages = (SortedMap<String, Package>) SerialisationUtils.readFromJSONFile(providersFileName);
|
|
|
87 |
logUniverse.info("\tDone, found " + allPackages.size() + " packages.");
|
|
|
88 |
}
|
|
|
89 |
|
| 82 |
} catch (ParseException e) {
|
90 |
} catch (ParseException e) {
|
| 83 |
System.err.println("CLI parsing failed. Reason: " + e.getMessage());
|
91 |
System.err.println("CLI parsing failed. Reason: " + e.getMessage());
|
| 84 |
// automatically generate the help statement
|
92 |
// automatically generate the help statement
|
| 85 |
HelpFormatter formatter = new HelpFormatter();
|
93 |
HelpFormatter formatter = new HelpFormatter();
|
| 86 |
formatter.printHelp("ant", options);
|
94 |
formatter.printHelp("ant", options);
|
| 87 |
return;
|
95 |
return;
|
| 88 |
}
|
96 |
}
|
| 89 |
|
97 |
|
|
|
98 |
long startTime = System.currentTimeMillis();
|
|
|
99 |
|
|
|
100 |
// search all packages
|
|
|
101 |
ArrayList<String> allPackageNames = VirtualPackageAlternativeIdentifier.searchPackages(aptitudeSearchCommand);
|
|
|
102 |
logUniverse.info("Found " + allPackageNames.size() + " packages.\n");
|
|
|
103 |
|
| 90 |
if (allPackages == null) {
|
104 |
if (allPackages == null) {
|
| 91 |
allPackages = new TreeMap<String, Package>();
|
105 |
allPackages = new TreeMap<String, Package>();
|
| 92 |
// allPackages = new HashMap<String, Package>(allPackageNames.size());
|
106 |
// allPackages = new HashMap<String, Package>(allPackageNames.size());
|
| 93 |
}
|
107 |
}
|
|
|
108 |
|
|
|
109 |
// read missing packages
|
| 94 |
|
110 |
|
| 95 |
// query the package information, in parallel manner
|
111 |
// query the package information, in parallel manner
|
| 96 |
int nrOfProcessors = Runtime.getRuntime().availableProcessors();
|
112 |
int nrOfProcessors = Runtime.getRuntime().availableProcessors();
|
| 97 |
ExecutorService eservice = Executors.newFixedThreadPool(nrOfProcessors);
|
113 |
ExecutorService eservice = Executors.newFixedThreadPool(nrOfProcessors);
|
| 98 |
logUniverse.info("*** Working with " + nrOfProcessors + " parallel processes");
|
114 |
logUniverse.info("*** Working with " + nrOfProcessors + " parallel processes");
|
|
... |
|
... |
| 112 |
// if (index == maxTestCount) {
|
128 |
// if (index == maxTestCount) {
|
| 113 |
// break;
|
129 |
// break;
|
| 114 |
// }
|
130 |
// }
|
| 115 |
}
|
131 |
}
|
| 116 |
|
132 |
|
|
|
133 |
// wait for all tasks to finish
|
| 117 |
eservice.shutdown();
|
134 |
eservice.shutdown();
|
| 118 |
try {
|
135 |
try {
|
| 119 |
eservice.awaitTermination(Long.MAX_VALUE, TimeUnit.NANOSECONDS);
|
136 |
eservice.awaitTermination(Long.MAX_VALUE, TimeUnit.NANOSECONDS);
|
| 120 |
} catch (InterruptedException e) {
|
137 |
} catch (InterruptedException e) {
|
| 121 |
System.out.println(e);
|
138 |
System.out.println(e);
|
| 122 |
}
|
139 |
}
|
| 123 |
|
140 |
|
|
|
141 |
// store the results - if there were any
|
|
|
142 |
if (futuresList.size() > 0) {
|
|
|
143 |
long endReadingPackagesTime = System.currentTimeMillis();
|
|
|
144 |
logUniverse.info("Reading packages took "
|
|
|
145 |
+ DurationFormatUtils.formatDurationHMS(endReadingPackagesTime - startTime));
|
|
|
146 |
|
| 124 |
for (Future<Package> future : futuresList) {
|
147 |
for (Future<Package> future : futuresList) {
|
| 125 |
Package pkg;
|
148 |
Package pkg;
|
|
|
149 |
try {
|
|
|
150 |
pkg = future.get();
|
|
|
151 |
allPackages.put(pkg.getName(), pkg);
|
|
|
152 |
} catch (InterruptedException e) {
|
|
|
153 |
e.printStackTrace();
|
|
|
154 |
} catch (ExecutionException e) {
|
|
|
155 |
e.printStackTrace();
|
|
|
156 |
}
|
|
|
157 |
}
|
|
|
158 |
|
|
|
159 |
// save results
|
|
|
160 |
final String fileName = "packageDetails-" + OSIdentifier.getEscapedOSString();
|
|
|
161 |
|
|
|
162 |
// object serialisation
|
|
|
163 |
logUniverse.info("Wrote packages to "
|
|
|
164 |
+ SerialisationUtils.writeAsBinary(allPackages,
|
|
|
165 |
VirtualPackageAlternativeIdentifier.PATH_KNOWDLEDGE_BASE + fileName).getAbsolutePath());
|
|
|
166 |
|
|
|
167 |
// save packages as JSON
|
| 126 |
try {
|
168 |
try {
|
| 127 |
pkg = future.get();
|
169 |
logUniverse.info("Wrote packages to "
|
| 128 |
allPackages.put(pkg.getName(), pkg);
|
170 |
+ SerialisationUtils.writeAsJSON(allPackages,
|
|
|
171 |
VirtualPackageAlternativeIdentifier.PATH_KNOWDLEDGE_BASE, fileName, true).getAbsolutePath());
|
| 129 |
} catch (InterruptedException e) {
|
172 |
} catch (FileNotFoundException e1) {
|
| 130 |
e.printStackTrace();
|
173 |
e1.printStackTrace();
|
|
|
174 |
}
|
|
|
175 |
|
|
|
176 |
// save packages in XML
|
|
|
177 |
try {
|
|
|
178 |
logUniverse.info("Wrote packages to "
|
|
|
179 |
+ SerialisationUtils.writeAsXML(allPackages,
|
|
|
180 |
VirtualPackageAlternativeIdentifier.PATH_KNOWDLEDGE_BASE, fileName, true).getAbsolutePath());
|
| 131 |
} catch (ExecutionException e) {
|
181 |
} catch (FileNotFoundException e1) {
|
| 132 |
e.printStackTrace();
|
182 |
e1.printStackTrace();
|
| 133 |
}
|
|
|
| 134 |
}
|
183 |
}
|
| 135 |
|
184 |
|
| 136 |
// save packages in XML
|
|
|
| 137 |
final String fileName = "packages-" + OSIdentifier.getEscapedOSString() + ".xml";
|
|
|
| 138 |
try {
|
185 |
}
|
| 139 |
logUniverse.info("Wrote packages to "
|
|
|
| 140 |
+ VirtualPackageAlternativeIdentifier.writeAsXML(allPackages, fileName).getAbsolutePath());
|
|
|
| 141 |
} catch (FileNotFoundException e1) {
|
|
|
| 142 |
// TODO Auto-generated catch block
|
|
|
| 143 |
e1.printStackTrace();
|
|
|
| 144 |
}
|
|
|
| 145 |
|
|
|
| 146 |
long endReadingPackagesTime = System.currentTimeMillis();
|
|
|
| 147 |
|
186 |
|
| 148 |
// process all packages
|
187 |
// process all packages
|
| 149 |
|
188 |
|
| 150 |
for (Package pkg : allPackages.values()) {
|
189 |
for (Package pkg : allPackages.values()) {
|
| 151 |
// process the lines
|
190 |
// process the lines
|
|
... |
|
... |
| 180 |
pkg.resolveLinks(allPackages);
|
219 |
pkg.resolveLinks(allPackages);
|
| 181 |
}
|
220 |
}
|
| 182 |
|
221 |
|
| 183 |
long totalEndTime = System.currentTimeMillis();
|
222 |
long totalEndTime = System.currentTimeMillis();
|
| 184 |
|
223 |
|
| 185 |
logUniverse.info("");
|
224 |
logUniverse.info("Total duration: " + DurationFormatUtils.formatDurationHMS(totalEndTime - startTime));
|
| 186 |
}
|
225 |
}
|
| 187 |
|
|
|
| 188 |
public static String inputStreamToString(Process process) throws IOException {
|
|
|
| 189 |
StringWriter writer = new StringWriter();
|
|
|
| 190 |
IOUtils.copy(process.getInputStream(), writer);
|
|
|
| 191 |
return writer.toString();
|
|
|
| 192 |
}
|
|
|
| 193 |
|
|
|
| 194 |
}
|
226 |
}
|
| 195 |
|
227 |
|
| 196 |
class PackgeDetailsTask implements Callable<Package> {
|
228 |
class PackgeDetailsTask implements Callable<Package> {
|
| 197 |
static final Logger logPackageDetails = Logger.getLogger("PackaDetailsQuery");
|
229 |
static final Logger logPackageDetails = Logger.getLogger("PackaDetailsQuery");
|
| 198 |
|
230 |
|