/**
* Copyright (c) 2013/2014 Verein zur Foerderung der IT-Sicherheit in Oesterreich (SBA).
* The work has been developed in the TIMBUS Project and the above-mentioned are Members of the TIMBUS Consortium.
* TIMBUS is supported by the European Union under the 7th Framework Programme for research and technological
* development and demonstration activities (FP7/2007-2013) under grant agreement no. 269940.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at: http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including without
* limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTIBITLY, or FITNESS FOR A PARTICULAR
* PURPOSE. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise,
* unless required by applicable law or agreed to in writing, shall any Contributor be liable for damages, including
* any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this
* License or out of the use or inability to use the Work.
* See the License for the specific language governing permissions and limitation under the License.
*/
package net.timbusproject.dpes.alternative.kb;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.ObjectInputStream;
import java.util.ArrayList;
import java.util.List;
import java.util.SortedMap;
import java.util.TreeMap;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.logging.Logger;
import org.apache.commons.cli.BasicParser;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.time.DurationFormatUtils;
/**
* @author Rudolf Mayer
*/
public class PackageUniverseBuilder {
public static final String aptitudeSearchCommand = "aptitude search -F %p \".*\"";
static final Logger logUniverse = Logger.getLogger("PackaUniverseBuilder");
public static void main(String[] args) throws IOException, ClassNotFoundException {
// this holds all the packages
SortedMap<String, Package> allPackages = new TreeMap<String, Package>();
Options options = new Options();
options.addOption(new Option("l", "loadPackages", true, "Load package information from a serialised file"));
options.addOption(new Option("x", "loadPackagesXML", true,
"Load package information from an XML serialised file"));
options.addOption(new Option("j", "loadPackagesJSON", true,
"Load package information from a JSON serialised file"));
CommandLineParser parser = new BasicParser();
try {
CommandLine cmd = parser.parse(options, args);
if (cmd.hasOption("loadPackages")) { // load the package information from the serialised file
String providersFileName = cmd.getOptionValue("loadPackages");
logUniverse.info("Trying to load package information from XML " + providersFileName);
allPackages = (SortedMap<String, Package>) new ObjectInputStream(new FileInputStream(providersFileName)).readObject();
logUniverse.info("\tDone, found " + allPackages.size() + " packages.");
}
if (cmd.hasOption("loadPackagesXML")) { // load the package information from the XML file
String providersFileName = cmd.getOptionValue("loadPackagesXML");
logUniverse.info("Trying to load package information from XML " + providersFileName);
allPackages = (SortedMap<String, Package>) SerialisationUtils.readFromXMLFile(providersFileName);
logUniverse.info("\tDone, found " + allPackages.size() + " packages.");
}
if (cmd.hasOption("loadPackagesJSON")) { // load the package information from the XML file
String providersFileName = cmd.getOptionValue("loadPackagesJSON");
logUniverse.info("Trying to load package information from JSON " + providersFileName);
allPackages = (SortedMap<String, Package>) SerialisationUtils.readFromJSONFile(providersFileName);
logUniverse.info("\tDone, found " + allPackages.size() + " packages.");
}
} catch (ParseException e) {
System.err.println("CLI parsing failed. Reason: " + e.getMessage());
// automatically generate the help statement
HelpFormatter formatter = new HelpFormatter();
formatter.printHelp("ant", options);
return;
}
long startTime = System.currentTimeMillis();
// search all packages
ArrayList<String> allPackageNames = VirtualPackageAlternativeIdentifier.searchPackages(aptitudeSearchCommand);
logUniverse.info("Found " + allPackageNames.size() + " packages.\n");
if (allPackages == null) {
allPackages = new TreeMap<String, Package>();
// allPackages = new HashMap<String, Package>(allPackageNames.size());
}
// read missing packages
// query the package information, in parallel manner
int nrOfProcessors = Runtime.getRuntime().availableProcessors();
ExecutorService eservice = Executors.newFixedThreadPool(nrOfProcessors);
logUniverse.info("*** Working with " + nrOfProcessors + " parallel processes");
List<Future<Package>> futuresList = new ArrayList<Future<Package>>();
int maxTestCount = 50;
int index = 0;
for (String packageName : allPackageNames) {
index++;
// check if we need to query this package
if (allPackages.containsKey(packageName)) {
logUniverse.info("Not querying already loaded package " + packageName);
} else {
futuresList.add(eservice.submit(new PackgeDetailsTask(packageName)));
}
// if (index == maxTestCount) {
// break;
// }
}
// wait for all tasks to finish
eservice.shutdown();
try {
eservice.awaitTermination(Long.MAX_VALUE, TimeUnit.NANOSECONDS);
} catch (InterruptedException e) {
System.out.println(e);
}
// store the results - if there were any
if (futuresList.size() > 0) {
long endReadingPackagesTime = System.currentTimeMillis();
logUniverse.info("Reading packages took "
+ DurationFormatUtils.formatDurationHMS(endReadingPackagesTime - startTime));
for (Future<Package> future : futuresList) {
Package pkg;
try {
pkg = future.get();
allPackages.put(pkg.getName(), pkg);
} catch (InterruptedException e) {
e.printStackTrace();
} catch (ExecutionException e) {
e.printStackTrace();
}
}
// save results
final String fileName = "packageDetails-" + OSIdentifier.getEscapedOSString();
// object serialisation
logUniverse.info("Wrote packages to "
+ SerialisationUtils.writeAsBinary(allPackages,
VirtualPackageAlternativeIdentifier.PATH_KNOWDLEDGE_BASE + fileName).getAbsolutePath());
// save packages as JSON
try {
logUniverse.info("Wrote packages to "
+ SerialisationUtils.writeAsJSON(allPackages,
VirtualPackageAlternativeIdentifier.PATH_KNOWDLEDGE_BASE, fileName, true).getAbsolutePath());
} catch (FileNotFoundException e1) {
e1.printStackTrace();
}
// save packages in XML
try {
logUniverse.info("Wrote packages to "
+ SerialisationUtils.writeAsXML(allPackages,
VirtualPackageAlternativeIdentifier.PATH_KNOWDLEDGE_BASE, fileName, true).getAbsolutePath());
} catch (FileNotFoundException e1) {
e1.printStackTrace();
}
}
// process all packages
for (Package pkg : allPackages.values()) {
// process the lines
for (String str : pkg.getPackageInformation()) {
str = str.trim();
String[] parts = str.split(":");
String key = parts[0].trim();
if (parts.length == 1) {
continue;
}
String value = str.substring(parts[0].length() + 1).trim();
if (key.equals("Package")) {
pkg.setName(parts[1]);
} else if (key.equals("Depends")) {
pkg.setDependsString(parts[1]);
} else if (key.equals("Conflicts")) {
pkg.setConflictsString(parts[1]);
} else if (key.equals("Recommends")) {
pkg.setRecommendsString(parts[1]);
} else if (key.equals("Version")) {
pkg.setVersionString(parts[1]);
}
}
if (allPackages.size() == 10) {
break;
}
}
// do post-processing ==> after all packages are read, we can resolve the links
for (Package pkg : allPackages.values()) {
pkg.resolveLinks(allPackages);
}
long totalEndTime = System.currentTimeMillis();
logUniverse.info("Total duration: " + DurationFormatUtils.formatDurationHMS(totalEndTime - startTime));
}
}
class PackgeDetailsTask implements Callable<Package> {
static final Logger logPackageDetails = Logger.getLogger("PackaDetailsQuery");
private final String packageName;
public PackgeDetailsTask(String packageName) {
this.packageName = packageName;
}
@Override
public Package call() {
Package pkg = new Package();
pkg.setName(packageName);
String aptShowCommand = "aptitude show " + packageName;
aptShowCommand = "apt-cache show " + packageName;
ProcessBuilder processBuilder = new ProcessBuilder(new String[] { "/bin/sh", "-c", aptShowCommand });
logPackageDetails.info("Querying " + packageName);
try {
Process process = processBuilder.start();
// read package details into a data structure
List<String> intialPackageLines;
intialPackageLines = IOUtils.readLines(process.getInputStream());
// process elements that span multi-lines
ArrayList<String> processedPackageLines = new ArrayList<String>();
for (String string : intialPackageLines) {
if (string.startsWith(" ")) {
int index = processedPackageLines.size() - 1;
String line = processedPackageLines.get(index) + System.getProperty("line.separator") + string;
processedPackageLines.remove(index);
processedPackageLines.add(line);
} else {
processedPackageLines.add(string);
}
}
pkg.setPackageInformation(processedPackageLines);
logPackageDetails.info("\tfinished " + packageName);
return pkg;
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
return null;
}
}
}