/**
* Copyright (c) 2013/2014 Verein zur Foerderung der IT-Sicherheit in Oesterreich (SBA).
* The work has been developed in the TIMBUS Project and the above-mentioned are Members of the TIMBUS Consortium.
* TIMBUS is supported by the European Union under the 7th Framework Programme for research and technological
* development and demonstration activities (FP7/2007-2013) under grant agreement no. 269940.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at: http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including without
* limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTIBITLY, or FITNESS FOR A PARTICULAR
* PURPOSE. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise,
* unless required by applicable law or agreed to in writing, shall any Contributor be liable for damages, including
* any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this
* License or out of the use or inability to use the Work.
* See the License for the specific language governing permissions and limitation under the License.
*/
package net.timbusproject.dpes.alternative.kb;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.PrintWriter;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Scanner;
import java.util.Set;
import org.apache.commons.cli.BasicParser;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
import org.apache.commons.lang3.StringUtils;
import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.ss.usermodel.Workbook;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
import org.semanticweb.owlapi.apibinding.OWLManager;
import org.semanticweb.owlapi.io.FileDocumentTarget;
import org.semanticweb.owlapi.io.OWLXMLOntologyFormat;
import org.semanticweb.owlapi.model.AddImport;
import org.semanticweb.owlapi.model.IRI;
import org.semanticweb.owlapi.model.OWLClass;
import org.semanticweb.owlapi.model.OWLDataFactory;
import org.semanticweb.owlapi.model.OWLNamedIndividual;
import org.semanticweb.owlapi.model.OWLObjectProperty;
import org.semanticweb.owlapi.model.OWLOntologyID;
import org.semanticweb.owlapi.model.OWLOntologyManager;
import org.semanticweb.owlapi.model.OWLOntologyStorageException;
import uk.ac.manchester.cs.owl.owlapi.OWLOntologyImpl;
/**
* @author Rudolf Mayer
*/
public class VirtualPackageAlternativeIdentifier {
public static final String PATH_KNOWDLEDGE_BASE = "knowledgeBases:softwareAlternatives:".replace(':', File.separatorChar);
public static final IRI IRI_CUDF = IRI.create("http://timbus.teco.edu/ontologies/DSOs/CUDF.owl");
public static final IRI IRI_CUDF_PACKAGE = IRI.create(IRI_CUDF.toString() + "#Package");
public static final IRI IRI_CUDF_VIRTUALPACKAGE = IRI.create(IRI_CUDF.toString() + "#VirtualPackage");
public static final IRI IRI_DIO_SYSTEM_SOFTWARE = IRI.create("http://timbus.teco.edu/ontologies/DIO.owl#SystemSoftware");
public static final OWLClass CLASS_CUDF_PACKAGE = OWLManager.getOWLDataFactory().getOWLClass(IRI_CUDF_PACKAGE);
public static final OWLClass CLASS_CUDF_VIRTUALPACKAGE = OWLManager.getOWLDataFactory().getOWLClass(
IRI_CUDF_VIRTUALPACKAGE);
public static final String IRIPREFIX_KB_PACKAGE_ALTERNATIVES = "http://timbus.teco.edu/ontologies/KB/PackageAlternatives";
public static final OWLObjectProperty OBJECTPROPERTY_CUDF_PROVIDES = OWLManager.getOWLDataFactory().getOWLObjectProperty(
IRI.create(IRI_CUDF.toString() + "#", "provides"));
private static final String completeOntologyKey = "complete";
public static PrintWriter log;
private VirtualPackageAlternativeIdentifier() throws FileNotFoundException {
log = new PrintWriter("softwareAlternatives.process.log");
}
public HashMap<String, HashMap<String, Set<String>>> getPackageProviders() throws FileNotFoundException {
System.out.println("Determining package providers.");
// Find all virtual packages with "aptitude search -F %p ~v"
String aptitudeSearchCommand = "aptitude search -F %p ~v";
ArrayList<String> allPackages = searchPackages(aptitudeSearchCommand);
System.out.println("Found " + allPackages.size() + " virtual packages.\n");
// split into subsets
final String ident64 = "64";
final String identI386 = ":i386";
final String identLib = "lib";
HashMap<String, ArrayList<String>> packageSubsets = new HashMap<String, ArrayList<String>>();
packageSubsets.put("", new ArrayList<String>());
packageSubsets.put(identLib, new ArrayList<String>());
packageSubsets.put(ident64, new ArrayList<String>());
packageSubsets.put(identI386, new ArrayList<String>());
packageSubsets.put(ident64 + identLib, new ArrayList<String>());
packageSubsets.put(identI386 + identLib, new ArrayList<String>());
ArrayList<String> packagesMain = new ArrayList<String>();
ArrayList<String> packagesLib = new ArrayList<String>();
ArrayList<String> packages64 = new ArrayList<String>();
ArrayList<String> packagesLib64 = new ArrayList<String>();
ArrayList<String> packages386 = new ArrayList<String>();
ArrayList<String> packagesLib386 = new ArrayList<String>();
LinkedHashMap<String, ArrayList<String>> packagesSubsets = new LinkedHashMap<String, ArrayList<String>>();
// packagesSubsets.put("main", packagesMain);
// packagesSubsets.put("lib", packagesLib);
packagesSubsets.put("amd64", packages64);
packagesSubsets.put("i386", packages386);
packagesSubsets.put("libamd64", packagesLib64);
packagesSubsets.put("libi386", packagesLib386);
for (String pkg : allPackages) {
final boolean is386 = pkg.endsWith(identI386);
final boolean isLib = pkg.startsWith(identLib);
if (isLib) {
packagesLib.add(pkg);
if (is386) {
packagesLib386.add(pkg);
} else {
packagesLib64.add(pkg);
}
} else {
packagesMain.add(pkg);
if (is386) {
packages386.add(pkg);
} else {
packages64.add(pkg);
}
}
}
System.out.println("Package statistics:");
System.out.println("\tMain packages: " + packagesMain.size());
System.out.println("\t\t" + "64: " + packages64.size() + ", i386: " + packages386.size() + "; checksum: "
+ (packages64.size() + packages386.size() == packagesMain.size() ? "OK" : "ERROR"));
System.out.println("\tLib packages: " + packagesLib.size());
System.out.println("\t\t" + "64: " + packagesLib64.size() + ", i386: " + packagesLib386.size() + " checksum: "
+ (packagesLib64.size() + packagesLib386.size() == packagesLib.size() ? "OK" : "ERROR"));
System.out.println();
HashMap<String, HashMap<String, Set<String>>> packageProviders = new LinkedHashMap<String, HashMap<String, Set<String>>>();
for (String key : packagesSubsets.keySet()) {
System.out.println("\n\n***** Fetching providers for " + key);
System.out.println("Start: " + new Date());
packageProviders.put(key, getProviders(packagesSubsets.get(key)));
System.out.println("Finished: " + new Date());
System.out.println("\n");
}
// merge in one section containing all packages
HashMap<String, Set<String>> hashmapAll = new HashMap<String, Set<String>>();
packageProviders.put("all", hashmapAll);
for (Entry<String, HashMap<String, Set<String>>> entry : packageProviders.entrySet()) {
hashmapAll.putAll(entry.getValue());
}
return packageProviders;
}
public static ArrayList<String> searchPackages(String aptitudeSearchCommand) {
ArrayList<String> allPackages = new ArrayList<String>();
Scanner vPackagesScanner = null;
try {
ProcessBuilder processBuilder = new ProcessBuilder(new String[] { "/bin/sh", "-c", aptitudeSearchCommand });
Process process = processBuilder.start();
InputStream stream = process.getInputStream();
log("Searching for packages using " + aptitudeSearchCommand + " ...\n");
log("... reading package names\n");
vPackagesScanner = new Scanner(stream);
// read packages into an array
while (vPackagesScanner.hasNext()) { // process each package
String pkg = vPackagesScanner.nextLine().trim();
allPackages.add(pkg);
}
log(" DONE!\n");
} catch (IOException e) {
e.printStackTrace();
} finally {
if (vPackagesScanner != null) {
vPackagesScanner.close();
}
}
return allPackages;
}
public void storeResults(HashMap<String, HashMap<String, Set<String>>> packagesProviders) throws IOException {
System.out.println("Storing results");
String osName = OSIdentifier.getEscapedOSString();
// convert to all subsets to one single XML
SerialisationUtils.writeAsXML(packagesProviders, PATH_KNOWDLEDGE_BASE, "providers-" + osName, false);
// One Excel sheet that will contain all sub-sections
Workbook wb = new XSSFWorkbook();
for (String key : packagesProviders.keySet()) {
Map<String, Set<String>> providers = packagesProviders.get(key);
storeSectionResult(providers, key, wb);
storeSectionResult(providers, "providers-" + osName + "-" + key);
}
try {
FileOutputStream fos = new FileOutputStream(PATH_KNOWDLEDGE_BASE + "providers-" + osName + ".xlsx");
wb.write(fos);
fos.close();
} catch (IOException e) {
e.printStackTrace();
}
}
public void storeSectionResult(final Map<String, Set<String>> providers, String sheetName, Workbook wb)
throws FileNotFoundException {
// spreadsheet output
Sheet sh = wb.createSheet(sheetName);
short rowCount = 0;
for (String pkg : providers.keySet()) {
String listOfProviders = StringUtils.join(providers.get(pkg), ",");
Row row = sh.createRow(rowCount++);
row.createCell(0).setCellValue(pkg);
row.createCell(1).setCellValue(providers.get(pkg).size());
row.createCell(2).setCellValue(listOfProviders);
}
}
public void storeSectionResult(final Map<String, Set<String>> providers, String fileName)
throws FileNotFoundException {
PrintWriter simpleOutput = new PrintWriter(fileName + ".txt");
PrintWriter csvOutput = new PrintWriter(fileName + ".csv");
PrintWriter xmlOutput = new PrintWriter(fileName + ".xml");
// convert to XML
String xml = SerialisationUtils.convertToXML(providers);
xmlOutput.println(xml);
System.out.println("\nResults:\n");
for (String pkg : providers.keySet()) {
String s = pkg + ": " + providers.get(pkg).size() + " providers";
System.out.println(s);
simpleOutput.println(s);
String listOfProviders = StringUtils.join(providers.get(pkg), ",");
csvOutput.println(pkg + ';' + providers.get(pkg).size() + ';' + listOfProviders);
}
simpleOutput.close();
csvOutput.close();
xmlOutput.close();
}
public HashMap<String, Set<String>> getProviders(ArrayList<String> packages) {
HashMap<String, Set<String>> providers = new HashMap<String, Set<String>>();
try {
for (int i = 0; i < packages.size(); i++) {
String pkg = packages.get(i);
// find packages that provide this virtual package
ProcessBuilder processBuilder = new ProcessBuilder(new String[] { "/bin/sh", "-c",
"aptitude search -F %p \"?provides(^" + pkg + "$)\"" });
Process process = processBuilder.start();
InputStream stream = process.getInputStream();
log("Fetching " + pkg + "'s providers...");
process.waitFor();
log(" DONE! (" + (i + 1) + "/" + packages.size() + ")\n");
// pkg = pkg.replaceAll(":i386$", "");
Scanner reader = new Scanner(stream);
if (providers.get(pkg) == null) {
providers.put(pkg, new HashSet<String>());
}
while (reader.hasNext()) {
String provider = reader.nextLine().trim();
providers.get(pkg).add(provider/* .replaceAll(":i386$", "") */);
}
reader.close();
}
} catch (InterruptedException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
}
return providers;
}
public static void log(String s) {
System.out.print(s);
if (log != null) {
log.print(s);
}
}
public static LinkedHashMap<String, OWLOntologyImpl> convertToOWL(
HashMap<String, HashMap<String, Set<String>>> packagesProviders, String ontologyPrefix) {
System.out.println("Converting to OWL.");
OWLDataFactory df = OWLManager.getOWLDataFactory();
OWLOntologyManager manager = OWLManager.createOWLOntologyManager(df);
LinkedHashMap<String, OWLOntologyImpl> resultMap = new LinkedHashMap<String, OWLOntologyImpl>();
IRI completeOntologyIRI = IRI.create(IRIPREFIX_KB_PACKAGE_ALTERNATIVES + ontologyPrefix + ".owl");
OWLOntologyImpl completeOntology = new OWLOntologyImpl(manager, new OWLOntologyID(completeOntologyIRI));
manager.applyChange(new AddImport(completeOntology, df.getOWLImportsDeclaration(IRI_CUDF)));
resultMap.put(completeOntologyKey, completeOntology);
for (String key : packagesProviders.keySet()) {
IRI providerOntologyIRI = IRI.create(IRIPREFIX_KB_PACKAGE_ALTERNATIVES + ontologyPrefix + "_" + key
+ ".owl");
OWLOntologyImpl providerOntology = new OWLOntologyImpl(manager, new OWLOntologyID(providerOntologyIRI));
manager.applyChange(new AddImport(providerOntology, df.getOWLImportsDeclaration(IRI_CUDF)));
for (Entry<String, Set<String>> entry : packagesProviders.get(key).entrySet()) {
// 1. create the virtual package
// 1. I.) individual creation & create class assertion (CUDF#VirtualPackage)
OWLNamedIndividual virtualPackageIndiv = OWLUtils.createAndRegisterIndividual(
providerOntologyIRI.toString(), entry.getKey(), CLASS_CUDF_VIRTUALPACKAGE, df, manager,
providerOntology);
OWLNamedIndividual virtualPackageIndivCompl = OWLUtils.createAndRegisterIndividual(
providerOntologyIRI.toString(), entry.getKey(), CLASS_CUDF_VIRTUALPACKAGE, df, manager,
completeOntology);
// 2. create & link the providing packages
for (String provider : entry.getValue()) {
// TODO: check if the package doesn't already exist.
// 2.I create the package individual
OWLNamedIndividual providingPackage = OWLUtils.createAndRegisterIndividual(
providerOntologyIRI.toString(), provider, CLASS_CUDF_PACKAGE, df, manager, providerOntology);
OWLNamedIndividual providingPackageCompl = OWLUtils.createAndRegisterIndividual(
providerOntologyIRI.toString(), provider, CLASS_CUDF_PACKAGE, df, manager, completeOntology);
// 2.II Link the event individual to the entities
OWLUtils.createRelation(providingPackage, virtualPackageIndiv, providerOntology, df, manager,
OBJECTPROPERTY_CUDF_PROVIDES);
OWLUtils.createRelation(providingPackageCompl, virtualPackageIndivCompl, completeOntology, df,
manager, OBJECTPROPERTY_CUDF_PROVIDES);
}
}
resultMap.put(key, providerOntology);
}
System.out.println("\tDone!");
return resultMap;
}
public static void main(String... args) throws IOException {
VirtualPackageAlternativeIdentifier virtualPackageProviderIdentifier = new VirtualPackageAlternativeIdentifier();
HashMap<String, HashMap<String, Set<String>>> packagesProviders = null;
Options options = new Options();
Option optLoadProviders = new Option("p", "providers", true, "Load the providers from an XML seralised file");
options.addOption(optLoadProviders);
CommandLineParser parser = new BasicParser();
try {
CommandLine cmd = parser.parse(options, args);
if (cmd.hasOption("providers")) { // load the providers from the file
// initialise the member variable
String providersFileName = cmd.getOptionValue("providers");
System.out.println("Trying to load package providers from XML " + providersFileName);
packagesProviders = (HashMap<String, HashMap<String, Set<String>>>) SerialisationUtils.readFromXMLFile(providersFileName);
System.out.println("\tDone!");
}
} catch (ParseException e) {
System.err.println("CLI parsing failed. Reason: " + e.getMessage());
// automatically generate the help statement
HelpFormatter formatter = new HelpFormatter();
formatter.printHelp("ant", options);
return;
}
if (packagesProviders == null) {
packagesProviders = virtualPackageProviderIdentifier.getPackageProviders();
virtualPackageProviderIdentifier.storeResults(packagesProviders);
}
String operatingSystemName = OSIdentifier.getEscapedOSString();
LinkedHashMap<String, OWLOntologyImpl> ontologies = convertToOWL(packagesProviders, operatingSystemName);
System.out.println("Saving to files ... ");
for (Entry<String, OWLOntologyImpl> entry : ontologies.entrySet()) {
OWLOntologyImpl ontology = entry.getValue();
String subPath = entry.getKey().equals(completeOntologyKey) ? "" : "subSections" + File.separatorChar;
File file = new File(PATH_KNOWDLEDGE_BASE + subPath + "providers-" + operatingSystemName + "-" + entry.getKey() + ".owl");
FileDocumentTarget fileDocumentTarget = new FileDocumentTarget(file);
try {
OWLManager.createOWLOntologyManager().saveOntology(ontology, new OWLXMLOntologyFormat(),
fileDocumentTarget);
System.out.println("\t" + file.getAbsolutePath());
} catch (OWLOntologyStorageException e) {
e.printStackTrace();
}
}
}
}