Switch to unified view

a/src/main/java/net/timbusproject/dpes/alternative/kb/PackageUniverseBuilder.java b/src/main/java/net/timbusproject/dpes/alternative/kb/PackageUniverseBuilder.java
...
...
16
 * See the License for the specific language governing permissions and limitation under the License.
16
 * See the License for the specific language governing permissions and limitation under the License.
17
 */
17
 */
18
18
19
package net.timbusproject.dpes.alternative.kb;
19
package net.timbusproject.dpes.alternative.kb;
20
20
21
import java.io.File;
21
import java.io.FileInputStream;
22
import java.io.FileInputStream;
22
import java.io.FileNotFoundException;
23
import java.io.FileNotFoundException;
23
import java.io.IOException;
24
import java.io.IOException;
24
import java.io.ObjectInputStream;
25
import java.io.ObjectInputStream;
25
import java.util.ArrayList;
26
import java.util.ArrayList;
27
import java.util.LinkedList;
26
import java.util.List;
28
import java.util.List;
29
import java.util.Map.Entry;
30
import java.util.Set;
27
import java.util.SortedMap;
31
import java.util.SortedMap;
28
import java.util.TreeMap;
32
import java.util.TreeMap;
29
import java.util.concurrent.Callable;
33
import java.util.concurrent.Callable;
30
import java.util.concurrent.ExecutionException;
34
import java.util.concurrent.ExecutionException;
31
import java.util.concurrent.ExecutorService;
35
import java.util.concurrent.ExecutorService;
...
...
41
import org.apache.commons.cli.Option;
45
import org.apache.commons.cli.Option;
42
import org.apache.commons.cli.Options;
46
import org.apache.commons.cli.Options;
43
import org.apache.commons.cli.ParseException;
47
import org.apache.commons.cli.ParseException;
44
import org.apache.commons.io.IOUtils;
48
import org.apache.commons.io.IOUtils;
45
import org.apache.commons.lang3.time.DurationFormatUtils;
49
import org.apache.commons.lang3.time.DurationFormatUtils;
50
import org.semanticweb.owlapi.apibinding.OWLManager;
51
import org.semanticweb.owlapi.io.FileDocumentTarget;
52
import org.semanticweb.owlapi.io.OWLXMLOntologyFormat;
53
import org.semanticweb.owlapi.model.AddImport;
54
import org.semanticweb.owlapi.model.IRI;
55
import org.semanticweb.owlapi.model.OWLDataFactory;
56
import org.semanticweb.owlapi.model.OWLNamedIndividual;
57
import org.semanticweb.owlapi.model.OWLObjectProperty;
58
import org.semanticweb.owlapi.model.OWLOntologyID;
59
import org.semanticweb.owlapi.model.OWLOntologyManager;
60
import org.semanticweb.owlapi.model.OWLOntologyStorageException;
61
62
import uk.ac.manchester.cs.owl.owlapi.OWLOntologyImpl;
46
63
47
/**
64
/**
48
 * @author Rudolf Mayer
65
 * @author Rudolf Mayer
49
 */
66
 */
50
public class PackageUniverseBuilder {
67
public class PackageUniverseBuilder {
51
68
69
    public static final OWLObjectProperty OBJECTPROPERTY_CUDF_DEPENDS = OWLManager.getOWLDataFactory().getOWLObjectProperty(
70
            IRI.create(VirtualPackageAlternativeIdentifier.IRI_CUDF.toString() + "#", "hasDependency"));
71
72
    public static final OWLObjectProperty OBJECTPROPERTY_CUDF_CONFLICTS = OWLManager.getOWLDataFactory().getOWLObjectProperty(
73
            IRI.create(VirtualPackageAlternativeIdentifier.IRI_CUDF.toString() + "#", "hasConflict"));
74
75
    public static final OWLObjectProperty OBJECTPROPERTY_CUDF_RECOMMENDS = OWLManager.getOWLDataFactory().getOWLObjectProperty(
76
            IRI.create(VirtualPackageAlternativeIdentifier.IRI_CUDF.toString() + "#", "hasRecommendation"));
77
52
    public static final String aptitudeSearchCommand = "aptitude search -F %p \".*\"";
78
    public static final String aptitudeSearchCommand = "aptitude search -F %p \".*\"";
53
79
54
    static final Logger logUniverse = Logger.getLogger("PackaUniverseBuilder");
80
    static final Logger logUniverse = Logger.getLogger("PackaUniverseBuilder");
55
81
56
    public static void main(String[] args) throws IOException, ClassNotFoundException {
82
    public static void main(String[] args) throws IOException, ClassNotFoundException {
...
...
58
        // this holds all the packages
84
        // this holds all the packages
59
        SortedMap<String, Package> allPackages = new TreeMap<String, Package>();
85
        SortedMap<String, Package> allPackages = new TreeMap<String, Package>();
60
86
61
        Options options = new Options();
87
        Options options = new Options();
62
        options.addOption(new Option("l", "loadPackages", true, "Load package information from a serialised file"));
88
        options.addOption(new Option("l", "loadPackages", true, "Load package information from a serialised file"));
63
        options.addOption(new Option("x", "loadPackagesXML", true,
89
        options.addOption(new Option("x", "loadPackagesXML", true, "Load package information from an XML serialised file"));
64
                "Load package information from an XML serialised file"));
90
        options.addOption(new Option("j", "loadPackagesJSON", true, "Load package information from a JSON serialised file"));
65
        options.addOption(new Option("j", "loadPackagesJSON", true,
66
                "Load package information from a JSON serialised file"));
67
91
68
        CommandLineParser parser = new BasicParser();
92
        CommandLineParser parser = new BasicParser();
69
        try {
93
        try {
70
            CommandLine cmd = parser.parse(options, args);
94
            CommandLine cmd = parser.parse(options, args);
71
            if (cmd.hasOption("loadPackages")) { // load the package information from the serialised file
95
            if (cmd.hasOption("loadPackages")) { // load the package information from the serialised file
72
                String providersFileName = cmd.getOptionValue("loadPackages");
96
                String providersFileName = cmd.getOptionValue("loadPackages");
73
                logUniverse.info("Trying to load package information from XML " + providersFileName);
97
                logUniverse.info("Trying to load package information from Binary " + providersFileName);
74
                allPackages = (SortedMap<String, Package>) new ObjectInputStream(new FileInputStream(providersFileName)).readObject();
98
                allPackages = (SortedMap<String, Package>) new ObjectInputStream(new FileInputStream(providersFileName)).readObject();
75
                logUniverse.info("\tDone, found " + allPackages.size() + " packages.");
99
                logUniverse.info("\tDone, found " + allPackages.size() + " packages.");
76
            }
100
            }
77
            if (cmd.hasOption("loadPackagesXML")) { // load the package information from the XML file
101
            if (cmd.hasOption("loadPackagesXML")) { // load the package information from the XML file
78
                String providersFileName = cmd.getOptionValue("loadPackagesXML");
102
                String providersFileName = cmd.getOptionValue("loadPackagesXML");
...
...
119
        int index = 0;
143
        int index = 0;
120
        for (String packageName : allPackageNames) {
144
        for (String packageName : allPackageNames) {
121
            index++;
145
            index++;
122
            // check if we need to query this package
146
            // check if we need to query this package
123
            if (allPackages.containsKey(packageName)) {
147
            if (allPackages.containsKey(packageName)) {
124
                logUniverse.info("Not querying already loaded package " + packageName);
148
                // logUniverse.info("Not querying already loaded package " + packageName);
125
            } else {
149
            } else {
126
                futuresList.add(eservice.submit(new PackgeDetailsTask(packageName)));
150
                futuresList.add(eservice.submit(new PackgeDetailsTask(packageName)));
127
            }
151
            }
128
            // if (index == maxTestCount) {
152
            // if (index == maxTestCount) {
129
            // break;
153
            // break;
...
...
139
        }
163
        }
140
164
141
        // store the results - if there were any
165
        // store the results - if there were any
142
        if (futuresList.size() > 0) {
166
        if (futuresList.size() > 0) {
143
            long endReadingPackagesTime = System.currentTimeMillis();
167
            long endReadingPackagesTime = System.currentTimeMillis();
144
            logUniverse.info("Reading packages took "
145
                    + DurationFormatUtils.formatDurationHMS(endReadingPackagesTime - startTime));
168
            logUniverse.info("Reading packages took " + DurationFormatUtils.formatDurationHMS(endReadingPackagesTime - startTime));
146
169
147
            for (Future<Package> future : futuresList) {
170
            for (Future<Package> future : futuresList) {
148
                Package pkg;
171
                Package pkg;
149
                try {
172
                try {
150
                    pkg = future.get();
173
                    pkg = future.get();
...
...
159
            // save results
182
            // save results
160
            final String fileName = "packageDetails-" + OSIdentifier.getEscapedOSString();
183
            final String fileName = "packageDetails-" + OSIdentifier.getEscapedOSString();
161
184
162
            // object serialisation
185
            // object serialisation
163
            logUniverse.info("Wrote packages to "
186
            logUniverse.info("Wrote packages to "
164
                    + SerialisationUtils.writeAsBinary(allPackages,
165
                            VirtualPackageAlternativeIdentifier.PATH_KNOWDLEDGE_BASE + fileName).getAbsolutePath());
187
                    + SerialisationUtils.writeAsBinary(allPackages, VirtualPackageAlternativeIdentifier.PATH_KNOWDLEDGE_BASE + fileName).getAbsolutePath());
166
188
167
            // save packages as JSON
189
            // save packages as JSON
168
            try {
190
            try {
169
                logUniverse.info("Wrote packages to "
191
                logUniverse.info("Wrote packages to "
170
                        + SerialisationUtils.writeAsJSON(allPackages,
171
                                VirtualPackageAlternativeIdentifier.PATH_KNOWDLEDGE_BASE, fileName, true).getAbsolutePath());
192
                        + SerialisationUtils.writeAsJSON(allPackages, VirtualPackageAlternativeIdentifier.PATH_KNOWDLEDGE_BASE, fileName, true).getAbsolutePath());
172
            } catch (FileNotFoundException e1) {
193
            } catch (FileNotFoundException e1) {
173
                e1.printStackTrace();
194
                e1.printStackTrace();
174
            }
195
            }
175
196
176
            // save packages in XML
197
            // save packages in XML
177
            try {
198
            try {
178
                logUniverse.info("Wrote packages to "
199
                logUniverse.info("Wrote packages to "
179
                        + SerialisationUtils.writeAsXML(allPackages,
180
                                VirtualPackageAlternativeIdentifier.PATH_KNOWDLEDGE_BASE, fileName, true).getAbsolutePath());
200
                        + SerialisationUtils.writeAsXML(allPackages, VirtualPackageAlternativeIdentifier.PATH_KNOWDLEDGE_BASE, fileName, true).getAbsolutePath());
181
            } catch (FileNotFoundException e1) {
201
            } catch (FileNotFoundException e1) {
182
                e1.printStackTrace();
202
                e1.printStackTrace();
183
            }
203
            }
184
204
185
        }
205
        }
...
...
187
        // process all packages
207
        // process all packages
188
208
189
        for (Package pkg : allPackages.values()) {
209
        for (Package pkg : allPackages.values()) {
190
            // process the lines
210
            // process the lines
191
            for (String str : pkg.getPackageInformation()) {
211
            for (String str : pkg.getPackageInformation()) {
212
                if (str.trim().length() == 0) {
213
                    continue;
214
                }
192
                str = str.trim();
215
                str = str.trim();
193
                String[] parts = str.split(":");
216
                int keyValueSeparator = str.indexOf(":");
194
                String key = parts[0].trim();
195
                if (parts.length == 1) {
217
                if (keyValueSeparator == -1) {
218
                    System.out.println("Skipping wrong str: " + str);
196
                    continue;
219
                    continue;
197
                }
220
                }
221
                // String[] parts = str.split(":");
222
                String key = str.substring(0, keyValueSeparator);
198
                String value = str.substring(parts[0].length() + 1).trim();
223
                String value = str.substring(keyValueSeparator + 1).trim();
199
                if (key.equals("Package")) {
224
                if (key.equals("Package")) {
200
                    pkg.setName(parts[1]);
225
                    pkg.setName(value);
201
                } else if (key.equals("Depends")) {
226
                } else if (key.equals("Depends")) {
202
                    pkg.setDependsString(parts[1]);
227
                    pkg.setDependsString(value);
203
                } else if (key.equals("Conflicts")) {
228
                } else if (key.equals("Conflicts")) {
204
                    pkg.setConflictsString(parts[1]);
229
                    pkg.setConflictsString(value);
205
                } else if (key.equals("Recommends")) {
230
                } else if (key.equals("Recommends")) {
206
                    pkg.setRecommendsString(parts[1]);
231
                    pkg.setRecommendsString(value);
207
                } else if (key.equals("Version")) {
232
                } else if (key.equals("Version")) {
208
                    pkg.setVersionString(parts[1]);
233
                    pkg.setVersionString(value);
209
                }
234
                }
210
            }
235
            }
211
236
212
            if (allPackages.size() == 10) {
237
            if (allPackages.size() == 10) {
213
                break;
238
                break;
214
            }
239
            }
215
        }
240
        }
241
242
        logUniverse.info("Parsed all packages");
216
243
217
        // do post-processing ==> after all packages are read, we can resolve the links
244
        // do post-processing ==> after all packages are read, we can resolve the links
218
        for (Package pkg : allPackages.values()) {
245
        for (Package pkg : allPackages.values()) {
219
            pkg.resolveLinks(allPackages);
246
            pkg.resolveLinks(allPackages);
220
        }
247
        }
221
248
249
        logUniverse.info("Resolved links between packages");
250
        final String ontologyPrefix = "packageUniverse-" + OSIdentifier.getEscapedOSString();
251
        OWLOntologyImpl ontology = convertToOWL(allPackages, ontologyPrefix);
252
253
        logUniverse.info("Converted to OWL");
254
255
        File file = new File(VirtualPackageAlternativeIdentifier.PATH_KNOWDLEDGE_BASE + ontologyPrefix + ".owl");
256
        FileDocumentTarget fileDocumentTarget = new FileDocumentTarget(file);
257
        try {
258
            OWLManager.createOWLOntologyManager().saveOntology(ontology, new OWLXMLOntologyFormat(), fileDocumentTarget);
259
            logUniverse.info("Saved to " + file.getAbsolutePath());
260
        } catch (OWLOntologyStorageException e) {
261
            e.printStackTrace();
262
        }
263
222
        long totalEndTime = System.currentTimeMillis();
264
        long totalEndTime = System.currentTimeMillis();
223
265
224
        logUniverse.info("Total duration: " + DurationFormatUtils.formatDurationHMS(totalEndTime - startTime));
266
        logUniverse.info("Total duration: " + DurationFormatUtils.formatDurationHMS(totalEndTime - startTime));
267
    }
268
269
    public static OWLOntologyImpl convertToOWL(SortedMap<String, Package> allPackages, String ontologyPrefix) {
270
271
        OWLDataFactory df = OWLManager.getOWLDataFactory();
272
        OWLOntologyManager manager = OWLManager.createOWLOntologyManager(df);
273
274
        IRI ontologyIRI = IRI.create(VirtualPackageAlternativeIdentifier.IRIPREFIX_KB_PACKAGE_ALTERNATIVES + ontologyPrefix + ".owl");
275
        OWLOntologyImpl ontology = new OWLOntologyImpl(manager, new OWLOntologyID(ontologyIRI));
276
        manager.applyChange(new AddImport(ontology, df.getOWLImportsDeclaration(VirtualPackageAlternativeIdentifier.IRI_CUDF)));
277
278
        final Set<Entry<String, Package>> entrySet = allPackages.entrySet();
279
        for (Entry<String, Package> entry : entrySet) {
280
            // 1. create the package
281
            // 1. I.) individual creation & create class assertion (CUDF#Package)
282
            OWLNamedIndividual packageIndiv = OWLUtils.createAndRegisterIndividual(ontologyIRI.toString(), entry.getKey(),
283
                    VirtualPackageAlternativeIdentifier.CLASS_CUDF_PACKAGE, df, manager, ontology);
284
285
            createRelations(df, manager, ontologyIRI, ontology, packageIndiv, entry.getValue().getDepends(), OBJECTPROPERTY_CUDF_DEPENDS);
286
            createRelations(df, manager, ontologyIRI, ontology, packageIndiv, entry.getValue().getConflicts(), OBJECTPROPERTY_CUDF_CONFLICTS);
287
            createRelations(df, manager, ontologyIRI, ontology, packageIndiv, entry.getValue().getRecommends(), OBJECTPROPERTY_CUDF_RECOMMENDS);
288
        }
289
290
        // System.out.println("\tDone!");
291
292
        return ontology;
293
    }
294
295
    public static void createRelations(OWLDataFactory df, OWLOntologyManager manager, IRI ontologyIRI, OWLOntologyImpl ontology,
296
            OWLNamedIndividual packageIndiv, LinkedList<PackageRelation> depends, OWLObjectProperty objectproperty) {
297
        for (PackageRelation relation : depends) {
298
            // 2.I create the package individual
299
            OWLNamedIndividual dependendPackageIndiv = OWLUtils.createAndRegisterIndividual(ontologyIRI.toString(), relation.getTargetPackageName(),
300
                    VirtualPackageAlternativeIdentifier.CLASS_CUDF_PACKAGE, df, manager, ontology);
301
302
            // 2.II Link the event individual to the entities
303
            OWLUtils.createRelation(packageIndiv, dependendPackageIndiv, ontology, df, manager, objectproperty);
304
        }
225
    }
305
    }
226
}
306
}
227
307
228
class PackgeDetailsTask implements Callable<Package> {
308
class PackgeDetailsTask implements Callable<Package> {
229
    static final Logger logPackageDetails = Logger.getLogger("PackaDetailsQuery");
309
    static final Logger logPackageDetails = Logger.getLogger("PackaDetailsQuery");