--- a
+++ b/src/main/java/org/sba_research/timbus/kb/importer/FreebaseImporter.java
@@ -0,0 +1,176 @@
+/**
+ * Copyright (c) 2013/2014 Verein zur Foerderung der IT-Sicherheit in Oesterreich (SBA).
+ * The work has been developed in the TIMBUS Project and the above-mentioned are Members of the TIMBUS Consortium.
+ * TIMBUS is supported by the European Union under the 7th Framework Programme for research and technological
+ * development and demonstration activities (FP7/2007-2013) under grant agreement no. 269940.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at: http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including without
+ * limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTIBITLY, or FITNESS FOR A PARTICULAR
+ * PURPOSE. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise,
+ * unless required by applicable law or agreed to in writing, shall any Contributor be liable for damages, including
+ * any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this
+ * License or out of the use or inability to use the Work.
+ * See the License for the specific language governing permissions and limitation under the License.
+ */
+package org.sba_research.timbus.kb.importer;
+
+import com.google.api.client.http.*;
+import com.google.api.client.http.javanet.NetHttpTransport;
+import com.jayway.jsonpath.JsonPath;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.lang.StringUtils;
+import org.json.simple.JSONArray;
+import org.json.simple.JSONObject;
+import org.json.simple.parser.JSONParser;
+import org.json.simple.parser.ParseException;
+import org.sbaresearch.owl.OwlApiFacade;
+import org.sbaresearch.owl.OwlElementNotFoundException;
+import org.semanticweb.owlapi.model.OWLNamedIndividual;
+import org.semanticweb.owlapi.vocab.OWL2Datatype;
+import uk.ac.manchester.cs.owl.owlapi.OWL2DatatypeImpl;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.logging.Logger;
+
+public class FreebaseImporter implements DataImporter {
+
+ private static final Logger LOG = Logger.getLogger(FreebaseImporter.class.getName());
+ private OwlApiFacade owl;
+ private String kb;
+
+ /**
+ * If the usage limit is exceeded an API key has to be provided.
+ * @see FreebaseImporter::recreateCache
+ * @see <a href="https://developers.google.com/freebase/usage-limits">Freebase usage limits</a>
+ */
+ //private static Properties properties = new Properties();
+
+ @Override
+ public void populate(OwlApiFacade owl, String kb) throws IOException, OwlElementNotFoundException, DataImporterException {
+ LOG.info("loading data...");
+ File file = new File("cache_freebase.json");
+ if (!file.exists()) {
+ recreateCache(file);
+ }
+ this.owl = owl;
+ this.kb = kb;
+
+ LOG.info("populating owl...");
+ JSONArray results = loadCache(file);
+ for (Object result : results) {
+ try {
+ addFormat(result);
+ } catch (InvalidInputException e) {
+ LOG.severe(e.getMessage());
+ }
+ }
+ }
+
+ private void addFormat(Object result) throws OwlElementNotFoundException, InvalidInputException {
+ String formatName = cleanName(jsonGet(result, "$.name"));
+ LOG.fine(String.format("%s, %s, %s, %s", formatName, jsonGet(result, "$.extension[*]"), jsonGet(result, "$.written_by"), jsonGet(result, "$.read_by")));
+ OWLNamedIndividual format = addFormat(owl, formatName);
+ OWLNamedIndividual registry = addRegistry(formatName, format);
+ safeAddEntryExtensionToRegistry(formatName, getExtensions((JSONObject) result), registry);
+ for (Object app : (JSONArray) ((JSONObject) result).get("read_by")) {
+ addToolAndAction(formatName, format, cleanName(app), "read", kb + "#isReading");
+ }
+ for (Object app : (JSONArray) ((JSONObject) result).get("written_by")) {
+ addToolAndAction(formatName, format, cleanName(app), "write", kb + "#isWriting");
+ }
+ }
+
+ private OWLNamedIndividual addRegistry(String formatName, OWLNamedIndividual format) {
+ OWLNamedIndividual registry = owl.addIndividual("registry_format_" + formatName + "_freebase", kb + "#FormatRegistry");
+ owl.addObjectProperty(format, kb + "#isIdentifiedBy", registry);
+ return registry;
+ }
+
+ private String getExtensions(JSONObject result) {
+ JSONArray extensionsList = (JSONArray) result.get("extension");
+ return StringUtils.join(extensionsList, ", ");
+ }
+
+ private JSONArray loadCache(File file) throws IOException, DataImporterException {
+ JSONParser parser = new JSONParser();
+ JSONObject response;
+ try {
+ response = (JSONObject) parser.parse(FileUtils.readFileToString(file));
+ } catch (ParseException e) {
+ throw new DataImporterException(e);
+ }
+ return (JSONArray) response.get("result");
+ }
+
+ private void safeAddEntryExtensionToRegistry(String formatName, String extensions, OWLNamedIndividual registry) {
+ if (extensions.trim().isEmpty()) return;
+ OWLNamedIndividual registryEntryExtension = owl.addIndividual("registry_format_" + formatName + "_freebase_extension", kb + "#RegistryEntry");
+ owl.addDataProperty(registryEntryExtension, kb + "#hasKey", owl.getOWLLiteral("extension", OWL2DatatypeImpl.getDatatype(OWL2Datatype.XSD_STRING)));
+ for (String ext : extensions.split(" ")) {
+ ext = StringUtils.strip(ext, " .,").toLowerCase();
+ owl.addDataProperty(registryEntryExtension, kb + "#hasValue", owl.getOWLLiteral(ext, OWL2DatatypeImpl.getDatatype(OWL2Datatype.XSD_STRING)));
+ }
+ owl.addObjectProperty(registry, kb + "#isConsistingOf", registryEntryExtension);
+ }
+
+ private String cleanName(Object app) {
+ return app.toString().replace(" ", "-").replace("|", "");
+ }
+
+ private OWLNamedIndividual addFormat(OwlApiFacade owl, String formatName) {
+ return owl.addIndividual(formatName, kb + "#FileFormat");
+ }
+
+ private void addToolAndAction(String formatName, OWLNamedIndividual formatIndiv, String appName, String action, String actionProperty) throws OwlElementNotFoundException {
+ OWLNamedIndividual toolAction = owl.addIndividual(String.format("action_" + action + "_%s_%s", formatName, appName), kb + "#ToolAction");
+ String abstractActionName = String.format("action_" + action + "_%s", formatName);
+ OWLNamedIndividual abstractAction = safeAddAbstractAction(formatIndiv, abstractActionName, actionProperty);
+ owl.addObjectProperty(toolAction, kb + "#isProviding", abstractAction);
+
+ OWLNamedIndividual tool = safeAddTool(appName);
+ owl.addObjectProperty(tool, kb + "#isProviding", toolAction);
+ }
+
+ private OWLNamedIndividual safeAddAbstractAction(OWLNamedIndividual formatIndiv, String abstractActionName, String actionProperty) throws OwlElementNotFoundException {
+ if (!owl.containsIndividual(abstractActionName)) {
+ OWLNamedIndividual tmp = owl.addIndividual(abstractActionName, kb + "#AbstractAction");
+ owl.addObjectProperty(tmp, actionProperty, formatIndiv);
+ }
+ return owl.getIndividual(abstractActionName);
+ }
+
+ private OWLNamedIndividual safeAddTool(String appName) throws OwlElementNotFoundException {
+ if (!owl.containsIndividual(appName)) {
+ OWLNamedIndividual tool = owl.addIndividual(appName, kb + "#Tool");
+ OWLNamedIndividual toolRegistry = owl.addIndividual(String.format("registry_tool_%s_%s", appName, "Freebase"), kb + "#ToolRegistry");
+ owl.addObjectProperty(tool, kb + "#isIdentifiedBy", toolRegistry);
+ }
+ return owl.getIndividual(appName);
+ }
+
+ private void recreateCache(File file) throws IOException {
+ // properties.load(new FileInputStream("freebase.properties"));
+ HttpTransport httpTransport = new NetHttpTransport();
+ HttpRequestFactory requestFactory = httpTransport.createRequestFactory();
+ String query = "[{\"id\":null,\"name\":null,\"extension\":[],\"written_by\":[],\"read_by\":[],\"type\":\"/computer/file_format\",\"limit\":4000}]";
+ GenericUrl url = new GenericUrl("https://www.googleapis.com/freebase/v1/mqlread");
+ url.put("query", query);
+ // url.put("key", properties.get("API_KEY"));
+ HttpRequest request = requestFactory.buildGetRequest(url);
+ HttpResponse httpResponse = request.execute();
+ FileUtils.writeLines(file, Arrays.asList(httpResponse.parseAsString()));
+ }
+
+ private String jsonGet(Object result, String jsonPath) throws InvalidInputException {
+ try {
+ return JsonPath.read(result, jsonPath).toString();
+ } catch (NullPointerException e) {
+ throw new InvalidInputException(e);
+ }
+ }
+}