src/main/java/net/timbusproject/extractors/modules
File was removed.
src/main/java/net/timbusproject/extractors/modules/debiansoftwareextractor/absolute
File was removed.
src/main/java/net/timbusproject/extractors/modules/debiansoftwareextractor/local/CommandManager.java
File was removed.
src/main/java/net/timbusproject/extractors/modules/debiansoftwareextractor/remote/DebianSoftwareExtractor.java
File was removed.
src/main/java/net/timbusproject/extractors/modules/debiansoftwareextractor/CLI.java
File was removed.
src/main/resources/META-INF
File was removed.
src/main/resources/META-INF/spring
File was removed.
src/test
File was removed.
src/test/resources
File was removed.
extraction.json
File was removed.
jarArguments.txt
File was removed.
src/main/java/net/timbusproject/extractors/modules/debiansoftwareextractor/absolute/Engine.java to src/main/java/net/timbusproject/extractors/debiansoftwareextractor/Engine.java
--- a/src/main/java/net/timbusproject/extractors/modules/debiansoftwareextractor/absolute/Engine.java +++ b/src/main/java/net/timbusproject/extractors/debiansoftwareextractor/Engine.java @@ -1,236 +1,161 @@ -/** - * Copyright (c) 2013, Caixa Magica Software Lda (CMS). - * The work has been developed in the TIMBUS Project and the above-mentioned are Members of the TIMBUS Consortium. - * TIMBUS is supported by the European Union under the 7th Framework Programme for research and technological - * development and demonstration activities (FP7/2007-2013) under grant agreement no. 269940. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at: http://www.apache.org/licenses/LICENSE-2.0 - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including without - * limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTIBITLY, or FITNESS FOR A PARTICULAR - * PURPOSE. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, - * unless required by applicable law or agreed to in writing, shall any Contributor be liable for damages, including - * any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this - * License or out of the use or inability to use the Work. - * See the License for the specific language governing permissions and limitation under the License. - */ +package net.timbusproject.extractors.debiansoftwareextractor; -package net.timbusproject.extractors.modules.debiansoftwareextractor.absolute; - +import ch.qos.logback.classic.Level; +import ch.qos.logback.classic.joran.JoranConfigurator; import com.jcraft.jsch.JSchException; +import org.apache.commons.io.IOUtils; import org.codehaus.jettison.json.JSONArray; import org.codehaus.jettison.json.JSONException; import org.codehaus.jettison.json.JSONObject; -import org.osgi.service.log.LogService; -import org.springframework.beans.factory.annotation.Autowired; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -import java.io.IOException; -import java.io.InputStream; -import java.text.ParseException; -import java.util.Scanner; +import java.io.*; +import java.util.*; +import java.util.regex.Matcher; +import java.util.regex.Pattern; public class Engine { - @Autowired - private LogService log; + private final Logger log = LoggerFactory.getLogger(Engine.class); + private final Properties commands = new Properties(); + private final SSHManager sshManager; - public JSONArray run(SSHManager instance) throws JSchException, IOException, JSONException, ParseException { - String dpkg = "dpkg -l | grep ^ii | awk '{print $2;}'"; - String dpkgStatus = "dpkg --status "; - - String pkgList; - if (instance != null) { - instance.connect(); - // call sendCommand for each command and the output (without prompts) is returned - pkgList = instance.sendCommand(dpkg); - } else - pkgList = doLocalCommand(dpkg); - - Scanner scanner = new Scanner(pkgList); - JSONArray jsonArray = new JSONArray(); - while (scanner.hasNextLine()) { - String pkg = scanner.nextLine(); - if (instance != null) - jsonArray.put(parser(instance.sendCommand(dpkgStatus + pkg))); - else - jsonArray.put(parser(doLocalCommand(dpkgStatus + pkg))); - //jsonArray.put(parser(instance.sendCommand(dpkgStatus + "chromium-browser"))); - - // line for testing purposes. delete it to retrieve more packages - //break; - } - - // close only after all commands are sent - if (instance != null) - instance.close(); - -// writeToFile(jsonArray); - return jsonArray; + public Engine() throws IOException { this((SSHManager) null); } + public Engine(SSHManager ssh) throws IOException { this(ssh, Level.INFO); } + public Engine(Level logLevel) throws IOException { this(null, logLevel); } + public Engine(SSHManager ssh, Level logLevel) throws IOException { + ((ch.qos.logback.classic.Logger) log).setLevel(logLevel); + log.info("Initializing engine..."); + sshManager = ssh; + commands.setProperty("is-command-available", "command -v %s"); + commands.setProperty("dpkg-status", "cat /var/lib/dpkg/status"); + commands.setProperty("licensecheck", "licensecheck -r -copyright %s 2>/dev/null"); } - public JSONArray run() throws ParseException, JSchException, JSONException, IOException { - return run(null); + public JSONArray run() throws JSchException, IOException, InterruptedException, JSONException { + if (isSsh()) { + log.info("Connecting..."); + try { + sshManager.connect(); + } catch (JSchException e) { + log.info("Connection failed."); + throw e; + } + } + log.info("Starting extraction..."); + + log.info("Extracting installed packages..."); + Hashtable<String, JSONObject> table = extractPackages(); + log.info("Extracting licenses..."); + extractLicenses(table); + + log.info("Extraction finished."); + log.info("Closing connection..."); + if (isSsh()) sshManager.disconnect(); + + return new JSONArray(table.values()); } - public String doLocalCommand(String command) throws IOException { - StringBuilder outputBuffer = new StringBuilder(); - InputStream commandOutput; - ProcessBuilder processBuilder = new ProcessBuilder("/bin/sh", "-c", command); - Process p = processBuilder.start(); - try { - p.waitFor(); - } catch (InterruptedException e) { - e.printStackTrace(); + private Hashtable<String, JSONObject> extractPackages() throws JSONException, InterruptedException, JSchException, IOException { + String dpkg = doCommand(commands.getProperty("dpkg-status")).getProperty("stdout"); + Hashtable<String, JSONObject> table = new Hashtable<String, JSONObject>(); + for (String control : dpkg.split("\\n\\n")) { + JSONObject object = extractPackage(control); + log.debug("Extracted package: " + object.getString("Package")); + table.put(object.getString("Package"), object); } - commandOutput = p.getInputStream(); - - int readByte = commandOutput.read(); - while (readByte != 0xffffffff) { - outputBuffer.append((char) readByte); - readByte = commandOutput.read(); - } - commandOutput.close(); - p.destroy(); - return outputBuffer.toString(); + return table; } - public JSONObject parser(String pkg) throws JSONException { - Scanner scanner = new Scanner(pkg); - JSONObject jsonObject = new JSONObject(); - String key = ""; - while (scanner.hasNextLine()) { - if (scanner.hasNext()) { - String tmp = scanner.next(); - if (tmp.endsWith(":")) { - key = tmp.replace(':', ' ').trim(); - } - switch (key.toLowerCase()) { - case "description": - StringBuilder stringBuilder = new StringBuilder(); - while (scanner.hasNextLine() && !tmp.contains(key)) { - String str = scanner.nextLine(); - if (str.length() == 0) { - // string is a blank line - } else if (!(Character.isUpperCase(str.charAt(0)) && str.contains(":"))) { - stringBuilder.append(str); - } - } - jsonObject.put(key, stringBuilder); - break; - case "conffiles": - getConffiles(scanner, jsonObject, tmp, key); - break; - // fields with | - case "pre-depends": - case "recommends": - case "suggests": - case "depends": - // fields without | - case "provides": - case "conflicts": - case "replaces": - JSONArray jsonArrayAnd = getDepends(scanner); - jsonObject.put(key, jsonArrayAnd); - break; - default: - jsonObject.put(key, scanner.nextLine().trim()); - break; - } - } + private JSONObject extractPackage(String control) throws JSONException { + final Pattern fieldPattern = Pattern.compile("([^:]+):[ \\n](\\p{all}+)"); + JSONObject object = new JSONObject(); + for (String field : control.split("\\n(?=\\w)")) { + Matcher matcher = fieldPattern.matcher(field); + matcher.find(); + if (matcher.group(1).matches("(?i)conffiles")) { + object.put(matcher.group(1), extractConffiles(matcher.group(2).trim())); + } else if (matcher.group(1).matches("(?i)breaks|conflicts|enhances|provides|replaces") // list fields + || matcher.group(1).matches("(?i)depends|pre-depends|recommends|suggests")) { // formula fields + object.put(matcher.group(1), extractListOrFormulaField(matcher.group(2))); + } else + object.put(matcher.group(1), matcher.group(2)); } - return jsonObject; + return object; } - private JSONArray getDepends(Scanner scanner) throws JSONException { - String[] depends = scanner.nextLine().trim().split(","); - // main array - JSONArray jsonArrayAnd = new JSONArray(); - for (String s : depends) { - JSONArray jsonArrayOr = new JSONArray(); - - if (s.contains("|")) { - JSONArray result = orDependency(s); - jsonArrayOr.put(result); -// System.out.println("Result " + result.toString()); - } else { - String[] temp = s.trim().split(" "); - for (String pkgOr : temp) { - JSONObject jsonDependency = new JSONObject(); - String[] split = pkgOr.trim().split(" "); - for (String string : split) { - if (string.contains("(")) { - jsonDependency.put("Comparator", string.replace("(", "")); - } else if (string.contains(")")) { - jsonDependency.put("Version", string.replace(")", "")); - } else if (string.length() > 0) { - jsonDependency.put("Package", string.trim()); - } -// System.out.println("STRING" + string); - } - jsonArrayOr.put(jsonDependency); - } - } - if (jsonArrayOr.length() == 1) - jsonArrayAnd.put(jsonArrayOr.get(0)); - else - jsonArrayOr.put(jsonArrayOr); - } - return jsonArrayAnd; + private JSONArray extractListOrFormulaField(String list) throws JSONException { + JSONArray array = new JSONArray(); + for (String element : list.trim().split(", ")) + if (element.contains(" | ")) { + JSONArray formula = new JSONArray(); + for (String sentence : element.split(" \\| ")) + formula.put(extractInlinePackage(sentence)); + array.put(formula); + } else + array.put(extractInlinePackage(element)); + return array; } - private JSONArray orDependency(String s) throws JSONException { - String[] temp = s.trim().split("\\|"); - JSONArray orDependencies = new JSONArray(); - for (String pkgOr : temp) { - JSONObject jsonDependency = new JSONObject(); - String[] split = pkgOr.trim().split(" "); - for (String string : split) { - if (string.contains("(")) { - jsonDependency.put("Comparator", string.replace("(", "")); - } else if (string.contains(")")) { - jsonDependency.put("Version", string.replace(")", "")); - } else if (string.length() > 0) { - jsonDependency.put("Package", string.trim()); - } - } - orDependencies.put(jsonDependency); - } - return orDependencies; + private JSONObject extractInlinePackage(String element) throws JSONException { + final Pattern pattern = Pattern.compile("(\\S+)(?: \\((\\S+) (.+)\\))?"); + Matcher matcher = pattern.matcher(element); + matcher.find(); + JSONObject object = new JSONObject().put("Package", matcher.group(1)); + if (matcher.groupCount() > 1) + object.put("Comparator", matcher.group(2)).put("Version", matcher.group(3)); + return object; } - private JSONArray andDependency(String s) { - return new JSONArray(); + private JSONArray extractConffiles(String list) throws JSONException { + JSONArray array = new JSONArray(); + for (String conffile : list.split("\\n ")) { + String[] values = conffile.split(" "); + array.put(new JSONObject().put("file", values[0]).put("hash", values[1])); + } + return array; } - - private void getConffiles(Scanner scanner, JSONObject jsonObject, String tmp, String key) throws JSONException { - JSONArray jsonArray = new JSONArray(); - JSONObject jsonObject1 = new JSONObject(); - String line = scanner.nextLine().trim(); - if (!tmp.contains(key)) { - jsonObject1.put("file", tmp); - jsonObject1.put("hash", line); - jsonArray.put(jsonObject1); - if (jsonObject.has(key)) { - jsonObject.getJSONArray(key).put(jsonObject1); - } else { - jsonObject.put(key, jsonArray); - } + private void extractLicenses(Hashtable<String, JSONObject> packages) throws InterruptedException, JSchException, IOException, JSONException { + if (!isCommandAvailable("licensecheck")) { log.info("Licenses could not be extracted."); return; } + final String path = "/usr/share/doc/"; + final Pattern licensePattern = Pattern.compile("([^/]+)[^:]+: (.+)"); + for (String line : doCommand(String.format(commands.getProperty("licensecheck"), path)).getProperty("stdout").split("\\n")) { + Matcher matcher = licensePattern.matcher(line.replace(path, "")); + matcher.find(); + String license = matcher.group(2).trim(); + if (license.matches("(\\*No copyright\\* .+)")) + license = license.replaceFirst("\\*No copyright\\* ", "").trim(); + if (license.equals("UNKNOWN")) + continue; + packages.get(matcher.group(1)).put("License", license); } } - /* - Gets all the packages from the Description field - Returns them in a JSONArray - */ - private JSONArray getDescription(Scanner scanner, String tmp, String key) { - JSONArray array = new JSONArray(); - if (!tmp.contains(key)) { - while (scanner.hasNextLine()) { - array.put(scanner.nextLine()); - } - } - return array; + private Properties doCommand(String command) throws IOException, JSchException, InterruptedException { + if (isSsh()) + return sshManager.sendCommand(command); + Properties properties = new Properties(); + Process process = new ProcessBuilder("/bin/sh", "-c", command).start(); + StringWriter writer = new StringWriter(); + IOUtils.copy(process.getInputStream(), writer); + properties.setProperty("stdout", writer.toString()); + IOUtils.copy(process.getErrorStream(), writer); + properties.setProperty("stderr", writer.toString()); + properties.setProperty("exit-value", String.valueOf(process.waitFor())); + process.destroy(); + if (Integer.parseInt(properties.getProperty("exit-value")) != 0) + log.warn("command \"" + command.split(" ")[0] + "\" failed with status: " + properties.getProperty("exit-value")); + return properties; } + + private boolean isCommandAvailable(String command) throws InterruptedException, JSchException, IOException { + log.debug("Testing command: " + command); + Properties commandProperties = doCommand(String.format(commands.getProperty("is-command-available"), command)); + return Integer.parseInt(commandProperties.getProperty("exit-value")) == 0; + } + + private boolean isSsh() { return sshManager != null; } + }
src/main/java/net/timbusproject/extractors/modules/debiansoftwareextractor/absolute/Engine2.java to cli/src/main/java/net/timbusproject/extractors/debiansoftwareextractor/CLI.java
--- a/src/main/java/net/timbusproject/extractors/modules/debiansoftwareextractor/absolute/Engine2.java +++ b/cli/src/main/java/net/timbusproject/extractors/debiansoftwareextractor/CLI.java @@ -1,224 +1,127 @@ -package net.timbusproject.extractors.modules.debiansoftwareextractor.absolute; +/** + * Copyright (c) 2013, Caixa Magica Software Lda (CMS). + * The work has been developed in the TIMBUS Project and the above-mentioned are Members of the TIMBUS Consortium. + * TIMBUS is supported by the European Union under the 7th Framework Programme for research and technological + * development and demonstration activities (FP7/2007-2013) under grant agreement no. 269940. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at: http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including without + * limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTIBITLY, or FITNESS FOR A PARTICULAR + * PURPOSE. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, + * unless required by applicable law or agreed to in writing, shall any Contributor be liable for damages, including + * any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this + * License or out of the use or inability to use the Work. + * See the License for the specific language governing permissions and limitation under the License. + */ +package net.timbusproject.extractors.debiansoftwareextractor; + +import ch.qos.logback.classic.joran.JoranConfigurator; +import com.fasterxml.uuid.Generators; import com.jcraft.jsch.JSchException; +import net.timbusproject.extractors.debiansoftwareextractor.Engine; +import net.timbusproject.extractors.debiansoftwareextractor.SSHManager; +import org.apache.commons.cli.*; import org.codehaus.jettison.json.JSONArray; import org.codehaus.jettison.json.JSONException; import org.codehaus.jettison.json.JSONObject; import java.io.*; -import java.text.ParseException; -import java.util.LinkedList; -import java.util.Scanner; -public class Engine2 { +public class CLI { - public JSONArray run(SSHManager instance) throws JSchException, IOException, JSONException { - String dpkgStatus = "less /var/lib/dpkg/status"; - String result; - if (instance != null) { - instance.connect(); - result = instance.sendCommand(dpkgStatus); - } else - result = doLocalCommand(dpkgStatus); + private static final String formatUUID = "d17250e8-af6e-5b84-8fab-404d5ecee47f"; - StringBuilder stringBuilder = new StringBuilder(result); - Scanner scanner = new Scanner(stringBuilder.toString()); - LinkedList<String> packgeList = buildList(scanner); - JSONArray array = parseList(packgeList); + public static void main(String... args) throws ParseException, IOException, JSONException, JSchException, java.text.ParseException, InterruptedException { + new JoranConfigurator(); + final Options options = new Options(); + final CommandLineParser parser = new PosixParser(); + //Available options + options.addOption("r", "remote", true, "[user@]hostname]"); + options.addOption("l", "local", false, "Extract locally. Prints to screen by default"); + options.addOption("f", "output-file", true, "[filename]"); + options.addOption("h", "help", false, "Prints this help message"); + final CommandLine line = parser.parse(options, args); - // printList(packgeList); + //Print help if there are no arguments + checkOptions(options, line); - writeToFile(array.toString(2)); - if (instance != null) - instance.close(); + if (line.hasOption("h")) { + printHelp(options); + System.exit(0); + } - return array; + JSONObject finalResult; + JSONArray result = null; + String user; + String fqdn; + if (line.hasOption("r")) { + String extract = getOption("r", line); + if (extract.isEmpty()) { + System.exit(0); + } else { + String[] split = extract.split("@"); + //user@hostname + if (split.length == 2) { + user = split[0]; + fqdn = split[1]; + SSHManager manager = new SSHManager(user, fqdn); + System.out.print("Enter password: "); + manager.setPassword(new String(System.console().readPassword())); + + result = new Engine(manager).run(); +// System.out.println("Extracting..."); + // System.out.print(jsonArray.toString(2)); //TODO save to file + } + } + } else if (line.hasOption("l")) { + result = new Engine().run(); + } + finalResult = new JSONObject().put("extractor", "debian-software-extractor") + .put("format", new JSONObject().put("id", formatUUID).put("multiple", false)) + .put("uuid", Generators.timeBasedGenerator().generate()) + .put("result", result); + if (line.hasOption("f")) { + String fileName = getOption("f", line); + if (result != null) { + if (!fileName.isEmpty()) + writeToFile(fileName, finalResult); + else + writeToFile("output_local_extraction.json", finalResult); + } + } else { + if (result != null) + System.out.println(finalResult.toString()); + } } - public JSONArray run() throws ParseException, JSchException, JSONException, IOException { - return run(null); + private final static String getOption(String option, CommandLine line) { + if (line.hasOption(option)) { + return line.getOptionValue(option); + } + + return ""; } - public String doLocalCommand(String command) throws IOException { - StringBuilder outputBuffer = new StringBuilder(); - InputStream commandOutput; - ProcessBuilder processBuilder = new ProcessBuilder("/bin/sh", "-c", command); - Process p = processBuilder.start(); - commandOutput = p.getInputStream(); - - int readByte = commandOutput.read(); - while (readByte != 0xffffffff) { - outputBuffer.append((char) readByte); - readByte = commandOutput.read(); - } - commandOutput.close(); - p.destroy(); - return outputBuffer.toString(); - } - - - private void printList(LinkedList<String> packgeList) throws IOException { - StringBuilder stringBuilder = new StringBuilder(); - for (String str : packgeList) - stringBuilder.append(str); - - writeToFile(stringBuilder.toString()); + private final static void checkOptions(Options options, CommandLine line) { + HelpFormatter formatter = new HelpFormatter(); + if (line.getOptions().length == 0) + formatter.printHelp("launch", options); } - private JSONArray parseList(LinkedList<String> packgeList) throws JSONException { - - JSONArray array = new JSONArray(); - for (String str : packgeList) { - array.put(parser(str)); - } - return array; + private final static void printHelp(Options options) { + HelpFormatter formatter = new HelpFormatter(); + formatter.printHelp("launch", options); + System.exit(0); } - private void writeToFile(String result) throws IOException { - File file = new File("intel-extraction.json"); - if (file.exists()) - file.createNewFile(); - - FileWriter fileWriter = new FileWriter(file.getAbsoluteFile()); - BufferedWriter bufWriter = new BufferedWriter(fileWriter); - bufWriter.write(result); - bufWriter.close(); - + public static void writeToFile(String fileName, JSONObject output) throws FileNotFoundException, UnsupportedEncodingException, JSONException { + PrintWriter writer = new PrintWriter(fileName, "UTF-8"); + writer.write(output.toString(2)); + writer.close(); } - private LinkedList<String> buildList(Scanner scanner) { - LinkedList<String> linkedList = new LinkedList<>(); - StringBuilder stringBuilder = new StringBuilder(); - String tmp; - while (scanner.hasNextLine()) { - tmp = scanner.nextLine(); - stringBuilder.append(tmp + "\n"); - - if (tmp.isEmpty()) { - linkedList.add(stringBuilder.toString()); - stringBuilder = new StringBuilder(); - } - } - return linkedList; - } - - - public JSONObject parser(String pkg) throws JSONException { - Scanner scanner = new Scanner(pkg); - JSONObject jsonObject = new JSONObject(); - String key = ""; - while (scanner.hasNextLine()) { - String tmp = ""; - - if (scanner.hasNext()) - tmp = scanner.next(); - - if (tmp.endsWith(":")) { - key = tmp.replace(':', ' ').trim(); - } - - switch (key.toLowerCase()) { - case "conffiles": - getConffiles(scanner, jsonObject, tmp, key); - break; - - // fields with | - case "pre-depends": - case "recommends": - case "suggests": - case "depends": - // fields without | - case "provides": - case "conflicts": - case "replaces": - JSONArray jsonArrayAnd = getDepends(scanner); - jsonObject.put(key, jsonArrayAnd); - break; - default: - if (scanner.hasNextLine()) - jsonObject.put(key, scanner.nextLine().trim()); - break; - } - - } - return jsonObject; - } - - private JSONArray getDepends(Scanner scanner) throws JSONException { - String[] depends = scanner.nextLine().trim().split(","); - // main array - JSONArray jsonArrayAnd = new JSONArray(); - for (String s : depends) { - JSONArray jsonArrayOr = new JSONArray(); - - if (s.contains("|")) { - JSONArray result = orDependency(s); - jsonArrayOr.put(result); -// System.out.println("Result " + result.toString()); - } else { - String[] temp = s.trim().split(" "); - JSONObject jsonDependency = new JSONObject(); - for (String pkgOr : temp) { - String[] split = pkgOr.trim().split(" "); - for (String string : split) { - if (string.contains("(")) { - jsonDependency.put("Comparator", string.replace("(", "")); - } else if (string.contains(")")) { - jsonDependency.put("Version", string.replace(")", "")); - } else if (string.length() > 0) { - jsonDependency.put("Package", string.trim()); - } -// System.out.println("STRING" + string); - } - } - jsonArrayOr.put(jsonDependency); - } - if (jsonArrayOr.length() == 1) - jsonArrayAnd.put(jsonArrayOr.get(0)); - else - jsonArrayAnd.put(jsonArrayOr); - } - return jsonArrayAnd; - } - - private JSONArray orDependency(String s) throws JSONException { - String[] temp = s.trim().split("\\|"); - JSONArray orDependencies = new JSONArray(); - for (String pkgOr : temp) { - JSONObject jsonDependency = new JSONObject(); - String[] split = pkgOr.trim().split(" "); - for (String string : split) { - if (string.contains("(")) { - jsonDependency.put("Comparator", string.replace("(", "")); - } else if (string.contains(")")) { - jsonDependency.put("Version", string.replace(")", "")); - } else if (string.length() > 0) { - jsonDependency.put("Package", string.trim()); - } - } - orDependencies.put(jsonDependency); - } - return orDependencies; - } - - - private void getConffiles(Scanner scanner, JSONObject jsonObject, String tmp, String key) throws JSONException { - JSONArray jsonArray = new JSONArray(); - JSONObject jsonObject1 = new JSONObject(); - String line = ""; - if (scanner.hasNextLine()) - line = scanner.nextLine().trim(); - - if (!tmp.contains(key)) { - jsonObject1.put("file", tmp); - jsonObject1.put("hash", line); - jsonArray.put(jsonObject1); - if (jsonObject.has(key)) { - jsonObject.getJSONArray(key).put(jsonObject1); - } else { - jsonObject.put(key, jsonArray); - } - } - } }
src/main/java/net/timbusproject/extractors/modules/debiansoftwareextractor/absolute/ReadJSON.java to src/main/java/net/timbusproject/extractors/debiansoftwareextractor/SSHManager.java
--- a/src/main/java/net/timbusproject/extractors/modules/debiansoftwareextractor/absolute/ReadJSON.java +++ b/src/main/java/net/timbusproject/extractors/debiansoftwareextractor/SSHManager.java @@ -15,29 +15,61 @@ * License or out of the use or inability to use the Work. * See the License for the specific language governing permissions and limitation under the License. */ -package net.timbusproject.extractors.modules.debiansoftwareextractor.absolute; +package net.timbusproject.extractors.debiansoftwareextractor; + +import com.jcraft.jsch.*; import org.apache.commons.io.IOUtils; -import org.codehaus.jettison.json.JSONArray; -import org.codehaus.jettison.json.JSONException; -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.io.StringWriter; +import java.io.*; +import java.util.Properties; -public class ReadJSON { +public class SSHManager { - private JSONArray jsonArray; + private Session session; - public ReadJSON(String path) throws IOException, JSONException { - StringWriter writer = new StringWriter(); - IOUtils.copy(new FileInputStream(new File(path)), writer); - this.jsonArray = new JSONArray(writer.toString()); + public SSHManager(String username, String fqdn) throws JSchException { this(username, fqdn, 22); } + public SSHManager(String username, String fqdn, int port) throws JSchException { + this(username, fqdn, port, null, null); + } + public SSHManager(String username, String fqdn, int port, String privateKey) throws JSchException { + this(username, fqdn, port, privateKey, null); + } + public SSHManager(String username, String fqdn, int port, InputStream knownHosts) throws JSchException { + this(username, fqdn, port, null, knownHosts); + } + public SSHManager(String username, String fqdn, int port, String privateKey, InputStream knownHosts) throws JSchException { + JSch jSch = new JSch(); + if (privateKey != null && !privateKey.isEmpty()) jSch.addIdentity(privateKey); + if (knownHosts != null) jSch.setKnownHosts(knownHosts); + session = jSch.getSession(username, fqdn, port); + // UNCOMMENT THIS FOR TESTING PURPOSES, BUT DO NOT USE IN PRODUCTION + session.setConfig("StrictHostKeyChecking", "no"); } - public JSONArray getJsonArray() { - return jsonArray; + public void setPassword(String password) { session.setPassword(password); } + + public void connect() throws JSchException { connect(60 * 1000); } + public void connect(int timeout) throws JSchException { session.connect(timeout); } + + public Properties sendCommand(String command) throws IOException, InterruptedException, JSchException { + Channel channel = session.openChannel("exec"); + ((ChannelExec) channel).setCommand(command); + channel.connect(); + Properties properties = new Properties(); + StringWriter writer = new StringWriter(); + IOUtils.copy(channel.getInputStream(), writer); + properties.setProperty("stdout", writer.toString()); + try { + IOUtils.copy(channel.getExtInputStream(), writer); + properties.setProperty("stderr", writer.toString()); + } catch (NullPointerException ignored) {} + new Thread(channel).join(); + properties.setProperty("exit-value", String.valueOf(channel.getExitStatus())); + channel.disconnect(); + return properties; } -} + public void disconnect() { session.disconnect(); } + +}
src/main/java/net/timbusproject/extractors/modules/debiansoftwareextractor/absolute/SSHManager.java to src/main/java/net/timbusproject/extractors/debiansoftwareextractor/DebianSoftwareExtractor.java
--- a/src/main/java/net/timbusproject/extractors/modules/debiansoftwareextractor/absolute/SSHManager.java +++ b/src/main/java/net/timbusproject/extractors/debiansoftwareextractor/DebianSoftwareExtractor.java @@ -16,109 +16,86 @@ * See the License for the specific language governing permissions and limitation under the License. */ -package net.timbusproject.extractors.modules.debiansoftwareextractor.absolute; +package net.timbusproject.extractors.debiansoftwareextractor; -import com.jcraft.jsch.*; +import com.fasterxml.uuid.Generators; +import net.timbusproject.extractors.core.*; +import org.codehaus.jettison.json.JSONArray; +import org.codehaus.jettison.json.JSONObject; +import org.osgi.framework.BundleContext; +import org.osgi.framework.Version; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -import java.io.IOException; -import java.io.InputStream; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.util.EnumSet; +import java.util.HashMap; -public class SSHManager { - private JSch jschSSHChannel; - private String strUserName; - private String strConnectionIP; - private int intConnectionPort; - private String strPassword; - private Session sesConnection; - private int intTimeOut; +public class DebianSoftwareExtractor implements IExtractor { - private void doCommonConstructorActions(String userName, String password, String connectionIP, - String knownHostsFileName, String privateKey) throws JSchException { - jschSSHChannel = new JSch(); - if (knownHostsFileName != null && knownHostsFileName.length() > 0) - jschSSHChannel.setKnownHosts(knownHostsFileName); + private static final String formatUUID = "d17250e8-af6e-5b84-8fab-404d5ecee47f"; - strUserName = userName; - strPassword = password; - strConnectionIP = connectionIP; - if (privateKey != null && privateKey.length() > 0) - jschSSHChannel.addIdentity(privateKey); + private BundleContext bundleContext; + private final Logger log = LoggerFactory.getLogger(DebianSoftwareExtractor.class); + + public DebianSoftwareExtractor(BundleContext bundleContext) { this.bundleContext = bundleContext; } + + @Override + public String getName() { + if(bundleContext != null) + return bundleContext.getBundle().getHeaders().get("Bundle-Name"); + return String.valueOf(""); } - public SSHManager(String userName, String password, String connectionIP, String knownHostsFileName, - String privateKey) throws JSchException { - doCommonConstructorActions(userName, password, connectionIP, knownHostsFileName, privateKey); - intConnectionPort = 22; - intTimeOut = 60000; + @Override + public String getSymbolicName() { + return bundleContext != null ? bundleContext.getBundle().getSymbolicName() : getClass().getCanonicalName(); } - public SSHManager(String userName, String password, String connectionIP, String knownHostsFileName, - int connectionPort, String privateKey) throws JSchException { - doCommonConstructorActions(userName, password, connectionIP, knownHostsFileName, privateKey); - intConnectionPort = connectionPort; - intTimeOut = 60000; + @Override + public Version getVersion() { + return bundleContext != null ? bundleContext.getBundle().getVersion() : Version.emptyVersion; } - public SSHManager(String userName, String password, String connectionIP, String knownHostsFileName, - String privateKey, int connectionPort, int timeOutMilliseconds) throws JSchException { - doCommonConstructorActions(userName, password, connectionIP, knownHostsFileName, privateKey); - intConnectionPort = connectionPort; - intTimeOut = timeOutMilliseconds; + @Override + public EnumSet<OperatingSystem> getSupportedOperatingSystems() { + return EnumSet.of(OperatingSystem.LINUX); } - public SSHManager(String userName, String password, String connectionIP, String knownHostsFileName, - int connectionPort, int timeOutMilliseconds, String privateKey) { - try { - doCommonConstructorActions(userName, password, connectionIP, knownHostsFileName, privateKey); - } catch (JSchException e) { - e.printStackTrace(); //To change body of catch statement use File | Settings | File Templates. - } - intConnectionPort = connectionPort; - intTimeOut = timeOutMilliseconds; + @Override + public HashMap<String, Parameter> getParameters() { + HashMap<String, Parameter> parameters = new HashMap<String, Parameter>(); + parameters.put("user", new Parameter(false)); + parameters.put("password", new Parameter(true)); + parameters.put("port", new Parameter(false, ParameterType.NUMBER)); + parameters.put("fqdn", new Parameter(false)); + return parameters; } - public String connect() throws JSchException { - String errorMessage = null; - sesConnection = jschSSHChannel.getSession(strUserName, strConnectionIP, intConnectionPort); - sesConnection.setPassword(strPassword); - // UNCOMMENT THIS FOR TESTING PURPOSES, BUT DO NOT USE IN PRODUCTION - sesConnection.setConfig("StrictHostKeyChecking", "no"); - sesConnection.connect(intTimeOut); - return errorMessage; + @Override + public String extract(Endpoint endpoint, boolean b) throws Exception { + FileInputStream knownHosts; + try { + knownHosts = endpoint.hasProperty("knownHosts") && endpoint.getProperty("knownHosts") != null + && !endpoint.getProperty("knownHosts").isEmpty() + ? new FileInputStream(endpoint.getProperty("knownHosts")) : null; + } catch (FileNotFoundException e) { knownHosts = null; } + SSHManager sshManager = new SSHManager( + endpoint.getProperty("user"), + endpoint.getProperty("fqdn"), + endpoint.hasProperty("port") ? Integer.parseInt(endpoint.getProperty("port")) : Endpoint.DEFAULT_SSH_PORT, + endpoint.getProperty("privateKey"), + knownHosts + ); + sshManager.setPassword(endpoint.getProperty("password")); + Engine engine = new Engine(sshManager); + JSONArray jsonArray = engine.run(); + return new JSONObject().put("extractor", getName()) + .put("format", new JSONObject().put("id", formatUUID).put("multiple", false)) + .put("uuid", Generators.timeBasedGenerator().generate()) + .put("result", jsonArray).toString(2); } - private String logError(String errorMessage) { - return errorMessage; - } +} - private String logWarning(String warnMessage) { - return warnMessage; - } - - public String sendCommand(String command) throws IOException, JSchException { - StringBuilder outputBuffer = new StringBuilder(); - try { - Channel channel = sesConnection.openChannel("exec"); - ((ChannelExec) channel).setCommand(command); - channel.connect(); - InputStream commandOutput = channel.getInputStream(); - int readByte = commandOutput.read(); - - while (readByte != 0xffffffff) { - outputBuffer.append((char) readByte); - readByte = commandOutput.read(); - } - - channel.disconnect(); - } catch (IOException ioX) { - logWarning(ioX.getMessage()); - return null; - } - return outputBuffer.toString(); - } - - public void close() { - sesConnection.disconnect(); - } - -}
src/main/java/net/timbusproject/extractors/modules/debiansoftwareextractor/local/CLI.java to cli/pom.xml
--- a/src/main/java/net/timbusproject/extractors/modules/debiansoftwareextractor/local/CLI.java +++ b/cli/pom.xml @@ -1,131 +1,82 @@ -/** - * Copyright (c) 2013, Caixa Magica Software Lda (CMS). - * The work has been developed in the TIMBUS Project and the above-mentioned are Members of the TIMBUS Consortium. - * TIMBUS is supported by the European Union under the 7th Framework Programme for research and technological - * development and demonstration activities (FP7/2007-2013) under grant agreement no. 269940. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at: http://www.apache.org/licenses/LICENSE-2.0 - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including without - * limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTIBITLY, or FITNESS FOR A PARTICULAR - * PURPOSE. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, - * unless required by applicable law or agreed to in writing, shall any Contributor be liable for damages, including - * any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this - * License or out of the use or inability to use the Work. - * See the License for the specific language governing permissions and limitation under the License. - */ +<?xml version="1.0" encoding="UTF-8"?> +<project xmlns="http://maven.apache.org/POM/4.0.0" + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> + <parent> + <groupId>net.timbusproject.extractors</groupId> + <artifactId>debian-software-extractor</artifactId> + <version>0.2.0-SNAPSHOT</version> + <relativePath>../pom.xml</relativePath> + </parent> + <modelVersion>4.0.0</modelVersion> -package net.timbusproject.extractors.modules.debiansoftwareextractor.local; + <artifactId>debian-software-extractor-cli</artifactId> + <version>${project.parent.version}</version> + <packaging>jar</packaging> + <name>Debian Software Extractor - command-line interface</name> -import com.fasterxml.uuid.Generators; -import com.jcraft.jsch.JSchException; -import net.timbusproject.extractors.modules.debiansoftwareextractor.absolute.Engine; -import net.timbusproject.extractors.modules.debiansoftwareextractor.absolute.Engine2; -import net.timbusproject.extractors.modules.debiansoftwareextractor.absolute.SSHManager; -import org.apache.commons.cli.*; -import org.codehaus.jettison.json.JSONArray; -import org.codehaus.jettison.json.JSONException; -import org.codehaus.jettison.json.JSONObject; + <properties> + <project.mainClass>net.timbusproject.extractors.debiansoftwareextractor.CLI</project.mainClass> + </properties> -import java.io.*; + <dependencies> + <dependency> + <groupId>${project.parent.groupId}</groupId> + <artifactId>${project.parent.artifactId}</artifactId> + <version>${project.parent.version}</version> + <scope>system</scope> + <systemPath>${project.basedir}/../pom.xml</systemPath> + </dependency> + <dependency> + <groupId>commons-cli</groupId> + <artifactId>commons-cli</artifactId> + <version>1.2</version> + </dependency> + </dependencies> -public class CLI { - private static final String formatUUID = "d17250e8-af6e-5b84-8fab-404d5ecee47f"; + <build> + <resources> + <resource> + <directory>../src/main/resources</directory> + <includes> + <include>**/*</include> + </includes> + </resource> + <resource> + <directory>src/main/resources</directory> + <includes> + <include>**/*</include> + </includes> + </resource> + </resources> + <plugins> + <plugin> + <groupId>org.codehaus.mojo</groupId> + <artifactId>build-helper-maven-plugin</artifactId> + <version>1.9</version> + <executions> + <execution> + <goals> + <goal>add-source</goal> + </goals> + <configuration> + <sources> + <source>../src/main/java</source> + </sources> + </configuration> + </execution> + </executions> + </plugin> + <plugin> + <artifactId>maven-assembly-plugin</artifactId> + <configuration> + <appendAssemblyId>false</appendAssemblyId> + <descriptorRefs> + <descriptorRef>jar-with-dependencies</descriptorRef> + </descriptorRefs> + </configuration> + </plugin> + </plugins> + </build> - public static void main(String... args) throws ParseException, IOException, JSONException, JSchException, java.text.ParseException { - final Options options = new Options(); - final CommandLineParser parser = new PosixParser(); - //Available options - options.addOption("e", "Extract remotely", true, "[user@]hostname]"); - options.addOption("l", "Extract locally", false, "Extract dependencies locally. Prints to screen by default"); - options.addOption("f", "Write output on file", true, "[filename]"); - options.addOption("h", "help", false, "Prints this help message"); - final CommandLine line = parser.parse(options, args); - - //Print help if there are no arguments - checkOptions(options, line); - - if (line.hasOption("h")) { - printHelp(options); - System.exit(0); - } - - JSONObject finalResult; - JSONArray result = null; - Engine2 engine = new Engine2(); - String user; - String fqdn; - if (line.hasOption("e")) { - String extract = getOption("e", line); - if (extract.isEmpty()) { - System.exit(0); - } else { - String[] split = extract.split("@"); - //user@hostname - if (split.length == 2) { - user = split[0]; - fqdn = split[1]; - // Scanner scanner = new Scanner(System.in); - // System.out.println("Please write your pass: "); - // String pass = scanner.nextLine(); - Console console = System.console(); - System.out.println("Please enter your password"); - char[] password = console.readPassword(); - String pass = new String(password); - SSHManager manager = new SSHManager(user, pass, fqdn, "", ""); - - result = engine.run(manager); - System.out.println("Extracting..."); - // System.out.print(jsonArray.toString(2)); //TODO save to file - } - } - } else if (line.hasOption("l")) { - result = engine.run(); - } - finalResult = new JSONObject().put("extractor", "debian-software-extractor") - .put("format", new JSONObject().put("id", formatUUID).put("multiple", false)) - .put("uuid", Generators.timeBasedGenerator().generate()) - .put("result", result); - if (line.hasOption("f")) { - String fileName = getOption("f", line); - if (result != null) { - if (!fileName.isEmpty()) - writeToFile(fileName, finalResult.toString()); - else - writeToFile("output_local_extraction.json", finalResult.toString()); - } - } else { - if (result != null) - System.out.println(finalResult.toString()); - } - } - - private final static String getOption(String option, CommandLine line) { - if (line.hasOption(option)) { - return line.getOptionValue(option); - } - - return ""; - } - - private final static void checkOptions(Options options, CommandLine line) { - HelpFormatter formatter = new HelpFormatter(); - if (line.getOptions().length == 0) - formatter.printHelp("launch", options); - - } - - private final static void printHelp(Options options) { - HelpFormatter formatter = new HelpFormatter(); - formatter.printHelp("launch", options); - System.exit(0); - } - - public static void writeToFile(String fileName, String output) throws FileNotFoundException, UnsupportedEncodingException, JSONException { - PrintWriter writer = new PrintWriter(fileName, "UTF-8"); - writer.write(output); - writer.close(); - } - -} +</project>
src/main/java/net/timbusproject/extractors/modules/debiansoftwareextractor/local/LocalCLI.java to bundle/pom.xml
--- a/src/main/java/net/timbusproject/extractors/modules/debiansoftwareextractor/local/LocalCLI.java +++ b/bundle/pom.xml @@ -1,66 +1,80 @@ -/** - * Copyright (c) 2013, Caixa Magica Software Lda (CMS). - * The work has been developed in the TIMBUS Project and the above-mentioned are Members of the TIMBUS Consortium. - * TIMBUS is supported by the European Union under the 7th Framework Programme for research and technological - * development and demonstration activities (FP7/2007-2013) under grant agreement no. 269940. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at: http://www.apache.org/licenses/LICENSE-2.0 - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including without - * limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTIBITLY, or FITNESS FOR A PARTICULAR - * PURPOSE. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, - * unless required by applicable law or agreed to in writing, shall any Contributor be liable for damages, including - * any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this - * License or out of the use or inability to use the Work. - * See the License for the specific language governing permissions and limitation under the License. - */ +<?xml version="1.0" encoding="UTF-8"?> +<project xmlns="http://maven.apache.org/POM/4.0.0" + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> + <parent> + <groupId>net.timbusproject.extractors</groupId> + <artifactId>debian-software-extractor</artifactId> + <version>0.2.0-SNAPSHOT</version> + <relativePath>../pom.xml</relativePath> + </parent> + <modelVersion>4.0.0</modelVersion> -package net.timbusproject.extractors.modules.debiansoftwareextractor.local; + <artifactId>debian-software-extractor-bundle</artifactId> + <version>${project.parent.version}</version> + <packaging>bundle</packaging> + <name>Debian Software Extractor - OSGi bundle</name> -import org.codehaus.jettison.json.JSONArray; -import org.codehaus.jettison.json.JSONException; + <properties> + <!--<bundle.embedded.dependencies/>--> + </properties> -import java.io.BufferedWriter; -import java.io.File; -import java.io.FileWriter; -import java.io.IOException; -import java.text.ParseException; + <dependencies> + <dependency> + <groupId>${project.parent.groupId}</groupId> + <artifactId>${project.parent.artifactId}</artifactId> + <version>${project.parent.version}</version> + <scope>system</scope> + <systemPath>${project.basedir}/../pom.xml</systemPath> + </dependency> + <dependency> + <groupId>org.springframework</groupId> + <artifactId>spring-context</artifactId> + <version>3.2.3.RELEASE</version> + </dependency> + </dependencies> -public class LocalCLI { + <build> + <resources> + <resource> + <directory>../src/main/resources</directory> + <includes> + <include>**/*</include> + </includes> + </resource> + <resource> + <directory>src/main/resources</directory> + <includes> + <include>**/*</include> + </includes> + </resource> + </resources> + <plugins> + <plugin> + <groupId>org.codehaus.mojo</groupId> + <artifactId>build-helper-maven-plugin</artifactId> + <version>1.9</version> + <executions> + <execution> + <goals> + <goal>add-source</goal> + </goals> + <configuration> + <sources> + <source>../src/main/java</source> + </sources> + </configuration> + </execution> + </executions> + </plugin> + <plugin> + <groupId>org.apache.felix</groupId> + <artifactId>maven-bundle-plugin</artifactId> + <configuration> + <excludeDependencies>${project.parent.artifactId}</excludeDependencies> + </configuration> + </plugin> + </plugins> + </build> - public static void main(String[] args) throws IOException, JSONException { - try { - CommandManager commandManager = new CommandManager(); - JSONArray array; - - array = commandManager.run(true); - if (args.length > 0) { - File f; - if (args[0].equals("y")) - f = new File("extraction.json"); - else - f = new File(args[0]); - BufferedWriter fileWriter = new BufferedWriter(new FileWriter(f)); - fileWriter.write(array.toString(2)); - fileWriter.close(); - if (args.length > 1) { - String command; - if (args[1].equals("y")) - command = "java -jar local-deb-software-converter-1.0-SNAPSHOT.jar"; - else - command = "java -jar " + args[1]; - if (args.length > 2){ - command = command + " " + args[2]; - if(args.length > 3) - command = command + " " + args[3]; - } - commandManager.doCommand(command); - } - } - } catch (ParseException e) { - e.printStackTrace(); - } - } -} - +</project>
src/main/resources/META-INF/spring/bundle-context-osgi.xml to bundle/src/main/resources/META-INF/spring/bundle-context-osgi.xml
--- a/src/main/resources/META-INF/spring/bundle-context-osgi.xml +++ b/bundle/src/main/resources/META-INF/spring/bundle-context-osgi.xml @@ -13,6 +13,4 @@ <osgi:service interface="net.timbusproject.extractors.core.IExtractor" ref="extractor"/> - <osgi:reference id="log" interface="org.osgi.service.log.LogService"/> - </beans>
src/main/resources/META-INF/spring/bundle-context.xml to bundle/src/main/resources/META-INF/spring/bundle-context.xml
--- a/src/main/resources/META-INF/spring/bundle-context.xml +++ b/bundle/src/main/resources/META-INF/spring/bundle-context.xml @@ -10,9 +10,9 @@ configuration file so that this file can easily be used for integration testing outside of an OSGi environment --> - <bean name="extractor" class="net.timbusproject.extractors.modules.debiansoftwareextractor.remote.DebianSoftwareExtractor"/> + <bean name="extractor" class="net.timbusproject.extractors.debiansoftwareextractor.Bundle"/> - <!-- Enable @AutoWire --> + <!-- Enable @Autowired --> <context:annotation-config/> </beans>