X-Git-Url: http://demsky.eecs.uci.edu/git/?a=blobdiff_plain;f=Code%2FProjects%2FPacketLevelSignatureExtractor%2Fsrc%2Fmain%2Fjava%2Fedu%2Fuci%2Fiotproject%2FSignatureGenerator.java;h=57e0a0e9ca2d4c5a07f6c49519d114837ddf16a2;hb=0e1a130642de8343806aa43e4cfc132ca2ae2a45;hp=164a3289e10a2568df1ea7aedb0efdccb38382f1;hpb=18b3c06f38f8083d485f232c51e78e0fa5066d53;p=pingpong.git diff --git a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/SignatureGenerator.java b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/SignatureGenerator.java index 164a328..57e0a0e 100644 --- a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/SignatureGenerator.java +++ b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/SignatureGenerator.java @@ -3,6 +3,7 @@ package edu.uci.iotproject; import static edu.uci.iotproject.analysis.UserAction.Type; import edu.uci.iotproject.analysis.*; +import edu.uci.iotproject.io.PrintWriterUtils; import edu.uci.iotproject.io.TriggerTimesFileReader; import edu.uci.iotproject.trafficreassembly.layer3.Conversation; import edu.uci.iotproject.trafficreassembly.layer3.TcpReassembler; @@ -13,7 +14,7 @@ import org.apache.commons.math3.stat.clustering.DBSCANClusterer; import org.pcap4j.core.*; import org.pcap4j.packet.namednumber.DataLinkType; -import java.io.EOFException; +import java.io.*; import java.net.UnknownHostException; import java.time.Duration; import java.time.Instant; @@ -34,12 +35,29 @@ import java.util.stream.Stream; */ public class SignatureGenerator { + /** + * If set to {@code true}, output written to the results file is also dumped to standard out. + */ + private static boolean DUPLICATE_OUTPUT_TO_STD_OUT = true; + /** + * File name for logging. + */ + private static String LOG_EXTENSION = "_signature-generation.log"; + /** + * Directory for logging. + */ + private static String LOG_DIRECTORY = "./"; - public static void main(String[] args) throws PcapNativeException, NotOpenException, EOFException, TimeoutException, UnknownHostException { + public static void main(String[] args) throws PcapNativeException, NotOpenException, EOFException, + TimeoutException, UnknownHostException, IOException { // ------------------------------------------------------------------------------------------------------------- // ------------ # Code for extracting traffic generated by a device within x seconds of a trigger # ------------ if (args.length < 11) { - String errMsg = String.format("Usage: %s inputPcapFile outputPcapFile triggerTimesFile deviceIp" + + String errMsg = String.format("SPECTO version 1.0\n" + + "Copyright (C) 2018-2019 Janus Varmarken and Rahmadi Trimananda.\n" + + "University of California, Irvine.\n" + + "All rights reserved.\n\n" + + "Usage: %s inputPcapFile outputPcapFile triggerTimesFile deviceIp" + " onSignatureFile offSignatureFile onClusterAnalysisFile offClusterAnalysisFile epsilon" + " deletedSequencesOn deletedSequencesOff" + "\n inputPcapFile: the target of the detection" + @@ -71,6 +89,12 @@ public class SignatureGenerator { final double eps = Double.parseDouble(args[8]); final String deletedSequencesOn = args[9]; final String deletedSequencesOff = args[10]; + final String logFile = inputPcapFile + LOG_EXTENSION; + + // Prepare file outputter. + File outputFile = new File(logFile); + outputFile.getParentFile().mkdirs(); + final PrintWriter resultsWriter = new PrintWriter(new FileWriter(outputFile)); // =========================================== TRAFFIC FILTERING ============================================ @@ -107,29 +131,33 @@ public class SignatureGenerator { // Group conversations by hostname. Map> convsByHostname = TcpConversationUtils.groupConversationsByHostname(allConversations, dnsMap); - System.out.println("Grouped conversations by hostname."); + PrintWriterUtils.println("Grouped conversations by hostname.", resultsWriter, DUPLICATE_OUTPUT_TO_STD_OUT); // For each hostname, count the frequencies of packet lengths exchanged with that hostname. final Map> pktLenFreqsByHostname = new HashMap<>(); convsByHostname.forEach((host, convs) -> pktLenFreqsByHostname.put(host, TcpConversationUtils.countPacketLengthFrequencies(convs))); - System.out.println("Counted frequencies of packet lengths exchanged with each hostname."); + PrintWriterUtils.println("Counted frequencies of packet lengths exchanged with each hostname.", + resultsWriter, DUPLICATE_OUTPUT_TO_STD_OUT); // For each hostname, count the frequencies of packet sequences (i.e., count how many // conversations exchange a sequence of packets of some specific lengths). final Map> pktSeqFreqsByHostname = new HashMap<>(); convsByHostname.forEach((host, convs) -> pktSeqFreqsByHostname.put(host, TcpConversationUtils.countPacketSequenceFrequencies(convs))); - System.out.println("Counted frequencies of packet sequences exchanged with each hostname."); + PrintWriterUtils.println("Counted frequencies of packet sequences exchanged with each hostname.", + resultsWriter, DUPLICATE_OUTPUT_TO_STD_OUT); // For each hostname, count frequencies of packet pairs exchanged // with that hostname across all conversations final Map> pktPairFreqsByHostname = TcpConversationUtils.countPacketPairFrequenciesByHostname(allConversations, dnsMap); - System.out.println("Counted frequencies of packet pairs per hostname"); + PrintWriterUtils.println("Counted frequencies of packet pairs per hostname.", + resultsWriter, DUPLICATE_OUTPUT_TO_STD_OUT); // For each user action, reassemble the set of TCP connections occurring shortly after final Map> userActionToConversations = trafficLabeler.getLabeledReassembledTcpTraffic(); final Map>> userActionsToConvsByHostname = trafficLabeler.getLabeledReassembledTcpTraffic(dnsMap); - System.out.println("Reassembled TCP conversations occurring shortly after each user event"); + PrintWriterUtils.println("Reassembled TCP conversations occurring shortly after each user event.", + resultsWriter, DUPLICATE_OUTPUT_TO_STD_OUT); /* * NOTE: no need to generate these more complex on/off maps that also contain mappings from hostname and @@ -188,6 +216,8 @@ public class SignatureGenerator { int numberOfEventsPerType = triggerTimes.size() / 2; int lowerBound = numberOfEventsPerType - (int)(numberOfEventsPerType * 0.1); int upperBound = numberOfEventsPerType + (int)(numberOfEventsPerType * 0.1); + //int lowerBound = numberOfEventsPerType - (int)(numberOfEventsPerType * 0.5); + //int upperBound = numberOfEventsPerType + (int)(numberOfEventsPerType * 0.5); int minPts = lowerBound; DBSCANClusterer onClusterer = new DBSCANClusterer<>(eps, minPts); List> onClusters = onClusterer.cluster(onPairs); @@ -197,16 +227,20 @@ public class SignatureGenerator { // Sort the conversations as reference List sortedAllConversation = TcpConversationUtils.sortConversationList(allConversations); // Output clusters - System.out.println("========================================"); - System.out.println(" Clustering results for ON "); - System.out.println(" Number of clusters: " + onClusters.size()); + PrintWriterUtils.println("========================================", resultsWriter, + DUPLICATE_OUTPUT_TO_STD_OUT); + PrintWriterUtils.println(" Clustering results for ON ", resultsWriter, + DUPLICATE_OUTPUT_TO_STD_OUT); + PrintWriterUtils.println(" Number of clusters: " + onClusters.size(), resultsWriter, + DUPLICATE_OUTPUT_TO_STD_OUT); int count = 0; List>> ppListOfListReadOn = new ArrayList<>(); List>> ppListOfListListOn = new ArrayList<>(); List>> corePointRangeSignatureOn = new ArrayList<>(); for (Cluster c : onClusters) { - System.out.println(String.format("<<< Cluster #%02d (%03d points) >>>", ++count, c.getPoints().size())); - System.out.print(PrintUtils.toSummaryString(c)); + PrintWriterUtils.println(String.format("<<< Cluster #%02d (%03d points) >>>", ++count, c.getPoints().size()), + resultsWriter, DUPLICATE_OUTPUT_TO_STD_OUT); + PrintWriterUtils.println(PrintUtils.toSummaryString(c), resultsWriter, DUPLICATE_OUTPUT_TO_STD_OUT); if(c.getPoints().size() > lowerBound && c.getPoints().size() < upperBound) { // Print to file List> ppListOfList = PcapPacketUtils.clusterToListOfPcapPackets(c); @@ -215,16 +249,20 @@ public class SignatureGenerator { ppListOfListListOn.add(ppListOfList); } } - System.out.println("========================================"); - System.out.println(" Clustering results for OFF "); - System.out.println(" Number of clusters: " + offClusters.size()); + PrintWriterUtils.println("========================================", resultsWriter, + DUPLICATE_OUTPUT_TO_STD_OUT); + PrintWriterUtils.println(" Clustering results for OFF ", resultsWriter, + DUPLICATE_OUTPUT_TO_STD_OUT); + PrintWriterUtils.println(" Number of clusters: " + offClusters.size(), resultsWriter, + DUPLICATE_OUTPUT_TO_STD_OUT); count = 0; List>> ppListOfListReadOff = new ArrayList<>(); List>> ppListOfListListOff = new ArrayList<>(); List>> corePointRangeSignatureOff = new ArrayList<>(); for (Cluster c : offClusters) { - System.out.println(String.format("<<< Cluster #%03d (%06d points) >>>", ++count, c.getPoints().size())); - System.out.print(PrintUtils.toSummaryString(c)); + PrintWriterUtils.println(String.format("<<< Cluster #%03d (%06d points) >>>", ++count, c.getPoints().size()), + resultsWriter, DUPLICATE_OUTPUT_TO_STD_OUT); + PrintWriterUtils.println(PrintUtils.toSummaryString(c), resultsWriter, DUPLICATE_OUTPUT_TO_STD_OUT); if(c.getPoints().size() > lowerBound && c.getPoints().size() < upperBound) { // Print to file List> ppListOfList = PcapPacketUtils.clusterToListOfPcapPackets(c); @@ -237,9 +275,6 @@ public class SignatureGenerator { // =========================================== SIGNATURE CREATION =========================================== // Concatenate ppListOfListListOn = PcapPacketUtils.concatSequences(ppListOfListListOn, sortedAllConversation); - // TODO: Need to remove sequence number 0 for TP-Link plug since it is not a good signature! - // TODO: This sequence actually belongs to the local communication between the plug and the phone -// PcapPacketUtils.removeSequenceFromSignature(ppListOfListListOn, 0); // Remove sequences in the list that have overlap StringTokenizer stringTokenizerOn = new StringTokenizer(deletedSequencesOn, ","); while(stringTokenizerOn.hasMoreTokens()) { @@ -249,13 +284,18 @@ public class SignatureGenerator { } PcapPacketUtils.removeSequenceFromSignature(ppListOfListListOn, sequenceToDelete); } + PrintWriterUtils.println("ON Sequences: ", resultsWriter, + DUPLICATE_OUTPUT_TO_STD_OUT); + for(List> listOfList : ppListOfListListOn) { + PrintWriterUtils.println(listOfList.get(0).get(0).length() + "...", resultsWriter, + DUPLICATE_OUTPUT_TO_STD_OUT); + } ppListOfListListOn = PcapPacketUtils.sortSequences(ppListOfListListOn); + PrintWriterUtils.println("Concatenated and sorted ON signature sequences...", resultsWriter, + DUPLICATE_OUTPUT_TO_STD_OUT); // Concatenate ppListOfListListOff = PcapPacketUtils.concatSequences(ppListOfListListOff, sortedAllConversation); - // TODO: Need to remove sequence number 0 for TP-Link plug since it is not a good signature! - // TODO: This sequence actually belongs to the local communication between the plug and the phone -// PcapPacketUtils.removeSequenceFromSignature(ppListOfListListOff, 0); // Remove sequences in the list that have overlap StringTokenizer stringTokenizerOff = new StringTokenizer(deletedSequencesOff, ","); while(stringTokenizerOff.hasMoreTokens()) { @@ -265,17 +305,34 @@ public class SignatureGenerator { } PcapPacketUtils.removeSequenceFromSignature(ppListOfListListOff, sequenceToDelete); } + PrintWriterUtils.println("OFF Sequences: ", resultsWriter, + DUPLICATE_OUTPUT_TO_STD_OUT); + for(List> listOfList : ppListOfListListOff) { + PrintWriterUtils.println(listOfList.get(0).get(0).length() + "...", resultsWriter, + DUPLICATE_OUTPUT_TO_STD_OUT); + } ppListOfListListOff = PcapPacketUtils.sortSequences(ppListOfListListOff); + PrintWriterUtils.println("Concatenated and sorted OFF signature sequences...", resultsWriter, + DUPLICATE_OUTPUT_TO_STD_OUT); // Write the signatures into the screen - System.out.println("========================================"); - System.out.println(" ON Signature "); - System.out.println("========================================"); - PcapPacketUtils.printSignatures(ppListOfListListOn); - System.out.println("========================================"); - System.out.println(" OFF Signature "); - System.out.println("========================================"); - PcapPacketUtils.printSignatures(ppListOfListListOff); + PrintWriterUtils.println("========================================", resultsWriter, + DUPLICATE_OUTPUT_TO_STD_OUT); + PrintWriterUtils.println(" ON Signature ", resultsWriter, + DUPLICATE_OUTPUT_TO_STD_OUT); + PrintWriterUtils.println("========================================", resultsWriter, + DUPLICATE_OUTPUT_TO_STD_OUT); + PcapPacketUtils.printSignatures(ppListOfListListOn, resultsWriter, DUPLICATE_OUTPUT_TO_STD_OUT); + PrintWriterUtils.println("========================================", resultsWriter, + DUPLICATE_OUTPUT_TO_STD_OUT); + PrintWriterUtils.println(" OFF Signature ", resultsWriter, + DUPLICATE_OUTPUT_TO_STD_OUT); + PrintWriterUtils.println("========================================", resultsWriter, + DUPLICATE_OUTPUT_TO_STD_OUT); + PcapPacketUtils.printSignatures(ppListOfListListOff, resultsWriter, DUPLICATE_OUTPUT_TO_STD_OUT); + // Clean signatures from null elements + PcapPacketUtils.cleanSignature(ppListOfListListOn); + PcapPacketUtils.cleanSignature(ppListOfListListOff); // Printing signatures into files PrintUtils.serializeIntoFile(onSignatureFile, ppListOfListListOn); PrintUtils.serializeIntoFile(offSignatureFile, ppListOfListListOff); @@ -283,7 +340,7 @@ public class SignatureGenerator { PrintUtils.serializeIntoFile(onClusterAnalysisFile, corePointRangeSignatureOn); PrintUtils.serializeIntoFile(offClusterAnalysisFile, corePointRangeSignatureOff); - // =========================================== SIGNATURE DURATION =========================================== + // =========================================== SIGNATURE DURATIONS ============================================= List firstSignatureTimestamps = new ArrayList<>(); List lastSignatureTimestamps = new ArrayList<>(); if (!ppListOfListListOn.isEmpty()) { @@ -301,7 +358,7 @@ public class SignatureGenerator { } } - if (!ppListOfListListOn.isEmpty()) { + if (!ppListOfListListOff.isEmpty()) { List> firstListOffSign = ppListOfListListOff.get(0); List> lastListOffSign = ppListOfListListOff.get(ppListOfListListOff.size() - 1); // Load OFF signature first and last packet's timestamps @@ -316,42 +373,41 @@ public class SignatureGenerator { } } // Sort the timestamps - firstSignatureTimestamps.sort((p1, p2) -> { - return p1.compareTo(p2); - }); - // Sort the timestamps - lastSignatureTimestamps.sort((p1, p2) -> { - return p1.compareTo(p2); - }); + firstSignatureTimestamps.sort(Comparator.comparing(Instant::toEpochMilli)); + lastSignatureTimestamps.sort(Comparator.comparing(Instant::toEpochMilli)); Iterator iterFirst = firstSignatureTimestamps.iterator(); Iterator iterLast = lastSignatureTimestamps.iterator(); long duration; long maxDuration = Long.MIN_VALUE; - System.out.println("========================================"); - System.out.println(" Signature Durations "); - System.out.println("========================================"); + PrintWriterUtils.println("========================================", resultsWriter, + DUPLICATE_OUTPUT_TO_STD_OUT); + PrintWriterUtils.println(" Signature Durations ", resultsWriter, + DUPLICATE_OUTPUT_TO_STD_OUT); + PrintWriterUtils.println("========================================", resultsWriter, + DUPLICATE_OUTPUT_TO_STD_OUT); while (iterFirst.hasNext() && iterLast.hasNext()) { - Instant firstInst = (Instant) iterFirst.next(); - Instant lastInst = (Instant) iterLast.next(); + Instant firstInst = iterFirst.next(); + Instant lastInst = iterLast.next(); Duration dur = Duration.between(firstInst, lastInst); duration = dur.toMillis(); // Check duration --- should be below 15 seconds if (duration > TriggerTrafficExtractor.INCLUSION_WINDOW_MILLIS) { while (duration > TriggerTrafficExtractor.INCLUSION_WINDOW_MILLIS && iterFirst.hasNext()) { // that means we have to move to the next trigger - firstInst = (Instant) iterFirst.next(); + firstInst = iterFirst.next(); + dur = Duration.between(firstInst, lastInst); + duration = dur.toMillis(); } - dur = Duration.between(firstInst, lastInst); - duration = dur.toMillis(); } else { // Below 0/Negative --- that means we have to move to the next signature - while (duration < 0 && iterLast.hasNext()) { // that means we have to move to the next trigger - lastInst = (Instant) iterLast.next(); + while (duration < 0 && iterLast.hasNext()) { + // that means we have to move to the next trigger + lastInst = iterLast.next(); + dur = Duration.between(firstInst, lastInst); + duration = dur.toMillis(); } - dur = Duration.between(firstInst, lastInst); - duration = dur.toMillis(); } - System.out.println(duration); + PrintWriterUtils.println(duration, resultsWriter, DUPLICATE_OUTPUT_TO_STD_OUT); // Update duration if this bigger than the max value and still less than the window inclusion time maxDuration = maxDuration < duration && duration <= TriggerTrafficExtractor.INCLUSION_WINDOW_MILLIS ? duration : maxDuration; @@ -360,12 +416,14 @@ public class SignatureGenerator { if (maxDuration == Long.MIN_VALUE) { maxDuration = 0; } - System.out.println("========================================"); - System.out.println("Max signature duration: " + maxDuration); - System.out.println("========================================"); - + PrintWriterUtils.println("========================================", resultsWriter, + DUPLICATE_OUTPUT_TO_STD_OUT); + PrintWriterUtils.println("Max signature duration: " + maxDuration, resultsWriter, + DUPLICATE_OUTPUT_TO_STD_OUT); + PrintWriterUtils.println("========================================", resultsWriter, + DUPLICATE_OUTPUT_TO_STD_OUT); + resultsWriter.flush(); + resultsWriter.close(); // ========================================================================================================== } - - -} +} \ No newline at end of file