Printing output of signature generation into a log file.

diff --git a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/SignatureGenerator.java b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/SignatureGenerator.java
index 164a3289e10a2568df1ea7aedb0efdccb38382f1..ae8e8063a09730e0f9f0a4815e25ff6b6947f927 100644 (file)
--- a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/SignatureGenerator.java
+++ b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/SignatureGenerator.java
@@ -3,6 +3,7 @@ package edu.uci.iotproject;
 import static edu.uci.iotproject.analysis.UserAction.Type;
 
 import edu.uci.iotproject.analysis.*;
+import edu.uci.iotproject.io.PrintWriterUtils;
 import edu.uci.iotproject.io.TriggerTimesFileReader;
 import edu.uci.iotproject.trafficreassembly.layer3.Conversation;
 import edu.uci.iotproject.trafficreassembly.layer3.TcpReassembler;
@@ -13,7 +14,7 @@ import org.apache.commons.math3.stat.clustering.DBSCANClusterer;
 import org.pcap4j.core.*;
 import org.pcap4j.packet.namednumber.DataLinkType;
 
-import java.io.EOFException;
+import java.io.*;
 import java.net.UnknownHostException;
 import java.time.Duration;
 import java.time.Instant;
@@ -34,8 +35,21 @@ import java.util.stream.Stream;
  */
 public class SignatureGenerator {
 
+    /**
+     * If set to {@code true}, output written to the results file is also dumped to standard out.
+     */
+    private static boolean DUPLICATE_OUTPUT_TO_STD_OUT = true;
+    /**
+     * File name for logging.
+     */
+    private static String LOG_EXTENSION = "_signature-generation.log";
+    /**
+     * Directory for logging.
+     */
+    private static String LOG_DIRECTORY = "./";
 
-    public static void main(String[] args) throws PcapNativeException, NotOpenException, EOFException, TimeoutException, UnknownHostException {
+    public static void main(String[] args) throws PcapNativeException, NotOpenException, EOFException,
+            TimeoutException, UnknownHostException, IOException {
         // -------------------------------------------------------------------------------------------------------------
         // ------------ # Code for extracting traffic generated by a device within x seconds of a trigger # ------------
         if (args.length < 11) {
@@ -71,6 +85,12 @@ public class SignatureGenerator {
         final double eps = Double.parseDouble(args[8]);
         final String deletedSequencesOn = args[9];
         final String deletedSequencesOff = args[10];
+        final String logFile = inputPcapFile + LOG_EXTENSION;
+
+        // Prepare file outputter.
+        File outputFile = new File(logFile);
+        outputFile.getParentFile().mkdirs();
+        final PrintWriter resultsWriter = new PrintWriter(new FileWriter(outputFile));
 
         // =========================================== TRAFFIC FILTERING ============================================
 
@@ -107,29 +127,33 @@ public class SignatureGenerator {
         // Group conversations by hostname.
         Map<String, List<Conversation>> convsByHostname =
                 TcpConversationUtils.groupConversationsByHostname(allConversations, dnsMap);
-        System.out.println("Grouped conversations by hostname.");
+        PrintWriterUtils.println("Grouped conversations by hostname.", resultsWriter, DUPLICATE_OUTPUT_TO_STD_OUT);
         // For each hostname, count the frequencies of packet lengths exchanged with that hostname.
         final Map<String, Map<Integer, Integer>> pktLenFreqsByHostname = new HashMap<>();
         convsByHostname.forEach((host, convs) -> pktLenFreqsByHostname.put(host,
                 TcpConversationUtils.countPacketLengthFrequencies(convs)));
-        System.out.println("Counted frequencies of packet lengths exchanged with each hostname.");
+        PrintWriterUtils.println("Counted frequencies of packet lengths exchanged with each hostname.",
+                resultsWriter, DUPLICATE_OUTPUT_TO_STD_OUT);
         // For each hostname, count the frequencies of packet sequences (i.e., count how many
         // conversations exchange a sequence of packets of some specific lengths).
         final Map<String, Map<String, Integer>> pktSeqFreqsByHostname = new HashMap<>();
         convsByHostname.forEach((host, convs) -> pktSeqFreqsByHostname.put(host,
                 TcpConversationUtils.countPacketSequenceFrequencies(convs)));
-        System.out.println("Counted frequencies of packet sequences exchanged with each hostname.");
+        PrintWriterUtils.println("Counted frequencies of packet sequences exchanged with each hostname.",
+                resultsWriter, DUPLICATE_OUTPUT_TO_STD_OUT);
         // For each hostname, count frequencies of packet pairs exchanged
         // with that hostname across all conversations
         final Map<String, Map<String, Integer>> pktPairFreqsByHostname =
                 TcpConversationUtils.countPacketPairFrequenciesByHostname(allConversations, dnsMap);
-        System.out.println("Counted frequencies of packet pairs per hostname");
+        PrintWriterUtils.println("Counted frequencies of packet pairs per hostname.",
+                resultsWriter, DUPLICATE_OUTPUT_TO_STD_OUT);
         // For each user action, reassemble the set of TCP connections occurring shortly after
         final Map<UserAction, List<Conversation>> userActionToConversations =
                 trafficLabeler.getLabeledReassembledTcpTraffic();
         final Map<UserAction, Map<String, List<Conversation>>> userActionsToConvsByHostname =
                 trafficLabeler.getLabeledReassembledTcpTraffic(dnsMap);
-        System.out.println("Reassembled TCP conversations occurring shortly after each user event");
+        PrintWriterUtils.println("Reassembled TCP conversations occurring shortly after each user event.",
+                resultsWriter, DUPLICATE_OUTPUT_TO_STD_OUT);
 
         /*
          * NOTE: no need to generate these more complex on/off maps that also contain mappings from hostname and
@@ -197,16 +221,20 @@ public class SignatureGenerator {
         // Sort the conversations as reference
         List<Conversation> sortedAllConversation = TcpConversationUtils.sortConversationList(allConversations);
         // Output clusters
-        System.out.println("========================================");
-        System.out.println("       Clustering results for ON        ");
-        System.out.println("       Number of clusters: " + onClusters.size());
+        PrintWriterUtils.println("========================================", resultsWriter,
+                DUPLICATE_OUTPUT_TO_STD_OUT);
+        PrintWriterUtils.println("       Clustering results for ON        ", resultsWriter,
+                DUPLICATE_OUTPUT_TO_STD_OUT);
+        PrintWriterUtils.println("       Number of clusters: " + onClusters.size(), resultsWriter,
+                DUPLICATE_OUTPUT_TO_STD_OUT);
         int count = 0;
         List<List<List<PcapPacket>>> ppListOfListReadOn = new ArrayList<>();
         List<List<List<PcapPacket>>> ppListOfListListOn = new ArrayList<>();
         List<List<List<PcapPacket>>> corePointRangeSignatureOn = new ArrayList<>();
         for (Cluster<PcapPacketPair> c : onClusters) {
-            System.out.println(String.format("<<< Cluster #%02d (%03d points) >>>", ++count, c.getPoints().size()));
-            System.out.print(PrintUtils.toSummaryString(c));
+            PrintWriterUtils.println(String.format("<<< Cluster #%02d (%03d points) >>>", ++count, c.getPoints().size()),
+                    resultsWriter, DUPLICATE_OUTPUT_TO_STD_OUT);
+            PrintWriterUtils.println(PrintUtils.toSummaryString(c), resultsWriter, DUPLICATE_OUTPUT_TO_STD_OUT);
             if(c.getPoints().size() > lowerBound && c.getPoints().size() < upperBound) {
                 // Print to file
                 List<List<PcapPacket>> ppListOfList = PcapPacketUtils.clusterToListOfPcapPackets(c);
@@ -215,16 +243,20 @@ public class SignatureGenerator {
                 ppListOfListListOn.add(ppListOfList);
             }
         }
-        System.out.println("========================================");
-        System.out.println("       Clustering results for OFF       ");
-        System.out.println("       Number of clusters: " + offClusters.size());
+        PrintWriterUtils.println("========================================", resultsWriter,
+                DUPLICATE_OUTPUT_TO_STD_OUT);
+        PrintWriterUtils.println("       Clustering results for OFF        ", resultsWriter,
+                DUPLICATE_OUTPUT_TO_STD_OUT);
+        PrintWriterUtils.println("       Number of clusters: " + offClusters.size(), resultsWriter,
+                DUPLICATE_OUTPUT_TO_STD_OUT);
         count = 0;
         List<List<List<PcapPacket>>> ppListOfListReadOff = new ArrayList<>();
         List<List<List<PcapPacket>>> ppListOfListListOff = new ArrayList<>();
         List<List<List<PcapPacket>>> corePointRangeSignatureOff = new ArrayList<>();
         for (Cluster<PcapPacketPair> c : offClusters) {
-            System.out.println(String.format("<<< Cluster #%03d (%06d points) >>>", ++count, c.getPoints().size()));
-            System.out.print(PrintUtils.toSummaryString(c));
+            PrintWriterUtils.println(String.format("<<< Cluster #%03d (%06d points) >>>", ++count, c.getPoints().size()),
+                    resultsWriter, DUPLICATE_OUTPUT_TO_STD_OUT);
+            PrintWriterUtils.println(PrintUtils.toSummaryString(c), resultsWriter, DUPLICATE_OUTPUT_TO_STD_OUT);
             if(c.getPoints().size() > lowerBound && c.getPoints().size() < upperBound) {
                 // Print to file
                 List<List<PcapPacket>> ppListOfList = PcapPacketUtils.clusterToListOfPcapPackets(c);
@@ -237,9 +269,6 @@ public class SignatureGenerator {
         // =========================================== SIGNATURE CREATION ===========================================
         // Concatenate
         ppListOfListListOn = PcapPacketUtils.concatSequences(ppListOfListListOn, sortedAllConversation);
-        // TODO: Need to remove sequence number 0 for TP-Link plug since it is not a good signature!
-        // TODO: This sequence actually belongs to the local communication between the plug and the phone
-//        PcapPacketUtils.removeSequenceFromSignature(ppListOfListListOn, 0);
         // Remove sequences in the list that have overlap
         StringTokenizer stringTokenizerOn = new StringTokenizer(deletedSequencesOn, ",");
         while(stringTokenizerOn.hasMoreTokens()) {
@@ -253,9 +282,6 @@ public class SignatureGenerator {
 
         // Concatenate
         ppListOfListListOff = PcapPacketUtils.concatSequences(ppListOfListListOff, sortedAllConversation);
-        // TODO: Need to remove sequence number 0 for TP-Link plug since it is not a good signature!
-        // TODO: This sequence actually belongs to the local communication between the plug and the phone
-//        PcapPacketUtils.removeSequenceFromSignature(ppListOfListListOff, 0);
         // Remove sequences in the list that have overlap
         StringTokenizer stringTokenizerOff = new StringTokenizer(deletedSequencesOff, ",");
         while(stringTokenizerOff.hasMoreTokens()) {
@@ -266,16 +292,21 @@ public class SignatureGenerator {
             PcapPacketUtils.removeSequenceFromSignature(ppListOfListListOff, sequenceToDelete);
         }
         ppListOfListListOff = PcapPacketUtils.sortSequences(ppListOfListListOff);
-
         // Write the signatures into the screen
-        System.out.println("========================================");
-        System.out.println("              ON Signature              ");
-        System.out.println("========================================");
-        PcapPacketUtils.printSignatures(ppListOfListListOn);
-        System.out.println("========================================");
-        System.out.println("              OFF Signature             ");
-        System.out.println("========================================");
-        PcapPacketUtils.printSignatures(ppListOfListListOff);
+        PrintWriterUtils.println("========================================", resultsWriter,
+                DUPLICATE_OUTPUT_TO_STD_OUT);
+        PrintWriterUtils.println("              ON Signature              ", resultsWriter,
+                DUPLICATE_OUTPUT_TO_STD_OUT);
+        PrintWriterUtils.println("========================================", resultsWriter,
+                DUPLICATE_OUTPUT_TO_STD_OUT);
+        PcapPacketUtils.printSignatures(ppListOfListListOn, resultsWriter, DUPLICATE_OUTPUT_TO_STD_OUT);
+        PrintWriterUtils.println("========================================", resultsWriter,
+                DUPLICATE_OUTPUT_TO_STD_OUT);
+        PrintWriterUtils.println("              OFF Signature             ", resultsWriter,
+                DUPLICATE_OUTPUT_TO_STD_OUT);
+        PrintWriterUtils.println("========================================", resultsWriter,
+                DUPLICATE_OUTPUT_TO_STD_OUT);
+        PcapPacketUtils.printSignatures(ppListOfListListOff, resultsWriter, DUPLICATE_OUTPUT_TO_STD_OUT);
         // Printing signatures into files
         PrintUtils.serializeIntoFile(onSignatureFile, ppListOfListListOn);
         PrintUtils.serializeIntoFile(offSignatureFile, ppListOfListListOff);
@@ -328,9 +359,12 @@ public class SignatureGenerator {
         Iterator<Instant> iterLast = lastSignatureTimestamps.iterator();
         long duration;
         long maxDuration = Long.MIN_VALUE;
-        System.out.println("========================================");
-        System.out.println("           Signature Durations          ");
-        System.out.println("========================================");
+        PrintWriterUtils.println("========================================", resultsWriter,
+                DUPLICATE_OUTPUT_TO_STD_OUT);
+        PrintWriterUtils.println("           Signature Durations          ", resultsWriter,
+                DUPLICATE_OUTPUT_TO_STD_OUT);
+        PrintWriterUtils.println("========================================", resultsWriter,
+                DUPLICATE_OUTPUT_TO_STD_OUT);
         while (iterFirst.hasNext() && iterLast.hasNext()) {
             Instant firstInst = (Instant) iterFirst.next();
             Instant lastInst = (Instant) iterLast.next();
@@ -351,7 +385,7 @@ public class SignatureGenerator {
                 dur = Duration.between(firstInst, lastInst);
                 duration = dur.toMillis();
             }
-            System.out.println(duration);
+            PrintWriterUtils.println(duration, resultsWriter, DUPLICATE_OUTPUT_TO_STD_OUT);
             // Update duration if this bigger than the max value and still less than the window inclusion time
             maxDuration = maxDuration < duration && duration <= TriggerTrafficExtractor.INCLUSION_WINDOW_MILLIS ?
                     duration : maxDuration;
@@ -360,12 +394,14 @@ public class SignatureGenerator {
         if (maxDuration == Long.MIN_VALUE) {
             maxDuration = 0;
         }
-        System.out.println("========================================");
-        System.out.println("Max signature duration: " + maxDuration);
-        System.out.println("========================================");
-
+        PrintWriterUtils.println("========================================", resultsWriter,
+                DUPLICATE_OUTPUT_TO_STD_OUT);
+        PrintWriterUtils.println("Max signature duration: " + maxDuration, resultsWriter,
+                DUPLICATE_OUTPUT_TO_STD_OUT);
+        PrintWriterUtils.println("========================================", resultsWriter,
+                DUPLICATE_OUTPUT_TO_STD_OUT);
+        resultsWriter.flush();
+        resultsWriter.close();
         // ==========================================================================================================
     }
-
-
-}
+}
\ No newline at end of file

diff --git a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/io/PrintWriterUtils.java b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/io/PrintWriterUtils.java
index 167993de175850a2cbd55ad5da2b1ad26e36dabf..50b6b13c3756a4a3bb87808d9928dd8e6fca0c6f 100644 (file)
--- a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/io/PrintWriterUtils.java
+++ b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/io/PrintWriterUtils.java
@@ -28,6 +28,20 @@ public final class PrintWriterUtils {
         writer.println(line);
     }
 
+    /**
+     * Invoke {@link PrintWriter#println(Object)} passing {@code line} as argument while also printing {@code line} to
+     * standard output if {@code duplicateToStdOut} is {@code true}.
+     * @param line The line to be printed.
+     * @param writer The {@link PrintWriter} that is to print {@code line}.
+     * @param duplicateToStdOut Set to {@code true} if {@code line} should also be printed in standard output.
+     */
+    public static void print(Object line, PrintWriter writer, boolean duplicateToStdOut) {
+        if (duplicateToStdOut) {
+            System.out.print(line);
+        }
+        writer.print(line);
+    }
+
     /**
      * Make writer (and standard output, if {@code duplicateToStdOut} is {@code true}) print an empty line.
      * @param writer The writer that {@link PrintWriter#println()} is to be invoked on.

diff --git a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/util/PcapPacketUtils.java b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/util/PcapPacketUtils.java
index 2f20945921b25c54f354835f1f0d28d03c7d2a35..b4535668e3fec80615742fbe0f2b6e26d70d01eb 100644 (file)
--- a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/util/PcapPacketUtils.java
+++ b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/util/PcapPacketUtils.java
@@ -1,5 +1,6 @@
 package edu.uci.iotproject.util;
 
+import edu.uci.iotproject.io.PrintWriterUtils;
 import edu.uci.iotproject.trafficreassembly.layer3.Conversation;
 import edu.uci.iotproject.analysis.PcapPacketPair;
 import edu.uci.iotproject.analysis.TcpConversationUtils;
@@ -11,6 +12,7 @@ import org.pcap4j.packet.IpV4Packet;
 import org.pcap4j.packet.TcpPacket;
 import org.pcap4j.util.MacAddress;
 
+import java.io.PrintWriter;
 import java.util.*;
 
 /**
@@ -431,26 +433,32 @@ public final class PcapPacketUtils {
      *
      * @param signatures A {@link List} of {@link List} of {@link List} of
      *          {@link PcapPacket} objects that needs to be printed.
+     * @param resultsWriter PrintWriter object to write into log file.
+     * @param printToOutput Boolean to decide whether to print out to screen or just log file.
      */
-    public static void printSignatures(List<List<List<PcapPacket>>> signatures) {
+    public static void printSignatures(List<List<List<PcapPacket>>> signatures, PrintWriter resultsWriter, boolean
+                                       printToOutput) {
 
         // Iterate over the list of all clusters/sequences
         int sequenceCounter = 0;
         for(List<List<PcapPacket>> listListPcapPacket : signatures) {
             // Iterate over every member of a cluster/sequence
-            System.out.print("====== SEQUENCE " + ++sequenceCounter);
-            System.out.println(" - " + listListPcapPacket.size() + " MEMBERS ======");
+            PrintWriterUtils.print("====== SEQUENCE " + ++sequenceCounter, resultsWriter, printToOutput);
+            PrintWriterUtils.println(" - " + listListPcapPacket.size() + " MEMBERS ======", resultsWriter,
+                    printToOutput);
             for(List<PcapPacket> listPcapPacket : listListPcapPacket) {
                 // Print out packet lengths in a sequence
                 int packetCounter = 0;
                 for(PcapPacket pcapPacket : listPcapPacket) {
                     if(pcapPacket != null) {
-                        System.out.print(pcapPacket.length());
+                        PrintWriterUtils.print(pcapPacket.length(), resultsWriter, printToOutput);
                     }
                     if(packetCounter < listPcapPacket.size() - 1) {
-                        System.out.print(" ");  // Provide space if not last packet
+                        // Provide space if not last packet
+                        PrintWriterUtils.print(" ", resultsWriter, printToOutput);
                     } else {
-                        System.out.println();      // Newline if last packet
+                        // Newline if last packet
+                        PrintWriterUtils.println("", resultsWriter, printToOutput);
                     }
                     packetCounter++;
                 }