X-Git-Url: http://demsky.eecs.uci.edu/git/?a=blobdiff_plain;f=Code%2FProjects%2FPacketLevelSignatureExtractor%2Fsrc%2Fmain%2Fjava%2Fedu%2Fuci%2Fiotproject%2Futil%2FPcapPacketUtils.java;h=c1a1a25150b112a6def2e09b867a12772027eeb3;hb=39172356d48f5cd574ef15ec276a33de9146155a;hp=de89dc4b3115a9b488b1f458328821452fe1fc26;hpb=b2ae840137933954780a83dccbf0642b3b88513f;p=pingpong.git diff --git a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/util/PcapPacketUtils.java b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/util/PcapPacketUtils.java index de89dc4..c1a1a25 100644 --- a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/util/PcapPacketUtils.java +++ b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/util/PcapPacketUtils.java @@ -1,5 +1,6 @@ package edu.uci.iotproject.util; +import edu.uci.iotproject.io.PrintWriterUtils; import edu.uci.iotproject.trafficreassembly.layer3.Conversation; import edu.uci.iotproject.analysis.PcapPacketPair; import edu.uci.iotproject.analysis.TcpConversationUtils; @@ -11,6 +12,7 @@ import org.pcap4j.packet.IpV4Packet; import org.pcap4j.packet.TcpPacket; import org.pcap4j.util.MacAddress; +import java.io.PrintWriter; import java.util.*; /** @@ -214,51 +216,51 @@ public final class PcapPacketUtils { } /** - * Merge signatures in {@code List} of {@code List} of {@code List} of {@code PcapPacket} objects. + * Concatenate sequences in {@code List} of {@code List} of {@code List} of {@code PcapPacket} objects. * We cross-check these with {@code List} of {@code Conversation} objects to see * if two {@code List} of {@code PcapPacket} objects actually belong to the same {@code Conversation}. * @param signatures A {@link List} of {@link List} of {@link List} of - * {@link PcapPacket} objects that needs to be checked and merged. - * @param conversations A {@link List} of {@link Conversation} objects as reference for merging. + * {@link PcapPacket} objects that needs to be checked and concatenated. + * @param conversations A {@link List} of {@link Conversation} objects as reference for concatenation. * @return A {@link List} of {@link List} of {@link List} of - * {@link PcapPacket} objects as the result of the merging. + * {@link PcapPacket} objects as the result of the concatenation. */ public static List>> - mergeSignatures(List>> signatures, List conversations) { + concatSequences(List>> signatures, List conversations) { // TODO: THIS IS NOT A DEEP COPY; IT BASICALLY CREATES A REFERENCE TO THE SAME LIST OBJECT // List>> copySignatures = new ArrayList<>(signatures); // Make a deep copy first. List>> copySignatures = new ArrayList<>(); listDeepCopy(copySignatures, signatures); - // Traverse and look into the pairs of signatures. + // Traverse and look into the pairs. for (int first = 0; first < signatures.size(); first++) { List> firstList = signatures.get(first); for (int second = first+1; second < signatures.size(); second++) { - int maxSignatureEl = 0; // Number of maximum signature elements. + int maxSignatureEl = 0; List> secondList = signatures.get(second); int initialSecondListMembers = secondList.size(); - // Iterate over the signatures in the first list. + // Iterate over the sequences in the first list. for (List signature : firstList) { signature.removeIf(el -> el == null); // Clean up null elements. - // Return the Conversation that the signature is part of. + // Return the Conversation that the sequence is part of. Conversation conv = TcpConversationUtils.returnConversation(signature, conversations); // Find the element of the second list that is a match for that Conversation. for (List ppList : secondList) { ppList.removeIf(el -> el == null); // Clean up null elements. - // Check if they are part of a Conversation and are adjacent to the first signature. + // Check if they are part of a Conversation and are adjacent to the first sequence. // If yes then merge into the first list. TcpConversationUtils.SignaturePosition position = TcpConversationUtils.isPartOfConversationAndAdjacent(signature, ppList, conv); if (position == TcpConversationUtils.SignaturePosition.LEFT_ADJACENT) { - // Merge to the left side of the first signature. + // Merge to the left side of the first sequence. ppList.addAll(signature); signature = ppList; maxSignatureEl = signature.size() > maxSignatureEl ? signature.size() : maxSignatureEl; secondList.remove(ppList); // Remove as we merge. break; } else if (position == TcpConversationUtils.SignaturePosition.RIGHT_ADJACENT) { - // Merge to the right side of the first signature. + // Merge to the right side of the first sequence. signature.addAll(ppList); maxSignatureEl = signature.size() > maxSignatureEl ? signature.size() : maxSignatureEl; secondList.remove(ppList); // Remove as we merge. @@ -269,16 +271,16 @@ public final class PcapPacketUtils { // Call it a successful merging if there are only less than 5 elements from the second list that // cannot be merged. if (secondList.size() < SIGNATURE_MERGE_THRESHOLD) { - // Prune the unsuccessfully merged signatures (i.e., these will have size() < maxSignatureEl). + // Prune the unsuccessfully merged sequences (i.e., these will have size() < maxSignatureEl). final int maxNumOfEl = maxSignatureEl; // TODO: DOUBLE CHECK IF WE REALLY NEED TO PRUNE FAILED BINDINGS // TODO: SOMETIMES THE SEQUENCES ARE JUST INCOMPLETE // TODO: AND BOTH THE COMPLETE AND INCOMPLETE SEQUENCES ARE VALID SIGNATURES! firstList.removeIf(el -> el.size() < maxNumOfEl); - // Remove the merged set of signatures when successful. + // Remove the merged set of sequences when successful. signatures.remove(secondList); } else if (secondList.size() < initialSecondListMembers) { - // If only some of the signatures from the second list are merged, this means UNSUCCESSFUL merging. + // If only some of the sequences from the second list are merged, this means UNSUCCESSFUL merging. // Return the original copy of the signatures object. return copySignatures; } @@ -310,9 +312,9 @@ public final class PcapPacketUtils { } /** - * Sort the signatures in the {@code List} of {@code List} of {@code List} of {@code PcapPacket} objects. - * The purpose of this is to sort the order of signatures in the signature list. For detection purposes, we need - * to know if one signature occurs earlier/later in time with respect to the other signatures for more confidence + * Sort the sequences in the {@code List} of {@code List} of {@code List} of {@code PcapPacket} objects. + * The purpose of this is to sort the order of sequences in the sequence list. For detection purposes, we need + * to know if one sequence occurs earlier/later in time with respect to the other sequences for more confidence * in detecting the occurrence of an event. * @param signatures A {@code List} of {@code List} of {@code List} of {@code PcapPacket} objects that needs sorting. * We assume that innermost {@code List} of {@code PcapPacket} objects have been sorted ascending @@ -320,7 +322,7 @@ public final class PcapPacketUtils { * {@code clusterToListOfPcapPackets} method. * @return A sorted {@code List} of {@code List} of {@code List} of {@code PcapPacket} objects. */ - public static List>> sortSignatures(List>> signatures) { + public static List>> sortSequences(List>> signatures) { // TODO: This is the simplest solution!!! Might not cover all corner cases. // TODO: Sort the list of lists based on the first packet's timestamps! // Collections.sort(signatures, (p1, p2) -> { @@ -346,17 +348,8 @@ public final class PcapPacketUtils { if (Math.abs(timestamp1 - timestamp2) < TriggerTrafficExtractor.INCLUSION_WINDOW_MILLIS) { // If these two are within INCLUSION_WINDOW_MILLIS window then compare! compare = p1.get(count1).get(0).getTimestamp().compareTo(p2.get(count2).get(0).getTimestamp()); -// if (comparePrev != 0) { // First time since it is 0 -// if (Integer.signum(compare) != Integer.signum(comparePrev)) { -// // Throw an exception if the order of the two signatures is not consistent, -// // E.g., 111, 222, 333 in one occassion and 222, 333, 111 in the other. -// throw new Error("OVERLAP WARNING: " + "" + -// "Please remove one of the sequences: " + -// p1.get(0).get(0).length() + "... OR " + -// p2.get(0).get(0).length() + "..."); -// } -// } - overlapChecking(compare, comparePrev, p1.get(count1), p2.get(count2)); + overlapChecking(compare, comparePrev, p1.get(count1), p2.get(count2), + signatures.indexOf(p1), signatures.indexOf(p2)); comparePrev = compare; count1++; count2++; @@ -380,8 +373,12 @@ public final class PcapPacketUtils { * @param comparePrev Previous comparison value between packet sequences p1 and p2 * @param sequence1 The packet sequence ({@link List} of {@link PcapPacket} objects). * @param sequence2 The packet sequence ({@link List} of {@link PcapPacket} objects). + * @param indexSequence1 The index of packet sequence ({@link List} of {@link PcapPacket} objects). + * @param indexSequence2 The index of packet sequence ({@link List} of {@link PcapPacket} objects). */ - private static void overlapChecking(int compare, int comparePrev, List sequence1, List sequence2) { + private static void overlapChecking(int compare, int comparePrev, + List sequence1, List sequence2, + int indexSequence1, int indexSequence2) { // Check if p1 occurs before p2 but both have same overlap if (comparePrev != 0) { // First time since it is 0 @@ -390,8 +387,8 @@ public final class PcapPacketUtils { // E.g., 111, 222, 333 in one occassion and 222, 333, 111 in the other. throw new Error("OVERLAP WARNING: " + "" + "Two sequences have some overlap. Please remove one of the sequences: " + - sequence1.get(0).length() + "... OR " + - sequence2.get(0).length() + "..."); + sequence1.get(0).length() + " with index " + indexSequence1 + " OR " + + sequence2.get(0).length() + " with index " + indexSequence2); } } // Check if p1 is longer than p2 and p2 occurs during the occurrence of p1 @@ -441,26 +438,32 @@ public final class PcapPacketUtils { * * @param signatures A {@link List} of {@link List} of {@link List} of * {@link PcapPacket} objects that needs to be printed. + * @param resultsWriter PrintWriter object to write into log file. + * @param printToOutput Boolean to decide whether to print out to screen or just log file. */ - public static void printSignatures(List>> signatures) { + public static void printSignatures(List>> signatures, PrintWriter resultsWriter, boolean + printToOutput) { // Iterate over the list of all clusters/sequences int sequenceCounter = 0; for(List> listListPcapPacket : signatures) { // Iterate over every member of a cluster/sequence - System.out.print("====== SEQUENCE " + ++sequenceCounter); - System.out.println(" - " + listListPcapPacket.size() + " MEMBERS ======"); + PrintWriterUtils.print("====== SEQUENCE " + ++sequenceCounter, resultsWriter, printToOutput); + PrintWriterUtils.println(" - " + listListPcapPacket.size() + " MEMBERS ======", resultsWriter, + printToOutput); for(List listPcapPacket : listListPcapPacket) { // Print out packet lengths in a sequence int packetCounter = 0; for(PcapPacket pcapPacket : listPcapPacket) { if(pcapPacket != null) { - System.out.print(pcapPacket.length()); + PrintWriterUtils.print(pcapPacket.length(), resultsWriter, printToOutput); } if(packetCounter < listPcapPacket.size() - 1) { - System.out.print(" "); // Provide space if not last packet + // Provide space if not last packet + PrintWriterUtils.print(" ", resultsWriter, printToOutput); } else { - System.out.println(); // Newline if last packet + // Newline if last packet + PrintWriterUtils.println("", resultsWriter, printToOutput); } packetCounter++; } @@ -674,7 +677,10 @@ public final class PcapPacketUtils { // Get the ranges of the two signatures List>> signatureRanges = getSequenceRanges(signature); List>> otherSignatureRanges = getSequenceRanges(otherSignature); - if (!isRangeBased(signatureRanges) && !isRangeBased(otherSignatureRanges)) { + if (signature.size() == 1 && signature.get(0).get(0).size() == 2) { + // The signature only has 2 packets + return true; + } else if (!isRangeBased(signatureRanges) && !isRangeBased(otherSignatureRanges)) { // Conservative checking when there is no range return true; } else if(signatureRanges.size() != otherSignatureRanges.size()) {