From 6d0bfe17c889d3b0538ce1cb078fbc4e2694475f Mon Sep 17 00:00:00 2001 From: rtrimana Date: Mon, 4 Feb 2019 17:42:11 -0800 Subject: [PATCH] Adding PacketLevelSignatureExtractor. --- .../edu/uci/iotproject/ConversationPair.java | 149 -------- .../edu/uci/iotproject/FlowPatternFinder.java | 357 ------------------ 2 files changed, 506 deletions(-) delete mode 100644 Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/ConversationPair.java delete mode 100644 Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/FlowPatternFinder.java diff --git a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/ConversationPair.java b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/ConversationPair.java deleted file mode 100644 index b864ee5..0000000 --- a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/ConversationPair.java +++ /dev/null @@ -1,149 +0,0 @@ -package edu.uci.iotproject; - -import org.pcap4j.core.PcapHandle; -import org.pcap4j.core.PcapPacket; - -import java.io.FileNotFoundException; -import java.io.PrintWriter; -import java.io.UnsupportedEncodingException; -import java.util.HashMap; -import java.util.Map; - -/** - * Models a (TCP) conversation/connection/session/flow (packet's belonging to the same session between a client and a - * server). - * Holds a pair of packet lengths from {@link PcapPacket}s identified as pertaining to the flow. - * Here we consider pairs of packet lengths, e.g., from device to cloud and cloud to device. - * We collect these pairs of data points as signatures that we can plot on a graph. - * - * @author Janus Varmarken {@literal } - * @author Rahmadi Trimananda {@literal } - */ -public class ConversationPair { - - /* Begin instance properties */ - /** - * The PrintWriter object that writes data points into file - */ - private PrintWriter pw; - - /** - * The direction of conversation - * true = device to server to device - */ - private Direction direction; - - /** - * If this is the first packet processed then the value is true (it is false otherwise). - */ - private boolean firstPacket; - - /** - * Count the frequencies of points - */ - private Map pointFreq; - private String dataPoint; - - /** - * Four possible directions of conversations. - * E.g., DEVICE_TO_SERVER means the conversation is started from - * a device-server packet and then a server-device as a response. - * SERVER_TO_DEVICE means the conversation is started from a - * server-device packet and then a device-server packet as a response. - * The same pattern applies to PHONE_TO_SERVER and SERVER_TO_PHONE - * directions. - */ - public enum Direction { - DEVICE_TO_SERVER, - SERVER_TO_DEVICE, - PHONE_TO_SERVER, - SERVER_TO_PHONE - } - - /** - * Constructs a ConversationPair object. - * @param fileName The file name to write data points into. - * @param direction The direction of the first packet of the pair. - */ - public ConversationPair(String fileName, Direction direction) { - try { - this.pw = new PrintWriter(fileName, "UTF-8"); - } catch(UnsupportedEncodingException | - FileNotFoundException e) { - e.printStackTrace(); - } - this.direction = direction; - this.firstPacket = true; - this.pointFreq = new HashMap<>(); - this.dataPoint = null; - } - - /** - * Writes conversation pair's packet lengths. - * @param packet The {@link PcapPacket} object that has packet information. - * @param fromClient If true then this packet comes from client, e.g., device. - * @param fromServer If true then this packet comes from server. - */ - public void writeConversationPair(PcapPacket packet, boolean fromClient, boolean fromServer) { - - // Write device data point first and then server - if (direction == Direction.DEVICE_TO_SERVER || direction == Direction.PHONE_TO_SERVER) { - if (fromClient && firstPacket) { // first packet - pw.print(packet.getTimestamp() + ", " + packet.getPayload().length() + ", "); - System.out.print(packet.getTimestamp() + ", " + packet.getPayload().length() + ", "); - dataPoint = Integer.toString(packet.getPayload().length()) + ", "; - firstPacket = false; - } else if (fromServer && !firstPacket) { // second packet - pw.println(packet.getPayload().length()); - System.out.println(packet.getPayload().length()); - dataPoint = dataPoint + Integer.toString(packet.getPayload().length()); - countFrequency(dataPoint); - firstPacket = true; - } - // Write server data point first and then device - } else if (direction == Direction.SERVER_TO_DEVICE || direction == Direction.SERVER_TO_PHONE) { - if (fromServer && firstPacket) { // first packet - pw.print(packet.getTimestamp() + ", " + packet.getPayload().length() + ", "); - dataPoint = Integer.toString(packet.getPayload().length()) + ", "; - firstPacket = false; - } else if (fromClient && !firstPacket) { // second packet - pw.println(packet.getPayload().length()); - dataPoint = dataPoint + Integer.toString(packet.getPayload().length()); - countFrequency(dataPoint); - firstPacket = true; - } - } - } - - /** - * Counts the frequencies of data points. - * @param dataPoint One data point for a conversation pair, e.g., 556, 1232. - */ - private void countFrequency(String dataPoint) { - - Integer freq = null; - if (pointFreq.containsKey(dataPoint)) { - freq = pointFreq.get(dataPoint); - } else { - freq = new Integer(0); - } - freq = freq + 1; - pointFreq.put(dataPoint, freq); - } - - /** - * Prints the frequencies of data points from the Map. - */ - public void printListFrequency() { - for(Map.Entry entry : pointFreq.entrySet()) { - System.out.println(entry.getKey() + " - " + entry.getValue()); - } - } - - /** - * Close the PrintWriter object. - */ - public void close() { - pw.close(); - } -} \ No newline at end of file diff --git a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/FlowPatternFinder.java b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/FlowPatternFinder.java deleted file mode 100644 index c384852..0000000 --- a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/FlowPatternFinder.java +++ /dev/null @@ -1,357 +0,0 @@ -package edu.uci.iotproject; - -import edu.uci.iotproject.comparison.ComparisonFunctions; -import edu.uci.iotproject.comparison.CompleteMatchPatternComparisonResult; -import edu.uci.iotproject.comparison.PatternComparisonTask; -import edu.uci.iotproject.trafficreassembly.layer3.Conversation; -import org.pcap4j.core.NotOpenException; -import org.pcap4j.core.PcapHandle; -import org.pcap4j.core.PcapNativeException; -import org.pcap4j.core.PcapPacket; -import org.pcap4j.packet.DnsPacket; -import org.pcap4j.packet.IpV4Packet; -import org.pcap4j.packet.TcpPacket; - -import java.io.*; -import java.net.UnknownHostException; -import java.text.DateFormat; -import java.text.SimpleDateFormat; -import java.util.*; -import java.util.concurrent.*; - - -/** - *

Provides functionality for searching for the presence of a {@link FlowPattern} in a PCAP trace.

- * - *

- * The (entire) PCAP trace is traversed and parsed on one thread (specifically, the thread that calls - * {@link #findFlowPattern()}). This thread builds a {@link DnsMap} using the DNS packets present in the trace and uses - * that {@code DnsMap} to reassemble {@link Conversation}s that potentially match the provided - * {@link FlowPattern} (in that one end/party of said conversations matches the hostname(s) specified by the given - * {@code FlowPattern}). - * These potential matches are then examined on background worker thread(s) to determine if they are indeed a (complete) - * match of the provided {@code FlowPattern}. - *

- * - * @author Janus Varmarken {@literal } - * @author Rahmadi Trimananda {@literal } - */ -public class FlowPatternFinder { - - /* Begin class properties */ - /** - * {@link ExecutorService} responsible for parallelizing pattern searches. - * Declared as static to allow for reuse of threads across different instances of {@code FlowPatternFinder} and to - * avoid the overhead of initializing a new thread pool for each {@code FlowPatternFinder} instance. - */ - private static final ExecutorService EXECUTOR_SERVICE = Executors.newCachedThreadPool(); - /* End class properties */ - - /* Begin instance properties */ - /** - * Holds a set of {@link Conversation}s that potentially match {@link #mPattern} since each individual - * {@code Conversation} is communication with the hostname identified by {@code mPattern.getHostname()}. - * Note that due to limitations of the {@link Set} interface (specifically, there is no {@code get(T t)} method), - * we have to resort to a {@link Map} (in which keys map to themselves) to "mimic" a set with {@code get(T t)} - * functionality. - * - * @see this question on StackOverflow.com - */ - private final Map mConversations; - - /** - * Holds a list of trigger times. - */ - private final List mTriggerTimes; - private static int triggerListCounter; - - private final DnsMap mDnsMap; - private final PcapHandle mPcap; - private final FlowPattern mPattern; - private final ConversationPair mConvPair; - private final String FILE = "./devices/tplink_switch/datapoints.csv"; - //private final String REF_FILE = "./devices/dlink_switch/dlink-june-26-2018.timestamps"; - private final String REF_FILE = "./devices/tplink_switch/tplink-june-14-2018.timestamps"; - //private final String REF_FILE = "./devices/tplink_switch/tplink-feb-13-2018.timestamps"; - // Router time is in CET and we use PST for the trigger times - // Difference is 7 hours x 3600 x 1000ms = 25,200,000ms - private final long TIME_OFFSET = 25200000; - - private final List> mPendingComparisons = new ArrayList<>(); - /* End instance properties */ - - /** - * Constructs a new {@code FlowPatternFinder}. - * @param pcap an open {@link PcapHandle} that provides access to the trace that is to be examined. - * @param pattern the {@link FlowPattern} to search for. - */ - public FlowPatternFinder(PcapHandle pcap, FlowPattern pattern) { - this.mConversations = new HashMap<>(); - this.mTriggerTimes = readTriggerTimes(REF_FILE); - triggerListCounter = 0; - this.mDnsMap = new DnsMap(); - this.mPcap = Objects.requireNonNull(pcap, - String.format("Argument of type '%s' cannot be null", PcapHandle.class.getSimpleName())); - this.mPattern = Objects.requireNonNull(pattern, - String.format("Argument of type '%s' cannot be null", FlowPattern.class.getSimpleName())); - this.mConvPair = new ConversationPair(FILE, ConversationPair.Direction.DEVICE_TO_SERVER); - } - - - private List readTriggerTimes(String refFileName) { - - List listTriggerTimes = new ArrayList<>(); - try { - File file = new File(refFileName); - BufferedReader br = new BufferedReader(new FileReader(file)); - String s; - while ((s = br.readLine()) != null) { - listTriggerTimes.add(timeToMillis(s, false)); - } - } catch (IOException e) { - e.printStackTrace(); - } - System.out.println("List has: " + listTriggerTimes.size()); - - return listTriggerTimes; - } - - /** - * Starts the pattern search. - */ - public void start() { - - //findFlowPattern(); - findSignatureBasedOnTimestamp(); - } - - /** - * Find patterns based on the FlowPattern object (run by a thread) - */ - private void findFlowPattern() { - try { - PcapPacket packet; -// TODO: The new comparison method is pending -// TODO: For now, just compare using one hostname and one list per FlowPattern -// List hostnameList = mPattern.getHostnameList(); -// int hostIndex = 0; - while ((packet = mPcap.getNextPacketEx()) != null) { - // Let DnsMap handle DNS packets. - if (packet.get(DnsPacket.class) != null) { - // Check if this is a valid DNS packet - mDnsMap.validateAndAddNewEntry(packet); - continue; - } - // For now, we only work support pattern search in TCP over IPv4. - final IpV4Packet ipPacket = packet.get(IpV4Packet.class); - final TcpPacket tcpPacket = packet.get(TcpPacket.class); - if (ipPacket == null || tcpPacket == null) { - continue; - } - - String srcAddress = ipPacket.getHeader().getSrcAddr().getHostAddress(); - String dstAddress = ipPacket.getHeader().getDstAddr().getHostAddress(); - int srcPort = tcpPacket.getHeader().getSrcPort().valueAsInt(); - int dstPort = tcpPacket.getHeader().getDstPort().valueAsInt(); - // Is this packet related to the pattern; i.e. is it going to (or coming from) the cloud server? - boolean fromServer = mDnsMap.isRelatedToCloudServer(srcAddress, mPattern.getHostname()); - boolean fromClient = mDnsMap.isRelatedToCloudServer(dstAddress, mPattern.getHostname()); -// String currentHostname = hostnameList.get(hostIndex); -// boolean fromServer = mDnsMap.isRelatedToCloudServer(srcAddress, currentHostname); -// boolean fromClient = mDnsMap.isRelatedToCloudServer(dstAddress, currentHostname); - if (!fromServer && !fromClient) { - // Packet not related to pattern, skip it. - continue; - } - - // Conversations (connections/sessions) are identified by the four-tuple - // (clientIp, clientPort, serverIp, serverPort) (see Conversation Javadoc). - // Create "dummy" conversation for looking up an existing entry. - Conversation conversation = fromClient ? new Conversation(srcAddress, srcPort, dstAddress, dstPort) : - new Conversation(dstAddress, dstPort, srcAddress, srcPort); - // Add the packet so that the "dummy" conversation can be immediately added to the map if no entry - // exists for the conversation that the current packet belongs to. - if (tcpPacket.getHeader().getFin()) { - // Record FIN packets. - conversation.addFinPacket(packet); - } - if (tcpPacket.getPayload() != null) { - // Record regular payload packets. - conversation.addPacket(packet, true); - } - // Note: does not make sense to call attemptAcknowledgementOfFin here as the new packet has no FINs - // in its list, so if this packet is an ACK, it would not be added anyway. - // Need to retain a final reference to get access to the packet in the lambda below. - final PcapPacket finalPacket = packet; - // Add the new conversation to the map if an equal entry is not already present. - // If an existing entry is already present, the current packet is simply added to that conversation. - mConversations.merge(conversation, conversation, (existingEntry, toMerge) -> { - // toMerge may not have any payload packets if the current packet is a FIN packet. - if (toMerge.getPackets().size() > 0) { - existingEntry.addPacket(toMerge.getPackets().get(0), true); - } - if (toMerge.getFinAckPairs().size() > 0) { - // Add the FIN packet to the existing entry. - existingEntry.addFinPacket(toMerge.getFinAckPairs().get(0).getFinPacket()); - } - if (finalPacket.get(TcpPacket.class).getHeader().getAck()) { - existingEntry.attemptAcknowledgementOfFin(finalPacket); - } - return existingEntry; - }); - // Refresh reference to point to entry in map (in case packet was added to existing entry). - conversation = mConversations.get(conversation); - if (conversation.isGracefullyShutdown()) { - // Conversation terminated gracefully, so we can now start analyzing it. - // Remove the Conversation from the map and start the analysis. - // Any future packets identified by the same four tuple will be tied to a new Conversation instance. - mConversations.remove(conversation); - // Create comparison task and send to executor service. - PatternComparisonTask comparisonTask = - new PatternComparisonTask<>(conversation, mPattern, ComparisonFunctions.SUB_SEQUENCE_COMPLETE_MATCH); - mPendingComparisons.add(EXECUTOR_SERVICE.submit(comparisonTask)); - // Increment hostIndex to find the next - - } - } - } catch (EOFException eofe) { - // TODO should check for leftover conversations in map here and fire tasks for those. - // TODO [cont'd] such tasks may be present if connections did not terminate gracefully or if there are longlived connections. - System.out.println("[ findFlowPattern ] Finished processing entire PCAP stream!"); - System.out.println("[ findFlowPattern ] Now waiting for comparisons to finish..."); - // Wait for all comparisons to finish, then output their results to std.out. - for(Future comparisonTask : mPendingComparisons) { - try { - // Blocks until result is ready. - CompleteMatchPatternComparisonResult comparisonResult = comparisonTask.get(); - if (comparisonResult.getResult()) { - System.out.println(comparisonResult.getTextualDescription()); - } - } catch (InterruptedException|ExecutionException e) { - e.printStackTrace(); - } - } - } catch (UnknownHostException | - PcapNativeException | - NotOpenException | - TimeoutException ex) { - ex.printStackTrace(); - } - } - - /** - * Find patterns based on the FlowPattern object (run by a thread) - */ - private void findSignatureBasedOnTimestamp() { - try { - PcapPacket packet; -// TODO: The new comparison method is pending -// TODO: For now, just compare using one hostname and one list per FlowPattern - while ((packet = mPcap.getNextPacketEx()) != null) { - // Let DnsMap handle DNS packets. - if (packet.get(DnsPacket.class) != null) { - // Check if this is a valid DNS packet - mDnsMap.validateAndAddNewEntry(packet); - continue; - } - // For now, we only work support pattern search in TCP over IPv4. - final IpV4Packet ipPacket = packet.get(IpV4Packet.class); - final TcpPacket tcpPacket = packet.get(TcpPacket.class); - if (ipPacket == null || tcpPacket == null) { - continue; - } - - String srcAddress = ipPacket.getHeader().getSrcAddr().getHostAddress(); - String dstAddress = ipPacket.getHeader().getDstAddr().getHostAddress(); - int srcPort = tcpPacket.getHeader().getSrcPort().valueAsInt(); - int dstPort = tcpPacket.getHeader().getDstPort().valueAsInt(); - //System.out.println("Timestamp packet: " + packet.getTimestamp()); - // Is this packet related to the pattern; i.e. is it going to (or coming from) the cloud server? - boolean fromServer = mDnsMap.isRelatedToCloudServer(srcAddress, mPattern.getHostname()); - boolean fromClient = mDnsMap.isRelatedToCloudServer(dstAddress, mPattern.getHostname()); - if (!fromServer && !fromClient) { - // Packet not related to pattern, skip it. - continue; - } - // Record the conversation pairs - if (tcpPacket.getPayload() != null && checkTimeStamp(packet)) { - //if (tcpPacket.getPayload() != null) { - mConvPair.writeConversationPair(packet, fromClient, fromServer); - } - } - } catch (EOFException eofe) { - triggerListCounter = 0; - mConvPair.close(); - System.out.println("[ findFlowPattern ] ConversationPair writer closed!"); - System.out.println("[ findFlowPattern ] Frequencies of data points:"); - mConvPair.printListFrequency(); - } catch (UnknownHostException | - PcapNativeException | - NotOpenException | - TimeoutException ex) { - ex.printStackTrace(); - } - } - - private boolean checkTimeStamp(PcapPacket packet) { - - // Extract time from the packet's timestamp - String timeStamp = packet.getTimestamp().toString(); - String timeString = timeStamp.substring(timeStamp.indexOf("T") + 1, timeStamp.indexOf(".")); - // Timestamps are in CET (ahead of PST) so it should be deducted by TIME_OFFSET - long time = timeToMillis(timeString, true) - TIME_OFFSET; - //long time = timeToMillis(timeString, true); - - //System.out.println("Gets here: " + time + " trigger time: " + mTriggerTimes.get(triggerListCounter)); - - // We accept packets that are at most 3 seconds away from the trigger time - if ((mTriggerTimes.get(triggerListCounter) <= time) && - (time <= mTriggerTimes.get(triggerListCounter) + 3000)) { - //System.out.println("Gets here 1: " + timeString + " index: " + triggerListCounter); - return true; - } else { - // Handle the case that the timestamp is > 3000, but < next timestamp - // in the list. We ignore these packets. - if (time < mTriggerTimes.get(triggerListCounter)) { - // Timestamp is smaller than trigger, ignore! - //System.out.println("Gets here 2: " + timeString + " index: " + triggerListCounter); - return false; - } else { // Timestamp is greater than trigger, increment! - triggerListCounter = triggerListCounter + 1; - //System.out.println("Gets here 3: " + timeString + " index: " + triggerListCounter); - //return false; - return checkTimeStamp(packet); - } - } - - //System.out.println("Timestamp: " + timeToMillis(time, true)); - //String time2 = "21:38:08"; - //System.out.println("Timestamp: " + timeToMillis(time2, true)); - } - - /** - * A private function that returns time in milliseconds. - * @param time The time in the form of String. - * @param is24Hr If true, then this is in 24-hour format. - */ - private long timeToMillis(String time, boolean is24Hr) { - - String format = null; - if (is24Hr) { - format = "hh:mm:ss"; - } else { // 12 Hr format - format = "hh:mm:ss aa"; - } - DateFormat sdf = new SimpleDateFormat(format); - Date date = null; - try { - date = sdf.parse(time); - } catch(Exception e) { - e.printStackTrace(); - } - if (date == null) - return 0; - return date.getTime(); - } - -} -- 2.34.1