From ec18f2088211f3e2ef5e10ae79c00e64c8ae5042 Mon Sep 17 00:00:00 2001 From: rtrimana Date: Fri, 4 May 2018 14:10:03 -0700 Subject: [PATCH] Adding feature to hold multiple hostnames and lists of packet orders in FlowPattern, but keeping the old method of searching patterns (i.e. just one hostname and one pattern) for now---need to think more carefully so that we won't break stuff. --- .../java/edu/uci/iotproject/FlowPattern.java | 169 ++++++++++++------ .../edu/uci/iotproject/FlowPatternFinder.java | 18 +- .../main/java/edu/uci/iotproject/Main.java | 18 +- .../comparison/PatternComparisonTask.java | 8 +- 4 files changed, 148 insertions(+), 65 deletions(-) diff --git a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/FlowPattern.java b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/FlowPattern.java index fb0433c..aca63ab 100644 --- a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/FlowPattern.java +++ b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/FlowPattern.java @@ -5,44 +5,24 @@ import org.pcap4j.packet.*; import org.pcap4j.packet.DnsPacket; import org.pcap4j.packet.namednumber.DnsResourceRecordType; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - import java.io.EOFException; +import java.net.InetAddress; import java.net.UnknownHostException; +import java.util.*; import java.util.concurrent.TimeoutException; /** * TODO add class documentation. + * TODO: At this point, this class is still in transition to having multiple hostnames and lists of packets * * @author Janus Varmarken */ public class FlowPattern { - static { - // TP-Link Local ON packet lengths (TCP payload only), extracted from ON event at Feb 13, 2018 13:38:04 - // of the 5 switch data collection: - // 517 1448 1448 1448 855 191 51 490 1027 31 - - ArrayList packetLengths = new ArrayList<>(); - packetLengths.addAll(Arrays.asList(new Integer[] {517, 1448, 1448, 1448, 855, 191, 51, 490, 1027, 31})); - TP_LINK_LOCAL_ON = new FlowPattern("TP_LINK_LOCAL_ON", "events.tplinkra.com", packetLengths); - } - - public static final FlowPattern TP_LINK_LOCAL_ON; - /** * Class properties */ - private final String patternId; - - /** - * The hostname that this {@code FlowPattern} is associated with. - */ + private final String mPatternId; private final String hostname; // The hostname that this {@code FlowPattern} is associated with. /** @@ -50,9 +30,10 @@ public class FlowPattern { * TODO: this is a simplified representation, we should also include information about direction of each packet. */ private final List flowPacketOrder; - - private final Map> hostnameToPacketOrderMap; - private final PcapHandle pcap; + private final Map> mHostnameToPacketLengthsMap; + private final List mHostnameList; + private final PcapHandle mPcap; + /** * Class constants @@ -62,15 +43,17 @@ public class FlowPattern { /** * Constructor #1 */ - public FlowPattern(String patternId, String hostname, PcapHandle pcap) { - this.patternId = patternId; + public FlowPattern(String mPatternId, String hostname, PcapHandle mPcap) { + this.mPatternId = mPatternId; this.hostname = hostname; - this.pcap = pcap; - this.hostnameToPacketOrderMap = null; + this.mHostnameList = null; + this.mPcap = mPcap; + this.mHostnameToPacketLengthsMap = null; this.flowPacketOrder = new ArrayList(); processPcap(); } + /** * Process the PcapHandle to strip off unnecessary packets and just get the integer array of packet lengths */ @@ -78,7 +61,7 @@ public class FlowPattern { PcapPacket packet; try { - while ((packet = pcap.getNextPacketEx()) != null) { + while ((packet = mPcap.getNextPacketEx()) != null) { // For now, we only work support pattern search in TCP over IPv4. IpV4Packet ipPacket = packet.get(IpV4Packet.class); TcpPacket tcpPacket = packet.get(TcpPacket.class); @@ -91,7 +74,7 @@ public class FlowPattern { } } catch (EOFException eofe) { System.out.println("[ FlowPattern ] Finished processing a training PCAP stream!"); - System.out.println("[ FlowPattern ] Pattern for " + patternId + ": " + Arrays.toString(flowPacketOrder.toArray())); + System.out.println("[ FlowPattern ] Pattern for " + mPatternId + ": " + Arrays.toString(flowPacketOrder.toArray())); } catch (PcapNativeException | TimeoutException | NotOpenException ex) { @@ -99,47 +82,119 @@ public class FlowPattern { } } + /** - * Constructor #2 - * - * @param patternId Label for this pattern - * @param hostname Hostname associated with this pattern - * @param flowPacketOrder List of packets in order + * Process the PcapHandle to strip off unnecessary packets. + * We then map list of hostnames to their respective arrays of packet lengths */ - public FlowPattern(String patternId, String hostname, List flowPacketOrder) { - this.patternId = patternId; - this.hostname = hostname; - this.hostnameToPacketOrderMap = null; - this.pcap = null; - this.flowPacketOrder = Collections.unmodifiableList(flowPacketOrder); + private void processPcapToMap() { + + PcapPacket packet; + try { + int hostIndex = -1; + Set addressSet = new HashSet<>(); + while ((packet = mPcap.getNextPacketEx()) != null) { + // For now, we only work support pattern search in TCP over IPv4. + IpV4Packet ipPacket = packet.get(IpV4Packet.class); + TcpPacket tcpPacket = packet.get(TcpPacket.class); + if (ipPacket == null || tcpPacket == null) { + continue; + } + if (tcpPacket.getPayload() == null) { + // We skip non-payload control packets as these are less predictable + continue; + } + // We assume that if it is not a local address then it is a cloud server address + InetAddress srcAddress = ipPacket.getHeader().getSrcAddr(); + InetAddress dstAddress = ipPacket.getHeader().getDstAddr(); + boolean fromServer = !srcAddress.isSiteLocalAddress(); + boolean fromClient = !dstAddress.isSiteLocalAddress(); + if (!fromServer && !fromClient) { + // Packet not related to pattern, skip it + continue; + } else { + // We relate and assume that this address is from our cloud server + String cloudAddress = null; + if (fromClient) { + cloudAddress = dstAddress.getHostAddress(); + } else { // fromServer + cloudAddress = srcAddress.getHostAddress(); + } + //System.out.println("\nCloud address: " + cloudAddress); + if (!addressSet.contains(cloudAddress)) { + addressSet.add(cloudAddress); + hostIndex++; + } + + String hostname = mHostnameList.get(hostIndex); + List packetLengthsList = mHostnameToPacketLengthsMap.containsKey(hostname) ? + mHostnameToPacketLengthsMap.get(hostname) : new ArrayList<>(); + int packetLength = tcpPacket.getPayload().length(); + packetLengthsList.add(packetLength); + mHostnameToPacketLengthsMap.put(hostname, packetLengthsList); + } + } + } catch (EOFException eofe) { + System.out.println("[ FlowPattern ] Finished processing a training PCAP stream!"); + System.out.println("[ FlowPattern ] Pattern for " + mPatternId + ": " + Arrays.toString(mHostnameToPacketLengthsMap.entrySet().toArray())); + } catch (PcapNativeException | + TimeoutException | + NotOpenException ex) { + ex.printStackTrace(); + } } + /** - * Constructor #3 + * Constructor #2 */ - public FlowPattern(String patternId, String hostname, Map> hostnameToPacketOrderMap) { - this.patternId = patternId; - this.hostname = hostname; - this.pcap = null; + public FlowPattern(String mPatternId, List mHostnameList, PcapHandle mPcap) { + this.mPatternId = mPatternId; + this.hostname = null; + this.mHostnameList = mHostnameList; + this.mPcap = mPcap; this.flowPacketOrder = null; - this.hostnameToPacketOrderMap = Collections.unmodifiableMap(hostnameToPacketOrderMap); + this.mHostnameToPacketLengthsMap = new HashMap<>(); + processPcapToMap(); } + public String getPatternId() { - return patternId; + return mPatternId; } + public String getHostname() { return hostname; } + /** - * Get the the sequence of packet lengths that defines this {@code FlowPattern}. + * Get the sequence of packet lengths that defines this {@code FlowPattern}. * @return the sequence of packet lengths that defines this {@code FlowPattern}. */ public List getPacketOrder() { return flowPacketOrder; } + + + /** + * Get the sequence of packet lengths based on input hostname. + * @return the sequence of packet lengths that defines this {@code FlowPattern}. + */ + public List getPacketOrder(String hostname) { + return mHostnameToPacketLengthsMap.get(hostname); + } + + + /** + * Get the list of associated hostnames. + * @return the associated hostnames that define this {@code FlowPattern}. + */ + public List getHostnameList() { + return mHostnameList; + } + /** * Get the length of the List of {@code FlowPattern}. @@ -147,6 +202,14 @@ public class FlowPattern { */ public int getLength() { return flowPacketOrder.size(); - } + } + + /** + * Get the length of the List of {@code FlowPattern}. + * @return the length of the List of {@code FlowPattern}. + */ + public int getLength(String hostname) { + return mHostnameToPacketLengthsMap.get(hostname).size(); + } } diff --git a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/FlowPatternFinder.java b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/FlowPatternFinder.java index 1152946..da0d3eb 100644 --- a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/FlowPatternFinder.java +++ b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/FlowPatternFinder.java @@ -90,6 +90,10 @@ public class FlowPatternFinder { private void findFlowPattern() { try { PcapPacket packet; +// TODO: The new comparison method is pending +// TODO: For now, just compare using one hostname and one list per FlowPattern +// List hostnameList = mPattern.getHostnameList(); +// int hostIndex = 0; int patternLength = mPattern.getLength(); while ((packet = mPcap.getNextPacketEx()) != null) { // Let DnsMap handle DNS packets. @@ -104,6 +108,10 @@ public class FlowPatternFinder { if (ipPacket == null || tcpPacket == null) { continue; } + if (tcpPacket.getPayload() == null) { + // We skip non-payload control packets as these are less predictable + continue; + } String srcAddress = ipPacket.getHeader().getSrcAddr().getHostAddress(); String dstAddress = ipPacket.getHeader().getDstAddr().getHostAddress(); int srcPort = tcpPacket.getHeader().getSrcPort().valueAsInt(); @@ -111,14 +119,13 @@ public class FlowPatternFinder { // Is this packet related to the pattern; i.e. is it going to (or coming from) the cloud server? boolean fromServer = mDnsMap.isRelatedToCloudServer(srcAddress, mPattern.getHostname()); boolean fromClient = mDnsMap.isRelatedToCloudServer(dstAddress, mPattern.getHostname()); +// String currentHostname = hostnameList.get(hostIndex); +// boolean fromServer = mDnsMap.isRelatedToCloudServer(srcAddress, currentHostname); +// boolean fromClient = mDnsMap.isRelatedToCloudServer(dstAddress, currentHostname); if (!fromServer && !fromClient) { // Packet not related to pattern, skip it. continue; } - if (tcpPacket.getPayload() == null) { - // We skip non-payload control packets as these are less predictable - continue; - } // Conversations (connections/sessions) are identified by the four-tuple // (clientIp, clientPort, serverIp, serverPort) (see Conversation Javadoc). // Create "dummy" conversation for looking up an existing entry. @@ -138,6 +145,7 @@ public class FlowPatternFinder { // Refresh reference to point to entry in map (in case packet was added to existing entry). conversation = mConversations.get(conversation); if (conversation.getPackets().size() == mPattern.getLength()) { +// if (conversation.getPackets().size() == mPattern.getLength(currentHostname)) { // Conversation reached a size that matches the expected size. // Remove the Conversation from the map and start the analysis. // Any future packets identified by the same four tuple will be tied to a new Conversation instance. @@ -147,6 +155,8 @@ public class FlowPatternFinder { PatternComparisonTask comparisonTask = new PatternComparisonTask<>(conversation, mPattern, ComparisonFunctions.COMPLETE_MATCH); mPendingComparisons.add(EXECUTOR_SERVICE.submit(comparisonTask)); + // Increment hostIndex to find the next + } } } catch (EOFException eofe) { diff --git a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/Main.java b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/Main.java index 239d8c1..b828323 100644 --- a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/Main.java +++ b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/Main.java @@ -4,6 +4,7 @@ import org.pcap4j.core.*; import java.io.EOFException; import java.net.UnknownHostException; +import java.util.*; import java.util.concurrent.TimeoutException; /** @@ -21,7 +22,8 @@ public class Main { public static void main(String[] args) throws PcapNativeException, NotOpenException, EOFException, TimeoutException, UnknownHostException { final String fileName = args.length > 0 ? args[0] : "/home/rtrimana/pcap_processing/smart_home_traffic/Code/Projects/SmartPlugDetector/pcap/wlan1.local.remote.dns.pcap"; - final String trainingFileName = "./pcap/TP_LINK_LOCAL_OFF.pcap"; + final String trainingFileName = "./pcap/TP_LINK_LOCAL_ON.pcap"; + //final String trainingFileName = "./pcap/TP_LINK_REMOTE_ON.pcap"; // ====== Debug code ====== PcapHandle handle; @@ -33,9 +35,17 @@ public class Main { handle = Pcaps.openOffline(fileName); trainingPcap = Pcaps.openOffline(trainingFileName); } - FlowPattern fp = new FlowPattern("TP_LINK_LOCAL_OFF", "events.tplinkra.com", trainingPcap); - - //FlowPatternFinder fpf = new FlowPatternFinder(handle, FlowPattern.TP_LINK_LOCAL_ON); + + // TODO: The followings are the way to extract multiple hostnames and their associated packet lengths lists + //List list = new ArrayList<>(); + //list.add("events.tplinkra.com"); + //FlowPattern fp = new FlowPattern("TP_LINK_LOCAL_ON", list, trainingPcap); + //List list2 = new ArrayList<>(); + //list2.add("devs.tplinkcloud.com"); + //list2.add("events.tplinkra.com"); + //FlowPattern fp3 = new FlowPattern("TP_LINK_REMOTE_ON", list2, trainingPcap); + + FlowPattern fp = new FlowPattern("TP_LINK_LOCAL_ON", "events.tplinkra.com", trainingPcap); FlowPatternFinder fpf = new FlowPatternFinder(handle, fp); fpf.start(); diff --git a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/comparison/PatternComparisonTask.java b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/comparison/PatternComparisonTask.java index 705646c..808dee7 100644 --- a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/comparison/PatternComparisonTask.java +++ b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/comparison/PatternComparisonTask.java @@ -17,7 +17,7 @@ public class PatternComparisonTask> private final Conversation mConversation; private final FlowPattern mFlowPattern; - private final BiFunction mComparitor; + private final BiFunction mComparator; /** * Create a new {@code PatternComparisonTask}. @@ -29,12 +29,12 @@ public class PatternComparisonTask> public PatternComparisonTask(Conversation conversation, FlowPattern pattern, BiFunction comparisonFunction) { this.mConversation = conversation; this.mFlowPattern = pattern; - this.mComparitor = comparisonFunction; + this.mComparator = comparisonFunction; } @Override public R call() throws Exception { - return mComparitor.apply(mConversation, mFlowPattern); + return mComparator.apply(mConversation, mFlowPattern); } -} \ No newline at end of file +} -- 2.34.1