- // Print out all the pairs into a file for ON events
- File fileOnEvents = new File(onPairsPath);
- PrintWriter pwOn = null;
- try {
- pwOn = new PrintWriter(fileOnEvents);
- } catch(Exception ex) {
- ex.printStackTrace();
- }
- for(Map.Entry<String, Map<String, List<Conversation>>> entry : ons.entrySet()) {
- Map<String, List<Conversation>> seqsToConvs = entry.getValue();
- for(Map.Entry<String, List<Conversation>> entryConv : seqsToConvs.entrySet()) {
- List<Conversation> listConv = entryConv.getValue();
- // Just get the first Conversation because all Conversations in this group
- // should have the same pairs of Application Data.
- for(Conversation conv : listConv) {
- // Process only if it is a TLS packet
- if (conv.isTls()) {
- List<PcapPacket> tlsAppDataList = conv.getTlsApplicationDataPackets();
- // Loop and print out packets
- int count = 0;
- for (PcapPacket pcap : tlsAppDataList) {
- boolean isPair = false;
- if (count % 2 == 0) {
- pwOn.print(pcap.length() + ", ");
- //System.out.print(pcap.length() + ", ");
- } else {// count%2 == 1
- isPair = true;
- pwOn.println(pcap.length());
- //System.out.println(pcap.length());
- }
- count++;
- // If we can't create a pair then just pad it with 0
- if (count == tlsAppDataList.size() && !isPair) {
- pwOn.println("0");
- //System.out.println("0");
- }
- }
- }
- }
- }
- }
- pwOn.close();
-
- // Print out all the pairs into a file for ON events
- File fileOffEvents = new File(offPairsPath);
- PrintWriter pwOff = null;
- try {
- pwOff = new PrintWriter(fileOffEvents);
- } catch(Exception ex) {
- ex.printStackTrace();
- }
- for(Map.Entry<String, Map<String, List<Conversation>>> entry : offs.entrySet()) {
- Map<String, List<Conversation>> seqsToConvs = entry.getValue();
- for(Map.Entry<String, List<Conversation>> entryConv : seqsToConvs.entrySet()) {
- List<Conversation> listConv = entryConv.getValue();
- // Just get the first Conversation because all Conversations in this group
- // should have the same pairs of Application Data.
- for(Conversation conv : listConv) {
- // Process only if it is a TLS packet
- if (conv.isTls()) {
- List<PcapPacket> tlsAppDataList = conv.getTlsApplicationDataPackets();
- // Loop and print out packets
- int count = 0;
- for (PcapPacket pcap : tlsAppDataList) {
- boolean isPair = false;
- if (count % 2 == 0) {
- pwOff.print(pcap.length() + ", ");
- //System.out.print(pcap.length() + ", ");
- } else {// count%2 == 1
- isPair = true;
- pwOff.println(pcap.length());
- //System.out.println(pcap.length());
- }
- count++;
- // If we can't create a pair then just pad it with 0
- if (count == tlsAppDataList.size() && !isPair) {
- pwOff.println("0");
- //System.out.println("0");
- }
- }
- }
- }
- }
- }
- pwOff.close();
- */
-
- // ================================================================================================
- // <<< Some work-in-progress/explorative code that extracts a "representative" sequence >>>
- //
- // Currently need to know relevant hostname in advance :(
- String hostname = "events.tplinkra.com";
-// String hostname = "rfe-us-west-1.dch.dlink.com";
- // Conversations with 'hostname' for ON events.
- List<Conversation> onsForHostname = new ArrayList<>();
- // Conversations with 'hostname' for OFF events.
- List<Conversation> offsForHostname = new ArrayList<>();
- // "Unwrap" sequence groupings in ons/offs maps.
- ons.get(hostname).forEach((k,v) -> onsForHostname.addAll(v));
- offs.get(hostname).forEach((k,v) -> offsForHostname.addAll(v));
-
-
- Map<String, List<Conversation>> onsForHostnameGroupedByTlsAppDataSequence = TcpConversationUtils.groupConversationsByTlsApplicationDataPacketSequence(onsForHostname);
-
-
- // Extract representative sequence for ON and OFF by providing the list of conversations with
- // 'hostname' observed for each event type (the training data).
- SequenceExtraction seqExtraction = new SequenceExtraction();
-// ExtractedSequence extractedSequenceForOn = seqExtraction.extract(onsForHostname);
-// ExtractedSequence extractedSequenceForOff = seqExtraction.extract(offsForHostname);
-
- ExtractedSequence extractedSequenceForOn = seqExtraction.extractByTlsAppData(onsForHostname);
- ExtractedSequence extractedSequenceForOff = seqExtraction.extractByTlsAppData(offsForHostname);
-
- // Let's check how many ONs align with OFFs and vice versa (that is, how many times an event is incorrectly
- // labeled).
- int onsLabeledAsOff = 0;
- Integer[] representativeOnSeq = TcpConversationUtils.getPacketLengthSequence(extractedSequenceForOn.getRepresentativeSequence());
- Integer[] representativeOffSeq = TcpConversationUtils.getPacketLengthSequence(extractedSequenceForOff.getRepresentativeSequence());
- SequenceAlignment<Integer> seqAlg = seqExtraction.getAlignmentAlgorithm();
- for (Conversation c : onsForHostname) {
- Integer[] onSeq = TcpConversationUtils.getPacketLengthSequence(c);
- if (seqAlg.calculateAlignment(representativeOffSeq, onSeq) <= extractedSequenceForOff.getMaxAlignmentCost()) {
- onsLabeledAsOff++;
+ * NOTE: no need to generate these more complex on/off maps that also contain mappings from hostname and
+ * sequence identifiers as we do not care about hostnames and sequences during clustering.
+ * We can simply use the UserAction->List<Conversation> map to generate ON/OFF groupings of conversations.
+ */
+
+// // Contains all ON events: hostname -> sequence identifier -> list of conversations with that sequence
+// Map<String, Map<String, List<Conversation>>> ons = new HashMap<>();
+// // Contains all OFF events: hostname -> sequence identifier -> list of conversations with that sequence
+// Map<String, Map<String, List<Conversation>>> offs = new HashMap<>();
+// userActionsToConvsByHostname.forEach((ua, hostnameToConvs) -> {
+// Map<String, Map<String, List<Conversation>>> outer = ua.getType() == Type.TOGGLE_ON ? ons : offs;
+// hostnameToConvs.forEach((host, convs) -> {
+// Map<String, List<Conversation>> seqsToConvs = TcpConversationUtils.
+// groupConversationsByPacketSequence(convs, verbose);
+// outer.merge(host, seqsToConvs, (oldMap, newMap) -> {
+// newMap.forEach((sequence, cs) -> oldMap.merge(sequence, cs, (list1, list2) -> {
+// list1.addAll(list2);
+// return list1;
+// }));
+// return oldMap;
+// });
+// });
+// });
+//
+// System.out.println("==== ON ====");
+// // Print out all the pairs into a file for ON events
+// File fileOnEvents = new File(onPairsPath);
+// PrintWriter pwOn = null;
+// try {
+// pwOn = new PrintWriter(fileOnEvents);
+// } catch(Exception ex) {
+// ex.printStackTrace();
+// }
+// for(Map.Entry<String, Map<String, List<Conversation>>> entry : ons.entrySet()) {
+// Map<String, List<Conversation>> seqsToConvs = entry.getValue();
+// for(Map.Entry<String, List<Conversation>> entryConv : seqsToConvs.entrySet()) {
+// List<Conversation> listConv = entryConv.getValue();
+// // Just get the first Conversation because all Conversations in this group
+// // should have the same pairs of Application Data.
+// for(Conversation conv : listConv) {
+// // Process only if it is a TLS packet
+// if (conv.isTls()) {
+// List<PcapPacketPair> tlsAppDataList = TcpConversationUtils.extractTlsAppDataPacketPairs(conv);
+// for(PcapPacketPair pair: tlsAppDataList) {
+// System.out.println(PrintUtils.toCsv(pair, dnsMap));
+// pwOn.println(PrintUtils.toCsv(pair, dnsMap));
+// }
+// } else { // Non-TLS conversations
+// List<PcapPacketPair> packetList = TcpConversationUtils.extractPacketPairs(conv);
+// for(PcapPacketPair pair: packetList) {
+// System.out.println(PrintUtils.toCsv(pair, dnsMap));
+// pwOn.println(PrintUtils.toCsv(pair, dnsMap));
+// }
+// }
+// }
+// }
+// }
+// pwOn.close();
+//
+// System.out.println("==== OFF ====");
+// // Print out all the pairs into a file for ON events
+// File fileOffEvents = new File(offPairsPath);
+// PrintWriter pwOff = null;
+// try {
+// pwOff = new PrintWriter(fileOffEvents);
+// } catch(Exception ex) {
+// ex.printStackTrace();
+// }
+// for(Map.Entry<String, Map<String, List<Conversation>>> entry : offs.entrySet()) {
+// Map<String, List<Conversation>> seqsToConvs = entry.getValue();
+// for(Map.Entry<String, List<Conversation>> entryConv : seqsToConvs.entrySet()) {
+// List<Conversation> listConv = entryConv.getValue();
+// // Just get the first Conversation because all Conversations in this group
+// // should have the same pairs of Application Data.
+// for(Conversation conv : listConv) {
+// // Process only if it is a TLS packet
+// if (conv.isTls()) {
+// List<PcapPacketPair> tlsAppDataList = TcpConversationUtils.extractTlsAppDataPacketPairs(conv);
+// for(PcapPacketPair pair: tlsAppDataList) {
+// System.out.println(PrintUtils.toCsv(pair, dnsMap));
+// pwOff.println(PrintUtils.toCsv(pair, dnsMap));
+// }
+// } else { // Non-TLS conversations
+// List<PcapPacketPair> packetList = TcpConversationUtils.extractPacketPairs(conv);
+// for (PcapPacketPair pair : packetList) {
+// System.out.println(PrintUtils.toCsv(pair, dnsMap));
+// pwOff.println(PrintUtils.toCsv(pair, dnsMap));
+// }
+// }
+// }
+// }
+// }
+// pwOff.close();
+
+
+ // ================================================ CLUSTERING ================================================
+ // Note: no need to use the more convoluted on/off maps; can simply use the UserAction->List<Conversation> map
+ // when don't care about hostnames and sequences (see comment earlier).
+ List<Conversation> onConversations = userActionToConversations.entrySet().stream().
+ filter(e -> e.getKey().getType() == Type.TOGGLE_ON). // drop all OFF events from stream
+ map(e -> e.getValue()). // no longer interested in the UserActions
+ flatMap(List::stream). // flatten List<List<T>> to a List<T>
+ collect(Collectors.toList());
+ List<Conversation> offConversations = userActionToConversations.entrySet().stream().
+ filter(e -> e.getKey().getType() == Type.TOGGLE_OFF).
+ map(e -> e.getValue()).
+ flatMap(List::stream).
+ collect(Collectors.toList());
+ //Collections.sort(onConversations, (c1, c2) -> c1.getPackets().)
+
+ List<PcapPacketPair> onPairs = onConversations.stream().
+ map(c -> c.isTls() ? TcpConversationUtils.extractTlsAppDataPacketPairs(c) :
+ TcpConversationUtils.extractPacketPairs(c)).
+ flatMap(List::stream). // flatten List<List<>> to List<>
+ collect(Collectors.toList());
+ List<PcapPacketPair> offPairs = offConversations.stream().
+ map(c -> c.isTls() ? TcpConversationUtils.extractTlsAppDataPacketPairs(c) :
+ TcpConversationUtils.extractPacketPairs(c)).
+ flatMap(List::stream). // flatten List<List<>> to List<>
+ collect(Collectors.toList());
+ // Note: need to update the DnsMap of all PcapPacketPairs if we want to use the IP/hostname-sensitive distance.
+ Stream.concat(Stream.of(onPairs), Stream.of(offPairs)).flatMap(List::stream).forEach(p -> p.setDnsMap(dnsMap));
+ // Perform clustering on conversation logged as part of all ON events.
+ DBSCANClusterer<PcapPacketPair> onClusterer = new DBSCANClusterer<>(10.0, 45);
+ //DBSCANClusterer<PcapPacketPair> onClusterer = new DBSCANClusterer<>(3, 45);
+ //DBSCANClusterer<PcapPacketPair> onClusterer = new DBSCANClusterer<>(10.0, 10);
+ List<Cluster<PcapPacketPair>> onClusters = onClusterer.cluster(onPairs);
+ // Perform clustering on conversation logged as part of all OFF events.
+ DBSCANClusterer<PcapPacketPair> offClusterer = new DBSCANClusterer<>(10.0, 45);
+ //DBSCANClusterer<PcapPacketPair> offClusterer = new DBSCANClusterer<>(3, 45);
+ //DBSCANClusterer<PcapPacketPair> offClusterer = new DBSCANClusterer<>(10.0, 10);
+ List<Cluster<PcapPacketPair>> offClusters = offClusterer.cluster(offPairs);
+ // Sort the conversations as reference
+ List<Conversation> sortedAllConversation = TcpConversationUtils.sortConversationList(allConversations);
+ // Output clusters
+ System.out.println("========================================");
+ System.out.println(" Clustering results for ON ");
+ System.out.println(" Number of clusters: " + onClusters.size());
+ int count = 0;
+ List<List<List<PcapPacket>>> ppListOfListReadOn = new ArrayList<>();
+ List<List<List<PcapPacket>>> ppListOfListListOn = new ArrayList<>();
+ for (Cluster<PcapPacketPair> c : onClusters) {
+ System.out.println(String.format("<<< Cluster #%02d (%03d points) >>>", ++count, c.getPoints().size()));
+ System.out.print(PrintUtils.toSummaryString(c));
+ if(c.getPoints().size() > 45 && c.getPoints().size() < 55) {
+ //if(c.getPoints().size() > 25) {
+ // Print to file
+ List<List<PcapPacket>> ppListOfList = PcapPacketUtils.clusterToListOfPcapPackets(c);
+ ppListOfListListOn.add(ppListOfList);