Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/evaluation/DetectionResultsAnalyzer.java

   1 package edu.uci.iotproject.evaluation;
   2
   3 import edu.uci.iotproject.analysis.TriggerTrafficExtractor;
   4 import edu.uci.iotproject.analysis.UserAction;
   5 import edu.uci.iotproject.io.PrintWriterUtils;
   6 import edu.uci.iotproject.io.TriggerTimesFileReader;
   7
   8 import java.io.*;
   9 import java.time.Instant;
  10 import java.util.ArrayList;
  11 import java.util.List;
  12 import java.util.Optional;
  13
  14 /**
  15  * Utility for comparing detected events to logged (actual) events.
  16  *
  17  * @author Janus Varmarken {@literal <jvarmark@uci.edu>}
  18  * @author Rahmadi Trimananda {@literal <rtrimana@uci.edu>}
  19  */
  20 public class DetectionResultsAnalyzer {
  21
  22     private static boolean DUPLICATE_OUTPUT_TO_STD_OUT = true;
  23     private static boolean DETECTED_EVENT_EXACT_MATCH;
  24
  25     public static void main(String[] args) throws IOException {
  26         if (args.length < 4) {
  27             String errMsg = String.format("Usage: %s triggerTimesFile detectionOutputFile [stdOut]" +
  28                             "\n - triggerTimesFile: the file that contains the timestamps for the user actions" +
  29                             "\n - detectionOutputFile: the file that contains the detected events" +
  30                             "\n - analysisResultsFile: where to write the results of the detection analysis" +
  31                             "\n - matchEventType: true/false literal indicating if a detected event should" +
  32                                 " have a matching type" +
  33                             "\n - stdOut: optional true/false literal indicating if output should also be printed to" +
  34                                 " std out; default is true",
  35                     DetectionResultsAnalyzer.class.getSimpleName());
  36             System.out.println(errMsg);
  37             return;
  38         }
  39         String triggerTimesFile = args[0];
  40         File detectionOutputFile = new File(args[1]);
  41         String analysisResultsFile = args[2];
  42         DETECTED_EVENT_EXACT_MATCH = Boolean.parseBoolean(args[3]);
  43         if (args.length > 4) {
  44             DUPLICATE_OUTPUT_TO_STD_OUT = Boolean.parseBoolean(args[4]);
  45         }
  46
  47         // -------------------------------------- Parse the input files --------------------------------------
  48
  49         // Read the trigger times.
  50         // The trigger times file does not contain event types as we initially assumed that we would just be alternating
  51         // between ON and OFF.
  52         List<Instant> triggerTimestamps = new TriggerTimesFileReader().readTriggerTimes(triggerTimesFile, false);
  53         // Now generate user actions based on this alternating ON/OFF pattern.
  54         List<UserAction> triggers = new ArrayList<>();
  55         for (int i = 0; i < triggerTimestamps.size(); i++) {
  56             // NOTE: assumes triggers alternate between ON and OFF
  57             UserAction.Type actionType = i % 2 == 0 ? UserAction.Type.TOGGLE_ON : UserAction.Type.TOGGLE_OFF;
  58             triggers.add(new UserAction(actionType, triggerTimestamps.get(i)));
  59         }
  60         // Read the detection output file, assuming a format as specified in UserAction.toString()
  61         List<UserAction> detectedEvents = new ArrayList<>();
  62         try (BufferedReader br = new BufferedReader(new FileReader(detectionOutputFile))) {
  63             String s;
  64             while ((s = br.readLine()) != null) {
  65                 if (s.startsWith("#")) {
  66                     // Ignore comments.
  67                     continue;
  68                 }
  69                 detectedEvents.add(UserAction.fromString(s));
  70             }
  71         }
  72
  73         // -----------------  Now ready to compare the detected events with the logged events -----------------
  74
  75         // To contain all detected events that could be mapped to a trigger
  76         List<UserAction> truePositives = new ArrayList<>();
  77         if (DETECTED_EVENT_EXACT_MATCH) {
  78             for (UserAction detectedEvent : detectedEvents) {
  79                 Optional<UserAction> matchingTrigger = triggers.stream()
  80                         .filter(t -> t.getType() == detectedEvent.getType() &&
  81                                 t.getTimestamp().isBefore(detectedEvent.getTimestamp()) &&
  82                                 t.getTimestamp().plusMillis(TriggerTrafficExtractor.INCLUSION_WINDOW_MILLIS).
  83                                         isAfter(detectedEvent.getTimestamp())
  84                         ).findFirst();
  85                 matchingTrigger.ifPresent(mt -> {
  86                     // We've consumed the trigger (matched it with a detected event), so remove it so we don't match with
  87                     // another detected event.
  88                     triggers.remove(mt);
  89                     // The current detected event was a true positive as we could match it with a trigger.
  90                     truePositives.add(detectedEvent);
  91                 });
  92             }
  93             // TODO: Experimental
  94         } else { // DETECTED_EVENT_EXACT_MATCH == false
  95             for (UserAction detectedEvent : detectedEvents) {
  96                 Optional<UserAction> matchingTrigger = triggers.stream()
  97                         .filter(t ->
  98                                 t.getTimestamp().isBefore(detectedEvent.getTimestamp()) &&
  99                                 t.getTimestamp().plusMillis(TriggerTrafficExtractor.INCLUSION_WINDOW_MILLIS).
 100                                         isAfter(detectedEvent.getTimestamp())
 101                         ).findFirst();
 102                 matchingTrigger.ifPresent(mt -> {
 103                     // We've consumed the trigger (matched it with a detected event), so remove it so we don't match with
 104                     // another detected event.
 105                     triggers.remove(mt);
 106                     // The current detected event was a true positive as we could match it with a trigger.
 107                     truePositives.add(detectedEvent);
 108                 });
 109             }
 110         }
 111         // Now the false positives are those elements in detectedEvents that are not in truePositives
 112         List<UserAction> falsePositives = new ArrayList<>();
 113         falsePositives.addAll(detectedEvents);
 114         falsePositives.removeAll(truePositives);
 115
 116         // Output the results...
 117         PrintWriter outputter = new PrintWriter(new FileWriter(analysisResultsFile));
 118         PrintWriterUtils.println("---------- False negatives (events that were not detected) ----------", outputter, DUPLICATE_OUTPUT_TO_STD_OUT);
 119         for (UserAction missing : triggers) {
 120             PrintWriterUtils.println(missing, outputter, DUPLICATE_OUTPUT_TO_STD_OUT);
 121         }
 122         PrintWriterUtils.println("Total of " + Integer.toString(triggers.size()), outputter, DUPLICATE_OUTPUT_TO_STD_OUT);
 123         PrintWriterUtils.printEmptyLine(outputter, DUPLICATE_OUTPUT_TO_STD_OUT);
 124         PrintWriterUtils.println("---------- False positives (detected, but no matching trigger) ----------", outputter, DUPLICATE_OUTPUT_TO_STD_OUT);
 125         for (UserAction fp : falsePositives) {
 126             PrintWriterUtils.println(fp, outputter, DUPLICATE_OUTPUT_TO_STD_OUT);
 127         }
 128         PrintWriterUtils.println("Total of " + Integer.toString(falsePositives.size()), outputter, DUPLICATE_OUTPUT_TO_STD_OUT);
 129         outputter.flush();
 130         outputter.close();
 131     }
 132
 133 }