From: adash <adash>
Date: Sat, 14 Nov 2009 02:10:08 +0000 (+0000)
Subject: bug fixes with string split and other changes to make a little more sophisticated
X-Git-Url: http://demsky.eecs.uci.edu/git/?a=commitdiff_plain;h=44c701de6ef6cefbdc6514692b71561f9c842a4a;p=IRC.git

bug fixes with string split and other changes to make a little more sophisticated
signature for hosts in an email
---

diff --git a/Robust/src/Benchmarks/Distributed/SpamFilter/Mail.java b/Robust/src/Benchmarks/Distributed/SpamFilter/Mail.java
index ea449188..85a02afa 100644
--- a/Robust/src/Benchmarks/Distributed/SpamFilter/Mail.java
+++ b/Robust/src/Benchmarks/Distributed/SpamFilter/Mail.java
@@ -33,7 +33,7 @@ public class Mail {
 
   public Mail(String fileName)  // read a mail from file
   {
-    //System.out.println("fileName= " + fileName);
+    System.out.println("fileName= " + fileName);
 
     FileInputStream fileinput = new FileInputStream(fileName);
     String line;
@@ -46,7 +46,7 @@ public class Mail {
       Vector splittedLine = line.split();
       if(((String)(splittedLine.elementAt(0))).equals("Spam:"))
       {
-        spam = (String)splittedLine.elementAt(1);
+        spam = (String)(splittedLine.elementAt(1));
       }
       else if(((String)(splittedLine.elementAt(0))).equals("Header:"))  // message id
       {
@@ -86,7 +86,6 @@ public class Mail {
       body += new String(readBody);
     }
 
-
     fileinput.close();
 
     MAX_TOKEN_SIZE = 1024;
@@ -291,11 +290,11 @@ public class Mail {
   public Vector returnEmail() {
     Vector myemail = new Vector();
     myemail.addElement(getCommonPart());
-    System.out.println("DEBUG: getCommonPart.size= " + getCommonPart().size());
+    //System.out.println("DEBUG: getCommonPart.size= " + getCommonPart().size());
     myemail.addElement(getURLs());
-    System.out.println("DEBUG: getURLs.size= " + getURLs().size());
+    //System.out.println("DEBUG: getURLs.size= " + getURLs().size());
     myemail.addElement(getSplittedBody(MAX_TOKEN_SIZE));
-    System.out.println("DEBUG: getSplittedBody.size= " + getSplittedBody(MAX_TOKEN_SIZE).size());
+    //System.out.println("DEBUG: getSplittedBody.size= " + getSplittedBody(MAX_TOKEN_SIZE).size());
     return myemail;
   }
 
@@ -307,7 +306,7 @@ public class Mail {
     // add URL and email in the body
     for(int i=0; i<splittedBody.size(); i++) 
     {
-      String segment = (String)splittedBody.elementAt(i);
+      String segment = (String)(splittedBody.elementAt(i));
       if(segment.startsWith("http://"))  // URL
       {
         returnStrings.addElement(segment);
@@ -336,11 +335,11 @@ public class Mail {
 
     Vector splittedBody = body.split();
 
-    System.out.println("DEBUG: splittedBody.size()= " + splittedBody.size());
+    //System.out.println("DEBUG: splittedBody.size()= " + splittedBody.size());
     for(int i=0; i< splittedBody.size();i ++)
     {
       String segment = (String)(splittedBody.elementAt(i));
-      System.out.println("DEBUG: segment= " + segment);
+      //System.out.println("DEBUG: segment= " + segment);
       
       if(!(segment.startsWith("http://") || isEmailAccount(segment)))
         noURLBody += segment;
@@ -359,7 +358,7 @@ public class Mail {
     String tmpStr = new String();
     tmpStr += charArray[0];
 
-    System.out.println("tmpStr= " + tmpStr);
+    //System.out.println("tmpStr= " + tmpStr);
 
     for(int i=1; i< noURLBody.length(); i++)
     {
diff --git a/Robust/src/Benchmarks/Distributed/SpamFilter/SignatureComputer.java b/Robust/src/Benchmarks/Distributed/SpamFilter/SignatureComputer.java
index 600dee42..4a79caf0 100644
--- a/Robust/src/Benchmarks/Distributed/SpamFilter/SignatureComputer.java
+++ b/Robust/src/Benchmarks/Distributed/SpamFilter/SignatureComputer.java
@@ -98,18 +98,23 @@ public class SignatureComputer {
           sig = computeSignature(engineNo,mail);
         } 
 
-        if(engineNo!=4 || engineNo!=8) {
-          System.out.println("Err: Couldn't find the signature engine: " + engineNo);
+        if(engineNo==8) {
+          continue;
+        }
+
+        if((engineNo!=4)) {
+          System.out.println("Err: Common part Couldn't find the signature engine: " + engineNo);
         }
 
         if (sig != null) {
           String hash = engineNo + ":" + sig;
           printableSigs.addElement(hash);
+          // System.out.println("mail= " +mail + " hash= " + hash);
         } else {
           // we didn't produce a signature for the mail. 
         }
       }//engine
-    }
+    }//common part
 
     /** 
      * Step -II 
@@ -119,7 +124,6 @@ public class SignatureComputer {
     for (int mailIndex = 0; mailIndex < getBodywithNoURLs.size(); mailIndex++) {
       String mail = (String) (getBodywithNoURLs.elementAt(mailIndex));
 
-      System.out.println("mail= " + mail);
 
       if (mail == null) continue;
 
@@ -128,8 +132,6 @@ public class SignatureComputer {
        */
       for (int engineIndex = 0; engineIndex < enginesToUseForCheck.length; engineIndex++) {
         int engineNo = enginesToUseForCheck[engineIndex];
-        if(engineNo==8)
-          continue;
         String sig = null;
 
         /* EphemeralSignature calculator */
@@ -137,15 +139,18 @@ public class SignatureComputer {
           sig = computeSignature(engineNo,mail);
         } 
 
-        if(engineNo!=4 || engineNo!=8) {
-          System.out.println("Err: Couldn't find the signature engine: " + engineNo);
+        if(engineNo==8)
+            continue;
+
+        if(engineNo!=4) {
+          System.out.println("Err: body parts without URL Couldn't find the signature engine: " + engineNo);
         }
 
         if (sig != null) {
           String hash = engineNo + ":" + sig;
           printableSigs.addElement(hash);
         } else {
-          // we didn't produce a signature for the mail. 
+           // we didn't produce a signature for the mail. 
         }
       }//engine
     }
@@ -157,17 +162,15 @@ public class SignatureComputer {
     Vector getURLs = (Vector)(EmailParts.elementAt(1));
     for (int mailIndex = 0; mailIndex < getURLs.size(); mailIndex++) {
       String mail = (String) (getURLs.elementAt(mailIndex));
-      System.out.println("mail= " + mail);
+      System.out.println("from GETURLS mail= " + mail);
 
       /*
        * Compute Sig for bodyparts that are cleaned.
        */
       for (int engineIndex = 0; engineIndex < enginesToUseForCheck.length; engineIndex++) {
         int engineNo = enginesToUseForCheck[engineIndex];
-
-        if(engineNo==4) {
-          continue;
-        }
+        if(engineNo==4)
+            continue;
 
         /* WhiplashSignature calculator */
         String[] hosts = null;
@@ -183,12 +186,12 @@ public class SignatureComputer {
               printableSigs.addElement(hash);
             }
           } else {
-            // we didn't produce a signature for the mail.
+             // we didn't produce a signature for the mail.
           }
         } 
 
-        if(engineNo!=4 || engineNo!=8) {
-          System.out.println("Err: Couldn't find the signature engine: " + engineNo);
+        if(engineNo!=8) {
+          System.out.println("Err: body parts with URL Couldn't find the signature engine: " + engineNo);
         }
 
         /*
@@ -196,11 +199,13 @@ public class SignatureComputer {
            String hash = engineNo + ":" + sig;
            printableSigs.addElement(hash);
            } else {
-        // we didn't produce a signature for the mail. 
+         we didn't produce a signature for the mail. 
         }
          */
       }//engine
     }
+
+    // OLD IMPLEMENTATION
 //
 //    for (int mailIndex = 0; mailIndex < EmailParts.size(); mailIndex++) {
 //      String mail = (String) (EmailParts.elementAt(mailIndex));
diff --git a/Robust/src/Benchmarks/Distributed/SpamFilter/SpamFilter.java b/Robust/src/Benchmarks/Distributed/SpamFilter/SpamFilter.java
index 8bed8448..a4bc9e23 100644
--- a/Robust/src/Benchmarks/Distributed/SpamFilter/SpamFilter.java
+++ b/Robust/src/Benchmarks/Distributed/SpamFilter/SpamFilter.java
@@ -56,7 +56,6 @@ public class SpamFilter extends Thread {
         // randomly pick emails
         pickemail+=1;
         Mail email = new Mail("emails/email"+pickemail);
-        System.out.println("DEBUG: emailid = email" + pickemail); 
         Vector signatures = email.checkMail(thid);
 
         //check with global data structure
diff --git a/Robust/src/Benchmarks/Distributed/SpamFilter/WhiplashSignature.java b/Robust/src/Benchmarks/Distributed/SpamFilter/WhiplashSignature.java
index d6fdb2ae..97a19770 100644
--- a/Robust/src/Benchmarks/Distributed/SpamFilter/WhiplashSignature.java
+++ b/Robust/src/Benchmarks/Distributed/SpamFilter/WhiplashSignature.java
@@ -19,13 +19,12 @@
  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
 
 
- $Id: WhiplashSignature.java,v 1.2 2009/11/13 01:27:02 adash Exp $
+ $Id: WhiplashSignature.java,v 1.3 2009/11/14 02:10:08 adash Exp $
  */
 public class WhiplashSignature {
   char[] b64table;
 
   public WhiplashSignature() {
-    System.out.println("Inside WhiplashSignature");
     b64table = new char[64];
 
     for (int i= 0; i <= 25; i++) {
@@ -43,11 +42,11 @@ public class WhiplashSignature {
 
   public String[] computeSignature(String text) {
 
-    System.out.println("Inside computeSignature");
+    //System.out.println("Inside computeSignature");
     //Current: Simplify the host extraction and signature computation
     String[] sigs = whiplash(text);
+    // TODO: Extract canonical domain name and convert to Base64
     /*
-      TODO: Extract canonical domain name and convert to Base64
     if(sigs != null) {
       for(int i = 0; i<sigs.length; i++) {
         sigs[i] = hexToBase64(sigs[i]);
@@ -136,7 +135,6 @@ public class WhiplashSignature {
 
   public String[] whiplash(String text) {
     
-    //System.out.println("Inside whiplash");
     if (text == null) {
       return null;
     }
@@ -155,6 +153,9 @@ public class WhiplashSignature {
       md.update(buf, len);
       md.md5final(sig);
       String signature = new String(sig);
+
+      // System.out.println("DEBUG: host= " + host + " whiplash sig= " + signature);
+
       sigs[i] = signature;
     }
     return sigs;
@@ -165,20 +166,34 @@ public class WhiplashSignature {
     Vector hosts = new Vector();
     String buf = new String(text);
 
-    System.out.println("buf= " + buf);
+    System.out.println("DEBUG: extractHosts() string= " + buf);
 
-    String strwww = new String("www.");
+    /* Extract hosts from http:// links */
     int idx;
+    String strwww = new String("www.");
     while ((idx = buf.indexOf(strwww)) != -1) {
       int startidx = idx + strwww.length();
-      //System.out.println("idx= " + idx + " startidx= " + startidx);
       String strcom = new String(".");
-      buf = buf.substring(startidx);
+      buf = buf.subString(startidx);
       int endidx = buf.indexOf(strcom);
-      String host = buf.substring(0, endidx);
-      System.out.println(host);
+      String host = buf.subString(0, endidx);
+      System.out.println("http links extracted host= " + host);
+      hosts.addElement(host);
+      buf = buf.subString(endidx+strcom.length());
+    }
+
+    /* Extract hosts from email addressess */
+    buf = new String(text);
+    String strrate = new String("@");
+    while ((idx = buf.indexOf(strrate)) != -1) {
+      int startidx = idx + strrate.length();
+      String strdot = new String(".");
+      buf = buf.subString(startidx);
+      int endidx = buf.indexOf(strdot);
+      String host = buf.subString(0, endidx);
+      System.out.println("email addr extracted host= " + host);
       hosts.addElement(host);
-      buf = buf.substring(endidx+strcom.length());
+      buf = buf.subString(endidx+strdot.length());
     }
 
     if (hosts.size() == 0) {
diff --git a/Robust/src/Benchmarks/Distributed/SpamFilter/emails/gen.c b/Robust/src/Benchmarks/Distributed/SpamFilter/emails/gen.c
index 195fb38f..07ed60f0 100644
--- a/Robust/src/Benchmarks/Distributed/SpamFilter/emails/gen.c
+++ b/Robust/src/Benchmarks/Distributed/SpamFilter/emails/gen.c
@@ -125,7 +125,7 @@ void generateEmails(int num_email,char** wl,int word_num,char** ul,int url_num,c
      fprintf(newFile,"\n");
 
      // write title
-     writeString(newFile,"Title: ",wl,word_num);
+     writeString(newFile,"Subject: ",wl,word_num);
      fprintf(newFile,"\n");
 
      // write Body