From 8b1991c42c6681cb02fde179940d6bb3382c8f93 Mon Sep 17 00:00:00 2001 From: jihoonl Date: Tue, 10 Nov 2009 23:11:14 +0000 Subject: [PATCH] Recovery --- .../FileSystem/DistributedHashMap.java | 172 +++++++++ .../Recovery/FileSystem/LookUpService.java | 176 +++++++++ .../Benchmarks/Recovery/FileSystem/makefile | 10 + .../MatrixMultiply/MatrixMultiplyN.java | 230 +++++++++++ .../Recovery/MatrixMultiply/dstm.conf | 9 + .../Recovery/MatrixMultiply/makefile | 11 + .../Recovery/Spider/GlobalQuery.java | 92 +++++ .../Recovery/Spider/LocalQuery.java | 95 +++++ .../Benchmarks/Recovery/Spider/QueryTask.java | 365 ++++++++++++++++++ .../Benchmarks/Recovery/Spider/Spider.java | 62 +++ .../src/Benchmarks/Recovery/Spider/dstm.conf | 6 + .../src/Benchmarks/Recovery/Spider/makefile | 12 + 12 files changed, 1240 insertions(+) create mode 100644 Robust/src/Benchmarks/Recovery/FileSystem/DistributedHashMap.java create mode 100644 Robust/src/Benchmarks/Recovery/FileSystem/LookUpService.java create mode 100644 Robust/src/Benchmarks/Recovery/FileSystem/makefile create mode 100644 Robust/src/Benchmarks/Recovery/MatrixMultiply/MatrixMultiplyN.java create mode 100644 Robust/src/Benchmarks/Recovery/MatrixMultiply/dstm.conf create mode 100644 Robust/src/Benchmarks/Recovery/MatrixMultiply/makefile create mode 100644 Robust/src/Benchmarks/Recovery/Spider/GlobalQuery.java create mode 100644 Robust/src/Benchmarks/Recovery/Spider/LocalQuery.java create mode 100644 Robust/src/Benchmarks/Recovery/Spider/QueryTask.java create mode 100644 Robust/src/Benchmarks/Recovery/Spider/Spider.java create mode 100644 Robust/src/Benchmarks/Recovery/Spider/dstm.conf create mode 100644 Robust/src/Benchmarks/Recovery/Spider/makefile diff --git a/Robust/src/Benchmarks/Recovery/FileSystem/DistributedHashMap.java b/Robust/src/Benchmarks/Recovery/FileSystem/DistributedHashMap.java new file mode 100644 index 00000000..0d0dfab0 --- /dev/null +++ b/Robust/src/Benchmarks/Recovery/FileSystem/DistributedHashMap.java @@ -0,0 +1,172 @@ +public class DistributedHashMap { + DistributedHashEntry[] table; + float loadFactor; + int secondcapacity; + + public DistributedHashMap(int initialCapacity, int secondcapacity, float loadFactor) { + init(initialCapacity, secondcapacity, loadFactor); + } + + private void init(int initialCapacity, int secondcapacity, float loadFactor) { + table = global new DistributedHashEntry[initialCapacity]; + this.loadFactor=loadFactor; + this.secondcapacity=secondcapacity; + } + + private static int hash1(int hashcode, int length) { + int value=hashcode%length; + if (value<0) + return -value; + else + return value; + } + + private static int hash2(int hashcode, int length1, int length2) { + int value=(hashcode*31)%length2; + if (value<0) + return -value; + else + return value; + } + + void resize(int index) { + DHashEntry[] oldtable=table[index].array; + int newCapacity=oldtable.length*2+1; + DHashEntry [] newtable=global new DHashEntry[newCapacity]; + table[index].array=newtable; + + for(int i=0; i(loadFactor*dhe.array.length)) { + //Resize the table + resize(index1); + } + return null; + } +} + + +class DistributedHashEntry { + public DistributedHashEntry(int capacity) { + array=global new DHashEntry[capacity]; + } + int count; + DHashEntry[] array; +} + + +class DHashEntry { + public DHashEntry() { + } + int hashval; + Object key; + Object value; + DHashEntry next; +} diff --git a/Robust/src/Benchmarks/Recovery/FileSystem/LookUpService.java b/Robust/src/Benchmarks/Recovery/FileSystem/LookUpService.java new file mode 100644 index 00000000..c61fa543 --- /dev/null +++ b/Robust/src/Benchmarks/Recovery/FileSystem/LookUpService.java @@ -0,0 +1,176 @@ +public class LookUpService extends Thread { + DistributedHashMap mydhmap; + /** + * The thread id involved + **/ + private int threadid; + /** + * The total number of threads + **/ + private int numthreads; + + /** + * The total number of transactions + **/ + private int numtrans; + + /** + * The total number of objects created + **/ + private int nobjs; + + /** + * The probability of initiating a look up + * the read probability % between 0-99 + **/ + private int rdprob; + + /** + * The number of look up operations + **/ + private int nLookUp; + + public LookUpService() { + } + + public LookUpService(DistributedHashMap dmap, int threadid, int numthreads, int nobjs, int numtrans, int rdprob, int nLookUp) { + mydhmap = dmap; + this.threadid = threadid; + this.numthreads = numthreads; + this.nobjs = nobjs; + this.numtrans = numtrans; + this.rdprob = rdprob; + this.nLookUp = nLookUp; + } + + public void run() { + int ntrans; + atomic { + ntrans = numtrans; + } + + // Do read/writes + Random rand = new Random(0); + + for (int i = 0; i < ntrans; i++) { + atomic { + for(int j = 0; j < nLookUp; j++) { + int rdwr = rand.nextInt(100); + int rwkey = rand.nextInt(nobjs); + Integer key = global new Integer(rwkey); + if (rdwr < rdprob) { + Object o3 = mydhmap.get(key); //Read + } else { + Integer val = global new Integer(j); + mydhmap.put(key, val); //Modify + } + } + } + } + } + + public static void main(String[] args) { + LookUpService ls = new LookUpService(); + LookUpService.parseCmdLine(args,ls); + + int nthreads = ls.numthreads; + int[] mid = new int[8]; + mid[0] = (128<<24)|(195<<16)|(136<<8)|162;//dc-1 + mid[1] = (128<<24)|(195<<16)|(136<<8)|163;//dc-2 + mid[2] = (128<<24)|(195<<16)|(136<<8)|164;//dc-3 + mid[3] = (128<<24)|(195<<16)|(136<<8)|165;//dc-4 + mid[4] = (128<<24)|(195<<16)|(136<<8)|166;//dc-5 + mid[5] = (128<<24)|(195<<16)|(136<<8)|167;//dc-6 + mid[6] = (128<<24)|(195<<16)|(136<<8)|168;//dc-7 + mid[7] = (128<<24)|(195<<16)|(136<<8)|169;//dc-8 + + LookUpService[] lus; + DistributedHashMap dhmap; + + atomic { + dhmap = global new DistributedHashMap(100, 100, 0.75f); + //Add to the hash map + for(int i = 0; i < ls.nobjs; i++) { + Integer key = global new Integer(i); + Integer val = global new Integer(i*i); + Object o1 = key; + Object o2 = val; + dhmap.put(o1, o2); + } + lus = global new LookUpService[nthreads]; + for(int i = 0; i -nEntry -nTrans -probRead -nLookUp \n"); + System.printString(" -N the number of threads\n"); + System.printString(" -nEntry the number of objects to be inserted into distributed hashmap\n"); + System.printString(" -nTrans the number of transactions to run\n"); + System.printString(" -probRead the probability of read given a transaction\n"); + System.printString(" -nLookUp the number of lookups per transaction\n"); + System.printString(" -h help with usage\n"); + } +} diff --git a/Robust/src/Benchmarks/Recovery/FileSystem/makefile b/Robust/src/Benchmarks/Recovery/FileSystem/makefile new file mode 100644 index 00000000..67165fe6 --- /dev/null +++ b/Robust/src/Benchmarks/Recovery/FileSystem/makefile @@ -0,0 +1,10 @@ +MAINCLASS=LookUpService +SRC1=${MAINCLASS}.java \ + DistributedHashMap.java +FLAGS= -dsm -recovery -optimize -mainclass ${MAINCLASS} +default: + ../../../buildscript ${FLAGS} -o ${MAINCLASS} ${SRC1} + +clean: + rm -rf tmpbuilddirectory + rm *.bin diff --git a/Robust/src/Benchmarks/Recovery/MatrixMultiply/MatrixMultiplyN.java b/Robust/src/Benchmarks/Recovery/MatrixMultiply/MatrixMultiplyN.java new file mode 100644 index 00000000..22843b25 --- /dev/null +++ b/Robust/src/Benchmarks/Recovery/MatrixMultiply/MatrixMultiplyN.java @@ -0,0 +1,230 @@ +public class MatrixMultiply extends Task { + MMul mmul; + int SIZE; + int increment; + + public MatrixMultiply(MMul mmul, int num_threads, int size) { + this.mmul = mmul; + + SIZE = size; + increment = 80; + + init(); + } + + public void init() { + todoList = global new Queue(); + doneList = global new Queue(); + + fillTodoList(); + } + + // fill up the Work Pool + public void fillTodoList() { + Segment seg; + int i; + + for(i = 0; i < SIZE; i +=increment) { + + if(i+increment > SIZE) { + seg = global new Segment(i,SIZE); + } + else { + seg = global new Segment(i, i + increment); + } + todoList.push(seg); + } + } + + public void execute() { + double la[][]; + double lc[][]; + double lb[][]; + double rowA[]; + double colB[]; + Segment seg; + + double innerproduct; + int i,j; + int x0; + int x1; + int size; + + // get matrix + atomic { + seg = (Segment)myWork; + x0 = seg.x0; // x start row + x1 = seg.x1; // x end row + la = mmul.a; // first mat + lb = mmul.btranspose; // second mat +// lc = mmul.c; // destination mat + size = SIZE; + } + + lc = new double[size][size]; + System.out.println("Seg x0 = " + x0 + " - x1 = " + x1); + + for(i = x0; i < x1 ; i++) { + System.printString("i = " + i + "\n"); + atomic { + rowA = la[i]; // grab first mat's row + + for(j = 0; j < size ; j++) { + colB = lb[j]; // grab second mat's col + + innerproduct = computeProduct(rowA,colB, size); // computes the value + + lc[i][j] = innerproduct; // store in dest mat + } // end of for j + } + } // end for i +// } + System.out.println("Finished comutation"); + + atomic { + for (i = x0; i < x1; i++) { + for (j = 0; j < size; j++) { + mmul.c[i][j] = lc[i][j]; + } + } + } + } + + public double computeProduct(double[] rowA,double[] colB, int size) + { + int i; + double sum = 0; + + for(i = 0 ;i < size; i++) { +// atomic { + sum += rowA[i] * colB[i]; +// } + } + + return sum; + } + + public void done(Object work) { + atomic { + ((Queue)doneList).push(work); + } + } + + public static void main(String[] args) { + int NUM_THREADS = 4; + int SIZE = 1600; + int i,j; + Work[] works; + MMul matrix; + MatrixMultiply mm; + Segment[] currentWorkList; + + if (args.length > 0) { + NUM_THREADS = Integer.parseInt(args[0]); + } + + int[] mid = new int[NUM_THREADS]; + mid[0] = (128<<24)|(195<<16)|(180<<8)|21; //dc1 + mid[1] = (128<<24)|(195<<16)|(180<<8)|24; //dc2 + mid[2] = (128<<24)|(195<<16)|(180<<8)|26; //dc3 + + atomic { + matrix = global new MMul(SIZE, SIZE, SIZE); + matrix.setValues(); + matrix.transpose(); + mm = global new MatrixMultiply(matrix, NUM_THREADS, SIZE); + + works = global new Work[NUM_THREADS]; + currentWorkList = global new Segment[NUM_THREADS]; + + for(i = 0; i < NUM_THREADS; i++) { + works[i] = global new Work(mm, NUM_THREADS, i,currentWorkList); + } + } + System.out.println("Finished to createObjects"); + + Work tmp; + for (i = 0; i < NUM_THREADS; i++) { + atomic { + tmp = works[i]; + } + Thread.myStart(tmp,mid[i]); + } + + for (i = 0; i < NUM_THREADS; i++) { + atomic { + tmp = works[i]; + } + tmp.join(); + } + + System.printString("Finished\n"); + } +} + +public class MMul{ + public int L, M, N; + public double[][] a; + public double[][] b; + public double[][] c; + public double[][] btranspose; + + public MMul(int L, int M, int N) { + this.L = L; + this.M = M; + this.N = N; + a = global new double[L][M]; + b = global new double[M][N]; + c = global new double[L][N]; + btranspose = global new double[N][M]; + } + + public void setValues() { + for(int i = 0; i < L; i++) { + double ai[] = a[i]; + for(int j = 0; j < M; j++) { + ai[j] = j+1; + } + } + + for(int i = 0; i < M; i++) { + double bi[] = b[i]; + for(int j = 0; j < N; j++) { + bi[j] = j+1; + } + } + + for(int i = 0; i < L; i++) { + double ci[] = c[i]; + for(int j = 0; j < N; j++) { + ci[j] = 0; + } + } + for(int i = 0; i < N; i++) { + double btransposei[] = btranspose[i]; + for(int j = 0; j < M; j++) { + btransposei[j] = 0; + } + } + } + + public void transpose() { + for(int row = 0; row < M; row++) { + double brow[] = b[row]; + for(int col = 0; col < N; col++) { + btranspose[col][row] = brow[col]; + } + } + } +} + +public class Segment { + int x0; + int x1; + + Segment (int x0, int x1) { + this.x0 = x0; + this.x1 = x1; + } +} + diff --git a/Robust/src/Benchmarks/Recovery/MatrixMultiply/dstm.conf b/Robust/src/Benchmarks/Recovery/MatrixMultiply/dstm.conf new file mode 100644 index 00000000..1ecc1322 --- /dev/null +++ b/Robust/src/Benchmarks/Recovery/MatrixMultiply/dstm.conf @@ -0,0 +1,9 @@ +#128.195.180.21 +#128.195.180.24 +#128.195.180.26 +128.195.136.162 +128.195.136.163 +128.195.136.164 +128.195.136.165 +128.195.136.166 +128.195.136.167 diff --git a/Robust/src/Benchmarks/Recovery/MatrixMultiply/makefile b/Robust/src/Benchmarks/Recovery/MatrixMultiply/makefile new file mode 100644 index 00000000..b78e006c --- /dev/null +++ b/Robust/src/Benchmarks/Recovery/MatrixMultiply/makefile @@ -0,0 +1,11 @@ +MAINCLASS=MatrixMultiply +SRC1=${MAINCLASS}N.java + +FLAGS= -recovery -dsm -32bit -nooptimize -recovery -debug -dsmtask -mainclass ${MAINCLASS} + +default: + ../../../buildscript ${FLAGS} -o ${MAINCLASS} ${SRC1} + +clean: + rm -rf tmpbuilddirectory + rm *.bin diff --git a/Robust/src/Benchmarks/Recovery/Spider/GlobalQuery.java b/Robust/src/Benchmarks/Recovery/Spider/GlobalQuery.java new file mode 100644 index 00000000..718728b8 --- /dev/null +++ b/Robust/src/Benchmarks/Recovery/Spider/GlobalQuery.java @@ -0,0 +1,92 @@ +public class GlobalQuery { + GlobalString hostname; + GlobalString path; + int depth; + + public GlobalQuery(GlobalString hostname, GlobalString path) { + this.hostname = hostname; + this.path = path; + this.depth = 0; + } + + public GlobalQuery(GlobalString hostname, GlobalString path, int depth) { + this.hostname = global new GlobalString(hostname); + this.path = global new GlobalString(path); + this.depth = depth; + } + + public int getDepth() { + return depth; + } + + public GlobalString getHostName() { + return hostname; + } + + public GlobalString getPath() { + return path; + } + + public GlobalString makewebcanonical(GlobalString page) { + GlobalStringBuffer b = global new GlobalStringBuffer(getHostName(page)); + b.append("/"); + b.append(getPathName(page)); + return b.toGlobalString(); + } + + public GlobalString getHostName(GlobalString page) { + GlobalString http = global new GlobalString("http://"); + GlobalString https = global new GlobalString("https://"); + int beginindex; + int endindex; + + if ((page.indexOf(http) == -1) && (page.indexOf(https) == -1)) { + return getHostName(); + } + else if (page.indexOf(https) != -1) { + beginindex = page.indexOf(https) + https.length(); + } + else { + beginindex = page.indexOf(http) + http.length(); + } + endindex = page.indexOf('/',beginindex+1); + + if ((beginindex == -1)) { + System.printString("ERROR"); + } + if (endindex == -1) + endindex = page.length(); + + return page.subString(beginindex, endindex); + } + + + public GlobalString getPathName(GlobalString page) { + GlobalString http = global new GlobalString("http://"); + GlobalString https = global new GlobalString("https://"); + int beginindex; + int nextindex; + + if ((page.indexOf(http) == -1) && (page.indexOf(https) == -1)) { + GlobalString path = getPath(); + int lastindex = path.lastindexOf('/'); + if (lastindex == -1) + return page; + + GlobalStringBuffer sb = global new GlobalStringBuffer(path.subString(0,lastindex+1)); + sb.append(page); + return sb.toGlobalString(); + } + else if (page.indexOf(https) != -1) { + beginindex = page.indexOf(https) + https.length(); + } + else { + beginindex = page.indexOf(http) + http.length(); + } + nextindex = page.indexOf('/',beginindex+1); + + if ((beginindex == -1) || (nextindex == -1)) + return global new GlobalString("index.html"); + return page.subString(nextindex+1, page.length()); + } +} diff --git a/Robust/src/Benchmarks/Recovery/Spider/LocalQuery.java b/Robust/src/Benchmarks/Recovery/Spider/LocalQuery.java new file mode 100644 index 00000000..1beeadbe --- /dev/null +++ b/Robust/src/Benchmarks/Recovery/Spider/LocalQuery.java @@ -0,0 +1,95 @@ +public class LocalQuery { + String hostname; + String path; + StringBuffer response; + int depth; + + public LocalQuery(String hostname, String path, int depth) { + this.hostname = new String(hostname); + this.path = new String(path); + response = new StringBuffer(); + this.depth = depth; + } + + public int getDepth() { + return depth; + } + + public String getHostName() { + return hostname; + } + + public String getPath() { + return path; + } + + public void outputFile() { + StringBuffer sb = new StringBuffer(hostname); + sb.append(path); + FileOutputStream fos = new FileOutputStream(sb.toString().replace('/','#')); + fos.write(response.toString().getBytes()); + fos.close(); + } + + public String makewebcanonical(String page) { + StringBuffer b = new StringBuffer(getHostName(page)); + b.append("/"); + b.append(getPathName(page)); + return b.toString(); + } + + public String getHostName(String page) { + String http = new String("http://"); + String https = new String("https://"); + int beginindex; + int endindex; + + if ((page.indexOf(http) == -1) && (page.indexOf(https) == -1)) { + return getHostName(); + } + else if (page.indexOf(https) != -1) { + beginindex = page.indexOf(https) + https.length(); + } + else { + beginindex = page.indexOf(http) + http.length(); + } + endindex = page.indexOf('/',beginindex+1); + + if ((beginindex == -1)) { + System.printString("ERROR"); + } + if (endindex == -1) + endindex = page.length(); + + return page.subString(beginindex, endindex); + } + + public String getPathName(String page) { + String http = new String("http://"); + String https = new String("https://"); + int beginindex; + int nextindex; + + if ((page.indexOf(http) == -1) && (page.indexOf(https) == -1)) { + String path = getPath(); + int lastindex = path.lastindexOf('/'); + if (lastindex == -1) + return page; + + StringBuffer sb = new StringBuffer(path.subString(0,lastindex+1)); + sb.append(page); + return sb.toString(); + } + else if (page.indexOf(https) != -1) { + beginindex = page.indexOf(https) + https.length(); + } + else { + beginindex = page.indexOf(http) + http.length(); + } + nextindex = page.indexOf('/',beginindex+1); + + if ((beginindex==-1) || (nextindex==-1)) + return new String("index.html"); + return page.subString(nextindex+1, page.length()); + } +} diff --git a/Robust/src/Benchmarks/Recovery/Spider/QueryTask.java b/Robust/src/Benchmarks/Recovery/Spider/QueryTask.java new file mode 100644 index 00000000..8064a624 --- /dev/null +++ b/Robust/src/Benchmarks/Recovery/Spider/QueryTask.java @@ -0,0 +1,365 @@ +public class QueryTask extends Task { + int maxDepth; + Queue toprocess; + DistributedHashMap results; + GlobalString gTitle; + GlobalString workingURL; + + public QueryTask(Queue todoList, DistributedHashMap doneList, int maxDepth, DistributedHashMap results) { + this.todoList = todoList; + this.doneList = doneList; + this.maxDepth = maxDepth; + this.results = results; + toprocess = global new Queue(); + } + + public void execute() { + int depth; + int max; + + atomic { + depth = ((GlobalQuery)myWork).getDepth(); + max = this.maxDepth; + } + + if (depth < max) { + /* global variables */ + GlobalQuery gq; + + /* local variables */ + LocalQuery lq; + String hostname; + String path; + String title; + + atomic { + gq = (GlobalQuery)myWork; + hostname = new String(GlobalString.toLocalCharArray(gq.getHostName())); + path = new String(GlobalString.toLocalCharArray(gq.getPath())); + + GlobalStringBuffer gsb = global new GlobalStringBuffer(hostname); + gsb.append("/"); + gsb.append(path); + workingURL = global new GlobalString(gsb.toGlobalString()); + gTitle = null; + } + lq = new LocalQuery(hostname, path, depth); + + System.printString("["+lq.getDepth()+"] "); + System.printString("Processing - Hostname : "); + System.printString(hostname); + System.printString(", Path : "); + System.printString(path); + System.printString("\n"); + + if (isDocument(path)) { + return; + } + + Socket s = new Socket(); + + if(s.connect(hostname, 80) == -1) { + return; + } + + requestQuery(hostname, path, s); + readResponse(lq, s); + + if ((title = grabTitle(lq)) != null) { + atomic { + gTitle = global new GlobalString(title); + } + atomic { + toprocess = processPage(lq); + } + } + s.close(); + } + } + + public static boolean isDocument(String str) { + int index = str.lastindexOf('.'); + + if (index != -1) { + if ((str.subString(index+1)).equals("pdf")) return true; + else if ((str.subString(index+1)).equals("ps")) return true; + else if ((str.subString(index+1)).equals("ppt")) return true; + else if ((str.subString(index+1)).equals("pptx")) return true; + else if ((str.subString(index+1)).equals("jpg")) return true; + else if ((str.subString(index+1)).equals("mp3")) return true; + else if ((str.subString(index+1)).equals("wmv")) return true; + else if ((str.subString(index+1)).equals("doc")) return true; + else if ((str.subString(index+1)).equals("docx")) return true; + else if ((str.subString(index+1)).equals("mov")) return true; + else if ((str.subString(index+1)).equals("flv")) return true; + else return false; + } + return false; + } + + public void done(Object obj) { + if ((gTitle != null) && (gTitle.length() > 0)) { + processList(); + } + + while(!toprocess.isEmpty()) { + GlobalQuery q = (GlobalQuery)toprocess.pop(); + + GlobalString hostname = global new GlobalString(q.getHostName()); + GlobalString path = global new GlobalString(q.getPath()); + + GlobalStringBuffer gsb = global new GlobalStringBuffer(hostname); + gsb.append("/"); + gsb.append(path); + + if (!doneList.containsKey(gsb.toGlobalString())) { + todoList.push(q); + + GlobalString str = global new GlobalString("1"); + doneList.put(gsb.toGlobalString(), str); + } + } + } + + public static String grabTitle(LocalQuery lq) { + String sBrace = new String("<"); + String strTitle = new String("title>"); + String searchstr = lq.response.toString(); + String title = null; + char ch; + + int mindex = -1; + int endquote = -1; + int i, j; + String tmp; + + for (i = 0; i < searchstr.length(); i++) { + if (searchstr.charAt(i) == '<') { + i++; + if (searchstr.length() > (i+strTitle.length())) { + tmp = searchstr.subString(i, i+strTitle.length()); + if (tmp.equalsIgnoreCase("title>")) { + mindex = i + tmp.length(); + for (j = mindex; j < searchstr.length(); j++) { + if (searchstr.charAt(j) == '<') { + j++; + tmp = searchstr.subString(j, j+strTitle.length()+1); + if (tmp.equalsIgnoreCase("/title>")) { + endquote = j - 1; + break; + } + } + } + } + } + } + } + + if (mindex != -1) { + title = searchstr.subString(mindex, endquote); + if (Character.isWhitespace(title.charAt(0))){ + mindex=0; + while (Character.isWhitespace(title.charAt(mindex++))); + mindex--; + if (mindex >= title.length()) return null; + title = new String(title.subString(mindex)); + } + + if (Character.isWhitespace(title.charAt(title.length()-1))) { + endquote=title.length()-1; + while (Character.isWhitespace(title.charAt(endquote--))); + endquote += 2; + if (mindex >= endquote) return null; + title = new String(title.subString(0, endquote)); + } + + if (isErrorPage(title)) { + return null; + } + } + + return title; + } + + public static boolean isErrorPage(String str) { + if (str.equals("301 Moved Permanently")) + return true; + else if (str.equals("302 Found")) + return true; + else if (str.equals("404 Not Found")) + return true; + else if (str.equals("403 Forbidden")) + return true; + else if (str.equals("404 File Not Found")) + return true; + else + return false; + } + + public static void requestQuery(String hostname, String path, Socket sock) { + StringBuffer req = new StringBuffer("GET "); + req.append("/"); + req.append(path); + req.append(" HTTP/1.0\r\nHost: "); + req.append(hostname); + req.append("\r\n\r\n"); + sock.write(req.toString().getBytes()); + } + + public static void readResponse(LocalQuery lq, Socket sock) { + // state 0 - nothing + // state 1 - \r + // state 2 - \r\n + // state 3 - \r\n\r + // state 4 - \r\n\r\n + byte[] buffer = new byte[1024]; + int numchars; + + do { + numchars = sock.read(buffer); + + String curr = (new String(buffer)).subString(0, numchars); + + lq.response.append(curr); + buffer = new byte[1024]; + } while(numchars > 0); + } + + public void processList() { + LinkedList ll; + GlobalString token = null; + int mindex = 0; + int endquote = 0; + + while (endquote != -1) { + endquote = gTitle.indexOf(' ', mindex); + + if (endquote != -1) { + token = gTitle.subString(mindex, endquote); + mindex = endquote + 1; + if (filter(token)) { + continue; + } + token = refine(token); + } + else { + token = gTitle.subString(mindex); + token = refine(token); + } + + Queue q = (Queue)results.get(token); + if (q == null) { + q = global new Queue(); + } + q.push(workingURL); + results.put(token, q); + } + } + + public boolean filter(GlobalString str) { + if (str.equals("of")) return true; + else if (str.equals("for")) return true; + else if (str.equals("a")) return true; + else if (str.equals("an")) return true; + else if (str.equals("the")) return true; + else if (str.equals("at")) return true; + else if (str.equals("and")) return true; + else if (str.equals("or")) return true; + else if (str.equals("but")) return true; + else if (str.equals("to")) return true; + else if (str.equals("The")) return true; + else if (str.length() == 1) { + if (str.charAt(0) == '.') return true; + else if (str.charAt(0) == '.') return true; + else if (str.charAt(0) == '-') return true; + else if (str.charAt(0) == '=') return true; + else if (str.charAt(0) == '_') return true; + else if (str.charAt(0) == ':') return true; + else if (str.charAt(0) == ';') return true; + else if (str.charAt(0) == '\'') return true; + else if (str.charAt(0) == '\"') return true; + else if (str.charAt(0) == '|') return true; + else if (str.charAt(0) == '@') return true; + else if (str.charAt(0) == '&') return true; + else if (str.charAt(0) == ' ') return true; + } + else return false; + } + + public GlobalString refine(GlobalString str) { + str = refinePrefix(str); + str = refinePostfix(str); + return str; + } + + public GlobalString refinePrefix(GlobalString str) { + if (str.charAt(0) == '&') { // & + return str.subString(1); + } + else if (str.charAt(0) == '/') { // & + return str.subString(1); + } + return str; + } + + public GlobalString refinePostfix(GlobalString str) { + if (str.charAt(str.length()-1) == ',') { // , + return str.subString(0, str.length()-1); + } + else if (str.charAt(str.length()-1) == ':') { // : + return str.subString(0, str.length()-1); + } + else if (str.charAt(str.length()-1) == ';') { // ; + return str.subString(0, str.length()-1); + } + else if (str.charAt(str.length()-1) == '!') { // ! + return str.subString(0, str.length()-1); + } + else if (str.charAt(str.length()-1) == 's') { // 's + if (str.charAt(str.length()-2) == '\'') + return str.subString(0, str.length()-2); + } + else if (str.charAt(str.length()-1) == '-') { + int index = str.length()-2; + while (Character.isWhitespace(str.charAt(index--))); + return str.subString(0, index+2); + } + return str; + } + + public static Queue processPage(LocalQuery lq) { + int index = 0; + String href = new String("href=\""); + String searchstr = lq.response.toString(); + int depth; + boolean cont = true; + Queue toprocess; + + depth = lq.getDepth() + 1; + + toprocess = global new Queue(); + while(cont) { + int mindex = searchstr.indexOf(href,index); + if (mindex != -1) { + int endquote = searchstr.indexOf('"', mindex+href.length()); + if (endquote != -1) { + String match = searchstr.subString(mindex+href.length(), endquote); + String match2 = lq.makewebcanonical(match); + + GlobalString ghostname; + GlobalString gpath; + + ghostname = global new GlobalString(lq.getHostName(match)); + gpath = global new GlobalString(lq.getPathName(match)); + + if (match2 != null) { + GlobalQuery gq = global new GlobalQuery(ghostname, gpath, depth); + toprocess.push(gq); + } + index = endquote; + } else cont = false; + } else cont = false; + } + return toprocess; + } +} diff --git a/Robust/src/Benchmarks/Recovery/Spider/Spider.java b/Robust/src/Benchmarks/Recovery/Spider/Spider.java new file mode 100644 index 00000000..49b25d89 --- /dev/null +++ b/Robust/src/Benchmarks/Recovery/Spider/Spider.java @@ -0,0 +1,62 @@ +public class Spider { + public static void main(String[] args) { + int NUM_THREADS = 3; + int maxDepth = 4; + int i, j; + Work[] works; + QueryTask[] qt; + GlobalQuery[] currentWorkList; + + NUM_THREADS = Integer.parseInt(args[0]); + + GlobalString firstmachine; + GlobalString firstpage; + + int mid[] = new int[NUM_THREADS]; + mid[0] = (128<<24)|(195<<16)|(136<<8)|162; + mid[1] = (128<<24)|(195<<16)|(136<<8)|163; + mid[2] = (128<<24)|(195<<16)|(136<<8)|164; + + atomic { + firstmachine = global new GlobalString(args[1]); + if (args.length == 3) { + firstpage = global new GlobalString(args[2]); + } + else + firstpage = global new GlobalString("");; + + works = global new Work[NUM_THREADS]; + qt = global new QueryTask[NUM_THREADS]; + currentWorkList = global new GlobalQuery[NUM_THREADS]; + + GlobalQuery firstquery = global new GlobalQuery(firstmachine, firstpage); + + Queue todoList = global new Queue(); + DistributedHashMap doneList = global new DistributedHashMap(500, 500, 0.75f); + DistributedHashMap results = global new DistributedHashMap(100, 100, 0.75f); + + todoList.push(firstquery); + + for (i = 0; i < NUM_THREADS; i++) { + qt[i] = global new QueryTask(todoList, doneList, maxDepth, results); + works[i] = global new Work(qt[i], NUM_THREADS, i, currentWorkList); + } + } + System.printString("Finished to create Objects\n"); + + Work tmp; + for (i = 0; i < NUM_THREADS; i++) { + atomic { + tmp = works[i]; + } + Thread.myStart(tmp, mid[i]); + } + + for (i = 0; i < NUM_THREADS; i++) { + atomic { + tmp = works[i]; + } + tmp.join(); + } + } +} diff --git a/Robust/src/Benchmarks/Recovery/Spider/dstm.conf b/Robust/src/Benchmarks/Recovery/Spider/dstm.conf new file mode 100644 index 00000000..eff447b8 --- /dev/null +++ b/Robust/src/Benchmarks/Recovery/Spider/dstm.conf @@ -0,0 +1,6 @@ +#128.195.180.21 +#128.195.180.24 +#128.195.180.26 +128.195.136.162 +128.195.136.163 +128.195.136.164 diff --git a/Robust/src/Benchmarks/Recovery/Spider/makefile b/Robust/src/Benchmarks/Recovery/Spider/makefile new file mode 100644 index 00000000..eafdf663 --- /dev/null +++ b/Robust/src/Benchmarks/Recovery/Spider/makefile @@ -0,0 +1,12 @@ +MAINCLASS=Spider +SUBCLASS=Query +SRC1=${MAINCLASS}.java +SRC2=Global${SUBCLASS}.java +SRC3=${SUBCLASS}Task.java +FLAGS= -recovery -dsmtask -dsm -dsmtask -32bit -nooptimize -debug -mainclass ${MAINCLASS} +default: + ../../../buildscript ${FLAGS} -o ${MAINCLASS} ${SRC2} ${SRC3} ${SRC1} + +clean: + rm -rf tmpbuilddirectory + rm *.bin -- 2.34.1