--- /dev/null
+public class DistributedHashMap {
+ DistributedHashEntry[] table;
+ float loadFactor;
+ int secondcapacity;
+
+ public DistributedHashMap(int initialCapacity, int secondcapacity, float loadFactor) {
+ init(initialCapacity, secondcapacity, loadFactor);
+ }
+
+ private void init(int initialCapacity, int secondcapacity, float loadFactor) {
+ table = global new DistributedHashEntry[initialCapacity];
+ this.loadFactor=loadFactor;
+ this.secondcapacity=secondcapacity;
+ }
+
+ private static int hash1(int hashcode, int length) {
+ int value=hashcode%length;
+ if (value<0)
+ return -value;
+ else
+ return value;
+ }
+
+ private static int hash2(int hashcode, int length1, int length2) {
+ int value=(hashcode*31)%length2;
+ if (value<0)
+ return -value;
+ else
+ return value;
+ }
+
+ void resize(int index) {
+ DHashEntry[] oldtable=table[index].array;
+ int newCapacity=oldtable.length*2+1;
+ DHashEntry [] newtable=global new DHashEntry[newCapacity];
+ table[index].array=newtable;
+
+ for(int i=0; i<oldtable.length; i++) {
+ DHashEntry e=oldtable[i];
+ while(e!=null) {
+ DHashEntry next=e.next;
+ int bin=hash2(e.hashval, table.length, newCapacity);
+ e.next=newtable[bin];
+ newtable[bin]=e;
+ e=next;
+ }
+ }
+ }
+
+ Object remove(Object key) {
+ int hashcode=key.hashCode();
+ int index1=hash1(hashcode, table.length);
+ DistributedHashEntry dhe=table[index1];
+ if (dhe==null)
+ return null;
+ int index2=hash2(hashcode, table.length, dhe.array.length);
+ DHashEntry ptr=dhe.array[index2];
+
+ if (ptr!=null) {
+ if (ptr.hashval==hashcode&&ptr.key.equals(key)) {
+ dhe.array[index2]=ptr.next;
+ dhe.count--;
+ return ptr.value;
+ }
+ while(ptr.next!=null) {
+ if (ptr.hashval==hashcode&&ptr.next.key.equals(key)) {
+ Object oldvalue=ptr.value;
+ ptr.next=ptr.next.next;
+ dhe.count--;
+ return oldvalue;
+ }
+ ptr=ptr.next;
+ }
+ }
+ return null;
+ }
+
+ Object get(Object key) {
+ int hashcode=key.hashCode();
+ int index1=hash1(hashcode, table.length);
+
+ DistributedHashEntry dhe=table[index1];
+ if (dhe==null)
+ return null;
+
+ int index2=hash2(hashcode, table.length, dhe.array.length);
+
+ DHashEntry ptr=dhe.array[index2];
+
+ while(ptr!=null) {
+ if (ptr.hashval==hashcode
+ &&ptr.key.equals(key)) {
+ return ptr.value;
+ }
+ ptr=ptr.next;
+ }
+ return null;
+ }
+
+ boolean containsKey(Object key) {
+ int hashcode=key.hashCode();
+ int index1=hash1(hashcode, table.length);
+ DistributedHashEntry dhe=table[index1];
+ if (dhe==null)
+ return false;
+ int index2=hash2(hashcode, table.length, dhe.array.length);
+ DHashEntry ptr=dhe.array[index2];
+
+ while(ptr!=null) {
+ if (ptr.hashval==hashcode
+ &&ptr.key.equals(key)) {
+ return true;
+ }
+ ptr=ptr.next;
+ }
+ return false;
+ }
+
+ Object put(Object key, Object value) {
+ int hashcode=key.hashCode();
+ int index1=hash1(hashcode, table.length);
+ DistributedHashEntry dhe=table[index1];
+ if (dhe==null) {
+ dhe=global new DistributedHashEntry(secondcapacity);
+ table[index1]=dhe;
+ }
+ int index2=hash2(hashcode, table.length, dhe.array.length);
+ DHashEntry ptr=dhe.array[index2];
+
+ while(ptr!=null) {
+ if (ptr.hashval==hashcode&&ptr.key.equals(key)) {
+ Object oldvalue=ptr.value;
+ ptr.value=value;
+ return oldvalue;
+ }
+ ptr=ptr.next;
+ }
+
+ DHashEntry he=global new DHashEntry();
+ he.value=value;
+ he.key=key;
+ he.hashval=hashcode;
+ he.next=dhe.array[index2];
+ dhe.array[index2]=he;
+
+ dhe.count++;
+ if (dhe.count>(loadFactor*dhe.array.length)) {
+ //Resize the table
+ resize(index1);
+ }
+ return null;
+ }
+}
+
+
+class DistributedHashEntry {
+ public DistributedHashEntry(int capacity) {
+ array=global new DHashEntry[capacity];
+ }
+ int count;
+ DHashEntry[] array;
+}
+
+
+class DHashEntry {
+ public DHashEntry() {
+ }
+ int hashval;
+ Object key;
+ Object value;
+ DHashEntry next;
+}
--- /dev/null
+public class LookUpService extends Thread {
+ DistributedHashMap mydhmap;
+ /**
+ * The thread id involved
+ **/
+ private int threadid;
+ /**
+ * The total number of threads
+ **/
+ private int numthreads;
+
+ /**
+ * The total number of transactions
+ **/
+ private int numtrans;
+
+ /**
+ * The total number of objects created
+ **/
+ private int nobjs;
+
+ /**
+ * The probability of initiating a look up
+ * the read probability % between 0-99
+ **/
+ private int rdprob;
+
+ /**
+ * The number of look up operations
+ **/
+ private int nLookUp;
+
+ public LookUpService() {
+ }
+
+ public LookUpService(DistributedHashMap dmap, int threadid, int numthreads, int nobjs, int numtrans, int rdprob, int nLookUp) {
+ mydhmap = dmap;
+ this.threadid = threadid;
+ this.numthreads = numthreads;
+ this.nobjs = nobjs;
+ this.numtrans = numtrans;
+ this.rdprob = rdprob;
+ this.nLookUp = nLookUp;
+ }
+
+ public void run() {
+ int ntrans;
+ atomic {
+ ntrans = numtrans;
+ }
+
+ // Do read/writes
+ Random rand = new Random(0);
+
+ for (int i = 0; i < ntrans; i++) {
+ atomic {
+ for(int j = 0; j < nLookUp; j++) {
+ int rdwr = rand.nextInt(100);
+ int rwkey = rand.nextInt(nobjs);
+ Integer key = global new Integer(rwkey);
+ if (rdwr < rdprob) {
+ Object o3 = mydhmap.get(key); //Read
+ } else {
+ Integer val = global new Integer(j);
+ mydhmap.put(key, val); //Modify
+ }
+ }
+ }
+ }
+ }
+
+ public static void main(String[] args) {
+ LookUpService ls = new LookUpService();
+ LookUpService.parseCmdLine(args,ls);
+
+ int nthreads = ls.numthreads;
+ int[] mid = new int[8];
+ mid[0] = (128<<24)|(195<<16)|(136<<8)|162;//dc-1
+ mid[1] = (128<<24)|(195<<16)|(136<<8)|163;//dc-2
+ mid[2] = (128<<24)|(195<<16)|(136<<8)|164;//dc-3
+ mid[3] = (128<<24)|(195<<16)|(136<<8)|165;//dc-4
+ mid[4] = (128<<24)|(195<<16)|(136<<8)|166;//dc-5
+ mid[5] = (128<<24)|(195<<16)|(136<<8)|167;//dc-6
+ mid[6] = (128<<24)|(195<<16)|(136<<8)|168;//dc-7
+ mid[7] = (128<<24)|(195<<16)|(136<<8)|169;//dc-8
+
+ LookUpService[] lus;
+ DistributedHashMap dhmap;
+
+ atomic {
+ dhmap = global new DistributedHashMap(100, 100, 0.75f);
+ //Add to the hash map
+ for(int i = 0; i < ls.nobjs; i++) {
+ Integer key = global new Integer(i);
+ Integer val = global new Integer(i*i);
+ Object o1 = key;
+ Object o2 = val;
+ dhmap.put(o1, o2);
+ }
+ lus = global new LookUpService[nthreads];
+ for(int i = 0; i<nthreads; i++) {
+ lus[i] = global new LookUpService(dhmap, i, ls.numthreads, ls.nobjs, ls.numtrans, ls.rdprob, ls.nLookUp);
+ }
+ }
+
+ LookUpService tmp;
+ /* Start threads */
+ for(int i = 0; i<nthreads; i++) {
+ atomic {
+ tmp = lus[i];
+ }
+ tmp.start(mid[i]);
+ }
+
+ /* Join threads */
+ for(int i = 0; i<nthreads; i++) {
+ atomic {
+ tmp = lus[i];
+ }
+ tmp.join();
+ }
+
+ System.printString("Finished\n");
+ }
+
+ /**
+ * Parse the command line options.
+ **/
+ public static void parseCmdLine(String args[], LookUpService lus) {
+ int i = 0;
+ String arg;
+ while(i < args.length && args[i].startsWith("-")) {
+ arg = args[i++];
+ //check options
+ if(arg.equals("-N")) {
+ if(i < args.length) {
+ lus.numthreads = new Integer(args[i++]).intValue();
+ }
+ } else if(arg.equals("-nEntry")) {
+ if(i < args.length) {
+ lus.nobjs = new Integer(args[i++]).intValue();
+ }
+ } else if (arg.equals("-nTrans")) {
+ if(i < args.length) {
+ lus.numtrans = new Integer(args[i++]).intValue();
+ }
+ } else if(arg.equals("-probRead")) {
+ if(i < args.length) {
+ lus.rdprob = new Integer(args[i++]).intValue();
+ }
+ } else if(arg.equals("-nLookUp")) {
+ if(i < args.length) {
+ lus.nLookUp = new Integer(args[i++]).intValue();
+ }
+ } else if(arg.equals("-h")) {
+ lus.usage();
+ }
+ }
+
+ if(lus.nobjs == 0 || lus.numtrans == 0)
+ lus.usage();
+ }
+
+ /**
+ * The usage routine which describes the program options.
+ **/
+ public void usage() {
+ System.printString("usage: ./LookUpServiceN.bin master -N <threads> -nEntry <objects in hashmap> -nTrans <number of transactions> -probRead <read probability> -nLookUp <number of lookups>\n");
+ System.printString(" -N the number of threads\n");
+ System.printString(" -nEntry the number of objects to be inserted into distributed hashmap\n");
+ System.printString(" -nTrans the number of transactions to run\n");
+ System.printString(" -probRead the probability of read given a transaction\n");
+ System.printString(" -nLookUp the number of lookups per transaction\n");
+ System.printString(" -h help with usage\n");
+ }
+}
--- /dev/null
+MAINCLASS=LookUpService
+SRC1=${MAINCLASS}.java \
+ DistributedHashMap.java
+FLAGS= -dsm -recovery -optimize -mainclass ${MAINCLASS}
+default:
+ ../../../buildscript ${FLAGS} -o ${MAINCLASS} ${SRC1}
+
+clean:
+ rm -rf tmpbuilddirectory
+ rm *.bin
--- /dev/null
+public class MatrixMultiply extends Task {
+ MMul mmul;
+ int SIZE;
+ int increment;
+
+ public MatrixMultiply(MMul mmul, int num_threads, int size) {
+ this.mmul = mmul;
+
+ SIZE = size;
+ increment = 80;
+
+ init();
+ }
+
+ public void init() {
+ todoList = global new Queue();
+ doneList = global new Queue();
+
+ fillTodoList();
+ }
+
+ // fill up the Work Pool
+ public void fillTodoList() {
+ Segment seg;
+ int i;
+
+ for(i = 0; i < SIZE; i +=increment) {
+
+ if(i+increment > SIZE) {
+ seg = global new Segment(i,SIZE);
+ }
+ else {
+ seg = global new Segment(i, i + increment);
+ }
+ todoList.push(seg);
+ }
+ }
+
+ public void execute() {
+ double la[][];
+ double lc[][];
+ double lb[][];
+ double rowA[];
+ double colB[];
+ Segment seg;
+
+ double innerproduct;
+ int i,j;
+ int x0;
+ int x1;
+ int size;
+
+ // get matrix
+ atomic {
+ seg = (Segment)myWork;
+ x0 = seg.x0; // x start row
+ x1 = seg.x1; // x end row
+ la = mmul.a; // first mat
+ lb = mmul.btranspose; // second mat
+// lc = mmul.c; // destination mat
+ size = SIZE;
+ }
+
+ lc = new double[size][size];
+ System.out.println("Seg x0 = " + x0 + " - x1 = " + x1);
+
+ for(i = x0; i < x1 ; i++) {
+ System.printString("i = " + i + "\n");
+ atomic {
+ rowA = la[i]; // grab first mat's row
+
+ for(j = 0; j < size ; j++) {
+ colB = lb[j]; // grab second mat's col
+
+ innerproduct = computeProduct(rowA,colB, size); // computes the value
+
+ lc[i][j] = innerproduct; // store in dest mat
+ } // end of for j
+ }
+ } // end for i
+// }
+ System.out.println("Finished comutation");
+
+ atomic {
+ for (i = x0; i < x1; i++) {
+ for (j = 0; j < size; j++) {
+ mmul.c[i][j] = lc[i][j];
+ }
+ }
+ }
+ }
+
+ public double computeProduct(double[] rowA,double[] colB, int size)
+ {
+ int i;
+ double sum = 0;
+
+ for(i = 0 ;i < size; i++) {
+// atomic {
+ sum += rowA[i] * colB[i];
+// }
+ }
+
+ return sum;
+ }
+
+ public void done(Object work) {
+ atomic {
+ ((Queue)doneList).push(work);
+ }
+ }
+
+ public static void main(String[] args) {
+ int NUM_THREADS = 4;
+ int SIZE = 1600;
+ int i,j;
+ Work[] works;
+ MMul matrix;
+ MatrixMultiply mm;
+ Segment[] currentWorkList;
+
+ if (args.length > 0) {
+ NUM_THREADS = Integer.parseInt(args[0]);
+ }
+
+ int[] mid = new int[NUM_THREADS];
+ mid[0] = (128<<24)|(195<<16)|(180<<8)|21; //dc1
+ mid[1] = (128<<24)|(195<<16)|(180<<8)|24; //dc2
+ mid[2] = (128<<24)|(195<<16)|(180<<8)|26; //dc3
+
+ atomic {
+ matrix = global new MMul(SIZE, SIZE, SIZE);
+ matrix.setValues();
+ matrix.transpose();
+ mm = global new MatrixMultiply(matrix, NUM_THREADS, SIZE);
+
+ works = global new Work[NUM_THREADS];
+ currentWorkList = global new Segment[NUM_THREADS];
+
+ for(i = 0; i < NUM_THREADS; i++) {
+ works[i] = global new Work(mm, NUM_THREADS, i,currentWorkList);
+ }
+ }
+ System.out.println("Finished to createObjects");
+
+ Work tmp;
+ for (i = 0; i < NUM_THREADS; i++) {
+ atomic {
+ tmp = works[i];
+ }
+ Thread.myStart(tmp,mid[i]);
+ }
+
+ for (i = 0; i < NUM_THREADS; i++) {
+ atomic {
+ tmp = works[i];
+ }
+ tmp.join();
+ }
+
+ System.printString("Finished\n");
+ }
+}
+
+public class MMul{
+ public int L, M, N;
+ public double[][] a;
+ public double[][] b;
+ public double[][] c;
+ public double[][] btranspose;
+
+ public MMul(int L, int M, int N) {
+ this.L = L;
+ this.M = M;
+ this.N = N;
+ a = global new double[L][M];
+ b = global new double[M][N];
+ c = global new double[L][N];
+ btranspose = global new double[N][M];
+ }
+
+ public void setValues() {
+ for(int i = 0; i < L; i++) {
+ double ai[] = a[i];
+ for(int j = 0; j < M; j++) {
+ ai[j] = j+1;
+ }
+ }
+
+ for(int i = 0; i < M; i++) {
+ double bi[] = b[i];
+ for(int j = 0; j < N; j++) {
+ bi[j] = j+1;
+ }
+ }
+
+ for(int i = 0; i < L; i++) {
+ double ci[] = c[i];
+ for(int j = 0; j < N; j++) {
+ ci[j] = 0;
+ }
+ }
+ for(int i = 0; i < N; i++) {
+ double btransposei[] = btranspose[i];
+ for(int j = 0; j < M; j++) {
+ btransposei[j] = 0;
+ }
+ }
+ }
+
+ public void transpose() {
+ for(int row = 0; row < M; row++) {
+ double brow[] = b[row];
+ for(int col = 0; col < N; col++) {
+ btranspose[col][row] = brow[col];
+ }
+ }
+ }
+}
+
+public class Segment {
+ int x0;
+ int x1;
+
+ Segment (int x0, int x1) {
+ this.x0 = x0;
+ this.x1 = x1;
+ }
+}
+
--- /dev/null
+#128.195.180.21
+#128.195.180.24
+#128.195.180.26
+128.195.136.162
+128.195.136.163
+128.195.136.164
+128.195.136.165
+128.195.136.166
+128.195.136.167
--- /dev/null
+MAINCLASS=MatrixMultiply
+SRC1=${MAINCLASS}N.java
+
+FLAGS= -recovery -dsm -32bit -nooptimize -recovery -debug -dsmtask -mainclass ${MAINCLASS}
+
+default:
+ ../../../buildscript ${FLAGS} -o ${MAINCLASS} ${SRC1}
+
+clean:
+ rm -rf tmpbuilddirectory
+ rm *.bin
--- /dev/null
+public class GlobalQuery {
+ GlobalString hostname;
+ GlobalString path;
+ int depth;
+
+ public GlobalQuery(GlobalString hostname, GlobalString path) {
+ this.hostname = hostname;
+ this.path = path;
+ this.depth = 0;
+ }
+
+ public GlobalQuery(GlobalString hostname, GlobalString path, int depth) {
+ this.hostname = global new GlobalString(hostname);
+ this.path = global new GlobalString(path);
+ this.depth = depth;
+ }
+
+ public int getDepth() {
+ return depth;
+ }
+
+ public GlobalString getHostName() {
+ return hostname;
+ }
+
+ public GlobalString getPath() {
+ return path;
+ }
+
+ public GlobalString makewebcanonical(GlobalString page) {
+ GlobalStringBuffer b = global new GlobalStringBuffer(getHostName(page));
+ b.append("/");
+ b.append(getPathName(page));
+ return b.toGlobalString();
+ }
+
+ public GlobalString getHostName(GlobalString page) {
+ GlobalString http = global new GlobalString("http://");
+ GlobalString https = global new GlobalString("https://");
+ int beginindex;
+ int endindex;
+
+ if ((page.indexOf(http) == -1) && (page.indexOf(https) == -1)) {
+ return getHostName();
+ }
+ else if (page.indexOf(https) != -1) {
+ beginindex = page.indexOf(https) + https.length();
+ }
+ else {
+ beginindex = page.indexOf(http) + http.length();
+ }
+ endindex = page.indexOf('/',beginindex+1);
+
+ if ((beginindex == -1)) {
+ System.printString("ERROR");
+ }
+ if (endindex == -1)
+ endindex = page.length();
+
+ return page.subString(beginindex, endindex);
+ }
+
+
+ public GlobalString getPathName(GlobalString page) {
+ GlobalString http = global new GlobalString("http://");
+ GlobalString https = global new GlobalString("https://");
+ int beginindex;
+ int nextindex;
+
+ if ((page.indexOf(http) == -1) && (page.indexOf(https) == -1)) {
+ GlobalString path = getPath();
+ int lastindex = path.lastindexOf('/');
+ if (lastindex == -1)
+ return page;
+
+ GlobalStringBuffer sb = global new GlobalStringBuffer(path.subString(0,lastindex+1));
+ sb.append(page);
+ return sb.toGlobalString();
+ }
+ else if (page.indexOf(https) != -1) {
+ beginindex = page.indexOf(https) + https.length();
+ }
+ else {
+ beginindex = page.indexOf(http) + http.length();
+ }
+ nextindex = page.indexOf('/',beginindex+1);
+
+ if ((beginindex == -1) || (nextindex == -1))
+ return global new GlobalString("index.html");
+ return page.subString(nextindex+1, page.length());
+ }
+}
--- /dev/null
+public class LocalQuery {
+ String hostname;
+ String path;
+ StringBuffer response;
+ int depth;
+
+ public LocalQuery(String hostname, String path, int depth) {
+ this.hostname = new String(hostname);
+ this.path = new String(path);
+ response = new StringBuffer();
+ this.depth = depth;
+ }
+
+ public int getDepth() {
+ return depth;
+ }
+
+ public String getHostName() {
+ return hostname;
+ }
+
+ public String getPath() {
+ return path;
+ }
+
+ public void outputFile() {
+ StringBuffer sb = new StringBuffer(hostname);
+ sb.append(path);
+ FileOutputStream fos = new FileOutputStream(sb.toString().replace('/','#'));
+ fos.write(response.toString().getBytes());
+ fos.close();
+ }
+
+ public String makewebcanonical(String page) {
+ StringBuffer b = new StringBuffer(getHostName(page));
+ b.append("/");
+ b.append(getPathName(page));
+ return b.toString();
+ }
+
+ public String getHostName(String page) {
+ String http = new String("http://");
+ String https = new String("https://");
+ int beginindex;
+ int endindex;
+
+ if ((page.indexOf(http) == -1) && (page.indexOf(https) == -1)) {
+ return getHostName();
+ }
+ else if (page.indexOf(https) != -1) {
+ beginindex = page.indexOf(https) + https.length();
+ }
+ else {
+ beginindex = page.indexOf(http) + http.length();
+ }
+ endindex = page.indexOf('/',beginindex+1);
+
+ if ((beginindex == -1)) {
+ System.printString("ERROR");
+ }
+ if (endindex == -1)
+ endindex = page.length();
+
+ return page.subString(beginindex, endindex);
+ }
+
+ public String getPathName(String page) {
+ String http = new String("http://");
+ String https = new String("https://");
+ int beginindex;
+ int nextindex;
+
+ if ((page.indexOf(http) == -1) && (page.indexOf(https) == -1)) {
+ String path = getPath();
+ int lastindex = path.lastindexOf('/');
+ if (lastindex == -1)
+ return page;
+
+ StringBuffer sb = new StringBuffer(path.subString(0,lastindex+1));
+ sb.append(page);
+ return sb.toString();
+ }
+ else if (page.indexOf(https) != -1) {
+ beginindex = page.indexOf(https) + https.length();
+ }
+ else {
+ beginindex = page.indexOf(http) + http.length();
+ }
+ nextindex = page.indexOf('/',beginindex+1);
+
+ if ((beginindex==-1) || (nextindex==-1))
+ return new String("index.html");
+ return page.subString(nextindex+1, page.length());
+ }
+}
--- /dev/null
+public class QueryTask extends Task {
+ int maxDepth;
+ Queue toprocess;
+ DistributedHashMap results;
+ GlobalString gTitle;
+ GlobalString workingURL;
+
+ public QueryTask(Queue todoList, DistributedHashMap doneList, int maxDepth, DistributedHashMap results) {
+ this.todoList = todoList;
+ this.doneList = doneList;
+ this.maxDepth = maxDepth;
+ this.results = results;
+ toprocess = global new Queue();
+ }
+
+ public void execute() {
+ int depth;
+ int max;
+
+ atomic {
+ depth = ((GlobalQuery)myWork).getDepth();
+ max = this.maxDepth;
+ }
+
+ if (depth < max) {
+ /* global variables */
+ GlobalQuery gq;
+
+ /* local variables */
+ LocalQuery lq;
+ String hostname;
+ String path;
+ String title;
+
+ atomic {
+ gq = (GlobalQuery)myWork;
+ hostname = new String(GlobalString.toLocalCharArray(gq.getHostName()));
+ path = new String(GlobalString.toLocalCharArray(gq.getPath()));
+
+ GlobalStringBuffer gsb = global new GlobalStringBuffer(hostname);
+ gsb.append("/");
+ gsb.append(path);
+ workingURL = global new GlobalString(gsb.toGlobalString());
+ gTitle = null;
+ }
+ lq = new LocalQuery(hostname, path, depth);
+
+ System.printString("["+lq.getDepth()+"] ");
+ System.printString("Processing - Hostname : ");
+ System.printString(hostname);
+ System.printString(", Path : ");
+ System.printString(path);
+ System.printString("\n");
+
+ if (isDocument(path)) {
+ return;
+ }
+
+ Socket s = new Socket();
+
+ if(s.connect(hostname, 80) == -1) {
+ return;
+ }
+
+ requestQuery(hostname, path, s);
+ readResponse(lq, s);
+
+ if ((title = grabTitle(lq)) != null) {
+ atomic {
+ gTitle = global new GlobalString(title);
+ }
+ atomic {
+ toprocess = processPage(lq);
+ }
+ }
+ s.close();
+ }
+ }
+
+ public static boolean isDocument(String str) {
+ int index = str.lastindexOf('.');
+
+ if (index != -1) {
+ if ((str.subString(index+1)).equals("pdf")) return true;
+ else if ((str.subString(index+1)).equals("ps")) return true;
+ else if ((str.subString(index+1)).equals("ppt")) return true;
+ else if ((str.subString(index+1)).equals("pptx")) return true;
+ else if ((str.subString(index+1)).equals("jpg")) return true;
+ else if ((str.subString(index+1)).equals("mp3")) return true;
+ else if ((str.subString(index+1)).equals("wmv")) return true;
+ else if ((str.subString(index+1)).equals("doc")) return true;
+ else if ((str.subString(index+1)).equals("docx")) return true;
+ else if ((str.subString(index+1)).equals("mov")) return true;
+ else if ((str.subString(index+1)).equals("flv")) return true;
+ else return false;
+ }
+ return false;
+ }
+
+ public void done(Object obj) {
+ if ((gTitle != null) && (gTitle.length() > 0)) {
+ processList();
+ }
+
+ while(!toprocess.isEmpty()) {
+ GlobalQuery q = (GlobalQuery)toprocess.pop();
+
+ GlobalString hostname = global new GlobalString(q.getHostName());
+ GlobalString path = global new GlobalString(q.getPath());
+
+ GlobalStringBuffer gsb = global new GlobalStringBuffer(hostname);
+ gsb.append("/");
+ gsb.append(path);
+
+ if (!doneList.containsKey(gsb.toGlobalString())) {
+ todoList.push(q);
+
+ GlobalString str = global new GlobalString("1");
+ doneList.put(gsb.toGlobalString(), str);
+ }
+ }
+ }
+
+ public static String grabTitle(LocalQuery lq) {
+ String sBrace = new String("<");
+ String strTitle = new String("title>");
+ String searchstr = lq.response.toString();
+ String title = null;
+ char ch;
+
+ int mindex = -1;
+ int endquote = -1;
+ int i, j;
+ String tmp;
+
+ for (i = 0; i < searchstr.length(); i++) {
+ if (searchstr.charAt(i) == '<') {
+ i++;
+ if (searchstr.length() > (i+strTitle.length())) {
+ tmp = searchstr.subString(i, i+strTitle.length());
+ if (tmp.equalsIgnoreCase("title>")) {
+ mindex = i + tmp.length();
+ for (j = mindex; j < searchstr.length(); j++) {
+ if (searchstr.charAt(j) == '<') {
+ j++;
+ tmp = searchstr.subString(j, j+strTitle.length()+1);
+ if (tmp.equalsIgnoreCase("/title>")) {
+ endquote = j - 1;
+ break;
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+
+ if (mindex != -1) {
+ title = searchstr.subString(mindex, endquote);
+ if (Character.isWhitespace(title.charAt(0))){
+ mindex=0;
+ while (Character.isWhitespace(title.charAt(mindex++)));
+ mindex--;
+ if (mindex >= title.length()) return null;
+ title = new String(title.subString(mindex));
+ }
+
+ if (Character.isWhitespace(title.charAt(title.length()-1))) {
+ endquote=title.length()-1;
+ while (Character.isWhitespace(title.charAt(endquote--)));
+ endquote += 2;
+ if (mindex >= endquote) return null;
+ title = new String(title.subString(0, endquote));
+ }
+
+ if (isErrorPage(title)) {
+ return null;
+ }
+ }
+
+ return title;
+ }
+
+ public static boolean isErrorPage(String str) {
+ if (str.equals("301 Moved Permanently"))
+ return true;
+ else if (str.equals("302 Found"))
+ return true;
+ else if (str.equals("404 Not Found"))
+ return true;
+ else if (str.equals("403 Forbidden"))
+ return true;
+ else if (str.equals("404 File Not Found"))
+ return true;
+ else
+ return false;
+ }
+
+ public static void requestQuery(String hostname, String path, Socket sock) {
+ StringBuffer req = new StringBuffer("GET ");
+ req.append("/");
+ req.append(path);
+ req.append(" HTTP/1.0\r\nHost: ");
+ req.append(hostname);
+ req.append("\r\n\r\n");
+ sock.write(req.toString().getBytes());
+ }
+
+ public static void readResponse(LocalQuery lq, Socket sock) {
+ // state 0 - nothing
+ // state 1 - \r
+ // state 2 - \r\n
+ // state 3 - \r\n\r
+ // state 4 - \r\n\r\n
+ byte[] buffer = new byte[1024];
+ int numchars;
+
+ do {
+ numchars = sock.read(buffer);
+
+ String curr = (new String(buffer)).subString(0, numchars);
+
+ lq.response.append(curr);
+ buffer = new byte[1024];
+ } while(numchars > 0);
+ }
+
+ public void processList() {
+ LinkedList ll;
+ GlobalString token = null;
+ int mindex = 0;
+ int endquote = 0;
+
+ while (endquote != -1) {
+ endquote = gTitle.indexOf(' ', mindex);
+
+ if (endquote != -1) {
+ token = gTitle.subString(mindex, endquote);
+ mindex = endquote + 1;
+ if (filter(token)) {
+ continue;
+ }
+ token = refine(token);
+ }
+ else {
+ token = gTitle.subString(mindex);
+ token = refine(token);
+ }
+
+ Queue q = (Queue)results.get(token);
+ if (q == null) {
+ q = global new Queue();
+ }
+ q.push(workingURL);
+ results.put(token, q);
+ }
+ }
+
+ public boolean filter(GlobalString str) {
+ if (str.equals("of")) return true;
+ else if (str.equals("for")) return true;
+ else if (str.equals("a")) return true;
+ else if (str.equals("an")) return true;
+ else if (str.equals("the")) return true;
+ else if (str.equals("at")) return true;
+ else if (str.equals("and")) return true;
+ else if (str.equals("or")) return true;
+ else if (str.equals("but")) return true;
+ else if (str.equals("to")) return true;
+ else if (str.equals("The")) return true;
+ else if (str.length() == 1) {
+ if (str.charAt(0) == '.') return true;
+ else if (str.charAt(0) == '.') return true;
+ else if (str.charAt(0) == '-') return true;
+ else if (str.charAt(0) == '=') return true;
+ else if (str.charAt(0) == '_') return true;
+ else if (str.charAt(0) == ':') return true;
+ else if (str.charAt(0) == ';') return true;
+ else if (str.charAt(0) == '\'') return true;
+ else if (str.charAt(0) == '\"') return true;
+ else if (str.charAt(0) == '|') return true;
+ else if (str.charAt(0) == '@') return true;
+ else if (str.charAt(0) == '&') return true;
+ else if (str.charAt(0) == ' ') return true;
+ }
+ else return false;
+ }
+
+ public GlobalString refine(GlobalString str) {
+ str = refinePrefix(str);
+ str = refinePostfix(str);
+ return str;
+ }
+
+ public GlobalString refinePrefix(GlobalString str) {
+ if (str.charAt(0) == '&') { // &
+ return str.subString(1);
+ }
+ else if (str.charAt(0) == '/') { // &
+ return str.subString(1);
+ }
+ return str;
+ }
+
+ public GlobalString refinePostfix(GlobalString str) {
+ if (str.charAt(str.length()-1) == ',') { // ,
+ return str.subString(0, str.length()-1);
+ }
+ else if (str.charAt(str.length()-1) == ':') { // :
+ return str.subString(0, str.length()-1);
+ }
+ else if (str.charAt(str.length()-1) == ';') { // ;
+ return str.subString(0, str.length()-1);
+ }
+ else if (str.charAt(str.length()-1) == '!') { // !
+ return str.subString(0, str.length()-1);
+ }
+ else if (str.charAt(str.length()-1) == 's') { // 's
+ if (str.charAt(str.length()-2) == '\'')
+ return str.subString(0, str.length()-2);
+ }
+ else if (str.charAt(str.length()-1) == '-') {
+ int index = str.length()-2;
+ while (Character.isWhitespace(str.charAt(index--)));
+ return str.subString(0, index+2);
+ }
+ return str;
+ }
+
+ public static Queue processPage(LocalQuery lq) {
+ int index = 0;
+ String href = new String("href=\"");
+ String searchstr = lq.response.toString();
+ int depth;
+ boolean cont = true;
+ Queue toprocess;
+
+ depth = lq.getDepth() + 1;
+
+ toprocess = global new Queue();
+ while(cont) {
+ int mindex = searchstr.indexOf(href,index);
+ if (mindex != -1) {
+ int endquote = searchstr.indexOf('"', mindex+href.length());
+ if (endquote != -1) {
+ String match = searchstr.subString(mindex+href.length(), endquote);
+ String match2 = lq.makewebcanonical(match);
+
+ GlobalString ghostname;
+ GlobalString gpath;
+
+ ghostname = global new GlobalString(lq.getHostName(match));
+ gpath = global new GlobalString(lq.getPathName(match));
+
+ if (match2 != null) {
+ GlobalQuery gq = global new GlobalQuery(ghostname, gpath, depth);
+ toprocess.push(gq);
+ }
+ index = endquote;
+ } else cont = false;
+ } else cont = false;
+ }
+ return toprocess;
+ }
+}
--- /dev/null
+public class Spider {
+ public static void main(String[] args) {
+ int NUM_THREADS = 3;
+ int maxDepth = 4;
+ int i, j;
+ Work[] works;
+ QueryTask[] qt;
+ GlobalQuery[] currentWorkList;
+
+ NUM_THREADS = Integer.parseInt(args[0]);
+
+ GlobalString firstmachine;
+ GlobalString firstpage;
+
+ int mid[] = new int[NUM_THREADS];
+ mid[0] = (128<<24)|(195<<16)|(136<<8)|162;
+ mid[1] = (128<<24)|(195<<16)|(136<<8)|163;
+ mid[2] = (128<<24)|(195<<16)|(136<<8)|164;
+
+ atomic {
+ firstmachine = global new GlobalString(args[1]);
+ if (args.length == 3) {
+ firstpage = global new GlobalString(args[2]);
+ }
+ else
+ firstpage = global new GlobalString("");;
+
+ works = global new Work[NUM_THREADS];
+ qt = global new QueryTask[NUM_THREADS];
+ currentWorkList = global new GlobalQuery[NUM_THREADS];
+
+ GlobalQuery firstquery = global new GlobalQuery(firstmachine, firstpage);
+
+ Queue todoList = global new Queue();
+ DistributedHashMap doneList = global new DistributedHashMap(500, 500, 0.75f);
+ DistributedHashMap results = global new DistributedHashMap(100, 100, 0.75f);
+
+ todoList.push(firstquery);
+
+ for (i = 0; i < NUM_THREADS; i++) {
+ qt[i] = global new QueryTask(todoList, doneList, maxDepth, results);
+ works[i] = global new Work(qt[i], NUM_THREADS, i, currentWorkList);
+ }
+ }
+ System.printString("Finished to create Objects\n");
+
+ Work tmp;
+ for (i = 0; i < NUM_THREADS; i++) {
+ atomic {
+ tmp = works[i];
+ }
+ Thread.myStart(tmp, mid[i]);
+ }
+
+ for (i = 0; i < NUM_THREADS; i++) {
+ atomic {
+ tmp = works[i];
+ }
+ tmp.join();
+ }
+ }
+}
--- /dev/null
+#128.195.180.21
+#128.195.180.24
+#128.195.180.26
+128.195.136.162
+128.195.136.163
+128.195.136.164
--- /dev/null
+MAINCLASS=Spider
+SUBCLASS=Query
+SRC1=${MAINCLASS}.java
+SRC2=Global${SUBCLASS}.java
+SRC3=${SUBCLASS}Task.java
+FLAGS= -recovery -dsmtask -dsm -dsmtask -32bit -nooptimize -debug -mainclass ${MAINCLASS}
+default:
+ ../../../buildscript ${FLAGS} -o ${MAINCLASS} ${SRC2} ${SRC3} ${SRC1}
+
+clean:
+ rm -rf tmpbuilddirectory
+ rm *.bin
projects / IRC.git / commitdiff