--- /dev/null
+/*\r
+Usage :\r
+ ./FileSystem.bin <num thread> <datafile prefix>\r
+*/\r
+\r
+\r
+\r
+public class FileSystem extends Thread {\r
+ DistributedHashMap dir;\r
+ DistributedHashMap fs;\r
+ GlobalString inputfile;\r
+ int mid;\r
+ \r
+ public FileSystem(DistributedHashMap dir, DistributedHashMap fs) {\r
+ this.dir = dir;\r
+ this.fs = fs;\r
+ }\r
+ \r
+ public FileSystem(DistributedHashMap dir, DistributedHashMap fs, String filename, int mid) {\r
+ this.dir = dir;\r
+ this.fs = fs;\r
+ this.mid = mid;\r
+ this.inputfile = global new GlobalString("data/"+filename + mid);\r
+ }\r
+\r
+ public void init() {\r
+ fillHashTable();\r
+ }\r
+ \r
+ public void fillHashTable() {\r
+ GlobalString path;\r
+ DistributedLinkedList list; \r
+\r
+ atomic {\r
+ path = global new GlobalString("/home/"); // root is 'home'\r
+ list = global new DistributedLinkedList();\r
+\r
+ dir.put(path, list);\r
+ }\r
+ }\r
+ \r
+ public static void fillTodoList(String file, LinkedList todoList) {\r
+ FileInputStream fis;\r
+ String comm;\r
+ char c;\r
+ String key;\r
+ String val;\r
+ Transaction t;\r
+\r
+ fis = new FileInputStream(file);\r
+\r
+ while ((comm = fis.readLine()) != null) { // 'command' 'path'\r
+ c = comm.charAt(0); // ex) w /home/abc.c \r
+ key = comm.subString(2);\r
+ t = new Transaction(c, key);\r
+ todoList.add(t);\r
+ }\r
+ }\r
+\r
+ public void run() {\r
+ Transaction t;\r
+\r
+ char command;\r
+ String key;\r
+ String val;\r
+ boolean isDir;\r
+\r
+ GlobalString gkey;\r
+ GlobalString gval;\r
+\r
+ int index;\r
+ String file;\r
+ atomic {\r
+ file = inputfile.toLocalString();\r
+ }\r
+\r
+ LinkedList todoList = new LinkedList();\r
+ LinkedList myDir = new LinkedList();\r
+ fillTodoList(file, todoList);\r
+\r
+ while (!todoList.isEmpty()) {\r
+ t = (Transaction)(todoList.removeFirst());\r
+\r
+ command = t.getCommand();\r
+ key = t.getKey();\r
+\r
+ index = key.lastindexOf('/');\r
+ if (index+1 == key.length()) \r
+ isDir = true;\r
+ else \r
+ isDir = false;\r
+ \r
+ atomic {\r
+ gkey = global new GlobalString(key);\r
+ }\r
+\r
+ if (command == 'r') {\r
+ System.out.println("["+command+"] ["+key+"]");\r
+ atomic {\r
+ if (isDir == true) {\r
+ readDirectory(gkey);\r
+ }\r
+ else {\r
+ readFile(gkey);\r
+ }\r
+ }\r
+ }\r
+ else if (command == 'c') {\r
+ System.out.println("["+command+"] ["+key+"]");\r
+ atomic {\r
+ if (isDir == true) {\r
+ createDirectory(gkey);\r
+ if(!myDir.contains(key)) {\r
+ myDir.add(key);\r
+ }\r
+ }\r
+ else {\r
+ val = t.getValue();\r
+ gval = global new GlobalString(val);\r
+ createFile(gkey, gval);\r
+ }\r
+ }\r
+ }\r
+ }\r
+ output(myDir);\r
+\r
+ RecoveryStat.printRecoveryStat();\r
+ }\r
+\r
+ public static void output(LinkedList myDir) { \r
+ Iterator iter;\r
+ String str;\r
+\r
+ iter = myDir.iterator();\r
+\r
+ while (iter.hasNext()) {\r
+ str = (String)(iter.next());\r
+ System.printString(str + "\n");\r
+ }\r
+ }\r
+\r
+ public void readFile(GlobalString gkey) {\r
+ GlobalString gval;\r
+\r
+ gval = (GlobalString)(fs.get(gkey));\r
+ if (gval != null) {\r
+// System.out.println("<"+gval.toLocalString()+">");\r
+ }\r
+ else {\r
+ System.out.println("No such file or directory");\r
+ }\r
+ }\r
+\r
+ public void readDirectory(GlobalString gkey) {\r
+ DistributedLinkedList list;\r
+ Iterator iter;\r
+ GlobalString gval;\r
+\r
+ list = (DistributedLinkedList)(dir.get(gkey));\r
+\r
+ if (list != null) {\r
+ iter = list.iterator();\r
+ while (iter.hasNext() == true) {\r
+ gval = (GlobalString)(iter.next());\r
+// System.out.print("["+gval.toLocalString()+"] ");\r
+ }\r
+// System.out.println("");\r
+ }\r
+ else {\r
+ System.out.println("No such file or directory");\r
+ }\r
+ }\r
+\r
+ public void createFile(GlobalString gkey, GlobalString gval) {\r
+ GlobalString path;\r
+ GlobalString target;\r
+ int index;\r
+ DistributedLinkedList list;\r
+\r
+ index = gkey.lastindexOf('/');\r
+ path = gkey.subString(0, index+1);\r
+ target = gkey.subString(index+1);\r
+\r
+ if (dir.containsKey(path)) {\r
+ list = (DistributedLinkedList)(dir.get(path));\r
+ list.push(target);\r
+ dir.put(path, list);\r
+ fs.put(gkey, gval);\r
+ }\r
+ else {\r
+ System.out.println("Cannot create file");\r
+ }\r
+ }\r
+\r
+ public void createDirectory(GlobalString gkey) {\r
+ int index;\r
+ GlobalString path;\r
+ GlobalString target;\r
+ DistributedLinkedList list;\r
+\r
+ index = gkey.lastindexOf('/', gkey.length()-2);\r
+\r
+ if (index != -1) {\r
+ path = gkey.subString(0, index+1);\r
+ target = gkey.subString(index+1);\r
+\r
+ if (dir.containsKey(path)) {\r
+ list = (DistributedLinkedList)(dir.get(path));\r
+ list.push(target);\r
+ dir.put(path, list);\r
+\r
+ list = global new DistributedLinkedList();\r
+ dir.put(gkey, list);\r
+ }\r
+ else {\r
+ System.out.println("Cannot create directory");\r
+ }\r
+ }\r
+ }\r
+ \r
+ public void createFile(GlobalString gkey) {\r
+ GlobalString path;\r
+ GlobalString target;\r
+ GlobalString gval;\r
+ int index;\r
+ DistributedLinkedList list;\r
+\r
+ index = gkey.lastindexOf('/');\r
+ path = gkey.subString(0, index+1);\r
+ target = gkey.subString(index+1);\r
+ gval = global new GlobalString();\r
+\r
+ if (dir.containsKey(path)) {\r
+ list = (DistributedLinkedList)(dir.get(path));\r
+ list.push(target);\r
+ dir.put(path, list);\r
+ fs.put(gkey, gval);\r
+ }\r
+ else {\r
+ System.out.println("Cannot create file");\r
+ }\r
+ }\r
+\r
+ public Object read(DistributedHashMap mydhmap, GlobalString key) {\r
+ Object obj = mydhmap.get(key); \r
+ \r
+ return obj;\r
+ }\r
+ \r
+ public static void main(String[] args) {\r
+ int NUM_THREADS = 3;\r
+ String filename;\r
+\r
+ NUM_THREADS = Integer.parseInt(args[0]);\r
+ filename = args[1];\r
+ \r
+ int[] mid = new int[8];\r
+/* mid[0] = (128<<24)|(195<<16)|(180<<8)|21;//dw-2\r
+ mid[1] = (128<<24)|(195<<16)|(180<<8)|26;//dw-7\r
+*/\r
+ mid[0] = (128<<24)|(195<<16)|(136<<8)|162;//dc-1\r
+ mid[1] = (128<<24)|(195<<16)|(136<<8)|163;//dc-2\r
+ mid[2] = (128<<24)|(195<<16)|(136<<8)|164;//dc-3\r
+ mid[3] = (128<<24)|(195<<16)|(136<<8)|165;//dc-4\r
+ mid[4] = (128<<24)|(195<<16)|(136<<8)|166;//dc-5\r
+ mid[5] = (128<<24)|(195<<16)|(136<<8)|167;//dc-6\r
+ mid[6] = (128<<24)|(195<<16)|(136<<8)|168;//dc-7\r
+ mid[7] = (128<<24)|(195<<16)|(136<<8)|169;//dc-8\r
+ \r
+ FileSystem[] lus;\r
+ FileSystem initLus;\r
+\r
+ Work[] works;\r
+ Transaction[] currentWorkList; // type might be something else\r
+ \r
+ atomic {\r
+ currentWorkList = global new Transaction[NUM_THREADS]; // something else\r
+ works = global new Work[NUM_THREADS];\r
+ \r
+ DistributedHashMap fs = global new DistributedHashMap(500, 500, 0.75f);\r
+ DistributedHashMap dir = global new DistributedHashMap(500, 500, 0.75f);\r
+ \r
+ initLus = global new FileSystem(dir, fs);\r
+ initLus.init();\r
+\r
+ lus = global new FileSystem[NUM_THREADS];\r
+ for(int i = 0; i < NUM_THREADS; i++) {\r
+ lus[i] = global new FileSystem(initLus.dir, initLus.fs, filename, i);\r
+ }\r
+ }\r
+\r
+ FileSystem tmp;\r
+ /* Start threads */\r
+ for(int i = 0; i < NUM_THREADS; i++) {\r
+ atomic {\r
+ tmp = lus[i];\r
+ }\r
+ Thread.myStart(tmp, mid[i]);\r
+ }\r
+ \r
+ /* Join threads */\r
+ for(int i = 0; i < NUM_THREADS; i++) {\r
+ atomic {\r
+ tmp = lus[i];\r
+ }\r
+ tmp.join();\r
+ }\r
+ \r
+ System.printString("Finished\n");\r
+ }\r
+}\r
+\r
+public class Transaction { // object for todoList\r
+ char command; // r: read, w: write\r
+ String key;\r
+ String val;\r
+ \r
+ Transaction (char c, String key) {\r
+ command = c;\r
+ \r
+ this.key = new String(key);\r
+ this.val = new String();\r
+ }\r
+ \r
+ Transaction (char c, String key, String val) {\r
+ command = c;\r
+ \r
+ this.key = new String(key);\r
+ this.val = new String(val);\r
+ }\r
+ \r
+ public char getCommand() {\r
+ return command;\r
+ }\r
+ \r
+ public String getKey() {\r
+ return key;\r
+ }\r
+ \r
+ public String getValue() {\r
+ return val;\r
+ }\r
+}\r
+++ /dev/null
-/*\r
-Usage :\r
- ./LookupService.bin <num thread> <datafile prefix>\r
-*/\r
-\r
-\r
-\r
-public class LookUpService extends Thread {\r
- DistributedHashMap dir;\r
- DistributedHashMap fs;\r
- GlobalString inputfile;\r
- int mid;\r
- \r
- public LookUpService(DistributedHashMap dir, DistributedHashMap fs) {\r
- this.dir = dir;\r
- this.fs = fs;\r
- }\r
- \r
- public LookUpService(DistributedHashMap dir, DistributedHashMap fs, String filename, int mid) {\r
- this.dir = dir;\r
- this.fs = fs;\r
- this.mid = mid;\r
- this.inputfile = global new GlobalString("data/"+filename + mid);\r
- }\r
-\r
- public void init() {\r
- fillHashTable();\r
- }\r
- \r
- public void fillHashTable() {\r
- GlobalString path;\r
- DistributedLinkedList list; \r
-\r
- atomic {\r
- path = global new GlobalString("/home/"); // root is 'home'\r
- list = global new DistributedLinkedList();\r
-\r
- dir.put(path, list);\r
- }\r
- }\r
- \r
- public static void fillTodoList(String file, LinkedList todoList) {\r
- FileInputStream fis;\r
- String comm;\r
- char c;\r
- String key;\r
- String val;\r
- Transaction t;\r
-\r
- fis = new FileInputStream(file);\r
-\r
- while ((comm = fis.readLine()) != null) { // 'command' 'path'\r
- c = comm.charAt(0); // ex) w /home/abc.c \r
- key = comm.subString(2);\r
- t = new Transaction(c, key);\r
- todoList.add(t);\r
- }\r
- }\r
-\r
- public void run() {\r
- Transaction t;\r
-\r
- char command;\r
- String key;\r
- String val;\r
- boolean isDir;\r
-\r
- GlobalString gkey;\r
- GlobalString gval;\r
-\r
- int index;\r
- String file;\r
- atomic {\r
- file = inputfile.toLocalString();\r
- }\r
-\r
- LinkedList todoList = new LinkedList();\r
- LinkedList myDir = new LinkedList();\r
- fillTodoList(file, todoList);\r
-\r
- while (!todoList.isEmpty()) {\r
- t = (Transaction)(todoList.removeFirst());\r
-\r
- command = t.getCommand();\r
- key = t.getKey();\r
-\r
- index = key.lastindexOf('/');\r
- if (index+1 == key.length()) \r
- isDir = true;\r
- else \r
- isDir = false;\r
- \r
- atomic {\r
- gkey = global new GlobalString(key);\r
- }\r
-\r
- if (command == 'r') {\r
- System.out.println("["+command+"] ["+key+"]");\r
- atomic {\r
- if (isDir == true) {\r
- readDirectory(gkey);\r
- }\r
- else {\r
- readFile(gkey);\r
- }\r
- }\r
- }\r
- else if (command == 'c') {\r
- System.out.println("["+command+"] ["+key+"]");\r
- atomic {\r
- if (isDir == true) {\r
- createDirectory(gkey);\r
- if(!myDir.contains(key)) {\r
- myDir.add(key);\r
- }\r
- }\r
- else {\r
- val = t.getValue();\r
- gval = global new GlobalString(val);\r
- createFile(gkey, gval);\r
- }\r
- }\r
- }\r
- }\r
- output(myDir);\r
-\r
- RecoveryStat.printRecoveryStat();\r
- }\r
-\r
- public static void output(LinkedList myDir) { \r
- Iterator iter;\r
- String str;\r
-\r
- iter = myDir.iterator();\r
-\r
- while (iter.hasNext()) {\r
- str = (String)(iter.next());\r
- System.printString(str + "\n");\r
- }\r
- }\r
-\r
- public void readFile(GlobalString gkey) {\r
- GlobalString gval;\r
-\r
- gval = (GlobalString)(fs.get(gkey));\r
- if (gval != null) {\r
-// System.out.println("<"+gval.toLocalString()+">");\r
- }\r
- else {\r
- System.out.println("No such file or directory");\r
- }\r
- }\r
-\r
- public void readDirectory(GlobalString gkey) {\r
- DistributedLinkedList list;\r
- Iterator iter;\r
- GlobalString gval;\r
-\r
- list = (DistributedLinkedList)(dir.get(gkey));\r
-\r
- if (list != null) {\r
- iter = list.iterator();\r
- while (iter.hasNext() == true) {\r
- gval = (GlobalString)(iter.next());\r
-// System.out.print("["+gval.toLocalString()+"] ");\r
- }\r
-// System.out.println("");\r
- }\r
- else {\r
- System.out.println("No such file or directory");\r
- }\r
- }\r
-\r
- public void createFile(GlobalString gkey, GlobalString gval) {\r
- GlobalString path;\r
- GlobalString target;\r
- int index;\r
- DistributedLinkedList list;\r
-\r
- index = gkey.lastindexOf('/');\r
- path = gkey.subString(0, index+1);\r
- target = gkey.subString(index+1);\r
-\r
- if (dir.containsKey(path)) {\r
- list = (DistributedLinkedList)(dir.get(path));\r
- list.push(target);\r
- dir.put(path, list);\r
- fs.put(gkey, gval);\r
- }\r
- else {\r
- System.out.println("Cannot create file");\r
- }\r
- }\r
-\r
- public void createDirectory(GlobalString gkey) {\r
- int index;\r
- GlobalString path;\r
- GlobalString target;\r
- DistributedLinkedList list;\r
-\r
- index = gkey.lastindexOf('/', gkey.length()-2);\r
-\r
- if (index != -1) {\r
- path = gkey.subString(0, index+1);\r
- target = gkey.subString(index+1);\r
-\r
- if (dir.containsKey(path)) {\r
- list = (DistributedLinkedList)(dir.get(path));\r
- list.push(target);\r
- dir.put(path, list);\r
-\r
- list = global new DistributedLinkedList();\r
- dir.put(gkey, list);\r
- }\r
- else {\r
- System.out.println("Cannot create directory");\r
- }\r
- }\r
- }\r
- \r
- public void createFile(GlobalString gkey) {\r
- GlobalString path;\r
- GlobalString target;\r
- GlobalString gval;\r
- int index;\r
- DistributedLinkedList list;\r
-\r
- index = gkey.lastindexOf('/');\r
- path = gkey.subString(0, index+1);\r
- target = gkey.subString(index+1);\r
- gval = global new GlobalString();\r
-\r
- if (dir.containsKey(path)) {\r
- list = (DistributedLinkedList)(dir.get(path));\r
- list.push(target);\r
- dir.put(path, list);\r
- fs.put(gkey, gval);\r
- }\r
- else {\r
- System.out.println("Cannot create file");\r
- }\r
- }\r
-\r
- public Object read(DistributedHashMap mydhmap, GlobalString key) {\r
- Object obj = mydhmap.get(key); \r
- \r
- return obj;\r
- }\r
- \r
- public static void main(String[] args) {\r
- int NUM_THREADS = 3;\r
- String filename;\r
-\r
- NUM_THREADS = Integer.parseInt(args[0]);\r
- filename = args[1];\r
- \r
- int[] mid = new int[8];\r
- mid[0] = (128<<24)|(195<<16)|(180<<8)|21;//dw-2\r
-// mid[1] = (128<<24)|(195<<16)|(180<<8)|24;//dw-5\r
- mid[1] = (128<<24)|(195<<16)|(180<<8)|26;//dw-7\r
-// mid[0] = (128<<24)|(195<<16)|(136<<8)|165;//dc-4\r
-// mid[1] = (128<<24)|(195<<16)|(136<<8)|166;//dc-5\r
-// mid[2] = (128<<24)|(195<<16)|(136<<8)|167;//dc-6\r
-// mid[3] = (128<<24)|(195<<16)|(136<<8)|168;//dc-7\r
- \r
- LookUpService[] lus;\r
- LookUpService initLus;\r
-\r
- Work[] works;\r
- Transaction[] currentWorkList; // type might be something else\r
- \r
- atomic {\r
- currentWorkList = global new Transaction[NUM_THREADS]; // something else\r
- works = global new Work[NUM_THREADS];\r
- \r
- DistributedHashMap fs = global new DistributedHashMap(500, 500, 0.75f);\r
- DistributedHashMap dir = global new DistributedHashMap(500, 500, 0.75f);\r
- \r
- initLus = global new LookUpService(dir, fs);\r
- initLus.init();\r
-\r
- lus = global new LookUpService[NUM_THREADS];\r
- for(int i = 0; i < NUM_THREADS; i++) {\r
- lus[i] = global new LookUpService(initLus.dir, initLus.fs, filename, i);\r
- }\r
- }\r
-\r
- LookUpService tmp;\r
- /* Start threads */\r
- for(int i = 0; i < NUM_THREADS; i++) {\r
- atomic {\r
- tmp = lus[i];\r
- }\r
- Thread.myStart(tmp, mid[i]);\r
- }\r
- \r
- /* Join threads */\r
- for(int i = 0; i < NUM_THREADS; i++) {\r
- atomic {\r
- tmp = lus[i];\r
- }\r
- tmp.join();\r
- }\r
- \r
- System.printString("Finished\n");\r
- }\r
-}\r
-\r
-public class Transaction { // object for todoList\r
- char command; // r: read, w: write\r
- String key;\r
- String val;\r
- \r
- Transaction (char c, String key) {\r
- command = c;\r
- \r
- this.key = new String(key);\r
- this.val = new String();\r
- }\r
- \r
- Transaction (char c, String key, String val) {\r
- command = c;\r
- \r
- this.key = new String(key);\r
- this.val = new String(val);\r
- }\r
- \r
- public char getCommand() {\r
- return command;\r
- }\r
- \r
- public String getKey() {\r
- return key;\r
- }\r
- \r
- public String getValue() {\r
- return val;\r
- }\r
-}\r
--- /dev/null
+128.195.180.21
+#128.195.180.24
+128.195.180.26
-MAINCLASS=LookUpService
+MAINCLASS=FileSystem
SRC1=${MAINCLASS}.java
FLAGS= -recoverystats -dsm -dsmtask -32bit -recovery -nooptimize -mainclass ${MAINCLASS}
default:
// Sanity check
// System.out.println((i)+"th iteration correct = " + correct + " Wrong = " + wrong + " percentage = " + ((float)correct/(float)nemails));
System.out.println("\n\n\n I'm Done\n\n\n");
+
+ RecoveryStat.printRecoveryStat();
}
public static void main(String[] args) {
s.close();
}
}
+
+ public void output() {
+
+ System.out.println("size = " + results.size());
+
+
+ }
+
public static boolean isDocument(String str) {
int index = str.lastindexOf('.');
else if ((str.subString(index+1)).equals("docx")) return true;
else if ((str.subString(index+1)).equals("mov")) return true;
else if ((str.subString(index+1)).equals("flv")) return true;
+ else if ((str.subString(index+1)).equals("tar")) return true;
+ else if ((str.subString(index+1)).equals("tgz")) return true;
else return false;
}
return false;
+/*
+Usage :
+ ./Spider.java master <num_thread> <first machine> <first page> <maxDepth>
+*/
+
+
public class Spider {
public static void main(String[] args) {
int NUM_THREADS = 3;
Work[] works;
QueryTask[] qt;
GlobalQuery[] currentWorkList;
+ String fm = "www.uci.edu";
+ String fp = "";
- NUM_THREADS = Integer.parseInt(args[0]);
+ if(args.length != 4) {
+ System.out.println("./Spider.java master <num_thread> <first machine> <first page> <maxDepth>");
+ System.exit(0);
+ }
+ else {
+ NUM_THREADS = Integer.parseInt(args[0]);
+ fm = args[1];
+ fp = args[2];
+ maxDepth = Integer.parseInt(args[3]);
+ }
GlobalString firstmachine;
GlobalString firstpage;
int mid[] = new int[8];
- mid[0] = (128<<24)|(195<<16)|(136<<8)|162;
+ mid[0] = (128<<24)|(195<<16)|(180<<8)|21;
+ mid[1] = (128<<24)|(195<<16)|(180<<8)|26;
+/* mid[0] = (128<<24)|(195<<16)|(136<<8)|162;
mid[1] = (128<<24)|(195<<16)|(136<<8)|163;
mid[2] = (128<<24)|(195<<16)|(136<<8)|164;
mid[3] = (128<<24)|(195<<16)|(136<<8)|165;
mid[5] = (128<<24)|(195<<16)|(136<<8)|167;
mid[6] = (128<<24)|(195<<16)|(136<<8)|168;
mid[7] = (128<<24)|(195<<16)|(136<<8)|169;
-
+ */
atomic {
- firstmachine = global new GlobalString(args[1]);
+ firstmachine = global new GlobalString(fm);
if (args.length == 3) {
- firstpage = global new GlobalString(args[2]);
+ firstpage = global new GlobalString(fp);
}
else
firstpage = global new GlobalString("");;
-
+
works = global new Work[NUM_THREADS];
qt = global new QueryTask[NUM_THREADS];
currentWorkList = global new GlobalQuery[NUM_THREADS];
-#128.195.180.21
+128.195.180.21
#128.195.180.24
-#128.195.180.26
-128.195.136.162
-128.195.136.163
-128.195.136.164
+128.195.180.26
+#128.195.136.162
+#128.195.136.163
+#128.195.136.164
SRC1=${MAINCLASS}.java
SRC2=Global${SUBCLASS}.java
SRC3=${SUBCLASS}Task.java
-FLAGS= -recovery -dsmtask -dsm -32bit -nooptimize -debug -mainclass ${MAINCLASS}
+FLAGS= -recoverystats -recovery -dsmtask -dsm -32bit -nooptimize -debug -mainclass ${MAINCLASS}
default:
../../../buildscript ${FLAGS} -o ${MAINCLASS} ${SRC2} ${SRC3} ${SRC1}
-MatrixMultiply:400 80
+FileSystem:data
# !/bin/sh
BASEDIR=`pwd`
ITERATIONS=10
-WAITTIME=30
+WAITTIME=300
# killClients <fileName> <# of machines>
function killclients {
projects / IRC.git / commitdiff