\r
\r
public class FileSystem extends Thread {\r
- DistributedHashMap dir;\r
- DistributedHashMap fs;\r
+ DistributedHashMap dir; // Directory \r
+ DistributedHashMap fs; // File \r
+ DistributedLinkedList dir_list;\r
GlobalString inputfile;\r
int mid;\r
\r
- public FileSystem(DistributedHashMap dir, DistributedHashMap fs) {\r
+ public FileSystem(DistributedHashMap dir, DistributedHashMap fs, DistributedLinkedList dir_list) {\r
this.dir = dir;\r
this.fs = fs;\r
+ this.dir_list = dir_list;\r
}\r
\r
- public FileSystem(DistributedHashMap dir, DistributedHashMap fs, String filename, int mid) {\r
+ public FileSystem(DistributedHashMap dir, DistributedHashMap fs, DistributedLinkedList dir_list, String filename, int mid) {\r
this.dir = dir;\r
this.fs = fs;\r
+ this.dir_list = dir_list;\r
+ this.mid = mid;\r
+ this.inputfile = global new GlobalString("data/"+filename + mid);\r
+ }\r
+\r
+\r
+ public void setInputFileName(String filename, int mid) {\r
this.mid = mid;\r
this.inputfile = global new GlobalString("data/"+filename + mid);\r
}\r
list = global new DistributedLinkedList();\r
\r
dir.put(path, list);\r
+ dir_list.add(path);\r
}\r
}\r
\r
}\r
\r
LinkedList todoList = new LinkedList();\r
- LinkedList myDir = new LinkedList();\r
fillTodoList(file, todoList);\r
\r
while (!todoList.isEmpty()) {\r
atomic {\r
if (isDir == true) {\r
createDirectory(gkey);\r
- if(!myDir.contains(key)) {\r
- myDir.add(key);\r
- }\r
}\r
else {\r
val = t.getValue();\r
}\r
}\r
}\r
- output(myDir);\r
+\r
+ sleep(3000000);\r
+ atomic {\r
+ output();\r
+ }\r
\r
RecoveryStat.printRecoveryStat();\r
}\r
\r
- public static void output(LinkedList myDir) { \r
+ public void output() { \r
Iterator iter;\r
- String str;\r
+ GlobalString gstr;\r
\r
- iter = myDir.iterator();\r
+ iter = dir_list.iterator();\r
\r
while (iter.hasNext()) {\r
- str = (String)(iter.next());\r
- System.printString(str + "\n");\r
+ gstr = (GlobalString)(iter.next());\r
+ System.printString(gstr.toLocalString() + "\n");\r
}\r
}\r
\r
\r
list = global new DistributedLinkedList();\r
dir.put(gkey, list);\r
+ dir_list.add(gkey);\r
}\r
else {\r
System.out.println("Cannot create directory");\r
}\r
}\r
\r
- public void createFile(GlobalString gkey) {\r
- GlobalString path;\r
- GlobalString target;\r
- GlobalString gval;\r
- int index;\r
- DistributedLinkedList list;\r
-\r
- index = gkey.lastindexOf('/');\r
- path = gkey.subString(0, index+1);\r
- target = gkey.subString(index+1);\r
- gval = global new GlobalString();\r
-\r
- if (dir.containsKey(path)) {\r
- list = (DistributedLinkedList)(dir.get(path));\r
- list.push(target);\r
- dir.put(path, list);\r
- fs.put(gkey, gval);\r
- }\r
- else {\r
- System.out.println("Cannot create file");\r
- }\r
- }\r
-\r
public Object read(DistributedHashMap mydhmap, GlobalString key) {\r
Object obj = mydhmap.get(key); \r
\r
filename = args[1];\r
\r
int[] mid = new int[8];\r
-/* mid[0] = (128<<24)|(195<<16)|(180<<8)|21;//dw-2\r
+ mid[0] = (128<<24)|(195<<16)|(180<<8)|21;//dw-2\r
mid[1] = (128<<24)|(195<<16)|(180<<8)|26;//dw-7\r
-*/\r
+/*\r
mid[0] = (128<<24)|(195<<16)|(136<<8)|162;//dc-1\r
mid[1] = (128<<24)|(195<<16)|(136<<8)|163;//dc-2\r
mid[2] = (128<<24)|(195<<16)|(136<<8)|164;//dc-3\r
mid[5] = (128<<24)|(195<<16)|(136<<8)|167;//dc-6\r
mid[6] = (128<<24)|(195<<16)|(136<<8)|168;//dc-7\r
mid[7] = (128<<24)|(195<<16)|(136<<8)|169;//dc-8\r
- \r
+ */\r
FileSystem[] lus;\r
FileSystem initLus;\r
\r
\r
DistributedHashMap fs = global new DistributedHashMap(500, 500, 0.75f);\r
DistributedHashMap dir = global new DistributedHashMap(500, 500, 0.75f);\r
+ DistributedLinkedList dir_list = global new DistributedLinkedList();\r
\r
- initLus = global new FileSystem(dir, fs);\r
+ initLus = global new FileSystem(dir, fs, dir_list);\r
initLus.init();\r
\r
lus = global new FileSystem[NUM_THREADS];\r
for(int i = 0; i < NUM_THREADS; i++) {\r
- lus[i] = global new FileSystem(initLus.dir, initLus.fs, filename, i);\r
+// lus[i] = initLus;\r
+// lus[i].setInputFileName(filename, i);\r
+ lus[i] = global new FileSystem(initLus.dir, initLus.fs, initLus.dir_list, filename, i);\r
}\r
}\r
\r
int maxSearchDepth;
Queue toprocess;
DistributedHashMap results;
+ DistributedLinkedList results_list;
DistributedHashMap visitedList;
GlobalString gTitle;
GlobalString workingURL;
- public QueryTask(Queue todoList, DistributedHashMap visitedList, int maxDepth, int maxSearchDepth, DistributedHashMap results) {
+ public QueryTask(Queue todoList, DistributedHashMap visitedList, int maxDepth, int maxSearchDepth, DistributedHashMap results, DistributedLinkedList results_list) {
this.todoList = todoList;
this.visitedList = visitedList;
this.maxDepth = maxDepth;
this.maxSearchDepth = maxSearchDepth;
this.results = results;
+ this.results_list = results_list;
toprocess = global new Queue();
}
}
lq = new LocalQuery(hostname, path, depth);
- System.printString("["+lq.getDepth()+"] ");
+/* System.printString("["+lq.getDepth()+"] ");
System.printString("Processing - Hostname : ");
System.printString(hostname);
System.printString(", Path : ");
System.printString(path);
System.printString("\n");
-
+*/
if (isDocument(path)) {
return;
}
}
}
- public void output() {
-
- System.out.println("size = " + results.size());
-
-
- }
-
public static boolean isDocument(String str) {
int index = str.lastindexOf('.');
GlobalString str = global new GlobalString("1");
visitedList.put(gsb.toGlobalString(), str);
+ results_list.add(gsb.toGlobalString());
searchCnt++;
}
}
}
+ public void output() {
+ String str;
+ Iterator iter = results_list.iterator();
+
+ while (iter.hasNext() == true) {
+ str = ((GlobalString)(iter.next())).toLocalString();
+ System.printString(str + "\n");
+ }
+ }
+
public static String grabTitle(LocalQuery lq) {
String sBrace = new String("<");
String strTitle = new String("title>");
public class Spider {
public static void main(String[] args) {
int NUM_THREADS = 3;
- int maxDepth = 4;
+ int maxDepth = 3;
int maxSearchDepth = 10;
int i, j;
Work[] works;
Queue todoList = global new Queue();
DistributedHashMap visitedList = global new DistributedHashMap(500, 500, 0.75f);
DistributedHashMap results = global new DistributedHashMap(100, 100, 0.75f);
+ DistributedLinkedList results_list = global new DistributedLinkedList();
todoList.push(firstquery);
for (i = 0; i < NUM_THREADS; i++) {
- qt[i] = global new QueryTask(todoList, visitedList, maxDepth, maxSearchDepth, results);
+ qt[i] = global new QueryTask(todoList, visitedList, maxDepth, maxSearchDepth, results, results_list);
works[i] = global new Work(qt[i], NUM_THREADS, i, currentWorkList);
}
}
projects / IRC.git / commitdiff