From: adash Date: Mon, 3 May 2010 19:25:16 +0000 (+0000) Subject: adding new changes to FileSystem benchmark X-Git-Url: http://demsky.eecs.uci.edu/git/?a=commitdiff_plain;h=32f5616e5e0f2e02a7616c924a42b36ab2145b27;p=IRC.git adding new changes to FileSystem benchmark --- diff --git a/Robust/src/Benchmarks/Recovery/FileSystem/data/README b/Robust/src/Benchmarks/Recovery/FileSystem/data/README new file mode 100644 index 00000000..89754118 --- /dev/null +++ b/Robust/src/Benchmarks/Recovery/FileSystem/data/README @@ -0,0 +1,12 @@ + +Use gen1.c for file generation + +compile: +gcc -g gen1.c + +run: +./a.out 8 300 newdata + +num threads = 8 +num of commands in each file = 300 +name of file = newdata diff --git a/Robust/src/Benchmarks/Recovery/FileSystem/data/gen.c b/Robust/src/Benchmarks/Recovery/FileSystem/data/gen.c index 4fd36790..3f98ec2e 100644 --- a/Robust/src/Benchmarks/Recovery/FileSystem/data/gen.c +++ b/Robust/src/Benchmarks/Recovery/FileSystem/data/gen.c @@ -13,15 +13,15 @@ #define DIR_FILE "dirname" #define FILE_NAME "file" -#define HOME_DIR "home" +#define HOME_DIR "tmp" #define STR_SIZE 256 #define CREAT_FILE "creates.txt" -#define FACTOR (0.05) +#define FACTOR (0.20) unsigned int num_lines; void generateCrData(char fileName[],int numCmd,char** wordList,int numWord); -void generateRdData(char *filename, int numRdCmd); +void generateRdData(char *filename, int numRdCmd, int numCrCmd); /* to read word tokens*/ char** readList(char* fileName,int* num); @@ -75,39 +75,36 @@ int main(int argn,char** argv) for (i = 0; i < numFile; i++) { printf("Generating Reads %s%d...\n",prefix,i); sprintf(fileName,"%s%d",prefix,i); - generateRdData(fileName, numRdCmd); + generateRdData(fileName, numRdCmd, numCrCmd); } - - } -void generateRdData(char *filename, int numRdCmd) +void generateRdData(char *filename, int numRdCmd, int numCrCmd) { - FILE *fp = fopen(filename, "a+"); - FILE *fp_creates = fopen(CREAT_FILE, "r"); - char *rd_data[num_lines]; + FILE *fp = fopen(filename, "r+"); + char *rd_data[numCrCmd]; int i; + int num_lines = 0; - if (!fp || !fp_creates) { + if (!fp) { printf("error"); return; } - for (i = 0; i < num_lines; i++) { + for (i = 0; i < numCrCmd; i++) { if ((rd_data[i] = (char *) calloc(sizeof(char), STR_SIZE)) < 0) { perror(""); printf("Error at %d\n"); } } - for (i = 0; i < num_lines; i++) { - fgets(rd_data[i], STR_SIZE, fp_creates); + for (i = 0; i < numCrCmd; i++) { + fgets(rd_data[i], STR_SIZE, fp); } for (i = 0; i < numRdCmd; i++) { - int idx = rand() % num_lines; + int idx = rand() % numCrCmd; rd_data[idx][0] = 'r'; fprintf(fp, "%s", rd_data[idx]); } fclose(fp); - fclose(fp_creates); return; } diff --git a/Robust/src/Benchmarks/Recovery/FileSystem/data/gen1.c b/Robust/src/Benchmarks/Recovery/FileSystem/data/gen1.c new file mode 100644 index 00000000..0a167d86 --- /dev/null +++ b/Robust/src/Benchmarks/Recovery/FileSystem/data/gen1.c @@ -0,0 +1,207 @@ +/* + It generates random data files which contain a certain amount of command + for File System benchmark + + usage : + ./gen + */ + +#include +#include +#include +#include + +#define DIR_FILE "dirname" +#define FILE_NAME "file" +#define HOME_DIR "home/adash" +#define STR_SIZE 256 +#define CREAT_FILE "creates.txt" +#define FACTOR (0.05) + +unsigned int num_lines; + +//void generateCrData(char fileName[],int numCmd,char** wordList,int numWord); +void generateCrData(int numCmd,char** wordList,int numWord); +void generateRdData(char *filename, int numRdCmd, int numCrCmd, int numCmd); + +/* to read word tokens*/ +char** readList(char* fileName,int* num); +void freeList(char** list,int num); + +int main(int argn,char** argv) +{ + char** wordList; + int numWord; + int numFile; + int numCmd; + char* prefix; + char fileName[256]; + int i; + int numCrCmd, numRdCmd; + srand(0); + + if(argn < 4) { + printf("Usage : ./gen <# of file to generate> <# of command> \n"); + printf(" Ex) ./gen 8 800 data\n"); + printf("It generates 8 files(data0...data7) which contain 800 commands\n"); + exit(0); + } + else { + sscanf(argv[1]," %d",&numFile); + sscanf(argv[2]," %d",&numCmd); + prefix = argv[3]; + } + + printf("# of file : %d\n",numFile); + printf("# of Command : %d\n",numCmd); + numCrCmd = (int) (numCmd * FACTOR); + numRdCmd = (int) (numCmd * (1.0 - FACTOR)); + printf("# of Read Command : %d\n",numRdCmd); + printf("# of Create Command : %d\n", numCrCmd); + + wordList = readList(DIR_FILE,&numWord); + + /* Truncate the file */ + FILE *fp = fopen(CREAT_FILE, "w+"); + fclose(fp); + + generateCrData(numCrCmd,wordList,numWord); + + freeList(wordList,numWord); + + for (i = 0; i < numFile; i++) { + printf("Generating Reads %s%d...\n",prefix,i); + sprintf(fileName,"%s%d",prefix,i); + generateRdData(fileName, numRdCmd, numCrCmd, numCmd); + } +} + +void generateRdData(char *filename, int numRdCmd, int numCrCmd, int numCmd) +{ + //FILE *fp = fopen(filename, "r+"); + FILE *fp = fopen(filename, "w"); + FILE *fp_creates = fopen(CREAT_FILE, "r+"); + char *rd_data[numCrCmd]; + int i; + int num_lines = 0; + + if (!fp) { + printf("error"); + return; + } + for (i = 0; i < numCrCmd; i++) { + if ((rd_data[i] = (char *) calloc(sizeof(char), STR_SIZE)) < 0) { + perror(""); + printf("Error at %d\n"); + } + } + for (i = 0; i < numCrCmd; i++) { + fgets(rd_data[i], STR_SIZE, fp_creates); + } + + for (i = 0; i < numCmd; i++) { + int idx = rand() % numCrCmd; + int test = rand() % 100; + if(test < 10 ) { + rd_data[idx][0] = 'c'; + fprintf(fp, "%s", rd_data[idx]); + } else { + rd_data[idx][0] = 'r'; + fprintf(fp, "%s", rd_data[idx]); + } + } + /* + for (i = 0; i < numRdCmd; i++) { + int idx = rand() % numCrCmd; + rd_data[idx][0] = 'r'; + fprintf(fp, "%s", rd_data[idx]); + } + */ + fclose(fp); + fclose(fp_creates); + + return; +} + +void generateCrData(int numCmd,char** wordList,int numWord) +{ + FILE* fp_creates = fopen(CREAT_FILE,"a+"); /* This is superset of all creates */ + char cmdString[STR_SIZE]; + char subCmdString[STR_SIZE]; + char* wordToken; + int rand_index; + int token; + int i; + + // create initial directory on home + sprintf(cmdString,"c /%s/%s/",HOME_DIR,"newdata"); + fprintf(fp_creates,"%s\n",cmdString); + num_lines++; + numCmd--; + + while(numCmd > 0) { + + // creating directory + wordToken = wordList[rand() % numWord]; + sprintf(subCmdString,"%s%s/",cmdString,wordToken); + fprintf(fp_creates,"%s\n",subCmdString); + num_lines++; + numCmd--; + + if(numCmd == 0 ) + break; + + rand_index = (rand() % numCmd); + rand_index /= 2; + + // creating files in the directory + for(i = 0;i 0;i++) { + sprintf(subCmdString,"%s%s/%s%d",cmdString,wordToken,FILE_NAME,i); + fprintf(fp_creates,"%s\n",subCmdString); + num_lines++; + numCmd--; + } + } + fclose(fp_creates); +} + +char** readList(char* fileName,int* num) +{ + char ** list; + int cnt = 0 ; + char buffer[100]; + int size; + + FILE* fp = fopen(fileName,"r"); + + while((fscanf(fp," %s",buffer)) != EOF) cnt++; // to count the number of elements + + list = (char**)malloc(sizeof(char*) * cnt); + + rewind(fp); + + *num = 0; + + while((fscanf(fp," %s",buffer)) != EOF) // read actual list + { + size = strlen(buffer); // to get length of the word,url, or account + list[*num] = (char*)malloc(sizeof(char) * size + 1); + strcpy(list[*num],buffer); + (*num)++; + } + + fclose(fp); + return list; +} + +void freeList(char** list,int num) +{ + int i; + + for(i=0;i//....... #### +### first generate the directories before creating files under it ##### +### To run this script: ./generate_dirs.sh creates.txt, where creates.txt in the file with the list of directories to be created ####### + +#!/bin/bash + +# Read from $1 and create dirs + +file=$1 + +for line in `cat $file | awk '{print $2}'` +do + dir=`echo $line | grep -v 'file[0-9]*'` + if [ $? -eq 0 ]; + then + echo "Creating $dir " + mkdir -p $dir + fi +done diff --git a/Robust/src/Benchmarks/Recovery/FileSystem/java/FileSystem.java b/Robust/src/Benchmarks/Recovery/FileSystem/java/FileSystem.java index 6c4c9a77..63168313 100644 --- a/Robust/src/Benchmarks/Recovery/FileSystem/java/FileSystem.java +++ b/Robust/src/Benchmarks/Recovery/FileSystem/java/FileSystem.java @@ -65,6 +65,8 @@ public class FileSystem { LinkedList todoList = new LinkedList(); fillTodoList(inputfile, todoList); + long st = System.currentTimeMillis(); + long fi; while (!todoList.isEmpty()) { t = (Transaction)(todoList.removeFirst()); @@ -78,7 +80,7 @@ public class FileSystem { isDir = false; if (command == 'r') { - System.out.println("["+command+"] ["+key+"]"); + //System.out.println("["+command+"] ["+key+"]"); if (isDir == true) { readDirectory(key); } @@ -87,7 +89,7 @@ public class FileSystem { } } else if (command == 'c') { - System.out.println("["+command+"] ["+key+"]"); + //System.out.println("["+command+"] ["+key+"]"); if (isDir == true) { createDirectory(key); } @@ -98,8 +100,10 @@ public class FileSystem { } } } + fi = System.currentTimeMillis(); + System.out.println("\n\n\n I'm done - Time Elapse : "+ ((double)(fi-st)/1000) + "\n\n\n"); - output(); + //output(); // RecoveryStat.printRecoveryStat(); } @@ -112,7 +116,7 @@ public class FileSystem { while (iter.hasNext()) { str = (String)(iter.next()); - System.printString(str + "\n"); + //System.printString(str + "\n"); } } @@ -198,12 +202,14 @@ public class FileSystem { public static void main(String[] args) { String filename; + int NUM_THREADS = 1; - if (args.length == 1) { - filename = args[0]; + if (args.length == 2) { + NUM_THREADS = Integer.parseInt(args[0]); + filename = args[1]; } else { - System.out.println("usage: ./FileSystem.bin "); + System.out.println("usage: ./FileSystem.bin "); System.exit(0); } diff --git a/Robust/src/Benchmarks/Recovery/FileSystem/recovery/FileSystem.java b/Robust/src/Benchmarks/Recovery/FileSystem/recovery/FileSystem.java index 62eb2f69..9e5508b9 100644 --- a/Robust/src/Benchmarks/Recovery/FileSystem/recovery/FileSystem.java +++ b/Robust/src/Benchmarks/Recovery/FileSystem/recovery/FileSystem.java @@ -1,10 +1,9 @@ /* + System.out.println(key); Usage : ./FileSystem.bin */ - - public class FileSystem extends Thread { DistributedHashMap dir; // Directory DistributedHashMap fs; // File @@ -43,7 +42,7 @@ public class FileSystem extends Thread { DistributedLinkedList list; atomic { - path = global new GlobalString("/home/"); // root is 'home' + path = global new GlobalString("/tmp/"); // root is 'tmp' list = global new DistributedLinkedList(); dir.put(path, list); @@ -69,110 +68,141 @@ public class FileSystem extends Thread { } } - public void run() { - - Transaction t; - - char command; - String key; - String val; - GlobalString gkey; - GlobalString gval; - boolean isDir; - - int index; - String file; - atomic { - file = inputfile.toLocalString(); - } - - LinkedList todoList = new LinkedList(); - fillTodoList(file, todoList); - - long st = System.currentTimeMillis(); - long fi; - - atomic { - while (!todoList.isEmpty()) { - t = (Transaction)(todoList.removeFirst()); - - command = t.getCommand(); - key = t.getKey(); - - gkey = global new GlobalString(key); - - - index = key.lastindexOf('/'); - if (index+1 == key.length()) - isDir = true; - else - isDir = false; - - if (command == 'r') { -// System.out.println("["+command+"] ["+key+"]"); - if (isDir == true) { - readDirectory(gkey); - } - else { - readFile(gkey); - } - } - else if (command == 'c') { -// System.out.println("["+command+"] ["+key+"]"); - if (isDir == true) { - createDirectory(gkey); - } - else { - val = t.getValue(); - gval = global new GlobalString(val); - createFile(gkey, gval); - } - } + public void run() { + System.out.println("Inside run method\n"); + Transaction t; + char command; + String key; + String val; + GlobalString gkey; + GlobalString gval; + boolean isDir; + + int index; + String file; + atomic { + System.out.println("trans1"); + file = inputfile.toLocalString(); + } + + System.out.println("file= " + file); + + LinkedList todoList = new LinkedList(); + fillTodoList(file, todoList); + + long st = System.currentTimeMillis(); + long fi; + long tot1; + + if(todoList.isEmpty()) + System.out.println("todoList is Empty\n"); + + int iter = 0; + while (!todoList.isEmpty()) { + //System.out.println("iter= " + iter + "\n"); + int count = 5; + atomic { + //System.out.println("trans2"); + while(count>0 && !todoList.isEmpty()) { //commit 5 transactions + t = (Transaction)(todoList.removeFirst()); + if(t==null) { + count--; + continue; + } + command = t.getCommand(); + key = t.getKey(); + //System.out.println(key + "\n"); + gkey = global new GlobalString(key); + + index = key.lastindexOf('/'); + if (index+1 == key.length()) + isDir = true; + else + isDir = false; + + long st1 = 0L; + long fi1 = 0L; + if (command == 'r') { + st1 = System.currentTimeMillis(); + //System.out.println("["+command+"] ["+key+"]"); + if (isDir == true) { + readDirectory(gkey); + } + else { + readFile(gkey); + } + fi1 = System.currentTimeMillis(); + } + tot1 += fi1 - st1; + if (command == 'c') { + //System.out.println("["+command+"] ["+key+"]"); + if (isDir == true) { + createDirectory(gkey); + } + else { + val = t.getValue(); + gval = global new GlobalString(val); + createFile(gkey, gval); + } + } + count--; + }//end of inside loop + }//end of atomic + iter++; + } + fi = System.currentTimeMillis(); + + // sleep(3000000); + // atomic { + // output(); + // } + + RecoveryStat.printRecoveryStat(); + + System.out.println("\n\n\n I'm done - Time Elapse : "+ ((double)(fi-st)/1000) + "\n\n\n"); + System.out.println("\n Reading - Time Elapse : "+ ((double)tot1/1000) + "\n"); + while(true) { + sleep(100000); + } } - } - fi = System.currentTimeMillis(); - - sleep(3000000); - atomic { - output(); - } - - RecoveryStat.printRecoveryStat(); - - System.out.println("\n\n\n I'm done - Time Elapse : "+ ((double)(fi-st)/1000) + "\n\n\n"); - - while(true) { - sleep(100000); - } - - } - + /* public void output() { Iterator iter; GlobalString gstr; -/* iter = dir_list.iterator(); while (iter.hasNext()) { gstr = (GlobalString)(iter.next()); System.printString(gstr.toLocalString() + "\n"); - }*/ + } } + */ public void readFile(GlobalString gkey) { - GlobalString gval; - String val; - - atomic { - gval = (GlobalString)(fs.get(gkey)); - val = gval.toLocalString(); - } + GlobalString gval=null; + String val=null; + + //atomic { + gval = (GlobalString)(fs.get(gkey)); + if(gval!=null) { + val = gval.toLocalString(); + //Add some useless extra work for now + //to increase read time + int hashVal = gval.hashCode(); + int a=0; + for(int t=0; t"); + //System.out.println("<"+val+">"); } else { -// System.out.println("No such file or directory"); + //System.out.println("No such file or directory"); } } @@ -187,12 +217,18 @@ public class FileSystem extends Thread { iter = list.iterator(); while (iter.hasNext() == true) { gval = (GlobalString)(iter.next()); -// System.out.print("["+gval.toLocalString()+"] "); + //System.out.print("["+gval.toLocalString()+"] "); + //Add some useless extra work for now + int hashVal = gval.hashCode(); + int a=0; + for(int t=0; t "); - System.exit(0); - } - - int[] mid = new int[8]; - /* - mid[0] = (128<<24)|(195<<16)|(180<<8)|21;//dw-2 - mid[1] = (128<<24)|(195<<16)|(180<<8)|26;//dw-7 - */ - mid[0] = (128<<24)|(195<<16)|(136<<8)|162;//dc-1 - mid[1] = (128<<24)|(195<<16)|(136<<8)|163;//dc-2 - mid[2] = (128<<24)|(195<<16)|(136<<8)|164;//dc-3 - mid[3] = (128<<24)|(195<<16)|(136<<8)|165;//dc-4 - mid[4] = (128<<24)|(195<<16)|(136<<8)|166;//dc-5 - mid[5] = (128<<24)|(195<<16)|(136<<8)|167;//dc-6 - mid[6] = (128<<24)|(195<<16)|(136<<8)|168;//dc-7 - mid[7] = (128<<24)|(195<<16)|(136<<8)|169;//dc-8 - - FileSystem[] lus; - FileSystem initLus; - - atomic { - DistributedHashMap fs = global new DistributedHashMap(500, 500, 0.75f); - DistributedHashMap dir = global new DistributedHashMap(500, 500, 0.75f); - DistributedLinkedList dir_list = global new DistributedLinkedList(); - - initLus = global new FileSystem(dir, fs, dir_list); - initLus.init(); - - lus = global new FileSystem[NUM_THREADS]; - for(int i = 0; i < NUM_THREADS; i++) { - lus[i] = global new FileSystem(initLus.dir, initLus.fs, initLus.dir_list, filename, i,mid[i]); - } - } - FileSystem tmp; - /* Start threads */ - for(int i = 0; i < NUM_THREADS; i++) { - atomic { - tmp = lus[i]; - } - Thread.myStart(tmp, mid[i]); - } - - /* Join threads */ - for(int i = 0; i < NUM_THREADS; i++) { - atomic { - tmp = lus[i]; - } - tmp.join(); - } - - System.printString("Finished\n"); - } + public static void main(String[] args) { + int NUM_THREADS = 3; + String filename = new String(); + + if (args.length == 2) { + NUM_THREADS = Integer.parseInt(args[0]); + filename = args[1]; + System.out.println("filename= " + filename); + } + else { + System.out.println("./FileSystem.bin master "); + System.exit(0); + } + + int[] mid = new int[8]; + /* + mid[0] = (128<<24)|(195<<16)|(180<<8)|21;//dw-2 + mid[1] = (128<<24)|(195<<16)|(180<<8)|26;//dw-7 + */ + mid[0] = (128<<24)|(195<<16)|(136<<8)|162;//dc-1 + mid[1] = (128<<24)|(195<<16)|(136<<8)|163;//dc-2 + mid[2] = (128<<24)|(195<<16)|(136<<8)|164;//dc-3 + mid[3] = (128<<24)|(195<<16)|(136<<8)|165;//dc-4 + mid[4] = (128<<24)|(195<<16)|(136<<8)|166;//dc-5 + mid[5] = (128<<24)|(195<<16)|(136<<8)|167;//dc-6 + mid[6] = (128<<24)|(195<<16)|(136<<8)|168;//dc-7 + mid[7] = (128<<24)|(195<<16)|(136<<8)|169;//dc-8 + + FileSystem[] lus; + FileSystem initLus; + + atomic { + DistributedHashMap fs = global new DistributedHashMap(500, 100, 0.75f); + DistributedHashMap dir = global new DistributedHashMap(500, 100, 0.75f); + DistributedLinkedList dir_list = global new DistributedLinkedList(); + + initLus = global new FileSystem(dir, fs, dir_list); + initLus.init(); + + lus = global new FileSystem[NUM_THREADS]; + for(int i = 0; i < NUM_THREADS; i++) { + lus[i] = global new FileSystem(initLus.dir, initLus.fs, initLus.dir_list, filename, i,mid[i]); + } + } + + FileSystem tmp; + /* Start threads */ + for(int i = 0; i < NUM_THREADS; i++) { + atomic { + tmp = lus[i]; + } + Thread.myStart(tmp, mid[i]); + } + + /* Join threads */ + for(int i = 0; i < NUM_THREADS; i++) { + atomic { + tmp = lus[i]; + } + tmp.join(); + } + + System.printString("Finished\n"); + } } public class Transaction { // object for todoList - char command; // r: read, w: write - String key; - String val; - - Transaction (char c, String key) { - command = c; - - this.key = new String(key); - this.val = new String(); - } - - Transaction (char c, String key, String val) { - command = c; - - this.key = new String(key); - this.val = new String(val); - } - - public char getCommand() { - return command; - } - - public String getKey() { - return key; - } - - public String getValue() { - return val; - } + char command; // r: read, w: write + String key; + String val; + + Transaction (char c, String key) { + command = c; + + this.key = new String(key); + this.val = new String(); + } + + Transaction (char c, String key, String val) { + command = c; + + this.key = new String(key); + this.val = new String(val); + } + + public char getCommand() { + return command; + } + + public String getKey() { + return key; + } + + public String getValue() { + return val; + } } diff --git a/Robust/src/Benchmarks/Recovery/FileSystem/recovery/FileSystem2.java b/Robust/src/Benchmarks/Recovery/FileSystem/recovery/FileSystem2.java index 5011fef7..2b185edc 100644 --- a/Robust/src/Benchmarks/Recovery/FileSystem/recovery/FileSystem2.java +++ b/Robust/src/Benchmarks/Recovery/FileSystem/recovery/FileSystem2.java @@ -42,10 +42,9 @@ public class FileSystem extends Thread { DistributedLinkedList list; atomic { - path = global new GlobalString("/tmp/"); // root is 'tmp' + path = global new GlobalString("/home/adash/"); // root is 'home' list = global new DistributedLinkedList(); dir.put(path, list); - dir_list.add(path); } } @@ -83,7 +82,7 @@ public class FileSystem extends Thread { System.out.println("todoList is Empty\n"); while (!todoList.isEmpty()) { - int count = 5; + int count = 10; atomic { while(count>0 && !todoList.isEmpty()) { //commit 5 transactions t = (Transaction)(todoList.removeFirst()); @@ -150,20 +149,14 @@ public class FileSystem extends Thread { val = gval.toLocalString(); //Add some useless extra work for now //to increase read time - int hashVal = gval.hashCode(); + int hashVal = val.hashCode(); int a=0; for(int t=0; t"); } - else { - //System.out.println("No such file or directory"); + if (val == null) { + System.out.println("No such file or directory"); } } @@ -178,18 +171,10 @@ public class FileSystem extends Thread { iter = list.iterator(); while (iter.hasNext() == true) { gval = (GlobalString)(iter.next()); - //System.out.print("["+gval.toLocalString()+"] "); - //Add some useless extra work for now - int hashVal = gval.hashCode(); - int a=0; - for(int t=0; t