From: Hamed Gorjiara <hgorjiar@uci.edu>
Date: Sun, 5 Aug 2018 08:56:02 +0000 (-0700)
Subject: Automatic result generator
X-Git-Url: http://demsky.eecs.uci.edu/git/?a=commitdiff_plain;h=a9e4e42b996ac854e975bbaef9f0e849ab39dc68;p=Benchmarks_CSolver.git

Automatic result generator
---

diff --git a/.gitignore b/.gitignore
index 50d481c5..723e8f98 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,6 +1,7 @@
 # Ignoring editor's config folder
 */nbproject/
 */dumps/
+*/temp.out
 # Other Benchmarks
 scratch/
 BlackBox/
diff --git a/hexiom/bench.sh b/hexiom/bench.sh
index 22ce8b26..324fa915 100755
--- a/hexiom/bench.sh
+++ b/hexiom/bench.sh
@@ -7,6 +7,8 @@ set -e
 #Variables
 DUMPDIR="dumps"
 PARAMETERS="36 38"
+TEMP="temp.out"
+RESULT="result.csv"
 
 #Checking the number of 
 if [ "$#" -lt 1 ]; then
@@ -15,25 +17,40 @@ if [ "$#" -lt 1 ]; then
 	exit 1
 fi
 
+rm -f *.csv
+
 for PAR in $PARAMETERS; do
-    for ((i=0; i<"$1"; i++)); do
-
-	echo "$i:$PAR"
-	if [ "$#" -eq 1 ]; then
-        	(./run.sh hexiom_solve.py $PAR)	
-	else
-		if [ $2 = '--dump' ]; then
-			./run.sh csolverHexiom.py $PAR --dump
-			mkdir -p $DUMPDIR
-			mv DUMP* $DUMPDIR
-		elif [ $2 = '--csolver' ]; then
-			./run.sh csolverHexiom.py $PAR
+	rm -f $TEMP
+	for ((i=0; i<"$1"; i++)); do
+		echo "begin iteration: $i" >>$TEMP
+		if [ "$#" -eq 1 ]; then
+        		START=$(date +%s.%N)
+			./run.sh hexiom_solve.py $PAR >> $TEMP
+			END=$(date +%s.%N)
+			DIFF=$(echo "$END - $START" | bc)
+			echo "Program Execution Time: $DIFF" >>$TEMP
 		else
-			echo "Unknown command : " $2
-			exit 1
+			if [ $2 = '--dump' ]; then
+				START=$(date +%s.%N)
+				./run.sh csolverHexiom.py $PAR --dump >> $TEMP
+				END=$(date +%s.%N)
+                                DIFF=$(echo "$END - $START" | bc)
+                                echo "Program Execution Time: $DIFF" >>$TEMP
+				mkdir -p $DUMPDIR
+				mv DUMP* $DUMPDIR
+			elif [ $2 = '--csolver' ]; then
+				START=$(date +%s.%N)
+				./run.sh csolverHexiom.py $PAR >> $TEMP
+				END=$(date +%s.%N)
+        	                DIFF=$(echo "$END - $START" | bc)
+	                        echo "Program Execution Time: $DIFF" >>$TEMP
+			else
+				echo "Unknown command : " $2
+				exit 1
+			fi
 		fi
-	fi
 
-    done
+	done
+	./parse.sh $TEMP $PAR
 done
 
diff --git a/hexiom/parse.sh b/hexiom/parse.sh
new file mode 100755
index 00000000..838e8502
--- /dev/null
+++ b/hexiom/parse.sh
@@ -0,0 +1,28 @@
+#!/bin/bash
+OFILE=result.csv
+BEGIN=*"begin iteration"*
+EXECTIME=*"Program Execution Time"*
+SATCSOLVTIME=*"SAT Solving time"*
+SATTIME=*"c CPU time"*
+
+row=""
+
+
+while IFS='' read -r line || [[ -n "$line" ]]; do
+	if [[ $line == $EXECTIME ]]; then
+		number=$(echo $line|grep -Eo '[+-]?[0-9]+([.][0-9]+)?')
+		row+=","$number
+	fi
+
+	if [[ $line == $SATCSOLVTIME ]]; then
+		number=$(echo $line|grep -Eo '[+-]?[0-9]+([.][0-9]+)?')
+		row+=","$number
+	fi
+	
+	if [[ $line == $SATTIME ]]; then
+		number=$(echo $line|grep -Eo '[+-]?[0-9]+([.][0-9]+)?')
+		row+=","$number
+	fi
+
+done < "$1"
+echo "$2$row" >> $OFILE
diff --git a/hexiom/result.csv b/hexiom/result.csv
new file mode 100644
index 00000000..5c7449e5
--- /dev/null
+++ b/hexiom/result.csv
@@ -0,0 +1,2 @@
+36,0.648016,1.260086877,0.656467,1.275278727
+38,73.835073,75.245803462,73.947124,75.364286322
diff --git a/killerSudoku/bench.sh b/killerSudoku/bench.sh
index 90da937f..bea9e508 100755
--- a/killerSudoku/bench.sh
+++ b/killerSudoku/bench.sh
@@ -6,7 +6,9 @@ set -e
 
 #Variables
 DUMPDIR="dumps"
-PROBLEMS=./dataset-cs/2/*.killer
+PROBLEMS=./testcase/*.killer
+TEMP="temp.out"
+RESULT="result.csv"
 
 #Checking the number of 
 if [ "$#" -lt 1 ]; then
@@ -15,24 +17,39 @@ if [ "$#" -lt 1 ]; then
 	exit 1
 fi
 
+rm -f *.csv
+
 for PAR in $PROBLEMS; do
-    for ((i=0; i<"$1"; i++)); do
-	echo "$i:$PAR"
-	if [ "$#" -eq 1 ]; then
-        	./run.sh python killerSolver.py --problem $PAR
-	else
-		if [ $2 = '--dump' ]; then
-			./run.sh python killerSolver.py --problem $PAR --dump
-			mkdir -p $DUMPDIR
-			mv DUMP* $DUMPDIR
-		elif [ $2 = '--csolver' ]; then
-			./run.sh python killerSolver.py --problem $PAR --csolver
+	rm -f $TEMP
+	for ((i=0; i<"$1"; i++)); do
+		echo "begin iteration: $i" >>$TEMP
+		if [ "$#" -eq 1 ]; then
+			START=$(date +%s.%N)
+			./run.sh python killerSolver.py --problem $PAR >> $TEMP
+			END=$(date +%s.%N)
+                        DIFF=$(echo "$END - $START" | bc)
+                        echo "Program Execution Time: $DIFF" >>$TEMP
 		else
-			echo "Unknown command : " $2
-			exit 1
+			if [ $2 = '--dump' ]; then
+				START=$(date +%s.%N)
+				./run.sh python killerSolver.py --problem $PAR --dump >> $TEMP
+				END=$(date +%s.%N)
+                                DIFF=$(echo "$END - $START" | bc)
+                                echo "Program Execution Time: $DIFF" >>$TEMP
+				mkdir -p $DUMPDIR
+				mv DUMP* $DUMPDIR
+			elif [ $2 = '--csolver' ]; then
+				START=$(date +%s.%N)
+				./run.sh python killerSolver.py --problem $PAR --csolver >> $TEMP
+				END=$(date +%s.%N)
+                                DIFF=$(echo "$END - $START" | bc)
+                                echo "Program Execution Time: $DIFF" >>$TEMP
+			else
+				echo "Unknown command : " $2
+				exit 1
+			fi
 		fi
-	fi
-
-    done
+	done
+	./parse.sh $TEMP $PAR
 done
 
diff --git a/killerSudoku/parse.sh b/killerSudoku/parse.sh
new file mode 100755
index 00000000..49c93f00
--- /dev/null
+++ b/killerSudoku/parse.sh
@@ -0,0 +1,28 @@
+#!/bin/bash
+OFILE=result.csv
+BEGIN=*"begin iteration"*
+EXECTIME=*"Program Execution Time:"*
+SATCSOLVTIME=*"SAT Solving time"*
+SATTIME=*"Time in Sat Solver:"*
+
+row=""
+
+
+while IFS='' read -r line || [[ -n "$line" ]]; do
+	if [[ $line == $EXECTIME ]]; then
+		number=$(echo $line|grep -Eo '[+-]?[0-9]+([.][0-9]+)?')
+		row+=","$number
+	fi
+
+	if [[ $line == $SATCSOLVTIME ]]; then
+		number=$(echo $line|grep -Eo '[+-]?[0-9]+([.][0-9]+)?')
+		row+=","$number
+	fi
+	
+	if [[ $line == $SATTIME ]]; then
+		number=$(echo $line|grep -Eo '[+-]?[0-9]+([.][0-9]+)?')
+		row+=","$number
+	fi
+
+done < "$1"
+echo "$2$row" >> $OFILE
diff --git a/killerSudoku/result b/killerSudoku/result
deleted file mode 100644
index 5f909e3c..00000000
--- a/killerSudoku/result
+++ /dev/null
@@ -1,11 +0,0 @@
-med@dw-7:/scratch/satcheck/satproject/constraint_compiler/src/Benchmarks/kilerSudoku$ python killerSolver.py hamed/2/
-0-9.killer   1-36.killer  2-25.killer  3-4.killer   4-16.killer  
-hamed@dw-7:/scratch/satcheck/satproject/constraint_compiler/src/Benchmarks/kilerSudoku$ python killerSolver.py hamed/2/3-4.killer 
-CORRECT 0.00090 135 647
-hamed@dw-7:/scratch/satcheck/satproject/constraint_compiler/src/Benchmarks/kilerSudoku$ python killerSolver.py hamed/2/0-9.killer 
-CORRECT 0.00552 1436 8888
-hamed@dw-7:/scratch/satcheck/satproject/constraint_compiler/src/Benchmarks/kilerSudoku$ python killerSolver.py hamed/2/2-25.killer 
-CORRECT 0.41982 31698 209198
-hamed@dw-7:/scratch/satcheck/satproject/constraint_compiler/src/Benchmarks/kilerSudoku$ python killerSolver.py hamed/2/1-36.killer 
-CORRECT 375.01679 93931 633067
-
diff --git a/killerSudoku/result.csv b/killerSudoku/result.csv
new file mode 100644
index 00000000..a33dd36c
--- /dev/null
+++ b/killerSudoku/result.csv
@@ -0,0 +1,2 @@
+./testcase/0-9.killer,0.002690,225902096
+./testcase/4-16.killer,0.034678,362376188
diff --git a/killerSudoku/testcase/0-9.killer b/killerSudoku/testcase/0-9.killer
new file mode 100644
index 00000000..d2bb20b2
--- /dev/null
+++ b/killerSudoku/testcase/0-9.killer
@@ -0,0 +1,44 @@
+17=(8, 1)+(8, 0)
+6=(0, 6)+(1, 6)
+4=(5, 7)+(5, 6)
+13=(8, 3)+(8, 2)
+5=(6, 4)+(6, 3)
+12=(2, 0)+(1, 0)
+7=(0, 3)+(0, 4)
+4=(7, 2)+(7, 1)
+14=(3, 8)+(3, 7)
+9=(6, 8)+(5, 8)
+8=(0, 0)+(0, 1)
+15=(2, 6)+(3, 6)
+4=(4, 1)+(4, 0)
+12=(4, 5)+(5, 5)
+8=(7, 0)+(6, 0)
+9=(2, 2)+(2, 1)
+15=(7, 6)+(7, 5)
+16=(5, 1)+(5, 0)
+4=(1, 8)+(1, 7)
+12=(2, 7)+(2, 8)
+8=(1, 5)+(2, 5)
+13=(8, 4)+(7, 4)
+3=(8, 8)+(8, 7)
+15=(0, 7)+(0, 8)
+7=(8, 6)+(8, 5)
+13=(7, 7)+(6, 7)
+11=(1, 1)+(1, 2)
+10=(4, 2)+(4, 3)
+10=(6, 5)+(6, 6)
+10=(5, 4)+(4, 4)
+16=(2, 4)+(1, 4)
+8=(0, 5)
+7=(7, 3)
+5=(0, 2)
+9=(3, 1)+(3, 0)
+4=(3, 4)+(3, 5)
+10=(5, 2)+(6, 2)
+6=(1, 3)+(2, 3)
+13=(4, 8)+(4, 7)
+4=(7, 8)
+5=(6, 1)
+11=(3, 3)+(3, 2)
+5=(4, 6)
+8=(5, 3)
diff --git a/killerSudoku/testcase/4-16.killer b/killerSudoku/testcase/4-16.killer
new file mode 100644
index 00000000..76ac15fa
--- /dev/null
+++ b/killerSudoku/testcase/4-16.killer
@@ -0,0 +1,142 @@
+12=(13, 4)+(12, 4)
+28=(11, 5)+(10, 5)
+18=(10, 4)+(9, 4)
+11=(10, 15)+(10, 14)
+15=(11, 14)+(12, 14)
+24=(7, 0)+(6, 0)
+15=(15, 8)+(15, 9)
+26=(15, 5)+(14, 5)
+19=(3, 6)+(4, 6)
+20=(9, 9)+(10, 9)
+17=(7, 15)+(8, 15)
+8=(0, 1)+(1, 1)
+14=(13, 3)+(14, 3)
+23=(5, 15)+(5, 14)
+22=(4, 9)+(5, 9)
+11=(0, 5)+(0, 4)
+16=(6, 12)+(5, 12)
+14=(1, 4)+(2, 4)
+21=(2, 1)+(2, 2)
+17=(7, 8)+(7, 7)
+14=(13, 7)+(14, 7)
+29=(13, 9)+(13, 10)
+29=(15, 2)+(15, 1)
+29=(0, 14)+(0, 13)
+7=(2, 10)+(2, 11)
+17=(11, 8)+(11, 7)
+10=(5, 3)+(4, 3)
+22=(10, 6)+(11, 6)
+25=(9, 11)+(9, 12)
+8=(14, 10)+(15, 10)
+21=(12, 9)+(12, 10)
+11=(8, 6)+(8, 7)
+12=(12, 0)+(13, 0)
+12=(8, 1)+(9, 1)
+17=(4, 7)+(4, 8)
+31=(12, 12)+(12, 13)
+28=(14, 4)+(15, 4)
+21=(0, 10)+(0, 9)
+21=(10, 1)+(10, 2)
+3=(10, 10)+(10, 11)
+13=(5, 1)+(5, 2)
+15=(12, 2)+(11, 2)
+11=(4, 4)+(4, 5)
+31=(3, 3)+(3, 2)
+23=(14, 0)+(14, 1)
+16=(14, 6)+(13, 6)
+3=(9, 7)+(9, 6)
+12=(14, 12)+(15, 12)
+22=(13, 11)+(12, 11)
+3=(8, 3)+(8, 2)
+21=(7, 5)+(7, 6)
+20=(5, 10)+(6, 10)
+27=(3, 1)+(3, 0)
+16=(14, 2)+(13, 2)
+3=(3, 12)+(3, 13)
+20=(6, 6)+(5, 6)
+20=(13, 8)+(14, 8)
+23=(6, 3)+(6, 2)
+10=(1, 2)+(0, 2)
+12=(0, 3)+(1, 3)
+10=(11, 12)+(10, 12)
+20=(1, 15)+(2, 15)
+25=(8, 12)+(8, 11)
+6=(7, 13)+(6, 13)
+28=(9, 13)+(8, 13)
+17=(2, 7)+(2, 8)
+19=(9, 15)+(9, 14)
+10=(8, 0)+(9, 0)
+10=(7, 11)+(6, 11)
+11=(2, 13)+(2, 12)
+22=(9, 10)+(8, 10)
+18=(5, 7)+(6, 7)
+22=(0, 8)+(1, 8)
+5=(15, 13)+(15, 14)
+26=(4, 11)+(5, 11)
+14=(5, 5)+(6, 5)
+17=(8, 9)+(7, 9)
+10=(0, 6)+(1, 6)
+17=(10, 8)+(10, 7)
+14=(1, 0)+(2, 0)
+17=(12, 7)+(12, 8)
+6=(15, 15)+(14, 15)
+6=(15, 11)+(14, 11)
+19=(1, 13)+(1, 12)
+8=(7, 10)
+24=(10, 3)+(11, 3)
+19=(4, 15)+(3, 15)
+29=(7, 2)+(7, 1)
+12=(6, 14)+(7, 14)
+16=(1, 5)+(2, 5)
+22=(12, 15)+(13, 15)
+28=(1, 11)+(0, 11)
+25=(7, 4)+(7, 3)
+10=(13, 1)+(12, 1)
+13=(0, 12)
+10=(12, 5)+(13, 5)
+10=(2, 14)+(3, 14)
+3=(6, 9)+(6, 8)
+7=(6, 15)
+16=(15, 0)
+16=(2, 9)+(1, 9)
+13=(8, 14)
+12=(2, 3)
+12=(10, 13)+(11, 13)
+1=(12, 3)
+19=(3, 5)+(3, 4)
+17=(3, 7)+(3, 8)
+12=(1, 7)+(0, 7)
+29=(4, 14)+(4, 13)
+11=(11, 11)+(11, 10)
+19=(4, 10)+(3, 10)
+21=(13, 13)+(13, 14)
+11=(11, 4)
+8=(3, 11)
+9=(6, 1)
+2=(11, 15)
+8=(4, 0)+(5, 0)
+1=(0, 0)
+26=(11, 0)+(10, 0)
+5=(4, 2)+(4, 1)
+14=(4, 12)
+15=(2, 6)
+16=(6, 4)+(5, 4)
+26=(8, 8)+(9, 8)
+13=(14, 14)+(14, 13)
+19=(15, 6)+(15, 7)
+12=(9, 5)+(8, 5)
+9=(5, 13)
+12=(13, 12)
+14=(5, 8)
+8=(11, 9)
+11=(9, 3)+(9, 2)
+15=(1, 10)
+7=(8, 4)
+13=(15, 3)
+3=(14, 9)
+6=(3, 9)
+11=(1, 14)
+16=(0, 15)
+6=(12, 6)
+2=(7, 12)
+16=(11, 1)
diff --git a/nqueens/bench.sh b/nqueens/bench.sh
index c3cb80c4..e9e278d3 100755
--- a/nqueens/bench.sh
+++ b/nqueens/bench.sh
@@ -7,6 +7,8 @@ set -e
 #Variables
 DUMPDIR="dumps"
 PARAMETERS="8 16"
+TEMP="temp.out"
+RESULT="result.csv"
 
 #Checking the number of 
 if [ "$#" -lt 1 ]; then
@@ -15,25 +17,39 @@ if [ "$#" -lt 1 ]; then
 	exit 1
 fi
 
-for PAR in $PARAMETERS; do
-    for ((i=0; i<"$1"; i++)); do
+rm -f *.csv
 
-	echo "$i:$PAR"
-	if [ "$#" -eq 1 ]; then
-        	(./run.sh nqueen $PAR)	
-	else
-		if [ $2 = '--dump' ]; then
-			./run.sh nqueen $PAR --dump
-			mkdir -p $DUMPDIR
-			mv DUMP* $DUMPDIR
-		elif [ $2 = '--csolver' ]; then
-			./run.sh nqueen $PAR --csolver
+for PAR in $PARAMETERS; do
+	rm -f $TEMP
+	for ((i=0; i<"$1"; i++)); do
+		echo "begin iteration: $i" >>$TEMP	
+		if [ "$#" -eq 1 ]; then
+			START=$(date +%s.%N)
+			./run.sh nqueen $PAR >> $TEMP
+			END=$(date +%s.%N)
+                        DIFF=$(echo "$END - $START" | bc)
+                        echo "Program Execution Time: $DIFF" >>$TEMP
 		else
-			echo "Unknown command : " $2
-			exit 1
+			if [ $2 = '--dump' ]; then
+				START=$(date +%s.%N)
+				./run.sh nqueen $PAR --dump >> $TEMP
+				END=$(date +%s.%N)
+                                DIFF=$(echo "$END - $START" | bc)
+                                echo "Program Execution Time: $DIFF" >>$TEMP
+				mkdir -p $DUMPDIR
+				mv DUMP* $DUMPDIR
+			elif [ $2 = '--csolver' ]; then
+				START=$(date +%s.%N)
+				./run.sh nqueen $PAR --csolver >> $TEMP
+				END=$(date +%s.%N)
+                                DIFF=$(echo "$END - $START" | bc)
+                                echo "Program Execution Time: $DIFF" >>$TEMP
+			else
+				echo "Unknown command : " $2
+				exit 1
+			fi
 		fi
-	fi
-
-    done
+	done
+	./parse.sh $TEMP $PAR
 done
 
diff --git a/nqueens/nqueens.cc b/nqueens/nqueens.cc
index 616600eb..43f9af6c 100644
--- a/nqueens/nqueens.cc
+++ b/nqueens/nqueens.cc
@@ -12,6 +12,8 @@
 #include <algorithm>
 #include <ctime>
 
+#define NANOSEC 1000000000.0
+
 using namespace std;
 
 void EqualOneToCNF(vector<int> literals, vector< vector<int> > & cnf){
@@ -144,6 +146,7 @@ void printSolution(int N, int *table, int size){
 	printf("\n");
 }
 
+
 void originalNqueensEncoding(int N){
 	int numVars = N*N;
 	int kk=1;
@@ -229,10 +232,10 @@ void originalNqueensEncoding(int N){
 		addArrayClauseLiteral(solver, cnf[i].size(), cnf[i].data());
 	}
 	finishedClauses(solver);
-	int start_s=clock();
+	long long start = getTimeNano();
 	int result = solve(solver);
-	int stop_s=clock();
-	cout << "SAT Solving time: " << (stop_s-start_s)/double(CLOCKS_PER_SEC)*1000 << " ms" << endl;
+	long long stop = getTimeNano();
+	cout << "SAT Solving time: " << (stop-start)/NANOSEC << endl;
 	switch(result){
 		case IS_UNSAT:
 			printf("Problem is unsat\n");
diff --git a/nqueens/parse.sh b/nqueens/parse.sh
new file mode 100755
index 00000000..49c93f00
--- /dev/null
+++ b/nqueens/parse.sh
@@ -0,0 +1,28 @@
+#!/bin/bash
+OFILE=result.csv
+BEGIN=*"begin iteration"*
+EXECTIME=*"Program Execution Time:"*
+SATCSOLVTIME=*"SAT Solving time"*
+SATTIME=*"Time in Sat Solver:"*
+
+row=""
+
+
+while IFS='' read -r line || [[ -n "$line" ]]; do
+	if [[ $line == $EXECTIME ]]; then
+		number=$(echo $line|grep -Eo '[+-]?[0-9]+([.][0-9]+)?')
+		row+=","$number
+	fi
+
+	if [[ $line == $SATCSOLVTIME ]]; then
+		number=$(echo $line|grep -Eo '[+-]?[0-9]+([.][0-9]+)?')
+		row+=","$number
+	fi
+	
+	if [[ $line == $SATTIME ]]; then
+		number=$(echo $line|grep -Eo '[+-]?[0-9]+([.][0-9]+)?')
+		row+=","$number
+	fi
+
+done < "$1"
+echo "$2$row" >> $OFILE
diff --git a/nqueens/result.csv b/nqueens/result.csv
new file mode 100644
index 00000000..8d131500
--- /dev/null
+++ b/nqueens/result.csv
@@ -0,0 +1,2 @@
+8,0.000484,005518349
+16,0.002712,020521631
diff --git a/sudoku-csolver/Sudoku.py b/sudoku-csolver/Sudoku.py
index e07154d1..d84b126b 100644
--- a/sudoku-csolver/Sudoku.py
+++ b/sudoku-csolver/Sudoku.py
@@ -202,7 +202,7 @@ def solve(grid):
 	start = time.time()
 	sol = set(pycosat.solve(clauses))
 	end = time.time()
-	print("Time: "+str(end - start))
+	print("SUDOKU SAT SOLVING TIME: "+str(end - start))
     
 	def read_cell(i, j):
 		# return the digit of cell i, j according to the solution
diff --git a/sudoku-csolver/bench.sh b/sudoku-csolver/bench.sh
index 683d2c3b..72a693fd 100755
--- a/sudoku-csolver/bench.sh
+++ b/sudoku-csolver/bench.sh
@@ -6,7 +6,9 @@ set -e
 
 #Variables
 DUMPDIR="dumps"
-PROBLEMS=./problems/*.problem
+PROBLEMS=./testcase/*.problem
+TEMP="temp.out"
+RESULT="result.csv"
 
 #Checking the number of 
 if [ "$#" -lt 1 ]; then
@@ -15,25 +17,39 @@ if [ "$#" -lt 1 ]; then
 	exit 1
 fi
 
-for PAR in $PROBLEMS; do
-    for ((i=0; i<"$1"; i++)); do
+rm -f *.csv
 
-	echo "$i:$PAR"
-	if [ "$#" -eq 1 ]; then
-        	(./run.sh python Sudoku.py --file $PAR)	
-	else
-		if [ $2 = '--dump' ]; then
-			./run.sh python Sudoku.py --file $PAR --dump
-			mkdir -p $DUMPDIR
-			mv DUMP* $DUMPDIR
-		elif [ $2 = '--csolver' ]; then
-			./run.sh python Sudoku.py --file $PAR --csolver
+for PAR in $PROBLEMS; do
+	rm -f $TEMP
+	for ((i=0; i<"$1"; i++)); do
+		echo "begin iteration: $i" >>$TEMP
+		if [ "$#" -eq 1 ]; then
+			START=$(date +%s.%N)
+			./run.sh python Sudoku.py --file $PAR >> $TEMP
+			END=$(date +%s.%N)
+                        DIFF=$(echo "$END - $START" | bc)
+                        echo "Program Execution Time: $DIFF" >>$TEMP
 		else
-			echo "Unknown command : " $2
-			exit 1
+			if [ $2 = '--dump' ]; then
+				START=$(date +%s.%N)
+				./run.sh python Sudoku.py --file $PAR --dump >> $TEMP
+				END=$(date +%s.%N)
+                                DIFF=$(echo "$END - $START" | bc)
+                                echo "Program Execution Time: $DIFF" >>$TEMP
+				mkdir -p $DUMPDIR
+				mv DUMP* $DUMPDIR
+			elif [ $2 = '--csolver' ]; then
+				START=$(date +%s.%N)
+				./run.sh python Sudoku.py --file $PAR --csolver >> $TEMP
+				END=$(date +%s.%N)
+                                DIFF=$(echo "$END - $START" | bc)
+                                echo "Program Execution Time: $DIFF" >>$TEMP
+			else
+				echo "Unknown command : " $2
+				exit 1
+			fi
 		fi
-	fi
-
-    done
+	done
+	./parse.sh $TEMP $PAR
 done
 
diff --git a/sudoku-csolver/parse.sh b/sudoku-csolver/parse.sh
new file mode 100755
index 00000000..3da443bd
--- /dev/null
+++ b/sudoku-csolver/parse.sh
@@ -0,0 +1,28 @@
+#!/bin/bash
+OFILE=result.csv
+BEGIN=*"begin iteration"*
+EXECTIME=*"Program Execution Time:"*
+SATCSOLVTIME=*"SAT Solving time"*
+SATTIME=*"SUDOKU SAT SOLVING TIME"*
+
+row=""
+
+
+while IFS='' read -r line || [[ -n "$line" ]]; do
+	if [[ $line == $EXECTIME ]]; then
+		number=$(echo $line|grep -Eo '[+-]?[0-9]+([.][0-9]+)?')
+		row+=","$number
+	fi
+
+	if [[ $line == $SATCSOLVTIME ]]; then
+		number=$(echo $line|grep -Eo '[+-]?[0-9]+([.][0-9]+)?')
+		row+=","$number
+	fi
+	
+	if [[ $line == $SATTIME ]]; then
+		number=$(echo $line|grep -Eo '[+-]?[0-9]+([.][0-9]+)?')
+		row+=","$number
+	fi
+
+done < "$1"
+echo "$2$row" >> $OFILE
diff --git a/sudoku-csolver/result.csv b/sudoku-csolver/result.csv
new file mode 100644
index 00000000..95f04b8f
--- /dev/null
+++ b/sudoku-csolver/result.csv
@@ -0,0 +1,3 @@
+./testcase/16x16-12.problem,0.000035,120090005
+./testcase/25x25-12.problem,0.000043,335585845
+./testcase/9x9-20.problem,0.000061,059602763
diff --git a/sudoku-csolver/testcase/16x16-12.problem b/sudoku-csolver/testcase/16x16-12.problem
new file mode 100644
index 00000000..2a43484c
--- /dev/null
+++ b/sudoku-csolver/testcase/16x16-12.problem
@@ -0,0 +1,16 @@
+1.600000000000000000e+01 1.500000000000000000e+01 1.400000000000000000e+01 1.300000000000000000e+01 1.200000000000000000e+01 1.100000000000000000e+01 1.000000000000000000e+01 9.000000000000000000e+00 8.000000000000000000e+00 0.000000000000000000e+00 6.000000000000000000e+00 5.000000000000000000e+00 4.000000000000000000e+00 3.000000000000000000e+00 2.000000000000000000e+00 1.000000000000000000e+00
+1.200000000000000000e+01 1.100000000000000000e+01 1.000000000000000000e+01 9.000000000000000000e+00 1.600000000000000000e+01 1.500000000000000000e+01 1.400000000000000000e+01 1.300000000000000000e+01 4.000000000000000000e+00 0.000000000000000000e+00 2.000000000000000000e+00 1.000000000000000000e+00 8.000000000000000000e+00 7.000000000000000000e+00 6.000000000000000000e+00 5.000000000000000000e+00
+8.000000000000000000e+00 7.000000000000000000e+00 6.000000000000000000e+00 5.000000000000000000e+00 4.000000000000000000e+00 3.000000000000000000e+00 2.000000000000000000e+00 1.000000000000000000e+00 1.600000000000000000e+01 1.500000000000000000e+01 1.400000000000000000e+01 1.300000000000000000e+01 1.200000000000000000e+01 1.100000000000000000e+01 1.000000000000000000e+01 9.000000000000000000e+00
+4.000000000000000000e+00 3.000000000000000000e+00 2.000000000000000000e+00 1.000000000000000000e+00 8.000000000000000000e+00 7.000000000000000000e+00 6.000000000000000000e+00 5.000000000000000000e+00 1.200000000000000000e+01 1.100000000000000000e+01 1.000000000000000000e+01 9.000000000000000000e+00 1.600000000000000000e+01 1.500000000000000000e+01 1.400000000000000000e+01 1.300000000000000000e+01
+1.500000000000000000e+01 1.600000000000000000e+01 1.300000000000000000e+01 1.400000000000000000e+01 1.100000000000000000e+01 1.200000000000000000e+01 9.000000000000000000e+00 1.000000000000000000e+01 7.000000000000000000e+00 8.000000000000000000e+00 5.000000000000000000e+00 6.000000000000000000e+00 3.000000000000000000e+00 4.000000000000000000e+00 1.000000000000000000e+00 2.000000000000000000e+00
+1.100000000000000000e+01 1.200000000000000000e+01 9.000000000000000000e+00 1.000000000000000000e+01 1.500000000000000000e+01 1.600000000000000000e+01 1.300000000000000000e+01 1.400000000000000000e+01 3.000000000000000000e+00 4.000000000000000000e+00 1.000000000000000000e+00 2.000000000000000000e+00 7.000000000000000000e+00 8.000000000000000000e+00 5.000000000000000000e+00 6.000000000000000000e+00
+7.000000000000000000e+00 8.000000000000000000e+00 5.000000000000000000e+00 6.000000000000000000e+00 3.000000000000000000e+00 4.000000000000000000e+00 1.000000000000000000e+00 2.000000000000000000e+00 1.500000000000000000e+01 1.600000000000000000e+01 1.300000000000000000e+01 1.400000000000000000e+01 1.100000000000000000e+01 1.200000000000000000e+01 9.000000000000000000e+00 1.000000000000000000e+01
+3.000000000000000000e+00 4.000000000000000000e+00 0.000000000000000000e+00 2.000000000000000000e+00 7.000000000000000000e+00 8.000000000000000000e+00 5.000000000000000000e+00 6.000000000000000000e+00 1.100000000000000000e+01 1.200000000000000000e+01 9.000000000000000000e+00 1.000000000000000000e+01 1.500000000000000000e+01 1.600000000000000000e+01 1.300000000000000000e+01 1.400000000000000000e+01
+1.400000000000000000e+01 1.300000000000000000e+01 1.600000000000000000e+01 1.500000000000000000e+01 1.000000000000000000e+01 9.000000000000000000e+00 1.200000000000000000e+01 1.100000000000000000e+01 6.000000000000000000e+00 5.000000000000000000e+00 8.000000000000000000e+00 7.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 4.000000000000000000e+00 3.000000000000000000e+00
+1.000000000000000000e+01 9.000000000000000000e+00 1.200000000000000000e+01 1.100000000000000000e+01 1.400000000000000000e+01 1.300000000000000000e+01 1.600000000000000000e+01 1.500000000000000000e+01 2.000000000000000000e+00 0.000000000000000000e+00 4.000000000000000000e+00 3.000000000000000000e+00 6.000000000000000000e+00 5.000000000000000000e+00 8.000000000000000000e+00 7.000000000000000000e+00
+6.000000000000000000e+00 5.000000000000000000e+00 0.000000000000000000e+00 7.000000000000000000e+00 2.000000000000000000e+00 1.000000000000000000e+00 4.000000000000000000e+00 3.000000000000000000e+00 1.400000000000000000e+01 1.300000000000000000e+01 1.600000000000000000e+01 1.500000000000000000e+01 1.000000000000000000e+01 9.000000000000000000e+00 1.200000000000000000e+01 1.100000000000000000e+01
+2.000000000000000000e+00 1.000000000000000000e+00 4.000000000000000000e+00 3.000000000000000000e+00 0.000000000000000000e+00 5.000000000000000000e+00 8.000000000000000000e+00 7.000000000000000000e+00 1.000000000000000000e+01 9.000000000000000000e+00 1.200000000000000000e+01 1.100000000000000000e+01 1.400000000000000000e+01 1.300000000000000000e+01 1.600000000000000000e+01 1.500000000000000000e+01
+1.300000000000000000e+01 1.400000000000000000e+01 0.000000000000000000e+00 1.600000000000000000e+01 9.000000000000000000e+00 1.000000000000000000e+01 1.100000000000000000e+01 0.000000000000000000e+00 5.000000000000000000e+00 6.000000000000000000e+00 7.000000000000000000e+00 8.000000000000000000e+00 1.000000000000000000e+00 2.000000000000000000e+00 3.000000000000000000e+00 4.000000000000000000e+00
+9.000000000000000000e+00 1.000000000000000000e+01 1.100000000000000000e+01 1.200000000000000000e+01 1.300000000000000000e+01 1.400000000000000000e+01 1.500000000000000000e+01 0.000000000000000000e+00 1.000000000000000000e+00 2.000000000000000000e+00 3.000000000000000000e+00 4.000000000000000000e+00 5.000000000000000000e+00 6.000000000000000000e+00 7.000000000000000000e+00 8.000000000000000000e+00
+5.000000000000000000e+00 6.000000000000000000e+00 7.000000000000000000e+00 8.000000000000000000e+00 1.000000000000000000e+00 2.000000000000000000e+00 3.000000000000000000e+00 4.000000000000000000e+00 1.300000000000000000e+01 1.400000000000000000e+01 1.500000000000000000e+01 1.600000000000000000e+01 9.000000000000000000e+00 1.000000000000000000e+01 1.100000000000000000e+01 1.200000000000000000e+01
+0.000000000000000000e+00 2.000000000000000000e+00 3.000000000000000000e+00 4.000000000000000000e+00 5.000000000000000000e+00 6.000000000000000000e+00 7.000000000000000000e+00 8.000000000000000000e+00 9.000000000000000000e+00 1.000000000000000000e+01 1.100000000000000000e+01 1.200000000000000000e+01 1.300000000000000000e+01 1.400000000000000000e+01 1.500000000000000000e+01 1.600000000000000000e+01
diff --git a/sudoku-csolver/testcase/25x25-12.problem b/sudoku-csolver/testcase/25x25-12.problem
new file mode 100644
index 00000000..8064964f
--- /dev/null
+++ b/sudoku-csolver/testcase/25x25-12.problem
@@ -0,0 +1,25 @@
+1.400000000000000000e+01 7.000000000000000000e+00 0.000000000000000000e+00 2.500000000000000000e+01 2.000000000000000000e+00 0.000000000000000000e+00 9.000000000000000000e+00 1.500000000000000000e+01 8.000000000000000000e+00 1.800000000000000000e+01 1.100000000000000000e+01 2.000000000000000000e+01 6.000000000000000000e+00 2.200000000000000000e+01 2.400000000000000000e+01 1.300000000000000000e+01 1.000000000000000000e+01 1.200000000000000000e+01 1.600000000000000000e+01 1.700000000000000000e+01 4.000000000000000000e+00 2.300000000000000000e+01 5.000000000000000000e+00 2.100000000000000000e+01 1.900000000000000000e+01
+5.000000000000000000e+00 1.800000000000000000e+01 2.400000000000000000e+01 1.000000000000000000e+01 4.000000000000000000e+00 1.700000000000000000e+01 2.000000000000000000e+00 2.000000000000000000e+01 2.300000000000000000e+01 3.000000000000000000e+00 1.900000000000000000e+01 1.200000000000000000e+01 2.100000000000000000e+01 1.300000000000000000e+01 1.600000000000000000e+01 6.000000000000000000e+00 2.200000000000000000e+01 1.400000000000000000e+01 2.500000000000000000e+01 7.000000000000000000e+00 1.000000000000000000e+00 9.000000000000000000e+00 8.000000000000000000e+00 0.000000000000000000e+00 1.100000000000000000e+01
+1.500000000000000000e+01 1.700000000000000000e+01 6.000000000000000000e+00 2.000000000000000000e+01 1.900000000000000000e+01 1.000000000000000000e+01 2.200000000000000000e+01 4.000000000000000000e+00 2.100000000000000000e+01 1.600000000000000000e+01 9.000000000000000000e+00 2.000000000000000000e+00 2.500000000000000000e+01 5.000000000000000000e+00 8.000000000000000000e+00 1.100000000000000000e+01 2.300000000000000000e+01 1.000000000000000000e+00 1.800000000000000000e+01 3.000000000000000000e+00 2.400000000000000000e+01 1.300000000000000000e+01 1.200000000000000000e+01 1.400000000000000000e+01 7.000000000000000000e+00
+2.300000000000000000e+01 1.200000000000000000e+01 1.100000000000000000e+01 1.600000000000000000e+01 9.000000000000000000e+00 2.400000000000000000e+01 1.900000000000000000e+01 7.000000000000000000e+00 1.300000000000000000e+01 5.000000000000000000e+00 1.500000000000000000e+01 1.800000000000000000e+01 3.000000000000000000e+00 1.400000000000000000e+01 1.000000000000000000e+00 2.000000000000000000e+00 4.000000000000000000e+00 2.000000000000000000e+01 8.000000000000000000e+00 2.100000000000000000e+01 2.200000000000000000e+01 6.000000000000000000e+00 1.700000000000000000e+01 1.000000000000000000e+01 2.500000000000000000e+01
+2.200000000000000000e+01 1.000000000000000000e+00 1.300000000000000000e+01 8.000000000000000000e+00 2.100000000000000000e+01 6.000000000000000000e+00 2.500000000000000000e+01 1.100000000000000000e+01 1.400000000000000000e+01 1.200000000000000000e+01 2.300000000000000000e+01 4.000000000000000000e+00 1.700000000000000000e+01 1.000000000000000000e+01 7.000000000000000000e+00 1.500000000000000000e+01 1.900000000000000000e+01 9.000000000000000000e+00 5.000000000000000000e+00 2.400000000000000000e+01 2.000000000000000000e+00 2.000000000000000000e+01 3.000000000000000000e+00 1.800000000000000000e+01 1.600000000000000000e+01
+1.000000000000000000e+00 9.000000000000000000e+00 2.000000000000000000e+00 1.200000000000000000e+01 6.000000000000000000e+00 8.000000000000000000e+00 2.300000000000000000e+01 1.400000000000000000e+01 1.800000000000000000e+01 2.100000000000000000e+01 1.600000000000000000e+01 3.000000000000000000e+00 2.000000000000000000e+01 1.700000000000000000e+01 1.000000000000000000e+01 4.000000000000000000e+00 7.000000000000000000e+00 5.000000000000000000e+00 1.900000000000000000e+01 1.300000000000000000e+01 2.500000000000000000e+01 1.100000000000000000e+01 2.400000000000000000e+01 2.200000000000000000e+01 1.500000000000000000e+01
+1.900000000000000000e+01 1.500000000000000000e+01 1.800000000000000000e+01 2.400000000000000000e+01 8.000000000000000000e+00 1.100000000000000000e+01 3.000000000000000000e+00 9.000000000000000000e+00 1.000000000000000000e+01 2.200000000000000000e+01 1.000000000000000000e+00 5.000000000000000000e+00 1.300000000000000000e+01 7.000000000000000000e+00 4.000000000000000000e+00 2.500000000000000000e+01 1.200000000000000000e+01 2.100000000000000000e+01 2.000000000000000000e+00 1.400000000000000000e+01 2.300000000000000000e+01 1.700000000000000000e+01 1.600000000000000000e+01 2.000000000000000000e+01 6.000000000000000000e+00
+2.100000000000000000e+01 2.500000000000000000e+01 0.000000000000000000e+00 7.000000000000000000e+00 1.400000000000000000e+01 1.900000000000000000e+01 1.700000000000000000e+01 6.000000000000000000e+00 2.000000000000000000e+01 4.000000000000000000e+00 2.000000000000000000e+00 2.300000000000000000e+01 1.500000000000000000e+01 2.400000000000000000e+01 9.000000000000000000e+00 1.000000000000000000e+00 8.000000000000000000e+00 1.800000000000000000e+01 1.100000000000000000e+01 2.200000000000000000e+01 3.000000000000000000e+00 5.000000000000000000e+00 1.000000000000000000e+01 1.200000000000000000e+01 1.300000000000000000e+01
+1.100000000000000000e+01 4.000000000000000000e+00 2.300000000000000000e+01 3.000000000000000000e+00 2.000000000000000000e+01 7.000000000000000000e+00 2.400000000000000000e+01 5.000000000000000000e+00 2.500000000000000000e+01 1.300000000000000000e+01 8.000000000000000000e+00 1.400000000000000000e+01 2.200000000000000000e+01 2.100000000000000000e+01 1.200000000000000000e+01 1.700000000000000000e+01 1.600000000000000000e+01 6.000000000000000000e+00 1.500000000000000000e+01 1.000000000000000000e+01 9.000000000000000000e+00 1.800000000000000000e+01 2.000000000000000000e+00 1.900000000000000000e+01 1.000000000000000000e+00
+1.300000000000000000e+01 2.200000000000000000e+01 5.000000000000000000e+00 1.700000000000000000e+01 1.000000000000000000e+01 1.200000000000000000e+01 1.600000000000000000e+01 1.000000000000000000e+00 1.500000000000000000e+01 2.000000000000000000e+00 2.500000000000000000e+01 1.900000000000000000e+01 1.800000000000000000e+01 1.100000000000000000e+01 6.000000000000000000e+00 3.000000000000000000e+00 2.400000000000000000e+01 2.300000000000000000e+01 2.000000000000000000e+01 0.000000000000000000e+00 1.400000000000000000e+01 2.100000000000000000e+01 7.000000000000000000e+00 4.000000000000000000e+00 8.000000000000000000e+00
+1.200000000000000000e+01 1.000000000000000000e+01 7.000000000000000000e+00 2.300000000000000000e+01 1.800000000000000000e+01 0.000000000000000000e+00 1.500000000000000000e+01 2.000000000000000000e+00 5.000000000000000000e+00 1.400000000000000000e+01 1.300000000000000000e+01 2.100000000000000000e+01 9.000000000000000000e+00 4.000000000000000000e+00 1.900000000000000000e+01 1.600000000000000000e+01 1.700000000000000000e+01 2.400000000000000000e+01 2.200000000000000000e+01 1.000000000000000000e+00 8.000000000000000000e+00 3.000000000000000000e+00 1.100000000000000000e+01 6.000000000000000000e+00 2.000000000000000000e+01
+2.000000000000000000e+00 2.100000000000000000e+01 4.000000000000000000e+00 6.000000000000000000e+00 2.400000000000000000e+01 9.000000000000000000e+00 7.000000000000000000e+00 1.200000000000000000e+01 2.200000000000000000e+01 1.100000000000000000e+01 3.000000000000000000e+00 1.600000000000000000e+01 1.000000000000000000e+01 1.000000000000000000e+00 1.700000000000000000e+01 2.000000000000000000e+01 1.500000000000000000e+01 8.000000000000000000e+00 1.300000000000000000e+01 1.900000000000000000e+01 1.800000000000000000e+01 2.500000000000000000e+01 1.400000000000000000e+01 2.300000000000000000e+01 5.000000000000000000e+00
+2.000000000000000000e+01 5.000000000000000000e+00 1.400000000000000000e+01 9.000000000000000000e+00 1.700000000000000000e+01 1.600000000000000000e+01 1.000000000000000000e+01 1.900000000000000000e+01 4.000000000000000000e+00 2.300000000000000000e+01 6.000000000000000000e+00 8.000000000000000000e+00 1.200000000000000000e+01 1.800000000000000000e+01 2.200000000000000000e+01 7.000000000000000000e+00 3.000000000000000000e+00 1.100000000000000000e+01 2.100000000000000000e+01 2.500000000000000000e+01 1.300000000000000000e+01 2.400000000000000000e+01 1.500000000000000000e+01 0.000000000000000000e+00 2.000000000000000000e+00
+1.600000000000000000e+01 1.100000000000000000e+01 2.200000000000000000e+01 1.000000000000000000e+00 1.500000000000000000e+01 3.000000000000000000e+00 1.300000000000000000e+01 1.800000000000000000e+01 1.700000000000000000e+01 8.000000000000000000e+00 2.400000000000000000e+01 7.000000000000000000e+00 5.000000000000000000e+00 2.000000000000000000e+01 2.500000000000000000e+01 1.400000000000000000e+01 2.000000000000000000e+00 4.000000000000000000e+00 6.000000000000000000e+00 2.300000000000000000e+01 1.000000000000000000e+01 1.200000000000000000e+01 1.900000000000000000e+01 9.000000000000000000e+00 2.100000000000000000e+01
+3.000000000000000000e+00 1.900000000000000000e+01 8.000000000000000000e+00 1.300000000000000000e+01 2.500000000000000000e+01 2.000000000000000000e+01 2.100000000000000000e+01 2.400000000000000000e+01 6.000000000000000000e+00 1.000000000000000000e+00 1.400000000000000000e+01 1.100000000000000000e+01 2.300000000000000000e+01 1.500000000000000000e+01 2.000000000000000000e+00 5.000000000000000000e+00 1.800000000000000000e+01 1.000000000000000000e+01 9.000000000000000000e+00 1.200000000000000000e+01 7.000000000000000000e+00 4.000000000000000000e+00 2.200000000000000000e+01 1.600000000000000000e+01 1.700000000000000000e+01
+8.000000000000000000e+00 2.300000000000000000e+01 1.000000000000000000e+00 1.800000000000000000e+01 7.000000000000000000e+00 1.400000000000000000e+01 4.000000000000000000e+00 2.100000000000000000e+01 2.400000000000000000e+01 1.900000000000000000e+01 2.200000000000000000e+01 9.000000000000000000e+00 1.600000000000000000e+01 3.000000000000000000e+00 5.000000000000000000e+00 1.200000000000000000e+01 2.500000000000000000e+01 1.300000000000000000e+01 1.700000000000000000e+01 2.000000000000000000e+01 1.500000000000000000e+01 2.000000000000000000e+00 6.000000000000000000e+00 1.100000000000000000e+01 1.000000000000000000e+01
+4.000000000000000000e+00 2.000000000000000000e+01 1.900000000000000000e+01 2.100000000000000000e+01 2.200000000000000000e+01 1.300000000000000000e+01 1.100000000000000000e+01 0.000000000000000000e+00 1.600000000000000000e+01 7.000000000000000000e+00 1.000000000000000000e+01 1.000000000000000000e+00 2.000000000000000000e+00 6.000000000000000000e+00 2.300000000000000000e+01 1.800000000000000000e+01 5.000000000000000000e+00 3.000000000000000000e+00 2.400000000000000000e+01 1.500000000000000000e+01 1.200000000000000000e+01 1.400000000000000000e+01 2.500000000000000000e+01 8.000000000000000000e+00 9.000000000000000000e+00
+6.000000000000000000e+00 1.300000000000000000e+01 9.000000000000000000e+00 5.000000000000000000e+00 1.100000000000000000e+01 1.800000000000000000e+01 2.000000000000000000e+01 1.000000000000000000e+01 1.200000000000000000e+01 1.500000000000000000e+01 1.700000000000000000e+01 2.500000000000000000e+01 2.400000000000000000e+01 8.000000000000000000e+00 1.400000000000000000e+01 1.900000000000000000e+01 1.000000000000000000e+00 1.600000000000000000e+01 4.000000000000000000e+00 2.000000000000000000e+00 2.100000000000000000e+01 7.000000000000000000e+00 2.300000000000000000e+01 3.000000000000000000e+00 2.200000000000000000e+01
+2.400000000000000000e+01 2.000000000000000000e+00 1.700000000000000000e+01 1.400000000000000000e+01 1.600000000000000000e+01 2.300000000000000000e+01 1.000000000000000000e+00 2.200000000000000000e+01 3.000000000000000000e+00 2.500000000000000000e+01 2.000000000000000000e+01 1.500000000000000000e+01 1.100000000000000000e+01 1.200000000000000000e+01 2.100000000000000000e+01 8.000000000000000000e+00 9.000000000000000000e+00 7.000000000000000000e+00 1.000000000000000000e+01 6.000000000000000000e+00 5.000000000000000000e+00 1.900000000000000000e+01 1.800000000000000000e+01 1.300000000000000000e+01 4.000000000000000000e+00
+1.000000000000000000e+01 3.000000000000000000e+00 2.500000000000000000e+01 1.500000000000000000e+01 1.200000000000000000e+01 2.000000000000000000e+00 5.000000000000000000e+00 8.000000000000000000e+00 9.000000000000000000e+00 6.000000000000000000e+00 4.000000000000000000e+00 1.300000000000000000e+01 7.000000000000000000e+00 1.900000000000000000e+01 1.800000000000000000e+01 2.100000000000000000e+01 1.400000000000000000e+01 2.200000000000000000e+01 2.300000000000000000e+01 1.100000000000000000e+01 2.000000000000000000e+01 1.600000000000000000e+01 1.000000000000000000e+00 1.700000000000000000e+01 2.400000000000000000e+01
+2.500000000000000000e+01 6.000000000000000000e+00 1.500000000000000000e+01 4.000000000000000000e+00 3.000000000000000000e+00 2.200000000000000000e+01 1.400000000000000000e+01 1.600000000000000000e+01 7.000000000000000000e+00 0.000000000000000000e+00 2.100000000000000000e+01 1.000000000000000000e+01 1.900000000000000000e+01 2.300000000000000000e+01 1.100000000000000000e+01 9.000000000000000000e+00 2.000000000000000000e+01 2.000000000000000000e+00 1.000000000000000000e+00 1.800000000000000000e+01 1.700000000000000000e+01 8.000000000000000000e+00 1.300000000000000000e+01 5.000000000000000000e+00 1.200000000000000000e+01
+9.000000000000000000e+00 1.400000000000000000e+01 1.200000000000000000e+01 2.000000000000000000e+00 2.300000000000000000e+01 2.100000000000000000e+01 8.000000000000000000e+00 3.000000000000000000e+00 1.100000000000000000e+01 1.000000000000000000e+01 5.000000000000000000e+00 2.200000000000000000e+01 1.000000000000000000e+00 1.600000000000000000e+01 1.300000000000000000e+01 2.400000000000000000e+01 6.000000000000000000e+00 1.700000000000000000e+01 7.000000000000000000e+00 4.000000000000000000e+00 1.900000000000000000e+01 1.500000000000000000e+01 2.000000000000000000e+01 2.500000000000000000e+01 1.800000000000000000e+01
+1.800000000000000000e+01 2.400000000000000000e+01 2.100000000000000000e+01 1.900000000000000000e+01 1.300000000000000000e+01 5.000000000000000000e+00 6.000000000000000000e+00 2.300000000000000000e+01 1.000000000000000000e+00 2.000000000000000000e+01 1.200000000000000000e+01 1.700000000000000000e+01 4.000000000000000000e+00 2.000000000000000000e+00 1.500000000000000000e+01 1.000000000000000000e+01 1.100000000000000000e+01 2.500000000000000000e+01 1.400000000000000000e+01 8.000000000000000000e+00 1.600000000000000000e+01 0.000000000000000000e+00 9.000000000000000000e+00 7.000000000000000000e+00 3.000000000000000000e+00
+1.700000000000000000e+01 1.600000000000000000e+01 0.000000000000000000e+00 2.200000000000000000e+01 1.000000000000000000e+00 1.500000000000000000e+01 1.800000000000000000e+01 1.300000000000000000e+01 2.000000000000000000e+00 9.000000000000000000e+00 7.000000000000000000e+00 6.000000000000000000e+00 8.000000000000000000e+00 2.500000000000000000e+01 0.000000000000000000e+00 2.300000000000000000e+01 2.100000000000000000e+01 1.900000000000000000e+01 1.200000000000000000e+01 5.000000000000000000e+00 1.100000000000000000e+01 1.000000000000000000e+01 4.000000000000000000e+00 2.400000000000000000e+01 1.400000000000000000e+01
+7.000000000000000000e+00 8.000000000000000000e+00 1.000000000000000000e+01 1.100000000000000000e+01 5.000000000000000000e+00 4.000000000000000000e+00 1.200000000000000000e+01 2.500000000000000000e+01 1.900000000000000000e+01 1.700000000000000000e+01 1.800000000000000000e+01 2.400000000000000000e+01 1.400000000000000000e+01 9.000000000000000000e+00 2.000000000000000000e+01 2.200000000000000000e+01 1.300000000000000000e+01 1.500000000000000000e+01 3.000000000000000000e+00 1.600000000000000000e+01 6.000000000000000000e+00 1.000000000000000000e+00 2.100000000000000000e+01 2.000000000000000000e+00 2.300000000000000000e+01
diff --git a/sudoku-csolver/testcase/9x9-20.problem b/sudoku-csolver/testcase/9x9-20.problem
new file mode 100644
index 00000000..1ca6f6b9
--- /dev/null
+++ b/sudoku-csolver/testcase/9x9-20.problem
@@ -0,0 +1,9 @@
+9.000000000000000000e+00 8.000000000000000000e+00 7.000000000000000000e+00 6.000000000000000000e+00 5.000000000000000000e+00 4.000000000000000000e+00 3.000000000000000000e+00 2.000000000000000000e+00 1.000000000000000000e+00
+0.000000000000000000e+00 0.000000000000000000e+00 4.000000000000000000e+00 3.000000000000000000e+00 2.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 7.000000000000000000e+00 8.000000000000000000e+00
+0.000000000000000000e+00 2.000000000000000000e+00 1.000000000000000000e+00 0.000000000000000000e+00 8.000000000000000000e+00 7.000000000000000000e+00 6.000000000000000000e+00 0.000000000000000000e+00 4.000000000000000000e+00
+8.000000000000000000e+00 9.000000000000000000e+00 0.000000000000000000e+00 7.000000000000000000e+00 4.000000000000000000e+00 5.000000000000000000e+00 2.000000000000000000e+00 1.000000000000000000e+00 3.000000000000000000e+00
+0.000000000000000000e+00 4.000000000000000000e+00 5.000000000000000000e+00 0.000000000000000000e+00 1.000000000000000000e+00 3.000000000000000000e+00 8.000000000000000000e+00 9.000000000000000000e+00 6.000000000000000000e+00
+2.000000000000000000e+00 1.000000000000000000e+00 3.000000000000000000e+00 0.000000000000000000e+00 9.000000000000000000e+00 6.000000000000000000e+00 7.000000000000000000e+00 4.000000000000000000e+00 5.000000000000000000e+00
+0.000000000000000000e+00 7.000000000000000000e+00 0.000000000000000000e+00 4.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 1.000000000000000000e+00 3.000000000000000000e+00 2.000000000000000000e+00
+4.000000000000000000e+00 0.000000000000000000e+00 2.000000000000000000e+00 0.000000000000000000e+00 3.000000000000000000e+00 9.000000000000000000e+00 5.000000000000000000e+00 8.000000000000000000e+00 7.000000000000000000e+00
+0.000000000000000000e+00 3.000000000000000000e+00 8.000000000000000000e+00 5.000000000000000000e+00 7.000000000000000000e+00 0.000000000000000000e+00 4.000000000000000000e+00 0.000000000000000000e+00 9.000000000000000000e+00