From fb2579892bf7acc552bd1dcc762aa497c71de81f Mon Sep 17 00:00:00 2001
From: bdemsky <bdemsky>
Date: Sat, 16 Aug 2008 19:39:57 +0000
Subject: [PATCH] try to lower lock overhead

---
 Robust/src/Runtime/DSTM/interface/dstmserver.c |  2 +-
 Robust/src/Runtime/DSTM/interface/sockpool.c   | 17 +++++++++--------
 Robust/src/Runtime/DSTM/interface/sockpool.h   |  2 ++
 Robust/src/Runtime/DSTM/interface/trans.c      |  6 +++---
 4 files changed, 15 insertions(+), 12 deletions(-)

diff --git a/Robust/src/Runtime/DSTM/interface/dstmserver.c b/Robust/src/Runtime/DSTM/interface/dstmserver.c
index e5a2beed..965b0a24 100644
--- a/Robust/src/Runtime/DSTM/interface/dstmserver.c
+++ b/Robust/src/Runtime/DSTM/interface/dstmserver.c
@@ -45,7 +45,7 @@ int dstmInit(void)
 		return 1; //failure
 
     //Initialize socket pool
-    if((transPResponseSocketPool = createSockPool(transPResponseSocketPool, 2*numHostsInSystem+1)) == NULL) {
+    if((transPResponseSocketPool = createSockPool(transPResponseSocketPool, DEFAULTSOCKPOOLSIZE)) == NULL) {
         printf("Error in creating new socket pool at  %s line %d\n", __FILE__, __LINE__);
         return 0;
     }
diff --git a/Robust/src/Runtime/DSTM/interface/sockpool.c b/Robust/src/Runtime/DSTM/interface/sockpool.c
index 91ac3fc0..f5fe25d3 100644
--- a/Robust/src/Runtime/DSTM/interface/sockpool.c
+++ b/Robust/src/Runtime/DSTM/interface/sockpool.c
@@ -21,7 +21,7 @@ inline void UnLock(volatile unsigned int *addr) {
 #   error need implementation of test_and_set
 #endif
 
-#define MAXSPINS 100
+#define MAXSPINS 4
 
 inline void Lock(volatile unsigned int *s) {
   while(test_and_set(s)) {
@@ -51,6 +51,7 @@ sockPoolHashTable_t *createSockPool(sockPoolHashTable_t * sockhash, unsigned int
   sockhash->table = nodelist;
   sockhash->inuse = NULL;
   sockhash->size = size;
+  sockhash->mask = size - 1;
   sockhash->mylock=0;
   
   return sockhash;
@@ -80,7 +81,7 @@ int createNewSocket(unsigned int mid) {
 
 int getSockWithLock(sockPoolHashTable_t *sockhash, unsigned int mid) {
   socknode_t **ptr;
-  int key = mid%(sockhash->size);
+  int key = mid&(sockhash->mask);
   int sd;
   
   Lock(&sockhash->mylock);
@@ -112,7 +113,7 @@ int getSockWithLock(sockPoolHashTable_t *sockhash, unsigned int mid) {
 
 int getSock(sockPoolHashTable_t *sockhash, unsigned int mid) {
   socknode_t **ptr;
-  int key = mid%(sockhash->size);
+  int key = mid&(sockhash->mask);
   int sd;
   
   ptr=&(sockhash->table[key]);
@@ -140,7 +141,7 @@ int getSock(sockPoolHashTable_t *sockhash, unsigned int mid) {
 
 int getSock2(sockPoolHashTable_t *sockhash, unsigned int mid) {
   socknode_t **ptr;
-  int key = mid%(sockhash->size);
+  int key = mid&(sockhash->mask);
   int sd;
   
   ptr=&(sockhash->table[key]);
@@ -164,7 +165,7 @@ int getSock2(sockPoolHashTable_t *sockhash, unsigned int mid) {
 /*socket pool with multiple TR threads asking to connect to same machine  */
 int getSock2WithLock(sockPoolHashTable_t *sockhash, unsigned int mid) {
   socknode_t **ptr;
-  int key = mid%(sockhash->size);
+  int key = mid&(sockhash->mask);
   int sd;
   
   Lock(&sockhash->mylock);
@@ -189,7 +190,7 @@ int getSock2WithLock(sockPoolHashTable_t *sockhash, unsigned int mid) {
 }
 
 void addSockWithLock(sockPoolHashTable_t *sockhash, socknode_t *ptr) {
-  int key = ptr->mid%(sockhash->size);
+  int key = ptr->mid&(sockhash->mask);
   Lock(&sockhash->mylock);
   ptr->next = sockhash->table[key];
   sockhash->table[key] = ptr;
@@ -204,7 +205,7 @@ void insToListWithLock(sockPoolHashTable_t *sockhash, socknode_t *inusenode) {
 } 
 
 void freeSock(sockPoolHashTable_t *sockhash, unsigned int mid, int sd) {
-    int key = mid%(sockhash->size);
+    int key = mid&(sockhash->mask);
     socknode_t *ptr = sockhash->inuse; 
     sockhash->inuse = ptr->next;
     ptr->mid = mid;
@@ -214,7 +215,7 @@ void freeSock(sockPoolHashTable_t *sockhash, unsigned int mid, int sd) {
 }
 
 void freeSockWithLock(sockPoolHashTable_t *sockhash, unsigned int mid, int sd) {
-  int key = mid%(sockhash->size);
+  int key = mid&(sockhash->mask);
   socknode_t *ptr;
   Lock(&sockhash->mylock);
   ptr = sockhash->inuse; 
diff --git a/Robust/src/Runtime/DSTM/interface/sockpool.h b/Robust/src/Runtime/DSTM/interface/sockpool.h
index 8a9a64aa..471774af 100644
--- a/Robust/src/Runtime/DSTM/interface/sockpool.h
+++ b/Robust/src/Runtime/DSTM/interface/sockpool.h
@@ -3,6 +3,7 @@
 
 #include "dstm.h"
 #include "ip.h"
+#define DEFAULTSOCKPOOLSIZE 128
 
 int test_and_set(volatile unsigned int *addr);
 void UnLock(volatile unsigned int *addr);
@@ -17,6 +18,7 @@ typedef struct sockPoolHashTable {
     socknode_t **table;
     socknode_t *inuse;
     unsigned int size;
+  unsigned int mask;
     volatile unsigned int mylock;
 } sockPoolHashTable_t;
 
diff --git a/Robust/src/Runtime/DSTM/interface/trans.c b/Robust/src/Runtime/DSTM/interface/trans.c
index 2f28d677..9003e8ab 100644
--- a/Robust/src/Runtime/DSTM/interface/trans.c
+++ b/Robust/src/Runtime/DSTM/interface/trans.c
@@ -186,9 +186,9 @@ int dstmStartup(const char * option) {
 #endif
   
   //Initialize socket pool
-  transReadSockPool = createSockPool(transReadSockPool, 2*numHostsInSystem+1);
-  transPrefetchSockPool = createSockPool(transPrefetchSockPool, 2*numHostsInSystem+1);
-  transRequestSockPool = createSockPool(transRequestSockPool, 2*numHostsInSystem+1);
+  transReadSockPool = createSockPool(transReadSockPool, DEFAULTSOCKPOOLSIZE);
+  transPrefetchSockPool = createSockPool(transPrefetchSockPool, DEFAULTSOCKPOOLSIZE);
+  transRequestSockPool = createSockPool(transRequestSockPool, DEFAULTSOCKPOOLSIZE);
   
   dstmInit();
   transInit();
-- 
2.34.1