Robust/src/Runtime/multicoretask.c

   1 #ifdef TASK
   2 #include "runtime.h"
   3 #ifndef RAW
   4 #include "structdefs.h"
   5 #include "mem.h"
   6 #include "checkpoint.h"
   7 #include "Queue.h"
   8 #include "SimpleHash.h"
   9 #include "GenericHashtable.h"
  10 #include <sys/select.h>
  11 #include <sys/types.h>
  12 #include <sys/mman.h>
  13 #include <string.h>
  14 #include <signal.h>
  15 #include <assert.h>
  16 #include <errno.h>
  17 #endif
  18 #ifdef RAW
  19 #include <raw.h>
  20 #include <raw_compiler_defs.h>
  21 //#include <libints.h>
  22 #elif defined THREADSIMULATE
  23 // use POSIX message queue
  24 // for each core, its message queue named as
  25 // /msgqueue_corenum
  26 #include <mqueue.h>
  27 #include <sys/stat.h>
  28 #endif
  29 /*
  30 extern int injectfailures;
  31 extern float failurechance;
  32 */
  33 extern int debugtask;
  34 extern int instaccum;
  35
  36 #ifdef RAW
  37 #define TOTALCORE raw_get_num_tiles()
  38 #endif
  39
  40 #ifdef CONSCHECK
  41 #include "instrument.h"
  42 #endif
  43
  44 struct genhashtable * activetasks;
  45 struct genhashtable * failedtasks;
  46 struct taskparamdescriptor * currtpd;
  47 #ifndef RAW
  48 struct RuntimeHash * forward;
  49 struct RuntimeHash * reverse;
  50 #endif
  51
  52 int corestatus[NUMCORES]; // records status of each core
  53                           // 1: running tasks
  54                                                   // 0: stall
  55 int numsendobjs[NUMCORES]; // records how many objects a core has sent out
  56 int numreceiveobjs[NUMCORES]; // records how many objects a core has received
  57 #ifdef RAW
  58 struct RuntimeHash locktable;
  59 static struct RuntimeHash* locktbl = &locktable;
  60 void * curr_heapbase=0;
  61 void * curr_heaptop=0;
  62 int self_numsendobjs;
  63 int self_numreceiveobjs;
  64 int lockobj;
  65 int lockresult;
  66 bool lockflag;
  67 #ifndef INTERRUPT
  68 bool reside;
  69 #endif
  70 struct Queue objqueue;
  71 int msgdata[30];
  72 int msgtype;
  73 int msgdataindex;
  74 int msglength;
  75 void calCoords(int core_num, int* coordY, int* coordX);
  76 void recvMsg(void);
  77 #elif defined THREADSIMULATE
  78 static struct RuntimeHash* locktbl;
  79 struct thread_data {
  80         int corenum;
  81         int argc;
  82         char** argv;
  83         int numsendobjs;
  84         int numreceiveobjs;
  85 };
  86 struct thread_data thread_data_array[NUMCORES];
  87 mqd_t mqd[NUMCORES];
  88 static pthread_key_t key;
  89 static pthread_rwlock_t rwlock_tbl;
  90 static pthread_rwlock_t rwlock_init;
  91
  92 void run(void * arg);
  93 #endif
  94
  95 bool transStallMsg(int targetcore);
  96 void transTerminateMsg(int targetcore);
  97 int receiveObject();
  98 bool getreadlock(void* ptr);
  99 void releasereadlock(void* ptr);
 100 #ifdef RAW
 101 bool getreadlock_I(void* ptr);
 102 void releasereadlock_I(void* ptr);
 103 #endif
 104 bool getwritelock(void* ptr);
 105 void releasewritelock(void* ptr);
 106
 107 void flushAll(void);
 108
 109 void flushAll(void) {
 110 #ifdef RAW
 111         int i = 0;
 112 #ifdef INTERRUPT
 113         raw_user_interrupts_off();
 114 #endif
 115         raw_test_pass(0xec00);
 116         for(i = 0; i < 512; ++i) {
 117                 int base = i << 5;
 118                 int off1 = 1 << 14;
 119                 flushCacheline(base);
 120                 flushCacheline(base|off1);
 121         }
 122 #ifdef INTERRUPT
 123         raw_user_interrupts_on();
 124 #endif
 125         raw_test_pass(0xec02);
 126 #endif
 127 }
 128
 129 #ifdef RAW
 130 void recvMsg() {
 131         //raw_test_pass(0xefee);
 132         //raw_user_interrupts_off();
 133         //raw_test_pass(0xef00);
 134         receiveObject();
 135         //raw_test_pass(0xefff);
 136         //raw_user_interrupts_on();
 137         //raw_test_pass(0xefef);
 138 }
 139
 140 void begin() {
 141 #else
 142 int main(int argc, char **argv) {
 143 #endif
 144 #ifdef RAW
 145         int i = 0;
 146         int argc = 1;
 147         char ** argv = NULL;
 148         bool sendStall = false;
 149         bool isfirst = true;
 150         bool tocontinue = false;
 151         struct QueueItem * objitem = NULL;
 152         struct transObjInfo * objInfo = NULL;
 153         int grount = 0;
 154         bool allStall = true;
 155         int sumsendobj = 0;
 156
 157         raw_test_pass_reg(&locktable);
 158         raw_test_pass_reg(&msglength);
 159         raw_test_pass_reg(&i);
 160         raw_test_pass(0xee01);
 161         corenum = raw_get_abs_pos_x() + 4 * raw_get_abs_pos_y();
 162
 163         // initialize the arrays
 164         if(STARTUPCORE == corenum) {
 165                 // startup core to initialize corestatus[]
 166                 for(i = 0; i < NUMCORES; ++i) {
 167                         corestatus[i] = 1;
 168                         numsendobjs[i] = 0; // assume all variables in RAW are local variables! MAY BE WRONG!!!
 169                         numreceiveobjs[i] = 0;
 170                 }
 171         }
 172         self_numsendobjs = 0;
 173         self_numreceiveobjs = 0;
 174         for(i = 0; i < 30; ++i) {
 175                 msgdata[i] = -1;
 176         }
 177         //msgdata = NULL;
 178         msgtype = -1;
 179         msgdataindex = 0;
 180         msglength = 30;
 181         raw_test_pass(0xee02);
 182
 183         // create the lock table, lockresult table and obj queue
 184         locktable.size = 20;
 185     locktable.bucket = (struct RuntimeNode **) RUNMALLOC_I(sizeof(struct RuntimeNode *)*20);
 186     /* Set allocation blocks*/
 187     locktable.listhead=NULL;
 188     locktable.listtail=NULL;
 189     /*Set data counts*/
 190     locktable.numelements = 0;
 191         lockobj = 0;
 192         lockresult = 0;
 193         lockflag = false;
 194 #ifndef INTERRUPT
 195         reside = false;
 196 #endif
 197         objqueue.head = NULL;
 198         objqueue.tail = NULL;
 199         raw_test_pass(0xee03);
 200
 201 #ifdef INTERRUPT
 202         if (corenum < NUMCORES) {
 203         // set up interrupts
 204         setup_ints();
 205         //setup_interrupts();
 206         //start_gdn_avail_ints(recvMsg);
 207         raw_user_interrupts_on();
 208         raw_test_pass(0xee04);
 209         }
 210 #endif
 211
 212 #elif defined THREADSIMULATE
 213         errno = 0;
 214         int tids[NUMCORES];
 215         int rc[NUMCORES];
 216         pthread_t threads[NUMCORES];
 217         int i = 0;
 218
 219         // initialize three arrays and msg queue array
 220         char * pathhead = "/msgqueue_";
 221         int targetlen = strlen(pathhead);
 222         for(i = 0; i < NUMCORES; ++i) {
 223                 corestatus[i] = 1;
 224                 numsendobjs[i] = 0;
 225                 numreceiveobjs[i] = 0;
 226
 227                 char corenumstr[3];
 228                 int sourcelen = 0;
 229                 if(i < 10) {
 230                         corenumstr[0] = i + '0';
 231                         corenumstr[1] = '\0';
 232                         sourcelen = 1;
 233                 } else if(i < 100) {
 234                         corenumstr[1] = i %10 + '0';
 235                         corenumstr[0] = (i / 10) + '0';
 236                         corenumstr[2] = '\0';
 237                         sourcelen = 2;
 238                 } else {
 239                         printf("Error: i >= 100\n");
 240                         fflush(stdout);
 241                         exit(-1);
 242                 }
 243                 char path[targetlen + sourcelen + 1];
 244                 strcpy(path, pathhead);
 245                 strncat(path, corenumstr, sourcelen);
 246                 int oflags = O_RDONLY|O_CREAT|O_NONBLOCK;
 247                 int omodes = S_IRWXU|S_IRWXG|S_IRWXO;
 248                 mq_unlink(path);
 249                 mqd[i]= mq_open(path, oflags, omodes, NULL);
 250                 if(mqd[i] == -1) {
 251                         printf("[Main] mq_open %s fails: %d, error: %s\n", path, mqd[i], strerror(errno));
 252                         exit(-1);
 253                 } else {
 254                         printf("[Main] mq_open %s returns: %d\n", path, mqd[i]);
 255                 }
 256         }
 257
 258         // create the key
 259         pthread_key_create(&key, NULL);
 260
 261         // create the lock table and initialize its mutex
 262         locktbl = allocateRuntimeHash(20);
 263         int rc_locktbl = pthread_rwlock_init(&rwlock_tbl, NULL);
 264         printf("[Main] initialize the rwlock for lock table: %d error: \n", rc_locktbl, strerror(rc_locktbl));
 265
 266         for(i = 0; i < NUMCORES; ++i) {
 267                 thread_data_array[i].corenum = i;
 268                 thread_data_array[i].argc = argc;
 269                 thread_data_array[i].argv = argv;
 270                 thread_data_array[i].numsendobjs = 0;
 271                 thread_data_array[i].numreceiveobjs = 0;
 272                 printf("[main] creating thread %d\n", i);
 273                 rc[i] = pthread_create(&threads[i], NULL, run, (void *)&thread_data_array[i]);
 274         if (rc[i]){
 275                         printf("[main] ERROR; return code from pthread_create() is %d\n", rc[i]);
 276                         fflush(stdout);
 277                         exit(-1);
 278                 }
 279         }
 280
 281         //pthread_exit(NULL);
 282         while(true) {}
 283 }
 284
 285 void run(void* arg) {
 286         struct thread_data * my_tdata = (struct thread_data *)arg;
 287         pthread_setspecific(key, (void *)my_tdata->corenum);
 288         int argc = my_tdata->argc;
 289         char** argv = my_tdata->argv;
 290         printf("[run, %d] Thread %d runs: %x\n", my_tdata->corenum, my_tdata->corenum, (int)pthread_self());
 291         fflush(stdout);
 292
 293 #endif
 294
 295 #ifdef BOEHM_GC
 296   GC_init(); // Initialize the garbage collector
 297 #endif
 298 #ifdef CONSCHECK
 299   initializemmap();
 300 #endif
 301 #ifndef RAW
 302   processOptions();
 303 #endif
 304   initializeexithandler();
 305 #ifdef RAW
 306   raw_test_pass(0xee05);
 307 #endif
 308   /* Create table for failed tasks */
 309 #ifdef RAW
 310   if(corenum > NUMCORES - 1) {
 311           failedtasks = NULL;
 312           activetasks = NULL;
 313           while(true) {
 314                   receiveObject();
 315           }
 316   } else {
 317           raw_test_pass(0xee06);
 318 #endif
 319   /*failedtasks=genallocatehashtable((unsigned int (*)(void *)) &hashCodetpd,
 320                                    (int (*)(void *,void *)) &comparetpd);*/
 321           failedtasks = NULL;
 322 #ifdef RAW
 323   raw_test_pass(0xee07);
 324 #endif
 325   /* Create queue of active tasks */
 326   activetasks=genallocatehashtable((unsigned int (*)(void *)) &hashCodetpd,
 327                                    (int (*)(void *,void *)) &comparetpd);
 328 #ifdef RAW
 329   raw_test_pass(0xee08);
 330 #endif
 331
 332   /* Process task information */
 333   processtasks();
 334 #ifdef RAW
 335   raw_test_pass(0xee09);
 336 #endif
 337
 338   /* Create startup object */
 339   createstartupobject(argc, argv);
 340 #ifdef RAW
 341   raw_test_pass(0xee0a);
 342 #endif
 343
 344 #ifdef RAW
 345   raw_test_pass(0xee0b);
 346
 347   while(true) {
 348 /*#ifndef INTERRUPT
 349           while(receiveObject() != -1) {
 350           }
 351 #endif*/
 352
 353           // check if there are new active tasks can be executed
 354           executetasks();
 355
 356 #ifndef INTERRUPT
 357           while(receiveObject() != -1) {
 358           }
 359 #endif
 360
 361           raw_test_pass(0xee0c);
 362
 363           // check if there are some pending objects, if yes, enqueue them and executetasks again
 364           tocontinue = false;
 365           raw_test_pass(0xee0d);
 366           while(!isEmpty(&objqueue)) {
 367                   void * obj = NULL;
 368 #ifdef INTERRUPT
 369                   raw_user_interrupts_off();
 370 #endif
 371                   raw_test_pass(0xeee1);
 372                   sendStall = false;
 373                   tocontinue = true;
 374                   objitem = getTail(&objqueue);
 375                   //obj = objitem->objectptr;
 376                   objInfo = (struct transObjInfo *)objitem->objectptr;
 377                   obj = objInfo->objptr;
 378                   raw_test_pass_reg((int)obj);
 379                   // grab lock and flush the obj
 380                   getreadlock_I(obj);
 381                   while(!lockflag) {
 382                           receiveObject();
 383                   }
 384                   grount = lockresult;
 385                   raw_test_pass_reg(grount);
 386
 387                   lockresult = 0;
 388                   lockobj = 0;
 389                   lockflag = false;
 390 #ifndef INTERRUPT
 391                   reside = false;
 392 #endif
 393
 394                   if(grount == 1) {
 395                           int k = 0;
 396                           // flush the obj
 397                           for(k = 0; k < classsize[((struct ___Object___ *)obj)->type]; ++k) {
 398                                   invalidateAddr(obj + k);
 399                           }
 400                           // enqueue the object
 401                           for(k = 0; k < objInfo->length; ++k) {
 402                                   int taskindex = objInfo->queues[2 * k];
 403                                   int paramindex = objInfo->queues[2 * k + 1];
 404                                   struct parameterwrapper ** queues = &(paramqueues[corenum][taskindex][paramindex]);
 405                                   raw_test_pass_reg(taskindex);
 406                                   raw_test_pass_reg(paramindex);
 407                                   enqueueObject_I(obj, queues, 1);
 408                           }
 409                           removeItem(&objqueue, objitem);
 410                           releasereadlock_I(obj);
 411                           RUNFREE(objInfo->queues);
 412                           RUNFREE(objInfo);
 413                           /*enqueueObject_I(obj, NULL, 0);
 414                           removeItem(&objqueue, objitem);
 415                           releasereadlock_I(obj);*/
 416                   } else {
 417                           // can not get lock
 418                           // put it at the end of the queue
 419                           // and try to execute active tasks already enqueued first
 420                           removeItem(&objqueue, objitem);
 421                           addNewItem_I(&objqueue, objInfo);
 422                           break;
 423                   }
 424 #ifdef INTERRUPT
 425                   raw_user_interrupts_on();
 426 #endif
 427                   raw_test_pass(0xee0e);
 428           }
 429           raw_test_pass(0xee0f);
 430
 431           if(!tocontinue) {
 432                   // check if stop
 433                   if(STARTUPCORE == corenum) {
 434                           if(isfirst) {
 435                                   raw_test_pass(0xee10);
 436                                   isfirst = false;
 437                           }
 438 #ifdef INTERRUPT
 439                           raw_user_interrupts_off();
 440 #endif
 441                           corestatus[corenum] = 0;
 442                           numsendobjs[corenum] = self_numsendobjs;
 443                           numreceiveobjs[corenum] = self_numreceiveobjs;
 444                           // check the status of all cores
 445                           allStall = true;
 446                           raw_test_pass_reg(NUMCORES);
 447                           for(i = 0; i < NUMCORES; ++i) {
 448                                   raw_test_pass(0xe000 + corestatus[i]);
 449                                   if(corestatus[i] != 0) {
 450                                           allStall = false;
 451                                           break;
 452                                   }
 453                           }
 454                           if(allStall) {
 455                                   // check if the sum of send objs and receive obj are the same
 456                                   // yes->terminate
 457                                   // no->go on executing
 458                                   sumsendobj = 0;
 459                                   for(i = 0; i < NUMCORES; ++i) {
 460                                           sumsendobj += numsendobjs[i];
 461                                           raw_test_pass(0xf000 + numsendobjs[i]);
 462                                   }
 463                                   for(i = 0; i < NUMCORES; ++i) {
 464                                           sumsendobj -= numreceiveobjs[i];
 465                                           raw_test_pass(0xf000 + numreceiveobjs[i]);
 466                                   }
 467                                   if(0 == sumsendobj) {
 468                                           // terminate
 469                                           raw_test_pass(0xee11);
 470                                           raw_test_done(1);     // All done.
 471                                   }
 472                           }
 473 #ifdef INTERRUPT
 474                           raw_user_interrupts_on();
 475 #endif
 476                   } else {
 477                           if(!sendStall) {
 478                                   raw_test_pass(0xee12);
 479                                   if(isfirst) {
 480                                           // wait for some time
 481                                           int halt = 10000;
 482                                           raw_test_pass(0xee13);
 483                                           while(halt--){}
 484                                           isfirst = false;
 485                                           raw_test_pass(0xee14);
 486                                   } else {
 487                                         // send StallMsg to startup core
 488                                         raw_test_pass(0xee15);
 489                                         sendStall = transStallMsg(STARTUPCORE);
 490                                         isfirst = true;
 491                                   }
 492                           } else {
 493                                   isfirst = true;
 494                                   raw_test_pass(0xee16);
 495                           }
 496                   }
 497           }
 498   }
 499   }
 500 #elif defined THREADSIMULATE
 501   /* Start executing the tasks */
 502   executetasks();
 503
 504   int i = 0;
 505   // check if there are new objects coming
 506   bool sendStall = false;
 507
 508   int numofcore = pthread_getspecific(key);
 509   while(true) {
 510           switch(receiveObject()) {
 511                   case 0: {
 512                                           printf("[run, %d] receive an object\n", numofcore);
 513                                           sendStall = false;
 514                                           // received an object
 515                                           // check if there are new active tasks can be executed
 516                                           executetasks();
 517                                           break;
 518                                   }
 519                   case 1: {
 520                                           //printf("[run, %d] no msg\n", numofcore);
 521                                           // no msg received
 522                                           if(STARTUPCORE == numofcore) {
 523                                                   corestatus[numofcore] = 0;
 524                                                   // check the status of all cores
 525                                                   bool allStall = true;
 526                                                   for(i = 0; i < NUMCORES; ++i) {
 527                                                           if(corestatus[i] != 0) {
 528                                                                   allStall = false;
 529                                                                   break;
 530                                                           }
 531                                                   }
 532                                                   if(allStall) {
 533                                                           // check if the sum of send objs and receive obj are the same
 534                                                           // yes->terminate
 535                                                           // no->go on executing
 536                                                           int sumsendobj = 0;
 537                                                           for(i = 0; i < NUMCORES; ++i) {
 538                                                                   sumsendobj += numsendobjs[i];
 539                                                           }
 540                                                           for(i = 0; i < NUMCORES; ++i) {
 541                                                                   sumsendobj -= numreceiveobjs[i];
 542                                                           }
 543                                                           if(0 == sumsendobj) {
 544                                                                   // terminate
 545
 546                                                                   // release all locks
 547                                                                   int rc_tbl = pthread_rwlock_wrlock(&rwlock_tbl);
 548                                                                   printf("[run, %d] getting the write lock for locktbl: %d error: \n", numofcore, rc_tbl, strerror(rc_tbl));
 549                                                                   struct RuntimeIterator* it_lock = RuntimeHashcreateiterator(locktbl);
 550                                                                   while(0 != RunhasNext(it_lock)) {
 551                                                                           int key = Runkey(it_lock);
 552                                                                           pthread_rwlock_t* rwlock_obj = (pthread_rwlock_t*)Runnext(it_lock);
 553                                                                           int rc_des = pthread_rwlock_destroy(rwlock_obj);
 554                                                                           printf("[run, %d] destroy the rwlock for object: %d error: \n", numofcore, key, strerror(rc_des));
 555                                                                           RUNFREE(rwlock_obj);
 556                                                                   }
 557                                                                   freeRuntimeHash(locktbl);
 558                                                                   locktbl = NULL;
 559                                                                   RUNFREE(it_lock);
 560
 561                                                                   // destroy all message queues
 562                                                                   char * pathhead = "/msgqueue_";
 563                                                                   int targetlen = strlen(pathhead);
 564                                                                   for(i = 0; i < NUMCORES; ++i) {
 565                                                                           char corenumstr[3];
 566                                                                           int sourcelen = 0;
 567                                                                           if(i < 10) {
 568                                                                                   corenumstr[0] = i + '0';
 569                                                                                   corenumstr[1] = '\0';
 570                                                                                   sourcelen = 1;
 571                                                                           } else if(i < 100) {
 572                                                                                   corenumstr[1] = i %10 + '0';
 573                                                                                   corenumstr[0] = (i / 10) + '0';
 574                                                                                   corenumstr[2] = '\0';
 575                                                                                   sourcelen = 2;
 576                                                                           } else {
 577                                                                                   printf("Error: i >= 100\n");
 578                                                                                   fflush(stdout);
 579                                                                                   exit(-1);
 580                                                                           }
 581                                                                           char path[targetlen + sourcelen + 1];
 582                                                                           strcpy(path, pathhead);
 583                                                                           strncat(path, corenumstr, sourcelen);
 584                                                                           mq_unlink(path);
 585                                                                   }
 586
 587                                                                   printf("[run, %d] terminate!\n", numofcore);
 588                                                                   fflush(stdout);
 589                                                                   exit(0);
 590                                                           }
 591                                                   }
 592                                           } else {
 593                                                   if(!sendStall) {
 594                                                           // send StallMsg to startup core
 595                                                           sendStall = transStallMsg(STARTUPCORE);
 596                                                   }
 597                                           }
 598                                           break;
 599                                   }
 600                   case 2: {
 601                                           printf("[run, %d] receive a stall msg\n", numofcore);
 602                                           // receive a Stall Msg, do nothing
 603                                           assert(STARTUPCORE == numofcore); // only startup core can receive such msg
 604                                           sendStall = false;
 605                                           break;
 606                                   }
 607                  /* case 3: {
 608                                           printf("[run, %d] receive a terminate msg\n", numofcore);
 609                                           // receive a terminate Msg
 610                                           assert(STARTUPCORE != corenum); // only non-startup core can receive such msg
 611                                           mq_close(mqd[corenum]);
 612                                           fflush(stdout);
 613                                           exit(0);
 614                                           break;
 615                                   }*/
 616                   default: {
 617                                            printf("[run, %d] Error: invalid message type.\n", numofcore);
 618                                            fflush(stdout);
 619                                            exit(-1);
 620                                            break;
 621                                    }
 622           }
 623   }
 624 #endif
 625 }
 626
 627 void createstartupobject(int argc, char ** argv) {
 628   int i;
 629
 630   /* Allocate startup object     */
 631 #ifdef PRECISE_GC
 632   struct ___StartupObject___ *startupobject=(struct ___StartupObject___*) allocate_new(NULL, STARTUPTYPE);
 633   struct ArrayObject * stringarray=allocate_newarray(NULL, STRINGARRAYTYPE, argc-1);
 634 #else
 635   struct ___StartupObject___ *startupobject=(struct ___StartupObject___*) allocate_new(STARTUPTYPE);
 636   struct ArrayObject * stringarray=allocate_newarray(STRINGARRAYTYPE, argc-1);
 637 #endif
 638   /* Build array of strings */
 639   startupobject->___parameters___=stringarray;
 640   for(i=1;i<argc;i++) {
 641     int length=strlen(argv[i]);
 642 #ifdef PRECISE_GC
 643     struct ___String___ *newstring=NewString(NULL, argv[i],length);
 644 #else
 645     struct ___String___ *newstring=NewString(argv[i],length);
 646 #endif
 647     ((void **)(((char *)& stringarray->___length___)+sizeof(int)))[i-1]=newstring;
 648   }
 649
 650   startupobject->isolate = 1;
 651   startupobject->version = 0;
 652
 653   /* Set initialized flag for startup object */
 654   flagorandinit(startupobject,1,0xFFFFFFFF);
 655   enqueueObject(startupobject, NULL, 0);
 656 #ifdef RAW
 657   flushAll();
 658 #endif
 659 }
 660
 661 int hashCodetpd(struct taskparamdescriptor *ftd) {
 662   int hash=(int)ftd->task;
 663   int i;
 664   for(i=0;i<ftd->numParameters;i++){
 665     hash^=(int)ftd->parameterArray[i];
 666   }
 667   return hash;
 668 }
 669
 670 int comparetpd(struct taskparamdescriptor *ftd1, struct taskparamdescriptor *ftd2) {
 671   int i;
 672   if (ftd1->task!=ftd2->task)
 673     return 0;
 674   for(i=0;i<ftd1->numParameters;i++)
 675     if(ftd1->parameterArray[i]!=ftd2->parameterArray[i])
 676       return 0;
 677   return 1;
 678 }
 679
 680 /* This function sets a tag. */
 681 #ifdef PRECISE_GC
 682 void tagset(void *ptr, struct ___Object___ * obj, struct ___TagDescriptor___ * tagd) {
 683 #else
 684 void tagset(struct ___Object___ * obj, struct ___TagDescriptor___ * tagd) {
 685 #endif
 686         struct ArrayObject * ao=NULL;
 687   struct ___Object___ * tagptr=obj->___tags___;
 688 #ifdef RAW
 689   raw_test_pass(0xebb0);
 690 #endif
 691   if (tagptr==NULL) {
 692 #ifdef RAW
 693           raw_test_pass(0xebb1);
 694 #endif
 695     obj->___tags___=(struct ___Object___ *)tagd;
 696   } else {
 697     /* Have to check if it is already set */
 698     if (tagptr->type==TAGTYPE) {
 699       struct ___TagDescriptor___ * td=(struct ___TagDescriptor___ *) tagptr;
 700 #ifdef RAW
 701           raw_test_pass(0xebb2);
 702 #endif
 703       if (td==tagd) {
 704 #ifdef RAW
 705                   raw_test_pass(0xebb3);
 706 #endif
 707         return;
 708           }
 709 #ifdef PRECISE_GC
 710       int ptrarray[]={2, (int) ptr, (int) obj, (int)tagd};
 711       struct ArrayObject * ao=allocate_newarray(&ptrarray,TAGARRAYTYPE,TAGARRAYINTERVAL);
 712       obj=(struct ___Object___ *)ptrarray[2];
 713       tagd=(struct ___TagDescriptor___ *)ptrarray[3];
 714       td=(struct ___TagDescriptor___ *) obj->___tags___;
 715 #else
 716 #ifdef RAW
 717           raw_test_pass(0xebb4);
 718 #endif
 719       ao=allocate_newarray(TAGARRAYTYPE,TAGARRAYINTERVAL);
 720 #endif
 721 #ifdef RAW
 722           raw_test_pass(0xebb5);
 723 #endif
 724       ARRAYSET(ao, struct ___TagDescriptor___ *, 0, td);
 725       ARRAYSET(ao, struct ___TagDescriptor___ *, 1, tagd);
 726       obj->___tags___=(struct ___Object___ *) ao;
 727       ao->___cachedCode___=2;
 728 #ifdef RAW
 729           raw_test_pass(0xebb6);
 730 #endif
 731     } else {
 732       /* Array Case */
 733       int i;
 734       struct ArrayObject *ao=(struct ArrayObject *) tagptr;
 735 #ifdef RAW
 736           raw_test_pass(0xebb7);
 737 #endif
 738       for(i=0;i<ao->___cachedCode___;i++) {
 739         struct ___TagDescriptor___ * td=ARRAYGET(ao, struct ___TagDescriptor___*, i);
 740 #ifdef RAW
 741                 raw_test_pass(0xebb8);
 742 #endif
 743         if (td==tagd) {
 744 #ifdef RAW
 745                 raw_test_pass(0xebb9);
 746 #endif
 747           return;
 748         }
 749       }
 750       if (ao->___cachedCode___<ao->___length___) {
 751 #ifdef RAW
 752                 raw_test_pass(0xebba);
 753 #endif
 754         ARRAYSET(ao, struct ___TagDescriptor___ *, ao->___cachedCode___, tagd);
 755         ao->___cachedCode___++;
 756 #ifdef RAW
 757         raw_test_pass(0xebbb);
 758 #endif
 759       } else {
 760 #ifdef PRECISE_GC
 761         int ptrarray[]={2,(int) ptr, (int) obj, (int) tagd};
 762         struct ArrayObject * aonew=allocate_newarray(&ptrarray,TAGARRAYTYPE,TAGARRAYINTERVAL+ao->___length___);
 763         obj=(struct ___Object___ *)ptrarray[2];
 764         tagd=(struct ___TagDescriptor___ *) ptrarray[3];
 765         ao=(struct ArrayObject *)obj->___tags___;
 766 #else
 767         struct ArrayObject * aonew=allocate_newarray(TAGARRAYTYPE,TAGARRAYINTERVAL+ao->___length___);
 768 #endif
 769 #ifdef RAW
 770                 raw_test_pass(0xebbc);
 771 #endif
 772         aonew->___cachedCode___=ao->___length___+1;
 773         for(i=0;i<ao->___length___;i++) {
 774 #ifdef RAW
 775                 raw_test_pass(0xebbd);
 776 #endif
 777           ARRAYSET(aonew, struct ___TagDescriptor___*, i, ARRAYGET(ao, struct ___TagDescriptor___*, i));
 778         }
 779 #ifdef RAW
 780                 raw_test_pass(0xebbe);
 781 #endif
 782         ARRAYSET(aonew, struct ___TagDescriptor___ *, ao->___length___, tagd);
 783 #ifdef RAW
 784         raw_test_pass(0xebbf);
 785 #endif
 786       }
 787     }
 788   }
 789
 790   {
 791     struct ___Object___ * tagset=tagd->flagptr;
 792 #ifdef RAW
 793         raw_test_pass(0xb008);
 794 #endif
 795     if(tagset==NULL) {
 796 #ifdef RAW
 797                 raw_test_pass(0xb009);
 798 #endif
 799       tagd->flagptr=obj;
 800     } else if (tagset->type!=OBJECTARRAYTYPE) {
 801 #ifdef PRECISE_GC
 802       int ptrarray[]={2, (int) ptr, (int) obj, (int)tagd};
 803       struct ArrayObject * ao=allocate_newarray(&ptrarray,OBJECTARRAYTYPE,OBJECTARRAYINTERVAL);
 804       obj=(struct ___Object___ *)ptrarray[2];
 805       tagd=(struct ___TagDescriptor___ *)ptrarray[3];
 806 #else
 807       struct ArrayObject * ao=allocate_newarray(OBJECTARRAYTYPE,OBJECTARRAYINTERVAL);
 808 #endif
 809       ARRAYSET(ao, struct ___Object___ *, 0, tagd->flagptr);
 810       ARRAYSET(ao, struct ___Object___ *, 1, obj);
 811       ao->___cachedCode___=2;
 812       tagd->flagptr=(struct ___Object___ *)ao;
 813 #ifdef RAW
 814           raw_test_pass(0xb00a);
 815 #endif
 816     } else {
 817       struct ArrayObject *ao=(struct ArrayObject *) tagset;
 818       if (ao->___cachedCode___<ao->___length___) {
 819 #ifdef RAW
 820                   raw_test_pass(0xb00b);
 821 #endif
 822         ARRAYSET(ao, struct ___Object___*, ao->___cachedCode___++, obj);
 823       } else {
 824         int i;
 825 #ifdef PRECISE_GC
 826         int ptrarray[]={2, (int) ptr, (int) obj, (int)tagd};
 827         struct ArrayObject * aonew=allocate_newarray(&ptrarray,OBJECTARRAYTYPE,OBJECTARRAYINTERVAL+ao->___length___);
 828         obj=(struct ___Object___ *)ptrarray[2];
 829         tagd=(struct ___TagDescriptor___ *)ptrarray[3];
 830         ao=(struct ArrayObject *)tagd->flagptr;
 831 #else
 832         struct ArrayObject * aonew=allocate_newarray(OBJECTARRAYTYPE,OBJECTARRAYINTERVAL);
 833 #endif
 834         aonew->___cachedCode___=ao->___cachedCode___+1;
 835         for(i=0;i<ao->___length___;i++) {
 836           ARRAYSET(aonew, struct ___Object___*, i, ARRAYGET(ao, struct ___Object___*, i));
 837         }
 838         ARRAYSET(aonew, struct ___Object___ *, ao->___cachedCode___, obj);
 839         tagd->flagptr=(struct ___Object___ *) aonew;
 840 #ifdef RAW
 841         raw_test_pass(0xb00c);
 842 #endif
 843       }
 844     }
 845   }
 846 }
 847
 848 /* This function clears a tag. */
 849 #ifdef PRECISE_GC
 850 void tagclear(void *ptr, struct ___Object___ * obj, struct ___TagDescriptor___ * tagd) {
 851 #else
 852 void tagclear(struct ___Object___ * obj, struct ___TagDescriptor___ * tagd) {
 853 #endif
 854   /* We'll assume that tag is alway there.
 855      Need to statically check for this of course. */
 856   struct ___Object___ * tagptr=obj->___tags___;
 857
 858   if (tagptr->type==TAGTYPE) {
 859     if ((struct ___TagDescriptor___ *)tagptr==tagd)
 860       obj->___tags___=NULL;
 861     else
 862 #ifndef RAW
 863       printf("ERROR 1 in tagclear\n");
 864 #endif
 865         ;
 866   } else {
 867     struct ArrayObject *ao=(struct ArrayObject *) tagptr;
 868     int i;
 869     for(i=0;i<ao->___cachedCode___;i++) {
 870       struct ___TagDescriptor___ * td=ARRAYGET(ao, struct ___TagDescriptor___ *, i);
 871       if (td==tagd) {
 872         ao->___cachedCode___--;
 873         if (i<ao->___cachedCode___)
 874           ARRAYSET(ao, struct ___TagDescriptor___ *, i, ARRAYGET(ao, struct ___TagDescriptor___ *, ao->___cachedCode___));
 875         ARRAYSET(ao, struct ___TagDescriptor___ *, ao->___cachedCode___, NULL);
 876         if (ao->___cachedCode___==0)
 877           obj->___tags___=NULL;
 878         goto PROCESSCLEAR;
 879       }
 880     }
 881 #ifndef RAW
 882     printf("ERROR 2 in tagclear\n");
 883 #endif
 884         ;
 885   }
 886  PROCESSCLEAR:
 887   {
 888     struct ___Object___ *tagset=tagd->flagptr;
 889     if (tagset->type!=OBJECTARRAYTYPE) {
 890       if (tagset==obj)
 891         tagd->flagptr=NULL;
 892       else
 893 #ifndef RAW
 894         printf("ERROR 3 in tagclear\n");
 895 #endif
 896           ;
 897     } else {
 898       struct ArrayObject *ao=(struct ArrayObject *) tagset;
 899       int i;
 900       for(i=0;i<ao->___cachedCode___;i++) {
 901         struct ___Object___ * tobj=ARRAYGET(ao, struct ___Object___ *, i);
 902         if (tobj==obj) {
 903           ao->___cachedCode___--;
 904           if (i<ao->___cachedCode___)
 905             ARRAYSET(ao, struct ___Object___ *, i, ARRAYGET(ao, struct ___Object___ *, ao->___cachedCode___));
 906           ARRAYSET(ao, struct ___Object___ *, ao->___cachedCode___, NULL);
 907           if (ao->___cachedCode___==0)
 908             tagd->flagptr=NULL;
 909           goto ENDCLEAR;
 910         }
 911       }
 912 #ifndef RAW
 913       printf("ERROR 4 in tagclear\n");
 914 #endif
 915     }
 916   }
 917  ENDCLEAR:
 918   return;
 919 }
 920
 921 /* This function allocates a new tag. */
 922 #ifdef PRECISE_GC
 923 struct ___TagDescriptor___ * allocate_tag(void *ptr, int index) {
 924   struct ___TagDescriptor___ * v=(struct ___TagDescriptor___ *) mygcmalloc((struct garbagelist *) ptr, classsize[TAGTYPE]);
 925 #else
 926 struct ___TagDescriptor___ * allocate_tag(int index) {
 927   struct ___TagDescriptor___ * v=FREEMALLOC(classsize[TAGTYPE]);
 928 #endif
 929   v->type=TAGTYPE;
 930   v->flag=index;
 931   return v;
 932 }
 933
 934
 935
 936 /* This function updates the flag for object ptr.  It or's the flag
 937    with the or mask and and's it with the andmask. */
 938
 939 void flagbody(struct ___Object___ *ptr, int flag, struct parameterwrapper ** queues, int length, bool isnew);
 940
 941  int flagcomp(const int *val1, const int *val2) {
 942    return (*val1)-(*val2);
 943  }
 944
 945 void flagorand(void * ptr, int ormask, int andmask, struct parameterwrapper ** queues, int length) {
 946     {
 947       int oldflag=((int *)ptr)[1];
 948       int flag=ormask|oldflag;
 949       flag&=andmask;
 950 #ifdef RAW
 951           raw_test_pass(0xaa000000 + oldflag);
 952           raw_test_pass(0xaa000000 + flag);
 953 #endif
 954           flagbody(ptr, flag, queues, length, false);
 955     }
 956 }
 957
 958 bool intflagorand(void * ptr, int ormask, int andmask) {
 959     {
 960       int oldflag=((int *)ptr)[1];
 961       int flag=ormask|oldflag;
 962       flag&=andmask;
 963       if (flag==oldflag) /* Don't do anything */
 964         return false;
 965       else {
 966                  flagbody(ptr, flag, NULL, 0, false);
 967                  return true;
 968           }
 969     }
 970 }
 971
 972 void flagorandinit(void * ptr, int ormask, int andmask) {
 973   int oldflag=((int *)ptr)[1];
 974   int flag=ormask|oldflag;
 975   flag&=andmask;
 976 #ifdef RAW
 977           raw_test_pass(0xaa100000 + oldflag);
 978           raw_test_pass(0xaa100000 + flag);
 979 #endif
 980   flagbody(ptr,flag,NULL,0,true);
 981 }
 982
 983 void flagbody(struct ___Object___ *ptr, int flag, struct parameterwrapper ** vqueues, int vlength, bool isnew) {
 984   struct parameterwrapper * flagptr = NULL;
 985   int i = 0;
 986   struct parameterwrapper ** queues = vqueues;
 987   int length = vlength;
 988   int next;
 989   int UNUSED, UNUSED2;
 990   int * enterflags = NULL;
 991   if((!isnew) && (queues == NULL)) {
 992 #ifdef THREADSIMULATE
 993           int numofcore = pthread_getspecific(key);
 994           queues = objectqueues[numofcore][ptr->type];
 995           length = numqueues[numofcore][ptr->type];
 996 #else
 997 #ifdef RAW
 998           if(corenum < NUMCORES) {
 999 #endif
1000           queues = objectqueues[corenum][ptr->type];
1001           length = numqueues[corenum][ptr->type];
1002 #ifdef RAW
1003           } else {
1004                   return;
1005           }
1006 #endif
1007 #endif
1008   }
1009   ptr->flag=flag;
1010 #ifdef RAW
1011   raw_test_pass(0xbb000000 + ptr->flag);
1012 #endif
1013
1014   /*Remove object from all queues */
1015   for(i = 0; i < length; ++i) {
1016           flagptr = queues[i];
1017     ObjectHashget(flagptr->objectset, (int) ptr, (int *) &next, (int *) &enterflags, &UNUSED, &UNUSED2);
1018     ObjectHashremove(flagptr->objectset, (int)ptr);
1019     if (enterflags!=NULL)
1020       RUNFREE(enterflags);
1021   }
1022  }
1023
1024  void enqueueObject(void * vptr, struct parameterwrapper ** vqueues, int vlength) {
1025    struct ___Object___ *ptr = (struct ___Object___ *)vptr;
1026
1027   {
1028     struct QueueItem *tmpptr;
1029         struct parameterwrapper * parameter=NULL;
1030         int j;
1031         int i;
1032     struct parameterwrapper * prevptr=NULL;
1033     struct ___Object___ *tagptr=NULL;
1034         struct parameterwrapper ** queues = vqueues;
1035         int length = vlength;
1036 #ifdef RAW
1037         if(corenum > NUMCORES - 1) {
1038                 return;
1039         }
1040 #endif
1041         if(queues == NULL) {
1042 #ifdef THREADSIMULATE
1043                 int numofcore = pthread_getspecific(key);
1044                 queues = objectqueues[numofcore][ptr->type];
1045                 length = numqueues[numofcore][ptr->type];
1046 #else
1047                 queues = objectqueues[corenum][ptr->type];
1048                 length = numqueues[corenum][ptr->type];
1049 #endif
1050         }
1051     tagptr=ptr->___tags___;
1052
1053     /* Outer loop iterates through all parameter queues an object of
1054        this type could be in.  */
1055     for(j = 0; j < length; ++j) {
1056                 parameter = queues[j];
1057       /* Check tags */
1058       if (parameter->numbertags>0) {
1059         if (tagptr==NULL)
1060           goto nextloop;//that means the object has no tag but that param needs tag
1061         else if(tagptr->type==TAGTYPE) {//one tag
1062           struct ___TagDescriptor___ * tag=(struct ___TagDescriptor___*) tagptr;
1063           for(i=0;i<parameter->numbertags;i++) {
1064             //slotid is parameter->tagarray[2*i];
1065             int tagid=parameter->tagarray[2*i+1];
1066             if (tagid!=tagptr->flag)
1067               goto nextloop; /*We don't have this tag */
1068            }
1069         } else {//multiple tags
1070           struct ArrayObject * ao=(struct ArrayObject *) tagptr;
1071           for(i=0;i<parameter->numbertags;i++) {
1072             //slotid is parameter->tagarray[2*i];
1073             int tagid=parameter->tagarray[2*i+1];
1074             int j;
1075             for(j=0;j<ao->___cachedCode___;j++) {
1076               if (tagid==ARRAYGET(ao, struct ___TagDescriptor___*, j)->flag)
1077                 goto foundtag;
1078             }
1079             goto nextloop;
1080           foundtag:
1081             ;
1082           }
1083         }
1084       }
1085
1086       /* Check flags */
1087       for(i=0;i<parameter->numberofterms;i++) {
1088         int andmask=parameter->intarray[i*2];
1089         int checkmask=parameter->intarray[i*2+1];
1090         if ((ptr->flag&andmask)==checkmask) {
1091 #ifdef RAW
1092                 raw_test_pass(0xcc000000 + andmask);
1093                 raw_test_pass_reg((int)ptr);
1094                 raw_test_pass(0xcc000000 + ptr->flag);
1095                 raw_test_pass(0xcc000000 + checkmask);
1096 #endif
1097           enqueuetasks(parameter, prevptr, ptr, NULL, 0);
1098           prevptr=parameter;
1099           break;
1100         }
1101       }
1102     nextloop:
1103           ;
1104     }
1105   }
1106 }
1107
1108 #ifdef RAW
1109 void enqueueObject_I(void * vptr, struct parameterwrapper ** vqueues, int vlength) {
1110    struct ___Object___ *ptr = (struct ___Object___ *)vptr;
1111
1112   {
1113     struct QueueItem *tmpptr;
1114         struct parameterwrapper * parameter=NULL;
1115         int j;
1116         int i;
1117     struct parameterwrapper * prevptr=NULL;
1118     struct ___Object___ *tagptr=NULL;
1119         struct parameterwrapper ** queues = vqueues;
1120         int length = vlength;
1121 #ifdef RAW
1122         if(corenum > NUMCORES - 1) {
1123                 return;
1124         }
1125 #endif
1126         if(queues == NULL) {
1127 #ifdef THREADSIMULATE
1128                 int numofcore = pthread_getspecific(key);
1129                 queues = objectqueues[numofcore][ptr->type];
1130                 length = numqueues[numofcore][ptr->type];
1131 #else
1132                 queues = objectqueues[corenum][ptr->type];
1133                 length = numqueues[corenum][ptr->type];
1134 #endif
1135         }
1136 #ifdef RAW
1137         raw_test_pass(0xeaa1);
1138         raw_test_pass_reg(queues);
1139         raw_test_pass_reg(length);
1140 #endif
1141     tagptr=ptr->___tags___;
1142
1143     /* Outer loop iterates through all parameter queues an object of
1144        this type could be in.  */
1145     for(j = 0; j < length; ++j) {
1146                 parameter = queues[j];
1147       /* Check tags */
1148       if (parameter->numbertags>0) {
1149 #ifdef RAW
1150         raw_test_pass(0xeaa2);
1151         raw_test_pass_reg(tagptr);
1152 #endif
1153         if (tagptr==NULL)
1154           goto nextloop;//that means the object has no tag but that param needs tag
1155         else if(tagptr->type==TAGTYPE) {//one tag
1156           struct ___TagDescriptor___ * tag=(struct ___TagDescriptor___*) tagptr;
1157 #ifdef RAW
1158         raw_test_pass(0xeaa3);
1159 #endif
1160           for(i=0;i<parameter->numbertags;i++) {
1161             //slotid is parameter->tagarray[2*i];
1162             int tagid=parameter->tagarray[2*i+1];
1163             if (tagid!=tagptr->flag) {
1164 #ifdef RAW
1165         raw_test_pass(0xeaa4);
1166 #endif
1167               goto nextloop; /*We don't have this tag */
1168                 }
1169            }
1170         } else {//multiple tags
1171           struct ArrayObject * ao=(struct ArrayObject *) tagptr;
1172 #ifdef RAW
1173         raw_test_pass(0xeaa5);
1174 #endif
1175           for(i=0;i<parameter->numbertags;i++) {
1176             //slotid is parameter->tagarray[2*i];
1177             int tagid=parameter->tagarray[2*i+1];
1178             int j;
1179             for(j=0;j<ao->___cachedCode___;j++) {
1180               if (tagid==ARRAYGET(ao, struct ___TagDescriptor___*, j)->flag) {
1181                 goto foundtag;
1182                   }
1183             }
1184 #ifdef RAW
1185         raw_test_pass(0xeaa6);
1186 #endif
1187             goto nextloop;
1188           foundtag:
1189             ;
1190           }
1191         }
1192       }
1193
1194       /* Check flags */
1195       for(i=0;i<parameter->numberofterms;i++) {
1196         int andmask=parameter->intarray[i*2];
1197         int checkmask=parameter->intarray[i*2+1];
1198 #ifdef RAW
1199         raw_test_pass(0xeaa7);
1200         raw_test_pass(0xcc000000 + andmask);
1201         raw_test_pass_reg(ptr);
1202         raw_test_pass(0xcc000000 + ptr->flag);
1203         raw_test_pass(0xcc000000 + checkmask);
1204 #endif
1205         if ((ptr->flag&andmask)==checkmask) {
1206 #ifdef RAW
1207         raw_test_pass(0xeaa8);
1208 #endif
1209           enqueuetasks_I(parameter, prevptr, ptr, NULL, 0);
1210           prevptr=parameter;
1211           break;
1212         }
1213       }
1214     nextloop:
1215           ;
1216     }
1217   }
1218 }
1219
1220 // helper function to compute the coordinates of a core from the core number
1221 void calCoords(int core_num, int* coordY, int* coordX) {
1222         *coordX = core_num % 4;
1223         *coordY = core_num / 4;
1224 }
1225 #endif
1226
1227 /* Message format for RAW version:
1228  *      type + Msgbody
1229  * type: 0 -- transfer object
1230  *       1 -- transfer stall msg
1231  *       2 -- lock request
1232  *       3 -- lock grount
1233  *       4 -- lock deny
1234  *       5 -- lock release
1235  *
1236  * ObjMsg: 0 + size of msg + obj's address + (task index + param index)+
1237  * StallMsg: 1 + corenum + sendobjs + receiveobjs (size is always 4 * sizeof(int))
1238  * LockMsg: 2 + lock type + obj pointer + request core (size is always 4 * sizeof(int))
1239  *          3/4/5 + lock type + obj pointer (size is always 3 * sizeof(int))
1240  *          lock type: 0 -- read; 1 -- write
1241  */
1242
1243 // transfer an object to targetcore
1244 // format: object
1245 void transferObject(struct transObjInfo * transObj) {
1246         void * obj = transObj->objptr;
1247         int type=((int *)obj)[0];
1248     int size=classsize[type];
1249         int targetcore = transObj->targetcore;
1250         //assert(type < NUMCLASSES); // can only transfer normal object
1251
1252 #ifdef RAW
1253         unsigned msgHdr;
1254         int self_y, self_x, target_y, target_x;
1255         //int isshared = 0;
1256         // for 32 bit machine, the size of fixed part is always 3 words
1257         //int msgsize = sizeof(int) * 2 + sizeof(void *);
1258         int msgsize = 3 + transObj->length * 2;
1259         int i = 0;
1260
1261         struct ___Object___ * newobj = (struct ___Object___ *)obj;
1262         /*if(0 == newobj->isolate) {
1263                 isshared = 1;
1264         }*/
1265
1266         calCoords(corenum, &self_y, &self_x);
1267         calCoords(targetcore, &target_y, &target_x);
1268         // Build the message header
1269     msgHdr = construct_dyn_hdr(0, msgsize, 0,           // msgsize word sent.
1270                                            self_y, self_x,
1271                                                            target_y, target_x);
1272         gdn_send(msgHdr);               // Send the message header to EAST to handle fab(n - 1).
1273         raw_test_pass(0xbbbb);
1274         raw_test_pass(0xb000 + targetcore); // targetcore
1275     gdn_send(0);
1276     raw_test_pass(0);
1277         gdn_send(msgsize);
1278         raw_test_pass_reg(msgsize);
1279     gdn_send(obj);
1280         raw_test_pass_reg(obj);
1281         //gdn_send(isshared);
1282         //raw_test_pass_reg(isshared);
1283         for(i = 0; i < transObj->length; ++i) {
1284                 int taskindex = transObj->queues[2*i];
1285                 int paramindex = transObj->queues[2*i+1];
1286                 gdn_send(taskindex);
1287                 raw_test_pass_reg(taskindex);
1288                 gdn_send(paramindex);
1289                 raw_test_pass_reg(paramindex);
1290         }
1291         raw_test_pass(0xffff);
1292         ++(self_numsendobjs);
1293 #elif defined THREADSIMULATE
1294         int numofcore = pthread_getspecific(key);
1295
1296         // use POSIX message queue to transfer objects between cores
1297         mqd_t mqdnum;
1298         char corenumstr[3];
1299         int sourcelen = 0;
1300         if(targetcore < 10) {
1301                 corenumstr[0] = targetcore + '0';
1302                 corenumstr[1] = '\0';
1303                 sourcelen = 1;
1304         } else if(targetcore < 100) {
1305                 corenumstr[1] = targetcore % 10 + '0';
1306                 corenumstr[0] = (targetcore / 10) + '0';
1307                 corenumstr[2] = '\0';
1308                 sourcelen = 2;
1309         } else {
1310                 printf("Error: targetcore >= 100\n");
1311                 fflush(stdout);
1312                 exit(-1);
1313         }
1314         char * pathhead = "/msgqueue_";
1315         int targetlen = strlen(pathhead);
1316         char path[targetlen + sourcelen + 1];
1317         strcpy(path, pathhead);
1318         strncat(path, corenumstr, sourcelen);
1319         int oflags = O_WRONLY|O_NONBLOCK;
1320         int omodes = S_IRWXU|S_IRWXG|S_IRWXO;
1321         mqdnum = mq_open(path, oflags, omodes, NULL);
1322         if(mqdnum==-1) {
1323                 printf("[transferObject, %d] mq_open %s fail: %d, error: %s\n", numofcore, path, mqdnum, strerror(errno));
1324                 fflush(stdout);
1325                 exit(-1);
1326         }
1327         /*struct ___Object___ * newobj = (struct ___Object___ *)obj;
1328         if(0 == newobj->isolate) {
1329                 newobj = RUNMALLOC(size);
1330                 memcpy(newobj, obj, size);
1331                 newobj->original=obj;
1332         }*/
1333         struct transObjInfo * tmptransObj = RUNMALLOC(sizeof(struct transObjInfo));
1334         memcpy(tmptransObj, transObj, sizeof(struct transObjInfo));
1335         int * tmpqueue = RUNMALLOC(sizeof(int)*2*tmptransObj->length);
1336         memcpy(tmpqueue, tmptransObj->queues, sizeof(int)*2*tmptransObj->length);
1337         tmptransObj->queues = tmpqueue;
1338         struct ___Object___ * newobj = RUNMALLOC(sizeof(struct ___Object___));
1339         newobj->type = ((struct ___Object___ *)obj)->type;
1340         newobj->original = (struct ___Object___ *)tmptransObj;
1341         int ret;
1342         do {
1343                 ret=mq_send(mqdnum, (void *)newobj, sizeof(struct ___Object___), 0); // send the object into the queue
1344                 if(ret != 0) {
1345                         printf("[transferObject, %d] mq_send to %s returned: %d, error: %s\n", numofcore, path, ret, strerror(errno));
1346                 }
1347         }while(ret!=0);
1348         RUNFREE(newobj);
1349         if(numofcore == STARTUPCORE) {
1350                 ++numsendobjs[numofcore];
1351         } else {
1352                 ++(thread_data_array[numofcore].numsendobjs);
1353         }
1354         printf("[transferObject, %d] mq_send to %s returned: $%x\n", numofcore, path, ret);
1355 #endif
1356 }
1357
1358 // send terminate message to targetcore
1359 // format: -1
1360 bool transStallMsg(int targetcore) {
1361 #ifdef RAW
1362         unsigned msgHdr;
1363         int self_y, self_x, target_y, target_x;
1364         // for 32 bit machine, the size is always 4 words
1365         //int msgsize = sizeof(int) * 4;
1366         int msgsize = 4;
1367
1368         calCoords(corenum, &self_y, &self_x);
1369         calCoords(targetcore, &target_y, &target_x);
1370         // Build the message header
1371     msgHdr = construct_dyn_hdr(0, msgsize, 0,           // msgsize word sent.
1372                                            self_y, self_x,
1373                                                            target_y, target_x);
1374         gdn_send(msgHdr);               // Send the message header to EAST to handle fab(n - 1).
1375         raw_test_pass(0xbbbb);
1376         raw_test_pass(0xb000 + targetcore); // targetcore
1377     gdn_send(1);
1378     raw_test_pass(1);
1379     gdn_send(corenum);
1380         raw_test_pass_reg(corenum);
1381         gdn_send(self_numsendobjs);
1382     raw_test_pass_reg(self_numsendobjs);
1383     gdn_send(self_numreceiveobjs);
1384         raw_test_pass_reg(self_numreceiveobjs);
1385         raw_test_pass(0xffff);
1386         return true;
1387 #elif defined THREADSIMULATE
1388         struct ___Object___ *newobj = RUNMALLOC(sizeof(struct ___Object___));
1389         // use the first four int field to hold msgtype/corenum/sendobj/receiveobj
1390         newobj->type = -1;
1391         int numofcore = pthread_getspecific(key);
1392         newobj->flag = numofcore;
1393         newobj->___cachedHash___ = thread_data_array[numofcore].numsendobjs;
1394         newobj->___cachedCode___ = thread_data_array[numofcore].numreceiveobjs;
1395
1396         // use POSIX message queue to send stall msg to startup core
1397         assert(targetcore == STARTUPCORE);
1398         mqd_t mqdnum;
1399         char corenumstr[3];
1400         int sourcelen = 0;
1401         if(targetcore < 10) {
1402                 corenumstr[0] = targetcore + '0';
1403                 corenumstr[1] = '\0';
1404                 sourcelen = 1;
1405         } else if(targetcore < 100) {
1406                 corenumstr[1] = targetcore % 10 + '0';
1407                 corenumstr[0] = (targetcore / 10) + '0';
1408                 corenumstr[2] = '\0';
1409                 sourcelen = 2;
1410         } else {
1411                 printf("Error: targetcore >= 100\n");
1412                 fflush(stdout);
1413                 exit(-1);
1414         }
1415         char * pathhead = "/msgqueue_";
1416         int targetlen = strlen(pathhead);
1417         char path[targetlen + sourcelen + 1];
1418         strcpy(path, pathhead);
1419         strncat(path, corenumstr, sourcelen);
1420         int oflags = O_WRONLY|O_NONBLOCK;
1421         int omodes = S_IRWXU|S_IRWXG|S_IRWXO;
1422         mqdnum = mq_open(path, oflags, omodes, NULL);
1423         if(mqdnum==-1) {
1424                 printf("[transStallMsg, %d] mq_open %s fail: %d, error: %s\n", numofcore, path, mqdnum, strerror(errno));
1425                 fflush(stdout);
1426                 exit(-1);
1427         }
1428         int ret;
1429         ret=mq_send(mqdnum, (void *)newobj, sizeof(struct ___Object___), 0); // send the object into the queue
1430         if(ret != 0) {
1431                 printf("[transStallMsg, %d] mq_send to %s returned: %d, error: %s\n", numofcore, path, ret, strerror(errno));
1432                 RUNFREE(newobj);
1433                 return false;
1434         } else {
1435                 printf("[transStallMsg, %d] mq_send to %s returned: $%x\n", numofcore, path, ret);
1436                 printf("<transStallMsg> to %s index: %d, sendobjs: %d, receiveobjs: %d\n", path, newobj->flag, newobj->___cachedHash___, newobj->___cachedCode___);
1437                 RUNFREE(newobj);
1438                 return true;
1439         }
1440 #endif
1441 }
1442
1443 // receive object transferred from other cores
1444 // or the terminate message from other cores
1445 // NOTICE: following format is for threadsimulate version only
1446 //         RAW version please see previous description
1447 // format: type + object
1448 // type: -1--stall msg
1449 //      !-1--object
1450 // return value: 0--received an object
1451 //               1--received nothing
1452 //               2--received a Stall Msg
1453 //               3--received a lock Msg
1454 //               RAW version: -1 -- received nothing
1455 //                            otherwise -- received msg type
1456 int receiveObject() {
1457 #ifdef RAW
1458         bool deny = false;
1459         unsigned msgHdr;
1460         int self_y, self_x, target_y, target_x;
1461         int targetcore = 0;
1462         if(gdn_input_avail() == 0) {
1463                 if(corenum < NUMCORES) {
1464                         raw_test_pass(0xd001);
1465                 }
1466                 return -1;
1467         }
1468 msg:
1469         raw_test_pass(0xcccc);
1470         while((gdn_input_avail() != 0) && (msgdataindex < msglength)) {
1471                 msgdata[msgdataindex] = gdn_receive();
1472                 if(msgdataindex == 0) {
1473                         if(msgdata[0] > 2) {
1474                                 msglength = 3;
1475                         } else if(msgdata[0] > 0) {
1476                                 msglength = 4;
1477                         }
1478                 } else if((msgdataindex == 1) && (msgdata[0] == 0)) {
1479                         msglength = msgdata[msgdataindex];
1480                 }
1481                 raw_test_pass_reg(msgdata[msgdataindex]);
1482                 msgdataindex++;
1483
1484                 /*if(msgdataindex == 0) {
1485                         // type
1486                         msgtype = gdn_receive();
1487                         if(msgtype > 2) {
1488                                 msglength = 3;
1489                         } else {
1490                                 msglength = 4;
1491                         }
1492                         if(msgtype != 0) {
1493                                 msgdata = (int *)RUNMALLOC_I(msglength * sizeof(int));
1494                                 msgdata[msgdataindex] = msgtype;
1495                         }
1496                         raw_test_pass_reg(msgtype);
1497                 } else if((msgdataindex == 1) && (msgtype == 0)) {
1498                         // object transfer msg
1499                         msglength = gdn_receive();
1500                         msgdata = (int *)RUNMALLOC_I(msglength * sizeof(int));
1501                         msgdata[0] = msgtype;
1502                         msgdata[msgdataindex] = msglength;
1503                         raw_test_pass_reg(msgdata[msgdataindex]);
1504                 } else {
1505                         msgdata[msgdataindex] = gdn_receive();
1506                         raw_test_pass_reg(msgdata[msgdataindex]);
1507                 }
1508                 msgdataindex++;*/
1509         }
1510         raw_test_pass(0xffff);
1511         if(msgdataindex == msglength) {
1512                 // received a whole msg
1513                 int type, data1, data2; // will receive at least 3 words including type
1514                 type = msgdata[0];
1515                 data1 = msgdata[1];
1516                 data2 = msgdata[2];
1517                 switch(type) {
1518                         case 0: {
1519                                                 // receive a object transfer msg
1520                                                 struct transObjInfo * transObj = RUNMALLOC_I(sizeof(struct transObjInfo));
1521                                                 int k = 0;
1522                                                 if(corenum > NUMCORES - 1) {
1523                                                         raw_test_done(0xa00a);
1524                                                 }
1525                                                 // store the object and its corresponding queue info, enqueue it later
1526                                                 transObj->objptr = (void *)data2; // data1 is now size of the msg
1527                                                 transObj->length = (msglength - 3) / 2;
1528                                                 transObj->queues = RUNMALLOC_I(sizeof(int)*(msglength - 3));
1529                                                 for(k = 0; k < transObj->length; ++k) {
1530                                                         transObj->queues[2*k] = msgdata[3+2*k];
1531                                                         raw_test_pass_reg(transObj->queues[2*k]);
1532                                                         transObj->queues[2*k+1] = msgdata[3+2*k+1];
1533                                                         raw_test_pass_reg(transObj->queues[2*k+1]);
1534                                                 }
1535                                                 //memcpy(transObj->queues, msgdata[3], sizeof(int)*(msglength - 3));
1536                                                 addNewItem_I(&objqueue, (void *)transObj);
1537                                                 ++(self_numreceiveobjs);
1538                                                 raw_test_pass(0xe881);
1539                                                 /*
1540                                                 addNewItem_I(&objqueue, (void *)data2);
1541                                                 ++(self_numreceiveobjs);
1542                                                 raw_test_pass(0xe881);*/
1543                                                 break;
1544                                         }
1545                         case 1: {
1546                                                 // receive a stall msg
1547                                                 if(corenum != STARTUPCORE) {
1548                                                         // non startup core can not receive stall msg
1549                                                         // return -1
1550                                                         raw_test_done(0xa001);
1551                                                 }
1552                                                 if(data1 < NUMCORES) {
1553                                                         raw_test_pass(0xe882);
1554                                                         corestatus[data1] = 0;
1555                                                         numsendobjs[data1] = data2;
1556                                                         numreceiveobjs[data1] = msgdata[3];
1557                                                 }
1558                                                 break;
1559                                         }
1560                         case 2: {
1561                                                 // receive lock request msg
1562                                                 // for 32 bit machine, the size is always 3 words
1563                                                 //int msgsize = sizeof(int) * 3;
1564                                                 int msgsize = 3;
1565                                                 // lock request msg, handle it right now
1566                                                 // check to see if there is a lock exist in locktbl for the required obj
1567                                                 int data3 = msgdata[3];
1568                                                 deny = false;
1569                                                 if(!RuntimeHashcontainskey(locktbl, data2)) {
1570                                                         // no locks for this object
1571                                                         // first time to operate on this shared object
1572                                                         // create a lock for it
1573                                                         // the lock is an integer: 0 -- stall, >0 -- read lock, -1 -- write lock
1574                                                         raw_test_pass(0xe883);
1575                                                         if(data1 == 0) {
1576                                                                 RuntimeHashadd_I(locktbl, data2, 1);
1577                                                         } else {
1578                                                                 RuntimeHashadd_I(locktbl, data2, -1);
1579                                                         }
1580                                                 } else {
1581                                                         int rwlock_obj = 0;
1582                                                         raw_test_pass(0xe884);
1583                                                         RuntimeHashget(locktbl, data2, &rwlock_obj);
1584                                                         raw_test_pass_reg(rwlock_obj);
1585                                                         if(0 == rwlock_obj) {
1586                                                                 if(data1 == 0) {
1587                                                                         rwlock_obj = 1;
1588                                                                 } else {
1589                                                                         rwlock_obj = -1;
1590                                                                 }
1591                                                                 RuntimeHashremovekey(locktbl, data2);
1592                                                                 RuntimeHashadd_I(locktbl, data2, rwlock_obj);
1593                                                         } else if((rwlock_obj > 0) && (data1 == 0)) {
1594                                                                 // read lock request and there are only read locks
1595                                                                 rwlock_obj++;
1596                                                                 RuntimeHashremovekey(locktbl, data2);
1597                                                                 RuntimeHashadd_I(locktbl, data2, rwlock_obj);
1598                                                         } else {
1599                                                                 deny = true;
1600                                                         }
1601                                                         raw_test_pass_reg(rwlock_obj);
1602                                                 }
1603                                                 targetcore = data3;
1604                                                 calCoords(corenum, &self_y, &self_x);
1605                                                 calCoords(targetcore, &target_y, &target_x);
1606                                                 // Build the message header
1607                                                 msgHdr = construct_dyn_hdr(0, msgsize, 0,               // msgsize word sent.
1608                                                                                    self_y, self_x,
1609                                                                                                    target_y, target_x);
1610                                                 gdn_send(msgHdr);               // Send the message header to EAST to handle fab(n - 1).
1611                                                 raw_test_pass(0xbbbb);
1612                                                 raw_test_pass(0xb000 + targetcore); // targetcore
1613                                                 if(deny == true) {
1614                                                         // deny the lock request
1615                                                         gdn_send(4); // lock request
1616                                                         raw_test_pass(4);
1617                                                 } else {
1618                                                         // grount the lock request
1619                                                         gdn_send(3); // lock request
1620                                                         raw_test_pass(3);
1621                                                 }
1622                                                 gdn_send(data1); // lock type
1623                                                 raw_test_pass_reg(data1);
1624                                                 gdn_send(data2); // lock target
1625                                                 raw_test_pass_reg(data2);
1626                                                 raw_test_pass(0xffff);
1627                                                 break;
1628                                         }
1629                         case 3: {
1630                                                 // receive lock grount msg
1631                                                 if(corenum > NUMCORES - 1) {
1632                                                         raw_test_done(0xa00b);
1633                                                 }
1634                                                 if(lockobj == data2) {
1635                                                         lockresult = 1;
1636                                                         lockflag = true;
1637 #ifndef INTERRUPT
1638                                                         reside = false;
1639 #endif
1640                                                 } else {
1641                                                         // conflicts on lockresults
1642                                                 raw_test_done(0xa002);
1643                                                 }
1644                                                 break;
1645                                         }
1646                         case 4: {
1647                                                 // receive lock grount/deny msg
1648                                                 if(corenum > NUMCORES - 1) {
1649                                                         raw_test_done(0xa00c);
1650                                                 }
1651                                                 if(lockobj == data2) {
1652                                                         lockresult = 0;
1653                                                         lockflag = true;
1654 #ifndef INTERRUPT
1655                                                         reside = false;
1656 #endif
1657                                                 } else {
1658                                                         // conflicts on lockresults
1659                                                         raw_test_done(0xa003);
1660                                                 }
1661                                                 break;
1662                                         }
1663                         case 5: {
1664                                                 // receive lock release msg
1665                                                 if(!RuntimeHashcontainskey(locktbl, data2)) {
1666                                                         // no locks for this object, something is wrong
1667                                                         raw_test_done(0xa004);
1668                                                 } else {
1669                                                         int rwlock_obj = 0;
1670                                                         RuntimeHashget(locktbl, data2, &rwlock_obj);
1671                                                         raw_test_pass(0xe885);
1672                                                         raw_test_pass_reg(rwlock_obj);
1673                                                         if(data1 == 0) {
1674                                                                 rwlock_obj--;
1675                                                         } else {
1676                                                                 rwlock_obj++;
1677                                                         }
1678                                                         RuntimeHashremovekey(locktbl, data2);
1679                                                         RuntimeHashadd_I(locktbl, data2, rwlock_obj);
1680                                                         raw_test_pass_reg(rwlock_obj);
1681                                                 }
1682                                                 break;
1683                                         }
1684                         default:
1685                                         break;
1686                 }
1687                 //RUNFREE(msgdata);
1688                 //msgdata = NULL;
1689                 for(msgdataindex--;msgdataindex > 0; --msgdataindex) {
1690                         msgdata[msgdataindex] = -1;
1691                 }
1692                 msgtype = -1;
1693                 //msgdataindex = 0;
1694                 msglength = 30;
1695                 raw_test_pass(0xe886);
1696                 if(gdn_input_avail() != 0) {
1697                         goto msg;
1698                 }
1699                 return type;
1700         } else {
1701                 // not a whole msg
1702                 raw_test_pass(0xe887);
1703                 return -2;
1704         }
1705 #elif defined THREADSIMULATE
1706         int numofcore = pthread_getspecific(key);
1707         // use POSIX message queue to transfer object
1708         int msglen = 0;
1709         struct mq_attr mqattr;
1710         mq_getattr(mqd[numofcore], &mqattr);
1711         void * msgptr =RUNMALLOC(mqattr.mq_msgsize);
1712         msglen=mq_receive(mqd[numofcore], msgptr, mqattr.mq_msgsize, NULL); // receive the object into the queue
1713         if(-1 == msglen) {
1714                 // no msg
1715                 free(msgptr);
1716                 return 1;
1717         }
1718         //printf("msg: %s\n",msgptr);
1719         if(((int*)msgptr)[0] == -1) {
1720                 // StallMsg
1721                 struct ___Object___ * tmpptr = (struct ___Object___ *)msgptr;
1722                 int index = tmpptr->flag;
1723                 corestatus[index] = 0;
1724                 numsendobjs[index] = tmpptr->___cachedHash___;
1725                 numreceiveobjs[index] = tmpptr->___cachedCode___;
1726                 printf("<receiveObject> index: %d, sendobjs: %d, reveiveobjs: %d\n", index, numsendobjs[index], numreceiveobjs[index]);
1727                 free(msgptr);
1728                 return 2;
1729         } /*else if(((int*)msgptr)[0] == -2) {
1730                 // terminate msg
1731                 return 3;
1732         } */else {
1733                 // an object
1734                 if(numofcore == STARTUPCORE) {
1735                         ++(numreceiveobjs[numofcore]);
1736                 } else {
1737                         ++(thread_data_array[numofcore].numreceiveobjs);
1738                 }
1739                 struct ___Object___ * tmpptr = (struct ___Object___ *)msgptr;
1740                 struct transObjInfo * transObj = (struct transObjInfo *)tmpptr->original;
1741                 tmpptr = (struct ___Object___ *)(transObj->objptr);
1742                 int type = tmpptr->type;
1743                 int size=classsize[type];
1744                 struct ___Object___ * newobj=RUNMALLOC(size);
1745                 memcpy(newobj, tmpptr, size);
1746                 if(0 == newobj->isolate) {
1747                         newobj->original=tmpptr;
1748                 }
1749                 RUNFREE(msgptr);
1750                 tmpptr = NULL;
1751                 int k = 0;
1752                 for(k = 0; k < transObj->length; ++k) {
1753                         int taskindex = transObj->queues[2 * k];
1754                         int paramindex = transObj->queues[2 * k + 1];
1755                         struct parameterwrapper ** queues = &(paramqueues[numofcore][taskindex][paramindex]);
1756                         enqueueObject(newobj, queues, 1);
1757                 }
1758                 RUNFREE(transObj->queues);
1759                 RUNFREE(transObj);
1760                 return 0;
1761         }
1762 #endif
1763 }
1764
1765 bool getreadlock(void * ptr) {
1766 #ifdef RAW
1767         unsigned msgHdr;
1768         int self_y, self_x, target_y, target_x;
1769         int targetcore = ((int)ptr) % TOTALCORE;
1770         // for 32 bit machine, the size is always 4 words
1771         //int msgsize = sizeof(int) * 4;
1772         int msgsize = 4;
1773
1774         lockobj = (int)ptr;
1775         lockflag = false;
1776 #ifndef INTERRUPT
1777         reside = false;
1778 #endif
1779         lockresult = 0;
1780
1781         if(targetcore == corenum) {
1782                 // reside on this core
1783                 bool deny = false;
1784 #ifdef INTERRUPT
1785                 raw_user_interrupts_off();
1786 #endif
1787                 if(!RuntimeHashcontainskey(locktbl, (int)ptr)) {
1788                         // no locks for this object
1789                         // first time to operate on this shared object
1790                         // create a lock for it
1791                         // the lock is an integer: 0 -- stall, >0 -- read lock, -1 -- write lock
1792                         RuntimeHashadd_I(locktbl, (int)ptr, 1);
1793                 } else {
1794                         int rwlock_obj = 0;
1795                         RuntimeHashget(locktbl, (int)ptr, &rwlock_obj);
1796                         if(-1 != rwlock_obj) {
1797                                 rwlock_obj++;
1798                                 RuntimeHashremovekey(locktbl, (int)ptr);
1799                                 RuntimeHashadd_I(locktbl, (int)ptr, rwlock_obj);
1800                         } else {
1801                                 deny = true;
1802                         }
1803                 }
1804 #ifdef INTERRUPT
1805                 raw_user_interrupts_on();
1806 #endif
1807                 if(lockobj == (int)ptr) {
1808                         if(deny) {
1809                                 lockresult = 0;
1810                         } else {
1811                                 lockresult = 1;
1812                         }
1813                         lockflag = true;
1814 #ifndef INTERRUPT
1815                         reside = true;
1816 #endif
1817                 } else {
1818                         // conflicts on lockresults
1819                         raw_test_done(0xa005);
1820                 }
1821                 return true;
1822         }
1823
1824         calCoords(corenum, &self_y, &self_x);
1825         calCoords(targetcore, &target_y, &target_x);
1826         // Build the message header
1827     msgHdr = construct_dyn_hdr(0, msgsize, 0,           // msgsize word sent.
1828                                            self_y, self_x,
1829                                                            target_y, target_x);
1830         gdn_send(msgHdr);               // Send the message header to EAST to handle fab(n - 1).
1831         raw_test_pass(0xbbbb);
1832         raw_test_pass(0xb000 + targetcore); // targetcore
1833     gdn_send(2); // lock request
1834     raw_test_pass(2);
1835         gdn_send(0); // read lock
1836     raw_test_pass(0);
1837     gdn_send(ptr);
1838         raw_test_pass_reg(ptr);
1839         gdn_send(corenum);
1840         raw_test_pass_reg(corenum);
1841         raw_test_pass(0xffff);
1842         return true;
1843 #elif defined THREADSIMULATE
1844         int numofcore = pthread_getspecific(key);
1845
1846         int rc = pthread_rwlock_tryrdlock(&rwlock_tbl);
1847         printf("[getreadlock, %d] getting the read lock for locktbl: %d error: \n", numofcore, rc, strerror(rc));
1848         if(0 != rc) {
1849                 return false;
1850         }
1851         if(!RuntimeHashcontainskey(locktbl, (int)ptr)) {
1852                 // no locks for this object
1853                 // first time to operate on this shared object
1854                 // create a lock for it
1855                 rc = pthread_rwlock_unlock(&rwlock_tbl);
1856                 printf("[getreadlock, %d] release the read lock for locktbl: %d error: \n", numofcore, rc, strerror(rc));
1857                 pthread_rwlock_t* rwlock = (pthread_rwlock_t *)RUNMALLOC(sizeof(pthread_rwlock_t));
1858                 memcpy(rwlock, &rwlock_init, sizeof(pthread_rwlock_t));
1859                 rc = pthread_rwlock_init(rwlock, NULL);
1860                 printf("[getreadlock, %d] initialize the rwlock for object %d: %d error: \n", numofcore, (int)ptr, rc, strerror(rc));
1861                 rc = pthread_rwlock_trywrlock(&rwlock_tbl);
1862                 printf("[getreadlock, %d] getting the write lock for locktbl: %d error: \n", numofcore, rc, strerror(rc));
1863                 if(0 != rc) {
1864                         RUNFREE(rwlock);
1865                         return false;
1866                 } else {
1867                         if(!RuntimeHashcontainskey(locktbl, (int)ptr)) {
1868                                 // check again
1869                                 RuntimeHashadd(locktbl, (int)ptr, (int)rwlock);
1870                         } else {
1871                                 RUNFREE(rwlock);
1872                                 RuntimeHashget(locktbl, (int)ptr, (int*)&rwlock);
1873                         }
1874                         rc = pthread_rwlock_unlock(&rwlock_tbl);
1875                                 printf("[getreadlock, %d] release the write lock for locktbl: %d error: \n", numofcore, rc, strerror(rc));
1876                 }
1877                 rc = pthread_rwlock_tryrdlock(rwlock);
1878                 printf("[getreadlock, %d] getting read lock for object %d: %d error: \n", numofcore, (int)ptr, rc, strerror(rc));
1879                 if(0 != rc) {
1880                         return false;
1881                 } else {
1882                         return true;
1883                 }
1884         } else {
1885                 pthread_rwlock_t* rwlock_obj = NULL;
1886                 RuntimeHashget(locktbl, (int)ptr, (int*)&rwlock_obj);
1887                 rc = pthread_rwlock_unlock(&rwlock_tbl);
1888                 printf("[getreadlock, %d] release the read lock for locktbl: %d error: \n", numofcore, rc, strerror(rc));
1889                 int rc_obj = pthread_rwlock_tryrdlock(rwlock_obj);
1890                 printf("[getreadlock, %d] getting read lock for object %d: %d error: \n", numofcore, (int)ptr, rc_obj, strerror(rc_obj));
1891                 if(0 != rc_obj) {
1892                         return false;
1893                 } else {
1894                         return true;
1895                 }
1896         }
1897 #endif
1898 }
1899
1900 void releasereadlock(void * ptr) {
1901 #ifdef RAW
1902         unsigned msgHdr;
1903         int self_y, self_x, target_y, target_x;
1904         int targetcore = ((int)ptr) % TOTALCORE;
1905         // for 32 bit machine, the size is always 3 words
1906         //int msgsize = sizeof(int) * 3;
1907         int msgsize = 3;
1908
1909         if(targetcore == corenum) {
1910 #ifdef INTERRUPT
1911                 raw_user_interrupts_off();
1912 #endif
1913                 // reside on this core
1914                 if(!RuntimeHashcontainskey(locktbl, (int)ptr)) {
1915                         // no locks for this object, something is wrong
1916                         raw_test_done(0xa006);
1917                 } else {
1918                         int rwlock_obj = 0;
1919                         RuntimeHashget(locktbl, (int)ptr, &rwlock_obj);
1920                         rwlock_obj--;
1921                         RuntimeHashremovekey(locktbl, (int)ptr);
1922                         RuntimeHashadd_I(locktbl, (int)ptr, rwlock_obj);
1923                 }
1924 #ifdef INTERRUPT
1925                 raw_user_interrupts_on();
1926 #endif
1927                 return;
1928         }
1929
1930         calCoords(corenum, &self_y, &self_x);
1931         calCoords(targetcore, &target_y, &target_x);
1932         // Build the message header
1933     msgHdr = construct_dyn_hdr(0, msgsize, 0,           // msgsize word sent.
1934                                            self_y, self_x,
1935                                                            target_y, target_x);
1936         gdn_send(msgHdr);               // Send the message header to EAST to handle fab(n - 1).
1937         raw_test_pass(0xbbbb);
1938         raw_test_pass(0xb000 + targetcore); // targetcore
1939     gdn_send(5); // lock release
1940     raw_test_pass(5);
1941         gdn_send(0); // read lock
1942     raw_test_pass(0);
1943     gdn_send(ptr);
1944         raw_test_pass_reg(ptr);
1945         raw_test_pass(0xffff);
1946 #elif defined THREADSIMULATE
1947         int numofcore = pthread_getspecific(key);
1948         int rc = pthread_rwlock_rdlock(&rwlock_tbl);
1949         printf("[releasereadlock, %d] getting the read lock for locktbl: %d error: \n", numofcore, rc, strerror(rc));
1950         if(!RuntimeHashcontainskey(locktbl, (int)ptr)) {
1951                 printf("[releasereadlock, %d] Error: try to release a lock without previously grab it\n", numofcore);
1952                 exit(-1);
1953         }
1954         pthread_rwlock_t* rwlock_obj = NULL;
1955         RuntimeHashget(locktbl, (int)ptr, (int*)&rwlock_obj);
1956         int rc_obj = pthread_rwlock_unlock(rwlock_obj);
1957         printf("[releasereadlock, %d] unlocked object %d: %d error: \n", numofcore, (int)ptr, rc_obj, strerror(rc_obj));
1958         rc = pthread_rwlock_unlock(&rwlock_tbl);
1959         printf("[releasereadlock, %d] release the read lock for locktbl: %d error: \n", numofcore, rc, strerror(rc));
1960 #endif
1961 }
1962
1963 #ifdef RAW
1964 bool getreadlock_I(void * ptr) {
1965         unsigned msgHdr;
1966         int self_y, self_x, target_y, target_x;
1967         int targetcore = ((int)ptr) % TOTALCORE;
1968         // for 32 bit machine, the size is always 4 words
1969         //int msgsize = sizeof(int) * 4;
1970         int msgsize = 4;
1971
1972         lockobj = (int)ptr;
1973         lockflag = false;
1974 #ifndef INTERRUPT
1975         reside = false;
1976 #endif
1977         lockresult = 0;
1978
1979         if(targetcore == corenum) {
1980                 // reside on this core
1981                 bool deny = false;
1982                 if(!RuntimeHashcontainskey(locktbl, (int)ptr)) {
1983                         // no locks for this object
1984                         // first time to operate on this shared object
1985                         // create a lock for it
1986                         // the lock is an integer: 0 -- stall, >0 -- read lock, -1 -- write lock
1987                         RuntimeHashadd_I(locktbl, (int)ptr, 1);
1988                 } else {
1989                         int rwlock_obj = 0;
1990                         RuntimeHashget(locktbl, (int)ptr, &rwlock_obj);
1991                         if(-1 != rwlock_obj) {
1992                                 rwlock_obj++;
1993                                 RuntimeHashremovekey(locktbl, (int)ptr);
1994                                 RuntimeHashadd_I(locktbl, (int)ptr, rwlock_obj);
1995                         } else {
1996                                 deny = true;
1997                         }
1998                 }
1999                 if(lockobj == (int)ptr) {
2000                         if(deny) {
2001                                 lockresult = 0;
2002                         } else {
2003                                 lockresult = 1;
2004                         }
2005                         lockflag = true;
2006 #ifndef INTERRUPT
2007                         reside = true;
2008 #endif
2009                 } else {
2010                         // conflicts on lockresults
2011                         raw_test_done(0xa005);
2012                 }
2013                 return true;
2014         }
2015
2016         calCoords(corenum, &self_y, &self_x);
2017         calCoords(targetcore, &target_y, &target_x);
2018         // Build the message header
2019     msgHdr = construct_dyn_hdr(0, msgsize, 0,           // msgsize word sent.
2020                                            self_y, self_x,
2021                                                            target_y, target_x);
2022         gdn_send(msgHdr);               // Send the message header to EAST to handle fab(n - 1).
2023         raw_test_pass(0xbbbb);
2024         raw_test_pass(0xb000 + targetcore); // targetcore
2025     gdn_send(2); // lock request
2026     raw_test_pass(2);
2027         gdn_send(0); // read lock
2028     raw_test_pass(0);
2029     gdn_send(ptr);
2030         raw_test_pass_reg(ptr);
2031         gdn_send(corenum);
2032         raw_test_pass_reg(corenum);
2033         raw_test_pass(0xffff);
2034         return true;
2035 }
2036
2037 void releasereadlock_I(void * ptr) {
2038         unsigned msgHdr;
2039         int self_y, self_x, target_y, target_x;
2040         int targetcore = ((int)ptr) % TOTALCORE;
2041         // for 32 bit machine, the size is always 3 words
2042         //int msgsize = sizeof(int) * 3;
2043         int msgsize = 3;
2044
2045         if(targetcore == corenum) {
2046                 // reside on this core
2047                 if(!RuntimeHashcontainskey(locktbl, (int)ptr)) {
2048                         // no locks for this object, something is wrong
2049                         raw_test_done(0xa006);
2050                 } else {
2051                         int rwlock_obj = 0;
2052                         RuntimeHashget(locktbl, (int)ptr, &rwlock_obj);
2053                         rwlock_obj--;
2054                         RuntimeHashremovekey(locktbl, (int)ptr);
2055                         RuntimeHashadd_I(locktbl, (int)ptr, rwlock_obj);
2056                 }
2057                 return;
2058         }
2059
2060         calCoords(corenum, &self_y, &self_x);
2061         calCoords(targetcore, &target_y, &target_x);
2062         // Build the message header
2063     msgHdr = construct_dyn_hdr(0, msgsize, 0,           // msgsize word sent.
2064                                            self_y, self_x,
2065                                                            target_y, target_x);
2066         gdn_send(msgHdr);               // Send the message header to EAST to handle fab(n - 1).
2067         raw_test_pass(0xbbbb);
2068         raw_test_pass(0xb000 + targetcore); // targetcore
2069     gdn_send(5); // lock release
2070     raw_test_pass(5);
2071         gdn_send(0); // read lock
2072     raw_test_pass(0);
2073     gdn_send(ptr);
2074         raw_test_pass_reg(ptr);
2075         raw_test_pass(0xffff);
2076 }
2077 #endif
2078
2079 bool getwritelock(void * ptr) {
2080 #ifdef RAW
2081         unsigned msgHdr;
2082         int self_y, self_x, target_y, target_x;
2083         int targetcore = ((int)ptr) % TOTALCORE;
2084         // for 32 bit machine, the size is always 4 words
2085         //int msgsize = sizeof(int) * 4;
2086         int msgsize= 4;
2087
2088         lockobj = (int)ptr;
2089         lockflag = false;
2090 #ifndef INTERRUPT
2091         reside = false;
2092 #endif
2093         lockresult = 0;
2094
2095         if(targetcore == corenum) {
2096                 // reside on this core
2097                 bool deny = false;
2098 #ifdef INTERRUPT
2099                 raw_user_interrupts_off();
2100 #endif
2101                 if(!RuntimeHashcontainskey(locktbl, (int)ptr)) {
2102                         // no locks for this object
2103                         // first time to operate on this shared object
2104                         // create a lock for it
2105                         // the lock is an integer: 0 -- stall, >0 -- read lock, -1 -- write lock
2106                         raw_test_pass(0xe552);
2107                         RuntimeHashadd_I(locktbl, (int)ptr, -1);
2108                 } else {
2109                         int rwlock_obj = 0;
2110                         RuntimeHashget(locktbl, (int)ptr, &rwlock_obj);
2111                         raw_test_pass(0xe553);
2112                         raw_test_pass_reg(rwlock_obj);
2113                         if(0 == rwlock_obj) {
2114                                 rwlock_obj = -1;
2115                                 RuntimeHashremovekey(locktbl, (int)ptr);
2116                                 RuntimeHashadd_I(locktbl, (int)ptr, rwlock_obj);
2117                         } else {
2118                                 deny = true;
2119                         }
2120                 }
2121 #ifdef INTERRUPT
2122                 raw_user_interrupts_on();
2123 #endif
2124                 raw_test_pass(0xe554);
2125                 raw_test_pass_reg(lockresult);
2126                 if(lockobj == (int)ptr) {
2127                         if(deny) {
2128                                 lockresult = 0;
2129                                 raw_test_pass(0);
2130                         } else {
2131                                 lockresult = 1;
2132                                 raw_test_pass(1);
2133                         }
2134                         lockflag = true;
2135 #ifndef INTERRUPT
2136                         reside = true;
2137 #endif
2138                 } else {
2139                         // conflicts on lockresults
2140                         raw_test_done(0xa007);
2141                 }
2142                 return true;
2143         }
2144
2145         raw_test_pass(0xe555);
2146         calCoords(corenum, &self_y, &self_x);
2147         calCoords(targetcore, &target_y, &target_x);
2148         // Build the message header
2149     msgHdr = construct_dyn_hdr(0, msgsize, 0,           // msgsize word sent.
2150                                            self_y, self_x,
2151                                                            target_y, target_x);
2152         gdn_send(msgHdr);               // Send the message header to EAST to handle fab(n - 1).
2153         raw_test_pass(0xbbbb);
2154         raw_test_pass(0xb000 + targetcore); // targetcore
2155     gdn_send(2); // lock request
2156     raw_test_pass(2);
2157         gdn_send(1); // write lock
2158     raw_test_pass(1);
2159     gdn_send(ptr);
2160         raw_test_pass_reg(ptr);
2161         gdn_send(corenum);
2162         raw_test_pass_reg(corenum);
2163         raw_test_pass(0xffff);
2164         return true;
2165 #elif defined THREADSIMULATE
2166         int numofcore = pthread_getspecific(key);
2167
2168         int rc = pthread_rwlock_tryrdlock(&rwlock_tbl);
2169         printf("[getwritelock, %d] getting the read lock for locktbl: %d error: \n", numofcore, rc, strerror(rc));
2170         if(0 != rc) {
2171                 return false;
2172         }
2173         if(!RuntimeHashcontainskey(locktbl, (int)ptr)) {
2174                 // no locks for this object
2175                 // first time to operate on this shared object
2176                 // create a lock for it
2177                 rc = pthread_rwlock_unlock(&rwlock_tbl);
2178                 printf("[getwritelock, %d] release the read lock for locktbl: %d error: \n", numofcore, rc, strerror(rc));
2179                 pthread_rwlock_t* rwlock = (pthread_rwlock_t *)RUNMALLOC(sizeof(pthread_rwlock_t));
2180                 memcpy(rwlock, &rwlock_init, sizeof(pthread_rwlock_t));
2181                 rc = pthread_rwlock_init(rwlock, NULL);
2182                 printf("[getwritelock, %d] initialize the rwlock for object %d: %d error: \n", numofcore, (int)ptr, rc, strerror(rc));
2183                 rc = pthread_rwlock_trywrlock(&rwlock_tbl);
2184                 printf("[getwritelock, %d] getting the write lock for locktbl: %d error: \n", numofcore, rc, strerror(rc));
2185                 if(0 != rc) {
2186                         pthread_rwlock_destroy(rwlock);
2187                         RUNFREE(rwlock);
2188                         return false;
2189                 } else {
2190                         if(!RuntimeHashcontainskey(locktbl, (int)ptr)) {
2191                                 // check again
2192                                 RuntimeHashadd(locktbl, (int)ptr, (int)rwlock);
2193                         } else {
2194                                 pthread_rwlock_destroy(rwlock);
2195                                 RUNFREE(rwlock);
2196                                 RuntimeHashget(locktbl, (int)ptr, (int*)&rwlock);
2197                         }
2198                         rc = pthread_rwlock_unlock(&rwlock_tbl);
2199                         printf("[getwritelock, %d] release the write lock for locktbl: %d error: \n", numofcore, rc, strerror(rc));
2200                 }
2201                 rc = pthread_rwlock_trywrlock(rwlock);
2202                 printf("[getwritelock, %d] getting write lock for object %d: %d error: \n", numofcore, (int)ptr, rc, strerror(rc));
2203                 if(0 != rc) {
2204                         return false;
2205                 } else {
2206                         return true;
2207                 }
2208         } else {
2209                 pthread_rwlock_t* rwlock_obj = NULL;
2210                 RuntimeHashget(locktbl, (int)ptr, (int*)&rwlock_obj);
2211                 rc = pthread_rwlock_unlock(&rwlock_tbl);
2212                 printf("[getwritelock, %d] release the read lock for locktbl: %d error: \n", numofcore, rc, strerror(rc));
2213                 int rc_obj = pthread_rwlock_trywrlock(rwlock_obj);
2214                 printf("[getwritelock, %d] getting write lock for object %d: %d error: \n", numofcore, (int)ptr, rc_obj, strerror(rc_obj));
2215                 if(0 != rc_obj) {
2216                         return false;
2217                 } else {
2218                         return true;
2219                 }
2220         }
2221
2222 #endif
2223 }
2224
2225 void releasewritelock(void * ptr) {
2226 #ifdef RAW
2227         unsigned msgHdr;
2228         int self_y, self_x, target_y, target_x;
2229         int targetcore = ((int)ptr) % TOTALCORE;
2230         // for 32 bit machine, the size is always 3 words
2231         //int msgsize = sizeof(int) * 3;
2232         int msgsize = 3;
2233
2234         if(targetcore == corenum) {
2235 #ifdef INTERRUPT
2236                 raw_user_interrupts_off();
2237 #endif
2238                 // reside on this core
2239                 if(!RuntimeHashcontainskey(locktbl, (int)ptr)) {
2240                         // no locks for this object, something is wrong
2241                         raw_test_done(0xa008);
2242                 } else {
2243                         int rwlock_obj = 0;
2244                         raw_test_pass(0xe662);
2245                         RuntimeHashget(locktbl, (int)ptr, &rwlock_obj);
2246                         raw_test_pass_reg(rwlock_obj);
2247                         rwlock_obj++;
2248                         RuntimeHashremovekey(locktbl, (int)ptr);
2249                         RuntimeHashadd_I(locktbl, (int)ptr, rwlock_obj);
2250                         raw_test_pass_reg(rwlock_obj);
2251                 }
2252 #ifdef INTERRUPT
2253                 raw_user_interrupts_on();
2254 #endif
2255                 return;
2256         }
2257
2258         raw_test_pass(0xe663);
2259         calCoords(corenum, &self_y, &self_x);
2260         calCoords(targetcore, &target_y, &target_x);
2261         // Build the message header
2262     msgHdr = construct_dyn_hdr(0, msgsize, 0,           // msgsize word sent.
2263                                            self_y, self_x,
2264                                                            target_y, target_x);
2265         gdn_send(msgHdr);               // Send the message header to EAST to handle fab(n - 1).
2266         raw_test_pass(0xbbbb);
2267         raw_test_pass(0xb000 + targetcore);
2268     gdn_send(5); // lock release
2269     raw_test_pass(5);
2270         gdn_send(1); // write lock
2271     raw_test_pass(1);
2272     gdn_send(ptr);
2273         raw_test_pass_reg(ptr);
2274         raw_test_pass(0xffff);
2275 #elif defined THREADSIMULATE
2276         int numofcore = pthread_getspecific(key);
2277         int rc = pthread_rwlock_rdlock(&rwlock_tbl);
2278         printf("[releasewritelock, %d] getting the read lock for locktbl: %d error: \n", numofcore, rc, strerror(rc));
2279         if(!RuntimeHashcontainskey(locktbl, (int)ptr)) {
2280                 printf("[releasewritelock, %d] Error: try to release a lock without previously grab it\n", numofcore);
2281                 exit(-1);
2282         }
2283         pthread_rwlock_t* rwlock_obj = NULL;
2284         RuntimeHashget(locktbl, (int)ptr, (int*)&rwlock_obj);
2285         int rc_obj = pthread_rwlock_unlock(rwlock_obj);
2286         printf("[releasewritelock, %d] unlocked object %d: %d error:\n", numofcore, (int)ptr, rc_obj, strerror(rc_obj));
2287         rc = pthread_rwlock_unlock(&rwlock_tbl);
2288         printf("[releasewritelock, %d] release the read lock for locktbl: %d error: \n", numofcore, rc, strerror(rc));
2289 #endif
2290 }
2291
2292 int enqueuetasks(struct parameterwrapper *parameter, struct parameterwrapper *prevptr, struct ___Object___ *ptr, int * enterflags, int numenterflags) {
2293   void * taskpointerarray[MAXTASKPARAMS];
2294   int j;
2295   int numparams=parameter->task->numParameters;
2296   int numiterators=parameter->task->numTotal-1;
2297   int retval=1;
2298   int addnormal=1;
2299   int adderror=1;
2300
2301   struct taskdescriptor * task=parameter->task;
2302
2303         ObjectHashadd(parameter->objectset, (int) ptr, 0, (int) enterflags, numenterflags, enterflags==NULL);//this add the object to parameterwrapper
2304
2305   /* Add enqueued object to parameter vector */
2306   taskpointerarray[parameter->slot]=ptr;
2307
2308   /* Reset iterators */
2309   for(j=0;j<numiterators;j++) {
2310     toiReset(&parameter->iterators[j]);
2311   }
2312
2313   /* Find initial state */
2314   for(j=0;j<numiterators;j++) {
2315   backtrackinit:
2316     if(toiHasNext(&parameter->iterators[j], taskpointerarray OPTARG(failed)))
2317       toiNext(&parameter->iterators[j], taskpointerarray OPTARG(failed));
2318     else if (j>0) {
2319       /* Need to backtrack */
2320       toiReset(&parameter->iterators[j]);
2321       j--;
2322       goto backtrackinit;
2323     } else {
2324       /* Nothing to enqueue */
2325       return retval;
2326     }
2327   }
2328
2329
2330   while(1) {
2331     /* Enqueue current state */
2332     int launch = 0;
2333     struct taskparamdescriptor *tpd=RUNMALLOC(sizeof(struct taskparamdescriptor));
2334     tpd->task=task;
2335     tpd->numParameters=numiterators+1;
2336     tpd->parameterArray=RUNMALLOC(sizeof(void *)*(numiterators+1));
2337     for(j=0;j<=numiterators;j++){
2338       tpd->parameterArray[j]=taskpointerarray[j];//store the actual parameters
2339     }
2340     /* Enqueue task */
2341     if ((/*!gencontains(failedtasks, tpd)&&*/!gencontains(activetasks,tpd))) {
2342       genputtable(activetasks, tpd, tpd);
2343     } else {
2344       RUNFREE(tpd->parameterArray);
2345       RUNFREE(tpd);
2346     }
2347
2348     /* This loop iterates to the next parameter combination */
2349     if (numiterators==0)
2350       return retval;
2351
2352     for(j=numiterators-1; j<numiterators;j++) {
2353     backtrackinc:
2354       if(toiHasNext(&parameter->iterators[j], taskpointerarray OPTARG(failed)))
2355         toiNext(&parameter->iterators[j], taskpointerarray OPTARG(failed));
2356       else if (j>0) {
2357         /* Need to backtrack */
2358         toiReset(&parameter->iterators[j]);
2359         j--;
2360         goto backtrackinc;
2361       } else {
2362         /* Nothing more to enqueue */
2363         return retval;
2364       }
2365     }
2366   }
2367   return retval;
2368 }
2369
2370 #ifdef RAW
2371 int enqueuetasks_I(struct parameterwrapper *parameter, struct parameterwrapper *prevptr, struct ___Object___ *ptr, int * enterflags, int numenterflags) {
2372   void * taskpointerarray[MAXTASKPARAMS];
2373   int j;
2374   int numparams=parameter->task->numParameters;
2375   int numiterators=parameter->task->numTotal-1;
2376   int retval=1;
2377   int addnormal=1;
2378   int adderror=1;
2379
2380   struct taskdescriptor * task=parameter->task;
2381
2382         ObjectHashadd_I(parameter->objectset, (int) ptr, 0, (int) enterflags, numenterflags, enterflags==NULL);//this add the object to parameterwrapper
2383
2384   /* Add enqueued object to parameter vector */
2385   taskpointerarray[parameter->slot]=ptr;
2386
2387   /* Reset iterators */
2388   for(j=0;j<numiterators;j++) {
2389     toiReset(&parameter->iterators[j]);
2390   }
2391
2392   /* Find initial state */
2393   for(j=0;j<numiterators;j++) {
2394   backtrackinit:
2395     if(toiHasNext(&parameter->iterators[j], taskpointerarray OPTARG(failed)))
2396       toiNext(&parameter->iterators[j], taskpointerarray OPTARG(failed));
2397     else if (j>0) {
2398       /* Need to backtrack */
2399       toiReset(&parameter->iterators[j]);
2400       j--;
2401       goto backtrackinit;
2402     } else {
2403       /* Nothing to enqueue */
2404       return retval;
2405     }
2406   }
2407
2408   while(1) {
2409     /* Enqueue current state */
2410     int launch = 0;
2411     struct taskparamdescriptor *tpd=RUNMALLOC_I(sizeof(struct taskparamdescriptor));
2412     tpd->task=task;
2413     tpd->numParameters=numiterators+1;
2414     tpd->parameterArray=RUNMALLOC_I(sizeof(void *)*(numiterators+1));
2415     for(j=0;j<=numiterators;j++){
2416       tpd->parameterArray[j]=taskpointerarray[j];//store the actual parameters
2417     }
2418     /* Enqueue task */
2419     if ((/*!gencontains(failedtasks, tpd)&&*/!gencontains(activetasks,tpd))) {
2420       genputtable_I(activetasks, tpd, tpd);
2421     } else {
2422       RUNFREE(tpd->parameterArray);
2423       RUNFREE(tpd);
2424     }
2425
2426     /* This loop iterates to the next parameter combination */
2427     if (numiterators==0)
2428       return retval;
2429
2430     for(j=numiterators-1; j<numiterators;j++) {
2431     backtrackinc:
2432       if(toiHasNext(&parameter->iterators[j], taskpointerarray OPTARG(failed)))
2433         toiNext(&parameter->iterators[j], taskpointerarray OPTARG(failed));
2434       else if (j>0) {
2435         /* Need to backtrack */
2436         toiReset(&parameter->iterators[j]);
2437         j--;
2438         goto backtrackinc;
2439       } else {
2440         /* Nothing more to enqueue */
2441         return retval;
2442       }
2443     }
2444   }
2445   return retval;
2446 }
2447 #endif
2448
2449 /* Handler for signals. The signals catch null pointer errors and
2450    arithmatic errors. */
2451 #ifndef RAW
2452 void myhandler(int sig, siginfo_t *info, void *uap) {
2453   sigset_t toclear;
2454 #ifdef DEBUG
2455   printf("sig=%d\n",sig);
2456   printf("signal\n");
2457 #endif
2458   sigemptyset(&toclear);
2459   sigaddset(&toclear, sig);
2460   sigprocmask(SIG_UNBLOCK, &toclear,NULL);
2461   longjmp(error_handler,1);
2462 }
2463 #endif
2464
2465 fd_set readfds;
2466 int maxreadfd;
2467 struct RuntimeHash *fdtoobject;
2468
2469 void addreadfd(int fd) {
2470   if (fd>=maxreadfd)
2471     maxreadfd=fd+1;
2472   FD_SET(fd, &readfds);
2473 }
2474
2475 void removereadfd(int fd) {
2476   FD_CLR(fd, &readfds);
2477   if (maxreadfd==(fd+1)) {
2478     maxreadfd--;
2479     while(maxreadfd>0&&!FD_ISSET(maxreadfd-1, &readfds))
2480       maxreadfd--;
2481   }
2482 }
2483
2484 #ifdef PRECISE_GC
2485 #define OFFSET 2
2486 #else
2487 #define OFFSET 0
2488 #endif
2489
2490 void executetasks() {
2491   void * taskpointerarray[MAXTASKPARAMS+OFFSET];
2492   int numparams=0;
2493   int numtotal=0;
2494   struct ___Object___ * tmpparam = NULL;
2495   struct parameterdescriptor * pd=NULL;
2496   struct parameterwrapper *pw=NULL;
2497   int j = 0;
2498   int x = 0;
2499
2500 #ifdef RAW
2501   int grount = 0;
2502   int andmask=0;
2503   int checkmask=0;
2504
2505   raw_test_pass(0xe991);
2506 #endif
2507
2508 #ifndef RAW
2509   /* Set up signal handlers */
2510   struct sigaction sig;
2511   sig.sa_sigaction=&myhandler;
2512   sig.sa_flags=SA_SIGINFO;
2513   sigemptyset(&sig.sa_mask);
2514
2515   /* Catch bus errors, segmentation faults, and floating point exceptions*/
2516   sigaction(SIGBUS,&sig,0);
2517   sigaction(SIGSEGV,&sig,0);
2518   sigaction(SIGFPE,&sig,0);
2519   sigaction(SIGPIPE,&sig,0);
2520 #endif
2521
2522 #ifndef RAW
2523   /* Zero fd set */
2524   FD_ZERO(&readfds);
2525 #endif
2526   maxreadfd=0;
2527 #ifndef RAW
2528   fdtoobject=allocateRuntimeHash(100);
2529 #endif
2530
2531 #ifndef RAW
2532   /* Map first block of memory to protected, anonymous page */
2533   mmap(0, 0x1000, 0, MAP_SHARED|MAP_FIXED|MAP_ANON, -1, 0);
2534 #endif
2535
2536   newtask:
2537   while((hashsize(activetasks)>0)||(maxreadfd>0)) {
2538
2539 #ifdef RAW
2540           raw_test_pass(0xe992);
2541 #else
2542     /* Check if any filedescriptors have IO pending */
2543     if (maxreadfd>0) {
2544       int i;
2545       struct timeval timeout={0,0};
2546       fd_set tmpreadfds;
2547       int numselect;
2548       tmpreadfds=readfds;
2549       numselect=select(maxreadfd, &tmpreadfds, NULL, NULL, &timeout);
2550       if (numselect>0) {
2551         /* Process ready fd's */
2552         int fd;
2553         for(fd=0;fd<maxreadfd;fd++) {
2554           if (FD_ISSET(fd, &tmpreadfds)) {
2555             /* Set ready flag on object */
2556             void * objptr;
2557             //      printf("Setting fd %d\n",fd);
2558             if (RuntimeHashget(fdtoobject, fd,(int *) &objptr)) {
2559               if(intflagorand(objptr,1,0xFFFFFFFF)) { /* Set the first flag to 1 */
2560                          enqueueObject(objptr, NULL, 0);
2561                   }
2562             }
2563           }
2564         }
2565       }
2566     }
2567 #endif
2568
2569     /* See if there are any active tasks */
2570     if (hashsize(activetasks)>0) {
2571       int i;
2572       currtpd=(struct taskparamdescriptor *) getfirstkey(activetasks);
2573       genfreekey(activetasks, currtpd);
2574
2575       /* Check if this task has failed, allow a task that contains optional objects to fire */
2576       /*if (gencontains(failedtasks, currtpd)) {
2577         // Free up task parameter descriptor
2578         RUNFREE(currtpd->parameterArray);
2579         RUNFREE(currtpd);
2580         goto newtask;
2581       }*/
2582       numparams=currtpd->task->numParameters;
2583       numtotal=currtpd->task->numTotal;
2584
2585 #ifdef THREADSIMULATE
2586           int isolateflags[numparams];
2587 #endif
2588       /* Make sure that the parameters are still in the queues */
2589       for(i=0;i<numparams;i++) {
2590         void * parameter=currtpd->parameterArray[i];
2591 #ifdef RAW
2592         raw_test_pass(0xe993);
2593         // require locks for this parameter
2594         getwritelock(parameter);
2595         grount = 0;
2596         raw_user_interrupts_off();
2597         while(!lockflag) {
2598 //#ifndef INTERRUPT
2599                 receiveObject();
2600 //#endif
2601         }
2602 #ifndef INTERRUPT
2603         if(reside) {
2604                 while(receiveObject() != -1) {
2605                 }
2606         }
2607 #endif
2608         grount = lockresult;
2609
2610         lockresult = 0;
2611         lockobj = 0;
2612         lockflag = false;
2613 #ifndef INTERRUPT
2614         reside = false;
2615 #endif
2616         raw_user_interrupts_on();
2617
2618         if(grount == 0) {
2619                 raw_test_pass(0xe994);
2620                 // can not get the lock, try later
2621                 for(j = 0; j < i; ++j) {
2622                         releasewritelock(taskpointerarray[j+OFFSET]);
2623                 }
2624                 genputtable(activetasks, currtpd, currtpd);
2625                 goto newtask;
2626         }
2627         // flush the object
2628         {
2629                 int tmp = 0;
2630                 for(tmp = 0; tmp < classsize[((struct ___Object___ *)parameter)->type]; ++tmp) {
2631                         invalidateAddr(parameter + tmp);
2632                 }
2633         }
2634 #endif
2635         tmpparam = (struct ___Object___ *)parameter;
2636 #ifdef THREADSIMULATE
2637         if(0 == tmpparam->isolate) {
2638                 isolateflags[i] = 0;
2639                 // shared object, need to flush with current value
2640                 //if(!getreadlock(tmpparam->original)) {
2641                 //      // fail to get read lock of the original object, try this task later
2642                 if(!getwritelock(tmpparam->original)) {
2643                         // fail to get write lock, release all obtained locks and try this task later
2644                         int j = 0;
2645                         for(j = 0; j < i; ++j) {
2646                                 if(0 == isolateflags[j]) {
2647                                         releasewritelock(((struct ___Object___ *)taskpointerarray[j+OFFSET])->original);
2648                                 }
2649                         }
2650                         genputtable(activetasks, currtpd, currtpd);
2651                         goto newtask;
2652                 }
2653                 if(tmpparam->version != tmpparam->original->version) {
2654                         // some task on another core has changed this object
2655                         // flush this object
2656                         //memcpy(tmpparam, tmpparam->original, classsize[tmpparam->type]);
2657                         // release all obtained locks
2658                         int j = 0;
2659                         for(j = 0; j < i; ++j) {
2660                                 if(0 == isolateflags[j]) {
2661                                         releasewritelock(((struct ___Object___ *)taskpointerarray[j+OFFSET])->original);
2662                                 }
2663                         }
2664                         releasewritelock(tmpparam->original);
2665
2666                         // dequeue this object
2667                         int numofcore = pthread_getspecific(key);
2668                         struct parameterwrapper ** queues = objectqueues[numofcore][tmpparam->type];
2669                         int length = numqueues[numofcore][tmpparam->type];
2670                         for(j = 0; j < length; ++j) {
2671                                 struct parameterwrapper * pw = queues[j];
2672                                 if(ObjectHashcontainskey(pw->objectset, (int)tmpparam)) {
2673                                         int next;
2674                                         int UNUSED, UNUSED2;
2675                                         int * enterflags;
2676                                         ObjectHashget(pw->objectset, (int) tmpparam, (int *) &next, (int *) &enterflags, &UNUSED, &UNUSED2);
2677                                         ObjectHashremove(pw->objectset, (int)tmpparam);
2678                                         if (enterflags!=NULL)
2679                                                 free(enterflags);
2680                                 }
2681                         }
2682                         // try to enqueue it again to check if it feeds other tasks;
2683                         //enqueueObject(tmpparam, NULL, 0);
2684                         // Free up task parameter descriptor
2685                         RUNFREE(currtpd->parameterArray);
2686                         RUNFREE(currtpd);
2687                         goto newtask;
2688                 }
2689         } else {
2690                 isolateflags[i] = 1;
2691         }
2692 #endif
2693         pd=currtpd->task->descriptorarray[i];
2694         pw=(struct parameterwrapper *) pd->queue;
2695         /* Check that object is still in queue */
2696         {
2697           if (!ObjectHashcontainskey(pw->objectset, (int) parameter)) {
2698 #ifdef RAW
2699                   raw_test_pass(0xe995);
2700 #endif
2701                 // release grabbed locks
2702                 for(j = 0; j < i; ++j) {
2703                         releasewritelock(taskpointerarray[j+OFFSET]);
2704                 }
2705                 releasewritelock(parameter);
2706             RUNFREE(currtpd->parameterArray);
2707             RUNFREE(currtpd);
2708             goto newtask;
2709           }
2710         }
2711 #ifdef RAW
2712         /* Check if the object's flags still meets requirements */
2713         {
2714                 int tmpi = 0;
2715                 bool ismet = false;
2716                 for(tmpi = 0; tmpi < pw->numberofterms; ++tmpi) {
2717                         andmask=pw->intarray[tmpi*2];
2718                         checkmask=pw->intarray[tmpi*2+1];
2719                         raw_test_pass(0xdd000000 + andmask);
2720                         raw_test_pass_reg((int)parameter);
2721                         raw_test_pass(0xdd000000 + ((struct ___Object___ *)parameter)->flag);
2722                         raw_test_pass(0xdd000000 + checkmask);
2723                         if((((struct ___Object___ *)parameter)->flag&andmask)==checkmask) {
2724                                 ismet = true;
2725                                 break;
2726                         }
2727                 }
2728                 if (!ismet) {
2729                         // flags are never suitable
2730                         // remove this obj from the queue
2731                         int next;
2732                         int UNUSED, UNUSED2;
2733                         int * enterflags;
2734                         raw_test_pass(0xe996);
2735                         ObjectHashget(pw->objectset, (int) parameter, (int *) &next, (int *) &enterflags, &UNUSED, &UNUSED2);
2736                         ObjectHashremove(pw->objectset, (int)parameter);
2737                         if (enterflags!=NULL)
2738                                 free(enterflags);
2739                         // release grabbed locks
2740                         for(j = 0; j < i; ++j) {
2741                                 releasewritelock(taskpointerarray[j+OFFSET]);
2742                         }
2743                         releasewritelock(parameter);
2744                         RUNFREE(currtpd->parameterArray);
2745                         RUNFREE(currtpd);
2746                         goto newtask;
2747                 }
2748         }
2749 #endif
2750       parameterpresent:
2751         ;
2752         /* Check that object still has necessary tags */
2753         for(j=0;j<pd->numbertags;j++) {
2754           int slotid=pd->tagarray[2*j]+numparams;
2755           struct ___TagDescriptor___ *tagd=currtpd->parameterArray[slotid];
2756           if (!containstag(parameter, tagd)) {
2757 #ifdef RAW
2758                   raw_test_pass(0xe997);
2759 #endif
2760             RUNFREE(currtpd->parameterArray);
2761             RUNFREE(currtpd);
2762             goto newtask;
2763           }
2764         }
2765
2766         taskpointerarray[i+OFFSET]=parameter;
2767       }
2768       /* Copy the tags */
2769       for(;i<numtotal;i++) {
2770         taskpointerarray[i+OFFSET]=currtpd->parameterArray[i];
2771       }
2772
2773 #ifdef THREADSIMULATE
2774          for(i = 0; i < numparams; ++i) {
2775                   if(0 == isolateflags[i]) {
2776                           struct ___Object___ * tmpparam = (struct ___Object___ *)taskpointerarray[i+OFFSET];
2777                           if(tmpparam != tmpparam->original) {
2778                                   taskpointerarray[i+OFFSET] = tmpparam->original;
2779                           }
2780                   }
2781           }
2782 #endif
2783
2784       {
2785 #if 0
2786 #ifndef RAW
2787         /* Checkpoint the state */
2788         forward=allocateRuntimeHash(100);
2789         reverse=allocateRuntimeHash(100);
2790         //void ** checkpoint=makecheckpoint(currtpd->task->numParameters, currtpd->parameterArray, forward, reverse);
2791 #endif
2792 #endif
2793         if (x=setjmp(error_handler)) {
2794           int counter;
2795           /* Recover */
2796 #ifndef RAW
2797 #ifdef DEBUG
2798           printf("Fatal Error=%d, Recovering!\n",x);
2799 #endif
2800 #endif
2801          /*
2802           genputtable(failedtasks,currtpd,currtpd);
2803           //restorecheckpoint(currtpd->task->numParameters, currtpd->parameterArray, checkpoint, forward, reverse);
2804
2805           freeRuntimeHash(forward);
2806           freeRuntimeHash(reverse);
2807           freemalloc();
2808           forward=NULL;
2809           reverse=NULL;
2810           */
2811           //fflush(stdout);
2812 #ifdef RAW
2813           raw_test_pass_reg(x);
2814           raw_test_done(0xa009);
2815 #else
2816           exit(-1);
2817 #endif
2818         } else {
2819           /*if (injectfailures) {
2820             if ((((double)random())/RAND_MAX)<failurechance) {
2821               printf("\nINJECTING TASK FAILURE to %s\n", currtpd->task->name);
2822               longjmp(error_handler,10);
2823             }
2824           }*/
2825           /* Actually call task */
2826 #ifdef PRECISE_GC
2827           ((int *)taskpointerarray)[0]=currtpd->numParameters;
2828           taskpointerarray[1]=NULL;
2829 #endif
2830           if(debugtask){
2831 #ifndef RAW
2832             printf("ENTER %s count=%d\n",currtpd->task->name, (instaccum-instructioncount));
2833 #endif
2834             ((void (*) (void **)) currtpd->task->taskptr)(taskpointerarray);
2835 #ifndef RAW
2836             printf("EXIT %s count=%d\n",currtpd->task->name, (instaccum-instructioncount));
2837 #endif
2838           } else
2839             ((void (*) (void **)) currtpd->task->taskptr)(taskpointerarray);
2840
2841 #ifdef RAW
2842           for(i = 0; i < numparams; ++i) {
2843                   int j = 0;
2844                   struct ___Object___ * tmpparam = (struct ___Object___ *)taskpointerarray[i+OFFSET];
2845                   raw_test_pass(0xe998);
2846                 raw_test_pass(0xdd100000 + tmpparam->flag);
2847                   releasewritelock(tmpparam);
2848           }
2849 #elif defined THREADSIMULATE
2850           for(i = 0; i < numparams; ++i) {
2851                   if(0 == isolateflags[i]) {
2852                           struct ___Object___ * tmpparam = (struct ___Object___ *)taskpointerarray[i+OFFSET];
2853                           releasewritelock(tmpparam);
2854                   }
2855           }
2856 #endif
2857
2858 #if 0
2859 #ifndef RAW
2860           freeRuntimeHash(forward);
2861           freeRuntimeHash(reverse);
2862 #endif
2863 #endif
2864           freemalloc();
2865           // Free up task parameter descriptor
2866           RUNFREE(currtpd->parameterArray);
2867           RUNFREE(currtpd);
2868 #if 0
2869 #ifndef RAW
2870           forward=NULL;
2871           reverse=NULL;
2872 #endif
2873 #endif
2874         }
2875       }
2876     }
2877   }
2878 #ifdef RAW
2879         raw_test_pass(0xe999);
2880 #endif
2881 }
2882
2883 /* This function processes an objects tags */
2884 void processtags(struct parameterdescriptor *pd, int index, struct parameterwrapper *parameter, int * iteratorcount, int *statusarray, int numparams) {
2885   int i;
2886
2887   for(i=0;i<pd->numbertags;i++) {
2888     int slotid=pd->tagarray[2*i];
2889     int tagid=pd->tagarray[2*i+1];
2890
2891     if (statusarray[slotid+numparams]==0) {
2892       parameter->iterators[*iteratorcount].istag=1;
2893       parameter->iterators[*iteratorcount].tagid=tagid;
2894       parameter->iterators[*iteratorcount].slot=slotid+numparams;
2895       parameter->iterators[*iteratorcount].tagobjectslot=index;
2896       statusarray[slotid+numparams]=1;
2897       (*iteratorcount)++;
2898     }
2899   }
2900 }
2901
2902
2903 void processobject(struct parameterwrapper *parameter, int index, struct parameterdescriptor *pd, int *iteratorcount, int * statusarray, int numparams) {
2904   int i;
2905   int tagcount=0;
2906   struct ObjectHash * objectset=((struct parameterwrapper *)pd->queue)->objectset;
2907
2908   parameter->iterators[*iteratorcount].istag=0;
2909   parameter->iterators[*iteratorcount].slot=index;
2910   parameter->iterators[*iteratorcount].objectset=objectset;
2911   statusarray[index]=1;
2912
2913   for(i=0;i<pd->numbertags;i++) {
2914     int slotid=pd->tagarray[2*i];
2915     int tagid=pd->tagarray[2*i+1];
2916     if (statusarray[slotid+numparams]!=0) {
2917       /* This tag has already been enqueued, use it to narrow search */
2918       parameter->iterators[*iteratorcount].tagbindings[tagcount]=slotid+numparams;
2919       tagcount++;
2920     }
2921   }
2922   parameter->iterators[*iteratorcount].numtags=tagcount;
2923
2924   (*iteratorcount)++;
2925 }
2926
2927 /* This function builds the iterators for a task & parameter */
2928
2929 void builditerators(struct taskdescriptor * task, int index, struct parameterwrapper * parameter) {
2930   int statusarray[MAXTASKPARAMS];
2931   int i;
2932   int numparams=task->numParameters;
2933   int iteratorcount=0;
2934   for(i=0;i<MAXTASKPARAMS;i++) statusarray[i]=0;
2935
2936   statusarray[index]=1; /* Initial parameter */
2937   /* Process tags for initial iterator */
2938
2939   processtags(task->descriptorarray[index], index, parameter, & iteratorcount, statusarray, numparams);
2940
2941   while(1) {
2942   loopstart:
2943     /* Check for objects with existing tags */
2944     for(i=0;i<numparams;i++) {
2945       if (statusarray[i]==0) {
2946         struct parameterdescriptor *pd=task->descriptorarray[i];
2947         int j;
2948         for(j=0;j<pd->numbertags;j++) {
2949           int slotid=pd->tagarray[2*j];
2950           if(statusarray[slotid+numparams]!=0) {
2951             processobject(parameter, i, pd, &iteratorcount, statusarray, numparams);
2952             processtags(pd, i, parameter, &iteratorcount, statusarray, numparams);
2953             goto loopstart;
2954           }
2955         }
2956       }
2957     }
2958
2959     /* Next do objects w/ unbound tags*/
2960
2961     for(i=0;i<numparams;i++) {
2962       if (statusarray[i]==0) {
2963         struct parameterdescriptor *pd=task->descriptorarray[i];
2964         if (pd->numbertags>0) {
2965           processobject(parameter, i, pd, &iteratorcount, statusarray, numparams);
2966           processtags(pd, i, parameter, &iteratorcount, statusarray, numparams);
2967           goto loopstart;
2968         }
2969       }
2970     }
2971
2972     /* Nothing with a tag enqueued */
2973
2974     for(i=0;i<numparams;i++) {
2975       if (statusarray[i]==0) {
2976         struct parameterdescriptor *pd=task->descriptorarray[i];
2977         processobject(parameter, i, pd, &iteratorcount, statusarray, numparams);
2978         processtags(pd, i, parameter, &iteratorcount, statusarray, numparams);
2979         goto loopstart;
2980       }
2981     }
2982
2983     /* Nothing left */
2984     return;
2985   }
2986 }
2987
2988  void printdebug() {
2989    int i;
2990    int j;
2991 #ifdef THREADSIMULATE
2992    int numofcore = pthread_getspecific(key);
2993    for(i=0;i<numtasks[numofcore];i++) {
2994            struct taskdescriptor * task=taskarray[numofcore][i];
2995 #else
2996 #ifdef RAW
2997            if(corenum > NUMCORES - 1) {
2998                    return;
2999            }
3000 #endif
3001    for(i=0;i<numtasks[corenum];i++) {
3002      struct taskdescriptor * task=taskarray[corenum][i];
3003 #endif
3004 #ifndef RAW
3005      printf("%s\n", task->name);
3006 #endif
3007      for(j=0;j<task->numParameters;j++) {
3008        struct parameterdescriptor *param=task->descriptorarray[j];
3009        struct parameterwrapper *parameter=param->queue;
3010        struct ObjectHash * set=parameter->objectset;
3011        struct ObjectIterator objit;
3012 #ifndef RAW
3013        printf("  Parameter %d\n", j);
3014 #endif
3015        ObjectHashiterator(set, &objit);
3016        while(ObjhasNext(&objit)) {
3017          struct ___Object___ * obj=(struct ___Object___ *)Objkey(&objit);
3018          struct ___Object___ * tagptr=obj->___tags___;
3019          int nonfailed=Objdata4(&objit);
3020          int numflags=Objdata3(&objit);
3021          int flags=Objdata2(&objit);
3022          Objnext(&objit);
3023 #ifndef RAW
3024          printf("    Contains %lx\n", obj);
3025          printf("      flag=%d\n", obj->flag);
3026 #endif
3027          if (tagptr==NULL) {
3028          } else if (tagptr->type==TAGTYPE) {
3029 #ifndef RAW
3030            printf("      tag=%lx\n",tagptr);
3031 #endif
3032            ;
3033          } else {
3034            int tagindex=0;
3035            struct ArrayObject *ao=(struct ArrayObject *)tagptr;
3036            for(;tagindex<ao->___cachedCode___;tagindex++) {
3037 #ifndef RAW
3038              printf("      tag=%lx\n",ARRAYGET(ao, struct ___TagDescriptor___*, tagindex));
3039 #endif
3040            }
3041          }
3042        }
3043      }
3044    }
3045  }
3046
3047
3048 /* This function processes the task information to create queues for
3049    each parameter type. */
3050
3051 void processtasks() {
3052   int i;
3053 #ifdef RAW
3054           if(corenum > NUMCORES - 1) {
3055                   return;
3056           }
3057 #endif
3058 #ifdef THREADSIMULATE
3059   int numofcore = pthread_getspecific(key);
3060   for(i=0;i<numtasks[numofcore];i++) {
3061           struct taskdescriptor *task=taskarray[numofcore][i];
3062 #else
3063   for(i=0;i<numtasks[corenum];i++) {
3064     struct taskdescriptor * task=taskarray[corenum][i];
3065 #endif
3066     int j;
3067
3068         /* Build objectsets */
3069         for(j=0;j<task->numParameters;j++) {
3070       struct parameterdescriptor *param=task->descriptorarray[j];
3071       struct parameterwrapper *parameter=param->queue;
3072           parameter->objectset=allocateObjectHash(10);
3073           parameter->task=task;
3074     }
3075
3076     /* Build iterators for parameters */
3077     for(j=0;j<task->numParameters;j++) {
3078       struct parameterdescriptor *param=task->descriptorarray[j];
3079       struct parameterwrapper *parameter=param->queue;
3080       builditerators(task, j, parameter);
3081     }
3082  }
3083 }
3084
3085 void toiReset(struct tagobjectiterator * it) {
3086   if (it->istag) {
3087     it->tagobjindex=0;
3088   } else if (it->numtags>0) {
3089     it->tagobjindex=0;
3090   } else {
3091     ObjectHashiterator(it->objectset, &it->it);
3092   }
3093 }
3094
3095 int toiHasNext(struct tagobjectiterator *it, void ** objectarray OPTARG(int * failed)) {
3096   if (it->istag) {
3097     /* Iterate tag */
3098     /* Get object with tags */
3099     struct ___Object___ *obj=objectarray[it->tagobjectslot];
3100     struct ___Object___ *tagptr=obj->___tags___;
3101     if (tagptr->type==TAGTYPE) {
3102       if ((it->tagobjindex==0)&& /* First object */
3103           (it->tagid==((struct ___TagDescriptor___ *)tagptr)->flag)) /* Right tag type */
3104         return 1;
3105       else
3106         return 0;
3107     } else {
3108       struct ArrayObject *ao=(struct ArrayObject *) tagptr;
3109       int tagindex=it->tagobjindex;
3110       for(;tagindex<ao->___cachedCode___;tagindex++) {
3111         struct ___TagDescriptor___ *td=ARRAYGET(ao, struct ___TagDescriptor___ *, tagindex);
3112         if (td->flag==it->tagid) {
3113           it->tagobjindex=tagindex; /* Found right type of tag */
3114           return 1;
3115         }
3116       }
3117       return 0;
3118     }
3119   } else if (it->numtags>0) {
3120     /* Use tags to locate appropriate objects */
3121     struct ___TagDescriptor___ *tag=objectarray[it->tagbindings[0]];
3122     struct ___Object___ *objptr=tag->flagptr;
3123     int i;
3124     if (objptr->type!=OBJECTARRAYTYPE) {
3125       if (it->tagobjindex>0)
3126         return 0;
3127       if (!ObjectHashcontainskey(it->objectset, (int) objptr))
3128         return 0;
3129       for(i=1;i<it->numtags;i++) {
3130         struct ___TagDescriptor___ *tag2=objectarray[it->tagbindings[i]];
3131         if (!containstag(objptr,tag2))
3132           return 0;
3133       }
3134       return 1;
3135     } else {
3136       struct ArrayObject *ao=(struct ArrayObject *) objptr;
3137       int tagindex;
3138       int i;
3139       for(tagindex=it->tagobjindex;tagindex<ao->___cachedCode___;tagindex++) {
3140         struct ___Object___ *objptr=ARRAYGET(ao, struct ___Object___*, tagindex);
3141         if (!ObjectHashcontainskey(it->objectset, (int) objptr))
3142           continue;
3143         for(i=1;i<it->numtags;i++) {
3144           struct ___TagDescriptor___ *tag2=objectarray[it->tagbindings[i]];
3145           if (!containstag(objptr,tag2))
3146             goto nexttag;
3147         }
3148         it->tagobjindex=tagindex;
3149         return 1;
3150       nexttag:
3151         ;
3152       }
3153       it->tagobjindex=tagindex;
3154       return 0;
3155     }
3156   } else {
3157     return ObjhasNext(&it->it);
3158   }
3159 }
3160
3161 int containstag(struct ___Object___ *ptr, struct ___TagDescriptor___ *tag) {
3162   int j;
3163   struct ___Object___ * objptr=tag->flagptr;
3164   if (objptr->type==OBJECTARRAYTYPE) {
3165     struct ArrayObject *ao=(struct ArrayObject *)objptr;
3166     for(j=0;j<ao->___cachedCode___;j++) {
3167       if (ptr==ARRAYGET(ao, struct ___Object___*, j))
3168         return 1;
3169     }
3170     return 0;
3171   } else
3172     return objptr==ptr;
3173 }
3174
3175 void toiNext(struct tagobjectiterator *it , void ** objectarray OPTARG(int * failed)) {
3176   /* hasNext has all of the intelligence */
3177   if(it->istag) {
3178     /* Iterate tag */
3179     /* Get object with tags */
3180     struct ___Object___ *obj=objectarray[it->tagobjectslot];
3181     struct ___Object___ *tagptr=obj->___tags___;
3182     if (tagptr->type==TAGTYPE) {
3183       it->tagobjindex++;
3184       objectarray[it->slot]=tagptr;
3185     } else {
3186       struct ArrayObject *ao=(struct ArrayObject *) tagptr;
3187       objectarray[it->slot]=ARRAYGET(ao, struct ___TagDescriptor___ *, it->tagobjindex++);
3188     }
3189   } else if (it->numtags>0) {
3190     /* Use tags to locate appropriate objects */
3191     struct ___TagDescriptor___ *tag=objectarray[it->tagbindings[0]];
3192     struct ___Object___ *objptr=tag->flagptr;
3193     if (objptr->type!=OBJECTARRAYTYPE) {
3194       it->tagobjindex++;
3195       objectarray[it->slot]=objptr;
3196     } else {
3197       struct ArrayObject *ao=(struct ArrayObject *) objptr;
3198       objectarray[it->slot]=ARRAYGET(ao, struct ___Object___ *, it->tagobjindex++);
3199     }
3200   } else {
3201     /* Iterate object */
3202     objectarray[it->slot]=(void *)Objkey(&it->it);
3203     Objnext(&it->it);
3204   }
3205 }
3206 #endif