2 #include "multicoregccompact.h"
3 #include "runtime_arch.h"
4 #include "multicoreruntime.h"
6 INLINE bool gc_checkCoreStatus() {
7 BAMBOO_ENTER_RUNTIME_MODE_FROM_CLIENT();
8 for(int i = 0; i < NUMCORES4GC; ++i) {
9 if(gccorestatus[i] != 0) {
10 BAMBOO_ENTER_CLIENT_MODE_FROM_RUNTIME();
14 BAMBOO_ENTER_CLIENT_MODE_FROM_RUNTIME();
18 INLINE void gc_resetCoreStatus() {
19 BAMBOO_ENTER_RUNTIME_MODE_FROM_CLIENT();
20 for(int i = 0; i < NUMCORES4GC; ++i) {
23 BAMBOO_ENTER_CLIENT_MODE_FROM_RUNTIME();
26 // should be invoked with interrupt closed
27 INLINE int assignSpareMem_I(unsigned int sourcecore,unsigned int * requiredmem,unsigned int * tomove,unsigned int * startaddr) {
29 BLOCKINDEX(gcloads[sourcecore], &b);
30 unsigned int boundptr = BOUNDPTR(b);
31 unsigned int remain = boundptr - gcloads[sourcecore];
32 unsigned int memneed = requiredmem + BAMBOO_CACHE_LINE_SIZE;
33 *startaddr = gcloads[sourcecore];
34 *tomove = gcfilledblocks[sourcecore] + 1;
35 if(memneed < remain) {
36 gcloads[sourcecore] += memneed;
39 // next available block
40 gcfilledblocks[sourcecore] += 1;
41 unsigned int newbase = 0;
42 BASEPTR(sourcecore, gcfilledblocks[sourcecore], &newbase);
43 gcloads[sourcecore] = newbase;
44 return requiredmem-remain;
48 INLINE int assignSpareMem(unsigned int sourcecore,unsigned int * requiredmem,unsigned int * tomove,unsigned int * startaddr) {
49 BAMBOO_ENTER_RUNTIME_MODE_FROM_CLIENT();
51 BLOCKINDEX(gcloads[sourcecore], &b);
52 unsigned int boundptr = BOUNDPTR(b);
53 unsigned int remain = boundptr - gcloads[sourcecore];
54 unsigned int memneed = requiredmem + BAMBOO_CACHE_LINE_SIZE;
55 *startaddr = gcloads[sourcecore];
56 *tomove = gcfilledblocks[sourcecore] + 1;
57 if(memneed < remain) {
58 gcloads[sourcecore] += memneed;
59 BAMBOO_ENTER_CLIENT_MODE_FROM_RUNTIME();
62 // next available block
63 gcfilledblocks[sourcecore] += 1;
64 unsigned int newbase = 0;
65 BASEPTR(sourcecore, gcfilledblocks[sourcecore], &newbase);
66 gcloads[sourcecore] = newbase;
67 BAMBOO_ENTER_CLIENT_MODE_FROM_RUNTIME();
68 return requiredmem-remain;
72 INLINE void compact2Heaptophelper_I(unsigned int coren,unsigned int* p,unsigned int* numblocks,unsigned int* remain) {
74 unsigned int memneed = gcrequiredmems[coren] + BAMBOO_CACHE_LINE_SIZE;
75 if(STARTUPCORE == coren) {
78 gcdstcore = gctopcore;
79 gcblock2fill = *numblocks + 1;
81 if(BAMBOO_CHECK_SEND_MODE()) {
82 cache_msg_4_I(coren,GCMOVESTART,gctopcore,*p,(*numblocks)+1);
84 send_msg_4_I(coren,GCMOVESTART,gctopcore,*p,(*numblocks)+1);
87 if(memneed < *remain) {
89 gcrequiredmems[coren] = 0;
90 gcloads[gctopcore] += memneed;
91 *remain = *remain - memneed;
93 // next available block
95 gcfilledblocks[gctopcore] += 1;
96 unsigned int newbase = 0;
97 BASEPTR(gctopcore, gcfilledblocks[gctopcore], &newbase);
98 gcloads[gctopcore] = newbase;
99 gcrequiredmems[coren] -= *remain - BAMBOO_CACHE_LINE_SIZE;
100 gcstopblock[gctopcore]++;
101 gctopcore = NEXTTOPCORE(gctopblock);
103 *numblocks = gcstopblock[gctopcore];
104 *p = gcloads[gctopcore];
106 *remain=GC_BLOCK_REMAIN_SIZE(b, (*p));
111 INLINE void compact2Heaptop() {
112 // no cores with spare mem and some cores are blocked with pending move
113 // find the current heap top and make them move to the heap top
115 unsigned int numblocks = gcfilledblocks[gctopcore];
116 p = gcloads[gctopcore];
119 unsigned int remain=GC_BLOCK_REMAIN_SIZE(b, p);
120 // check if the top core finishes
121 BAMBOO_ENTER_RUNTIME_MODE_FROM_CLIENT();
122 if(gccorestatus[gctopcore] != 0) {
123 // let the top core finishes its own work first
124 compact2Heaptophelper_I(gctopcore, &p, &numblocks, &remain);
125 BAMBOO_ENTER_CLIENT_MODE_FROM_RUNTIME();
128 BAMBOO_ENTER_CLIENT_MODE_FROM_RUNTIME();
130 for(int i = 0; i < NUMCORES4GC; i++) {
131 BAMBOO_ENTER_RUNTIME_MODE_FROM_CLIENT();
132 if((gccorestatus[i] != 0) && (gcrequiredmems[i] > 0)) {
133 compact2Heaptophelper_I(i, &p, &numblocks, &remain);
134 if(gccorestatus[gctopcore] != 0) {
135 BAMBOO_ENTER_CLIENT_MODE_FROM_RUNTIME();
136 // the top core is not free now
140 BAMBOO_ENTER_CLIENT_MODE_FROM_RUNTIME();
144 INLINE void resolvePendingMoveRequest() {
147 bool nosparemem = true;
148 bool haspending = false;
149 bool hasrunning = false;
150 bool noblock = false;
151 unsigned int dstcore = 0; // the core who need spare mem
152 unsigned int sourcecore = 0; // the core who has spare mem
153 for(i = j = 0; (i < NUMCORES4GC) && (j < NUMCORES4GC); ) {
155 // check if there are cores with spare mem
156 if(gccorestatus[i] == 0) {
157 // finished working, check if it still have spare mem
158 if(gcfilledblocks[i] < gcstopblock[i]) {
159 // still have spare mem
167 if(gccorestatus[j] != 0) {
168 // not finished, check if it has pending move requests
169 if((gcfilledblocks[j]==gcstopblock[j])&&(gcrequiredmems[j]>0)) {
178 if(!nosparemem && haspending) {
180 unsigned int tomove = 0;
181 unsigned int startaddr = 0;
182 gcrequiredmems[dstcore] = assignSpareMem(sourcecore,gcrequiredmems[dstcore],&tomove,&startaddr);
183 if(STARTUPCORE == dstcore) {
184 gcdstcore = sourcecore;
186 gcmovestartaddr = startaddr;
187 gcblock2fill = tomove;
189 send_msg_4(dstcore,GCMOVESTART,sourcecore,startaddr,tomove);
198 if(!hasrunning && !noblock) {
199 gcphase = SUBTLECOMPACTPHASE;
204 // If out of boundary of valid shared memory, return false, else return true
205 INLINE bool nextSBlock(struct moveHelper * orig) {
206 orig->blockbase = orig->blockbound;
208 bool sbchanged = false;
209 unsigned int origptr = orig->ptr;
210 unsigned int blockbase = orig->blockbase;
211 unsigned int blockbound = orig->blockbound;
212 unsigned int bound = orig->bound;
214 // check if across a big block
215 // TODO now do not zero out the whole memory, maybe the last two conditions
217 if((blockbase>=bound)||(origptr>=bound)||((origptr!=NULL)&&(*((int*)origptr))==0)||((*((int*)blockbase))==0)) {
219 // end of current heap block, jump to next one
221 BASEPTR(BAMBOO_NUM_OF_CORE, orig->numblocks, &(orig->base));
222 if(orig->base >= gcbaseva + BAMBOO_SHARED_MEM_SIZE) {
224 orig->ptr = orig->base; // set current ptr to out of boundary too
227 orig->blockbase = orig->base;
228 orig->sblockindex=(unsigned int)(orig->blockbase-gcbaseva)/BAMBOO_SMEM_SIZE;
230 unsigned int blocknum = 0;
231 BLOCKINDEX(orig->base, &blocknum);
232 if(bamboo_smemtbl[blocknum] == 0) {
234 goto innernextSBlock;
236 // check the bamboo_smemtbl to decide the real bound
237 orig->bound = orig->base + bamboo_smemtbl[blocknum];
238 } else if(0 == (orig->blockbase%BAMBOO_SMEM_SIZE)) {
239 orig->sblockindex += 1;
243 // check if this sblock should be skipped or have special start point
244 int sbstart = gcsbstarttbl[orig->sblockindex];
247 orig->sblockindex += 1;
248 orig->blockbase += BAMBOO_SMEM_SIZE;
249 goto outernextSBlock;
250 } else if((sbstart != 0) && (sbchanged)) {
251 // the first time to access this SBlock
252 // not start from the very beginning
253 orig->blockbase = sbstart;
256 // setup information for this sblock
257 orig->blockbound = orig->blockbase+(unsigned int)*((int*)(orig->blockbase));
258 orig->offset = BAMBOO_CACHE_LINE_SIZE;
259 orig->ptr = orig->blockbase + orig->offset;
260 if(orig->ptr >= orig->bound) {
261 // met a lobj, move to next block
262 goto innernextSBlock;
268 // return false if there are no available data to compact
269 INLINE bool initOrig_Dst(struct moveHelper * orig,struct moveHelper * to) {
272 to->top = to->offset = BAMBOO_CACHE_LINE_SIZE;
273 to->bound = BAMBOO_SMEM_SIZE_L;
274 BASEPTR(BAMBOO_NUM_OF_CORE, to->numblocks, &(to->base));
276 unsigned int tobase = to->base;
277 to->ptr = tobase + to->offset;
282 unsigned int blocknum = 0;
283 BLOCKINDEX(orig->base, &blocknum);
284 unsigned int origbase = orig->base;
285 // check the bamboo_smemtbl to decide the real bound
286 orig->bound = origbase + (unsigned int)bamboo_smemtbl[blocknum];
287 orig->blockbase = origbase;
288 orig->sblockindex = (unsigned int)(origbase - gcbaseva) / BAMBOO_SMEM_SIZE;
290 int sbstart = gcsbstarttbl[orig->sblockindex];
293 orig->blockbound=gcbaseva+BAMBOO_SMEM_SIZE*(orig->sblockindex+1);
294 return nextSBlock(orig);
295 } else if(sbstart != 0) {
296 orig->blockbase = sbstart;
298 orig->blockbound = orig->blockbase + *((int*)(orig->blockbase));
299 orig->offset = BAMBOO_CACHE_LINE_SIZE;
300 orig->ptr = orig->blockbase + orig->offset;
305 INLINE void nextBlock(struct moveHelper * to) {
306 to->top = to->bound + BAMBOO_CACHE_LINE_SIZE; // header!
307 to->bound += BAMBOO_SMEM_SIZE;
309 BASEPTR(BAMBOO_NUM_OF_CORE, to->numblocks, &(to->base));
310 to->offset = BAMBOO_CACHE_LINE_SIZE;
311 to->ptr = to->base + to->offset;
314 INLINE unsigned int findValidObj(struct moveHelper * orig,struct moveHelper * to,int * type) {
315 unsigned int size = 0;
317 CACHEADAPT_COMPLETE_PAGE_CONVERT(orig, to, to->ptr, false);
318 unsigned int origptr = (unsigned int)(orig->ptr);
319 unsigned int origbound = (unsigned int)orig->bound;
320 unsigned int origblockbound = (unsigned int)orig->blockbound;
321 if((origptr >= origbound) || (origptr == origblockbound)) {
322 if(!nextSBlock(orig)) {
323 // finished, no more data
328 // check the obj's type, size and mark flag
329 *type = ((int *)(origptr))[0];
332 // end of this block, go to next one
333 if(!nextSBlock(orig)) {
334 // finished, no more data
338 } else if(*type < NUMCLASSES) {
340 size = classsize[*type];
343 struct ArrayObject *ao=(struct ArrayObject *)(origptr);
344 unsigned int elementsize=classsize[*type];
345 unsigned int length=ao->___length___;
346 size=(unsigned int)sizeof(struct ArrayObject)
347 +(unsigned int)(length*elementsize);
353 // endaddr does not contain spaces for headers
354 INLINE bool moveobj(struct moveHelper * orig, struct moveHelper * to, unsigned int stopblock) {
360 unsigned int size = findValidObj(orig, to, &type);
361 unsigned int isize = 0;
364 // finished, no more data
367 ALIGNSIZE(size, &isize); // no matter is the obj marked or not
368 // should be able to across
369 unsigned int origptr = (unsigned int)(orig->ptr);
370 if(((struct ___Object___ *)origptr)->marked == MARKED) {
371 unsigned int totop = (unsigned int)to->top;
372 unsigned int tobound = (unsigned int)to->bound;
373 GCPROFILE_RECORD_LIVE_OBJ();
374 // marked obj, copy it to current heap top
375 // check to see if remaining space is enough
376 if((unsigned int)(totop + isize) > tobound) {
377 // fill 0 indicating the end of this block
378 BAMBOO_MEMSET_WH(to->ptr, '\0', tobound - totop);
379 // fill the header of this block and then go to next block
380 to->offset += tobound - totop;
381 CLOSEBLOCK(to->base, to->offset);
382 #ifdef GC_CACHE_ADAPT
383 unsigned int tmp_ptr = to->ptr;
386 #ifdef GC_CACHE_ADAPT
387 CACHEADAPT_COMPLETE_PAGE_CONVERT(orig, to, tmp_ptr, true);
389 if(stopblock == to->numblocks) {
390 // already fulfilled the block
394 // set the mark field to 2, indicating that this obj has been moved
395 // and need to be flushed
396 ((struct ___Object___ *)origptr)->marked = COMPACTED;
397 unsigned int toptr = (unsigned int)to->ptr;
398 if(toptr != origptr) {
399 if((unsigned int)(origptr) < (unsigned int)(toptr+size)) {
400 memmove(toptr, origptr, size);
402 memcpy(toptr, origptr, size);
404 // fill the remaining space with -2
405 BAMBOO_MEMSET_WH((unsigned int)(toptr+size), -2, isize-size);
407 // store mapping info
408 gcmappingtbl[OBJMAPPINGINDEX((unsigned int)origptr)]=(unsigned int)toptr;
409 gccurr_heaptop -= isize;
413 #ifdef GC_CACHE_ADAPT
414 unsigned int tmp_ptr = to->ptr;
415 #endif // GC_CACHE_ADAPT
416 if(to->top == to->bound) {
417 CLOSEBLOCK(to->base, to->offset);
420 #ifdef GC_CACHE_ADAPT
421 CACHEADAPT_COMPLETE_PAGE_CONVERT(orig, to, tmp_ptr, true);
428 return ((((unsigned int)(orig->ptr) > (unsigned int)(orig->bound))||((unsigned int)(orig->ptr) == (unsigned int)(orig->blockbound)))&&!nextSBlock(orig));
431 // should be invoked with interrupt closed
432 bool gcfindSpareMem_I(unsigned int * startaddr,unsigned int * tomove,unsigned int * dstcore,unsigned int requiredmem,unsigned int requiredcore) {
433 for(int k = 0; k < NUMCORES4GC; k++) {
434 if((gccorestatus[k] == 0) && (gcfilledblocks[k] < gcstopblock[k])) {
435 // check if this stopped core has enough mem
436 assignSpareMem_I(k, requiredmem, tomove, startaddr);
441 // if can not find spare mem right now, hold the request
442 gcrequiredmems[requiredcore] = requiredmem;
447 bool gcfindSpareMem(unsigned int * startaddr,unsigned int * tomove,unsigned int * dstcore,unsigned int requiredmem,unsigned int requiredcore) {
448 BAMBOO_ENTER_RUNTIME_MODE_FROM_CLIENT();
449 for(int k = 0; k < NUMCORES4GC; k++) {
450 if((gccorestatus[k] == 0) && (gcfilledblocks[k] < gcstopblock[k])) {
451 // check if this stopped core has enough mem
452 assignSpareMem_I(k, requiredmem, tomove, startaddr);
454 BAMBOO_ENTER_CLIENT_MODE_FROM_RUNTIME();
458 // if can not find spare mem right now, hold the request
459 gcrequiredmems[requiredcore] = requiredmem;
461 BAMBOO_ENTER_CLIENT_MODE_FROM_RUNTIME();
465 INLINE bool compacthelper(struct moveHelper * orig,struct moveHelper * to,int * filledblocks,unsigned int * heaptopptr,bool * localcompact) {
466 // scan over all objs in this block, compact the marked objs
467 // loop stop when finishing either scanning all active objs or
468 // fulfilled the gcstopblock
470 while((unsigned int)(orig->ptr) < (unsigned int)gcmarkedptrbound) {
471 if(moveobj(orig, to, gcblock2fill)) {
475 CACHEADAPT_SAMPLING_DATA_CONVERT(to->ptr);
476 // if no objs have been compact, do nothing,
477 // otherwise, fill the header of this block
478 if(to->offset > (unsigned int)BAMBOO_CACHE_LINE_SIZE) {
479 CLOSEBLOCK(to->base, to->offset);
483 to->top -= BAMBOO_CACHE_LINE_SIZE;
486 *heaptopptr = to->ptr;
487 *filledblocks = to->numblocks;
490 // send msgs to core coordinator indicating that the compact is finishing
491 // send compact finish message to core coordinator
492 if(STARTUPCORE == BAMBOO_NUM_OF_CORE) {
493 gcfilledblocks[BAMBOO_NUM_OF_CORE] = *filledblocks;
494 gcloads[BAMBOO_NUM_OF_CORE] = *heaptopptr;
495 if((unsigned int)(orig->ptr) < (unsigned int)gcmarkedptrbound) {
498 if(gcfindSpareMem(&gcmovestartaddr,&gcblock2fill,&gcdstcore,gccurr_heaptop,BAMBOO_NUM_OF_CORE)) {
504 gccorestatus[BAMBOO_NUM_OF_CORE] = 0;
509 if((unsigned int)(orig->ptr) < (unsigned int)gcmarkedptrbound) {
512 send_msg_5(STARTUPCORE,GCFINISHCOMPACT,BAMBOO_NUM_OF_CORE,*filledblocks,*heaptopptr,gccurr_heaptop);
515 send_msg_5(STARTUPCORE,GCFINISHCOMPACT,BAMBOO_NUM_OF_CORE,*filledblocks,*heaptopptr, 0);
519 if(orig->ptr < gcmarkedptrbound) {
520 // still have unpacked obj
525 to->ptr = gcmovestartaddr;
526 to->numblocks = gcblock2fill - 1;
527 to->bound = BLOCKBOUND(to->numblocks);
528 BASEPTR(gcdstcore, to->numblocks, &(to->base));
529 to->offset = to->ptr - to->base;
530 to->top=(to->numblocks==0)?(to->offset):(to->bound-BAMBOO_SMEM_SIZE+to->offset);
532 to->offset = BAMBOO_CACHE_LINE_SIZE;
533 to->ptr += to->offset; // for header
534 to->top += to->offset;
535 *localcompact = (gcdstcore == BAMBOO_NUM_OF_CORE);
536 CACHEADAPT_SAMPLING_DATA_REVISE_INIT();
543 BAMBOO_ASSERT(COMPACTPHASE == gcphase);
545 // initialize pointers for comapcting
546 struct moveHelper * orig = (struct moveHelper *)RUNMALLOC(sizeof(struct moveHelper));
547 struct moveHelper * to = (struct moveHelper *)RUNMALLOC(sizeof(struct moveHelper));
548 if(!initOrig_Dst(orig, to)) {
549 // no available data to compact
550 // send compact finish msg to STARTUP core
551 send_msg_5(STARTUPCORE,GCFINISHCOMPACT,BAMBOO_NUM_OF_CORE,0,to->base,0);
555 CACHEADAPT_SAMPLING_DATA_REVISE_INIT();
557 unsigned int filledblocks = 0;
558 unsigned int heaptopptr = 0;
559 bool localcompact = true;
560 compacthelper(orig, to, &filledblocks, &heaptopptr, &localcompact);
566 void compact_master(struct moveHelper * orig, struct moveHelper * to) {
567 // initialize pointers for comapcting
568 initOrig_Dst(orig, to);
569 CACHEADAPT_SAMPLING_DATA_REVISE_INIT();
570 int filledblocks = 0;
571 unsigned int heaptopptr = 0;
572 bool finishcompact = false;
573 bool iscontinue = true;
574 bool localcompact = true;
575 while((COMPACTPHASE == gcphase) || (SUBTLECOMPACTPHASE == gcphase)) {
576 if((!finishcompact) && iscontinue) {
577 finishcompact = compacthelper(orig,to,&filledblocks,&heaptopptr,&localcompact);
580 if(gc_checkCoreStatus()) {
581 // all cores have finished compacting restore the gcstatus of all cores
582 gc_resetCoreStatus();
585 // check if there are spare mem for pending move requires
586 if(COMPACTPHASE == gcphase) {
587 resolvePendingMoveRequest();
594 to->ptr = gcmovestartaddr;
595 to->numblocks = gcblock2fill - 1;
596 to->bound = BLOCKBOUND(to->numblocks);
597 BASEPTR(gcdstcore, to->numblocks, &(to->base));
598 to->offset = to->ptr - to->base;
599 to->top = (to->numblocks==0)?(to->offset):(to->bound-BAMBOO_SMEM_SIZE+to->offset);
601 to->offset = BAMBOO_CACHE_LINE_SIZE;
602 to->ptr += to->offset; // for header
603 to->top += to->offset;
604 localcompact = (gcdstcore == BAMBOO_NUM_OF_CORE);
607 } else if(!finishcompact) {
614 #endif // MULTICORE_GC