From: bdemsky Date: Wed, 27 Jul 2016 05:03:57 +0000 (-0700) Subject: change resize algorithm and slot usage X-Git-Url: http://demsky.eecs.uci.edu/git/?a=commitdiff_plain;h=1066f2fb29ce5133eaee35ee48a4c491190ccf13;p=iotcloud.git change resize algorithm and slot usage --- diff --git a/src/java/iotcloud/CloudComm.java b/src/java/iotcloud/CloudComm.java index 964b7c0..ac906b1 100644 --- a/src/java/iotcloud/CloudComm.java +++ b/src/java/iotcloud/CloudComm.java @@ -23,6 +23,7 @@ class CloudComm { SecureRandom random; static final int SALT_SIZE = 8; byte salt[]; + Table table; /** * Empty Constructor needed for child class. @@ -35,7 +36,8 @@ class CloudComm { * Constructor for actual use. Takes in the url and password. */ - CloudComm(String _baseurl, String _password) { + CloudComm(Table _table, String _baseurl, String _password) { + this.table=_table; this.baseurl=_baseurl; this.password = _password; this.random = new SecureRandom(); @@ -222,7 +224,7 @@ class CloudComm { data = decryptCipher.doFinal(data); - slots[i]=Slot.decode(data, mac); + slots[i]=Slot.decode(table, data, mac); } dis.close(); return slots; diff --git a/src/java/iotcloud/Slot.java b/src/java/iotcloud/Slot.java index ed577a6..c22d0d9 100644 --- a/src/java/iotcloud/Slot.java +++ b/src/java/iotcloud/Slot.java @@ -13,8 +13,6 @@ import java.util.Arrays; class Slot implements Liveness { /** Sets the slot size. */ static final int SLOT_SIZE=2048; - /** Sets how many bytes we reserve. */ - static final int RESERVED_SPACE=64; /** Sets the size for the HMAC. */ static final int HMAC_SIZE=32; @@ -35,8 +33,10 @@ class Slot implements Liveness { private boolean seqnumlive; /** Number of bytes of free space. */ private int freespace; + /** Reference to Table */ + private Table table; - Slot(long _seqnum, long _machineid, byte[] _prevhmac, byte[] _hmac) { + Slot(Table _table, long _seqnum, long _machineid, byte[] _prevhmac, byte[] _hmac) { seqnum=_seqnum; machineid=_machineid; prevhmac=_prevhmac; @@ -45,14 +45,15 @@ class Slot implements Liveness { livecount=1; seqnumlive=true; freespace = SLOT_SIZE - getBaseSize(); + table=_table; } - Slot(long _seqnum, long _machineid, byte[] _prevhmac) { - this(_seqnum, _machineid, _prevhmac, null); + Slot(Table _table, long _seqnum, long _machineid, byte[] _prevhmac) { + this(_table, _seqnum, _machineid, _prevhmac, null); } - Slot(long _seqnum, long _machineid) { - this(_seqnum, _machineid, new byte[HMAC_SIZE], null); + Slot(Table _table, long _seqnum, long _machineid) { + this(_table, _seqnum, _machineid, new byte[HMAC_SIZE], null); } byte[] getHMAC() { @@ -81,15 +82,6 @@ class Slot implements Liveness { * using its reserved space. */ boolean hasSpace(Entry e) { - int newfreespace = freespace - e.getSize(); - return newfreespace > RESERVED_SPACE; - } - - /** - * Returns true if the slot can fit the entry potentially using the - * reserved space. */ - - boolean canFit(Entry e) { int newfreespace = freespace - e.getSize(); return newfreespace >= 0; } @@ -98,7 +90,7 @@ class Slot implements Liveness { return entries; } - static Slot decode(byte[] array, Mac mac) { + static Slot decode(Table table, byte[] array, Mac mac) { mac.update(array, HMAC_SIZE, array.length-HMAC_SIZE); byte[] realmac=mac.doFinal(); @@ -113,7 +105,7 @@ class Slot implements Liveness { long seqnum=bb.getLong(); long machineid=bb.getLong(); int numentries=bb.getInt(); - Slot slot=new Slot(seqnum, machineid, prevhmac, hmac); + Slot slot=new Slot(table, seqnum, machineid, prevhmac, hmac); for(int i=0; i e=getLiveEntries(); + if (livecount==0) + table.decrementLiveCount(); } /** diff --git a/src/java/iotcloud/Table.java b/src/java/iotcloud/Table.java index 028025f..9e18553 100644 --- a/src/java/iotcloud/Table.java +++ b/src/java/iotcloud/Table.java @@ -5,6 +5,7 @@ import java.util.Iterator; import java.util.HashSet; import java.util.Arrays; import java.util.Vector; +import java.util.Random; /** * IoTTable data structure. Provides client inferface. @@ -12,7 +13,6 @@ import java.util.Vector; * @version 1.0 */ - final public class Table { private int numslots; private HashMap table=new HashMap(); @@ -24,20 +24,30 @@ final public class Table { private long localmachineid; private TableStatus lastTableStatus; static final int FREE_SLOTS = 10; - static final int FORCED_RESIZE_INCREMENT = 20; - + static final int SKIP_THRESHOLD = 10; + private long liveslotcount=0; + private int chance; + static final double RESIZE_MULTIPLE = 1.2; + static final double RESIZE_THRESHOLD = 0.75; + private int resizethreshold; + private long lastliveslotseqn; + private Random random; + public Table(String baseurl, String password, long _localmachineid) { localmachineid=_localmachineid; buffer = new SlotBuffer(); numslots = buffer.capacity(); + setResizeThreshold(); sequencenumber = 0; - cloud=new CloudComm(baseurl, password); + cloud=new CloudComm(this, baseurl, password); + lastliveslotseqn = 1; } public Table(CloudComm _cloud, long _localmachineid) { localmachineid=_localmachineid; buffer = new SlotBuffer(); numslots = buffer.capacity(); + setResizeThreshold(); sequencenumber = 0; cloud=_cloud; } @@ -63,7 +73,7 @@ final public class Table { public void initTable() { cloud.setSalt();//Set the salt - Slot s=new Slot(1, localmachineid); + Slot s=new Slot(this, 1, localmachineid); TableStatus status=new TableStatus(s, numslots); s.addEntry(status); Slot[] array=cloud.putSlot(s, numslots); @@ -92,37 +102,61 @@ final public class Table { } } - private boolean tryput(IoTString key, IoTString value, boolean forcedresize) { - Slot s=new Slot(sequencenumber+1, localmachineid, buffer.getSlot(sequencenumber).getHMAC()); - long seqn = buffer.getOldestSeqNum(); + void decrementLiveCount() { + liveslotcount--; + } + + private void setResizeThreshold() { + int resize_lower=(int) RESIZE_THRESHOLD * numslots; + resizethreshold=resize_lower-1+random.nextInt(numslots-resize_lower); + } + + private boolean tryput(IoTString key, IoTString value, boolean resize) { + Slot s=new Slot(this, sequencenumber+1, localmachineid, buffer.getSlot(sequencenumber).getHMAC()); + int newsize = 0; + if (liveslotcount > resizethreshold) { + resize=true; + newsize = (int) (numslots * RESIZE_MULTIPLE); + } - if (forcedresize) { - TableStatus status=new TableStatus(s, FORCED_RESIZE_INCREMENT + numslots); + + if (resize) { + newsize = (int) (numslots * RESIZE_MULTIPLE); + TableStatus status=new TableStatus(s, newsize); s.addEntry(status); } - if ((numslots - buffer.size()) < FREE_SLOTS) { - /* have to check whether we have enough free slots */ - long fullfirstseqn = buffer.getNewestSeqNum() + 1 - numslots; - seqn = fullfirstseqn < 1?1:fullfirstseqn; - for(int i=0; i < FREE_SLOTS; i++, seqn++) { - Slot prevslot=buffer.getSlot(seqn); - if (!prevslot.isLive()) - continue; - Vector liveentries = prevslot.getLiveEntries(); - for(Entry liveentry:liveentries) { - if (s.hasSpace(liveentry)) - s.addEntry(liveentry); - else if (i==0) { - if (s.canFit(liveentry)) - s.addEntry(liveentry); - else if (!forcedresize) { - return tryput(key, value, true); - } + long newestseqnum = buffer.getNewestSeqNum(); + long oldestseqnum = buffer.getOldestSeqNum(); + if (lastliveslotseqn < oldestseqnum) + lastliveslotseqn = oldestseqnum; + + long seqn = lastliveslotseqn; + boolean seenliveslot = false; + long firstiffull = newestseqnum + 1 - numslots; + long threshold = firstiffull + FREE_SLOTS; + + for(; seqn < threshold; seqn++) { + Slot prevslot=buffer.getSlot(seqn); + //Push slot number forward + if (!seenliveslot) + lastliveslotseqn = seqn; + + if (!prevslot.isLive()) + continue; + seenliveslot = true; + Vector liveentries = prevslot.getLiveEntries(); + for(Entry liveentry:liveentries) { + if (s.hasSpace(liveentry)) { + s.addEntry(liveentry); + } else if (seqn==firstiffull) { + if (!resize) { + return tryput(key, value, true); } } } } + KeyValue kv=new KeyValue(s, key, value); boolean insertedkv=false; if (s.hasSpace(kv)) { @@ -130,24 +164,32 @@ final public class Table { insertedkv=true; } - long newestseqnum=buffer.getNewestSeqNum(); -search: - for(; seqn<=newestseqnum; seqn++) { + int skipcount=0; + search: + for(; seqn <= newestseqnum; seqn++) { Slot prevslot=buffer.getSlot(seqn); + //Push slot number forward + if (!seenliveslot) + lastliveslotseqn = seqn; + if (!prevslot.isLive()) continue; + seenliveslot = true; Vector liveentries = prevslot.getLiveEntries(); for(Entry liveentry:liveentries) { if (s.hasSpace(liveentry)) s.addEntry(liveentry); - else - break search; + else { + skipcount++; + if (skipcount > SKIP_THRESHOLD) + break search; + } } } int max=0; - if (forcedresize) - max = numslots + FORCED_RESIZE_INCREMENT; + if (resize) + max = newsize; Slot[] array=cloud.putSlot(s, max); if (array == null) array = new Slot[] {s}; @@ -193,6 +235,7 @@ search: /* Commit new to slots. */ for(Slot slot:newslots) { buffer.putSlot(slot); + liveslotcount++; } sequencenumber = newslots[newslots.length - 1].getSequenceNumber(); } @@ -225,6 +268,7 @@ search: buffer.resize(currmaxsize); numslots=currmaxsize; + setResizeThreshold(); } private void processEntry(KeyValue entry, SlotIndexer indexer) {