From: navid Date: Wed, 18 Mar 2009 03:12:31 +0000 (+0000) Subject: *** empty log message *** X-Git-Url: http://demsky.eecs.uci.edu/git/?a=commitdiff_plain;h=bd4ec7df8583cfba6a2c1d8af26d5161e8de2a4f;p=IRC.git *** empty log message *** --- diff --git a/Robust/Transactions/tuplesoup/core/DualFileTable.java b/Robust/Transactions/tuplesoup/core/DualFileTable.java new file mode 100644 index 00000000..1aa30f62 --- /dev/null +++ b/Robust/Transactions/tuplesoup/core/DualFileTable.java @@ -0,0 +1,457 @@ +/* + * Copyright (c) 2007, Solido Systems + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * Neither the name of Solido Systems nor the names of its contributors may be + * used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ +package com.solidosystems.tuplesoup.core; + +import TransactionalIO.core.TransactionalFile; +import TransactionalIO.interfaces.IOOperations; +import java.io.*; +import java.util.*; +import java.nio.channels.*; +import com.solidosystems.tuplesoup.filter.*; +import dstm2.Configs; +import dstm2.SpecialTransactionalFile; +import dstm2.atomic; +import dstm2.Thread; +import dstm2.util.HashMap; +import dstm2.factory.Factory; +import java.util.concurrent.Callable; + +/** + * The table stores a group of rows. + * Every row must have a unique id within a table. + */ +public class DualFileTable implements Table { + + private int INDEXCACHESIZE = 8192; + private IOOperations fileastream = null; + private IOOperations filebstream = null; + private TableIndex index = null; + private FilePosition fileaposition; + private FilePosition filebposition; + private boolean rowswitch = true; + private String title; + private String location; + private TableIndexNode indexcachefirst; + private TableIndexNode indexcachelast; + private intField indexcacheusage; + //private Hashtable indexcache; + private HashMap indexcache; + static Factory filepositionfactory = Thread.makeFactory(FilePosition.class); + static Factory intfiledfactory = Thread.makeFactory(intField.class); + static Factory tableindexnodefactory = Thread.makeFactory(TableIndexNode.class); + + @atomic + public interface FilePosition { + + Long getFileposition(); + + void setFileposition(Long val); + } + + @atomic + public interface intField { + + int getintValue(); + + void setintValue(int val); + } + + /** + * Create a new table object with the default flat index model + */ + /** + * Create a new table object with a specific index model + */ + public DualFileTable(String title, String location, int indextype) throws IOException { + + fileaposition = filepositionfactory.create(); + filebposition = filepositionfactory.create(); + + fileaposition.setFileposition((long) 0); + filebposition.setFileposition((long) 0); + + indexcacheusage = intfiledfactory.create(); + + this.title = title; + this.location = location; + if (!this.location.endsWith(File.separator)) { + this.location += File.separator; + } + switch (indextype) { + case PAGED: + index = new PagedIndex(getFileName(INDEX)); + break; + + } + indexcachefirst = null; + indexcachelast = null; + indexcacheusage.setintValue(0); + indexcache = new HashMap(); + } + + /** + * Set the maximal allowable size of the index cache. + */ + public void setIndexCacheSize(int newsize) { + INDEXCACHESIZE = newsize; + } + + /** + * Close all open file streams + */ + public void close() { + try { + if (fileastream != null) { + fileastream.close(); + } + if (filebstream != null) { + filebstream.close(); + } + index.close(); + } catch (Exception e) { + } + } + + /** + * Returns the name of this table + */ + public String getTitle() { + return title; + } + + /** + * Returns the location of this tables datafiles + */ + public String getLocation() { + return location; + } + + protected String getFileName(int type) { + switch (type) { + case FILEB: + return location + title + ".a"; + case FILEA: + return location + title + ".b"; + case INDEX: + return location + title + ".index"; + } + return null; + } + + /** + * Delete the files created by this table object. + * Be aware that this will delete any data stored in this table! + */ + public void deleteFiles() { + try { + File ftest = new File(getFileName(FILEA)); + ftest.delete(); + } catch (Exception e) { + } + try { + File ftest = new File(getFileName(FILEB)); + ftest.delete(); + } catch (Exception e) { + } + try { + File ftest = new File(getFileName(INDEX)); + ftest.delete(); + } catch (Exception e) { + } + } + + private synchronized void openFile(final int type) throws IOException { + + switch (type) { + case FILEA: + if (fileastream == null) { + if (Configs.inevitable) + fileastream = new SpecialTransactionalFile/*RandomAccessFile*//*TransactionalFile*/(getFileName(FILEA), "rw"); + else + fileastream = new TransactionalFile(getFileName(FILEA), "rw"); + + File ftest = new File(getFileName(FILEA)); + fileaposition.setFileposition(ftest.length()); + fileastream.seek(fileaposition.getFileposition()); + + } + break; + case FILEB: + if (filebstream == null) { + if (Configs.inevitable) + filebstream = new SpecialTransactionalFile/*RandomAccessFile*//*TransactionalFile*/(getFileName(FILEB), "rw"); + else + filebstream = new TransactionalFile(getFileName(FILEB), "rw"); + + File ftest = new File(getFileName(FILEB)); + filebposition.setFileposition(ftest.length()); + filebstream.seek(filebposition.getFileposition()); + } + break; + } + + + } + + /** + * Adds a row of data to this table. + */ + public void addRow(Row row) throws IOException { + // Distribute new rows between the two datafiles by using the rowswitch, but don't spend time synchronizing... this does not need to be acurate! + + if (rowswitch) { + addRowA(row); + } else { + addRowB(row); + } + rowswitch = !rowswitch; + } + + private void addCacheEntry(TableIndexEntry entry) { + // synchronized (indexcache) { + if (indexcacheusage.getintValue() > INDEXCACHESIZE) { + // remove first entry + + TableIndexNode node = indexcachefirst; + indexcache.remove(node.getData().getId().hashCode()); + int tmp = indexcacheusage.getintValue(); + indexcacheusage.setintValue(tmp - 1); + // System.out.println("in the if " + Thread.currentThread()); + indexcachefirst = node.getNext(); + if (indexcachefirst == null) { + indexcachelast = null; + } else { + indexcachefirst.setPrevious(null); + } + } + // System.out.println("after first if " + Thread.currentThread() + " objetc " + Thread.getTransaction()); + //TableIndexNode node = new TableIndexNode(indexcachelast, entry); + TableIndexNode node = tableindexnodefactory.create(); + node.setPrevious(indexcachelast); + node.setData(entry); + node.setNext(null); + + if (indexcachelast != null) { + indexcachelast.setNext(node); + } + if (indexcachefirst == null) { + indexcachefirst = node; + } + indexcachelast = node; + indexcache.put(entry.getId().hashCode(), node); + + int tmp = indexcacheusage.getintValue(); + indexcacheusage.setintValue(tmp + 1); + + // } + } + + private void addRowA(final Row row) throws IOException { + // synchronized(objecta){ + openFile(FILEA); + Thread.doIt(new Callable() { + + public Boolean call() throws IOException { + + int pre = (int) fileastream.getFilePointer(); + + row.writeToStream(fileastream); + int post = (int) fileastream.getFilePointer(); + + index.addEntry(row.getId(), row.getSize(), FILEA, fileaposition.getFileposition()); + + if (INDEXCACHESIZE > 0) { + TableIndexEntry entry = new TableIndexEntry(row.getId(), row.getSize(), FILEA, fileaposition.getFileposition()); + addCacheEntry(entry); + } + long tmp = fileaposition.getFileposition(); + fileaposition.setFileposition(tmp + Row.calcSize(pre, post)); + return true; + } + }); + // } + } + + private void addRowB(final Row row) throws IOException { + //synchronized(objectb){ + openFile(FILEB); + Thread.doIt(new Callable() { + + public Boolean call() throws IOException { + + int pre = (int) filebstream.getFilePointer(); + + row.writeToStream(filebstream); + int post = (int) filebstream.getFilePointer(); + //filebstream.flush(); + // System.out.println(row); + + index.addEntry(row.getId(), row.getSize(), FILEB, filebposition.getFileposition()); + if (INDEXCACHESIZE > 0) { + TableIndexEntry entry = new TableIndexEntry(row.getId(), row.getSize(), FILEB, filebposition.getFileposition()); + addCacheEntry(entry); + } + long tmp = filebposition.getFileposition(); + filebposition.setFileposition(tmp + Row.calcSize(pre, post)); + return true; + } + }); + // } + } + + private TableIndexEntry getCacheEntry(final String id) { + // synchronized (indexcache) { + + if (indexcache.containsKey(id.hashCode())) { + TableIndexNode node = indexcache.get(id.hashCode()); + if (node != indexcachelast) { + if (node == indexcachefirst) { + indexcachefirst = node.getNext(); + } + //node.remove(); + remove(node); + indexcachelast.setNext(node); + node.setPrevious(indexcachelast); + node.setNext(null); + indexcachelast = node; + } + + return node.getData(); + } else { + return null; + } + + // } + } + + /** + * Returns a tuplestream containing the given list of rows + + public TupleStream getRows(List rows) throws IOException { + return new IndexedTableReader(this, index.scanIndex(rows)); + } + + /** + * Returns a tuplestream containing the rows matching the given rowmatcher + + public TupleStream getRows(RowMatcher matcher) throws IOException { + return new IndexedTableReader(this, index.scanIndex(), matcher); + }*/ + + /** + * Returns a tuplestream containing those rows in the given list that matches the given RowMatcher + + public TupleStream getRows(List rows, RowMatcher matcher) throws IOException { + return new IndexedTableReader(this, index.scanIndex(rows), matcher); + }*/ + + /** + * Returns a tuplestream of all rows in this table. + + public TupleStream getRows() throws IOException { + // return new TableReader(this); + return new IndexedTableReader(this, index.scanIndex()); + }*/ + + /** + * Returns a single row stored in this table. + * If the row does not exist in the table, null will be returned. + */ + public Row getRow(final String id) throws IOException { + TableIndexEntry entry = null; + // Handle index entry caching + if (INDEXCACHESIZE > 0) { + // synchronized(indexcache){ + entry = Thread.doIt(new Callable() { + + public TableIndexEntry call() throws IOException { + TableIndexEntry entry = getCacheEntry(id); + if (entry == null) { + entry = index.scanIndex(id); + if (entry != null) { + addCacheEntry(entry); + } + } + return entry; + } + }); + // } + } else { + entry = index.scanIndex(id); + } + if (entry != null) { + long dataoffset = 0; + DataInputStream data = null; + if (entry.location == Table.FILEA) { + data = new DataInputStream(new BufferedInputStream(new FileInputStream(getFileName(Table.FILEA)))); + } else if (entry.location == Table.FILEB) { + data = new DataInputStream(new BufferedInputStream(new FileInputStream(getFileName(Table.FILEB)))); + } + if (data != null) { + while (dataoffset != entry.position) { + dataoffset += data.skipBytes((int) (entry.position - dataoffset)); + } + Row row = Row.readFromStream(data); + data.close(); + + + return row; + } + + } + return null; + } + + public void remove(TableIndexNode node) { + if (node.getPrevious() != null) { + node.getPrevious().setNext(node.getNext()); + } + if (node.getNext() != null) { + node.getNext().setPrevious(node.getPrevious()); + } + } + + public TupleStream getRows(List rows) throws IOException { + throw new UnsupportedOperationException("Not supported yet."); + } + + public TupleStream getRows(RowMatcher matcher) throws IOException { + throw new UnsupportedOperationException("Not supported yet."); + } + + public TupleStream getRows(List rows, RowMatcher matcher) throws IOException { + throw new UnsupportedOperationException("Not supported yet."); + } + + public TupleStream getRows() throws IOException { + throw new UnsupportedOperationException("Not supported yet."); + } +} \ No newline at end of file diff --git a/Robust/Transactions/tuplesoup/core/PagedIndex.java b/Robust/Transactions/tuplesoup/core/PagedIndex.java new file mode 100644 index 00000000..3c9dd6d5 --- /dev/null +++ b/Robust/Transactions/tuplesoup/core/PagedIndex.java @@ -0,0 +1,125 @@ +/* + * Copyright (c) 2007, Solido Systems + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * Neither the name of Solido Systems nor the names of its contributors may be + * used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +package com.solidosystems.tuplesoup.core; + +import TransactionalIO.core.TransactionalFile; +import TransactionalIO.interfaces.IOOperations; +import dstm2.Configs; +import dstm2.SpecialTransactionalFile; +import java.io.*; +import java.util.*; +import java.nio.channels.*; + +public class PagedIndex implements TableIndex{ + protected static final int INITIALPAGEHASH=1024; + protected static final int PAGESIZE=2048; + + //private RandomAccessFile out=null; + private IOOperations out=null; + private String filename; + private TableIndexPage[] root=null; + + + public PagedIndex(String filename) throws IOException{ + this.filename=filename; + File ftest=new File(filename); + if(!ftest.exists())ftest.createNewFile(); + //out=new RandomAccessFile(filename,"rw"); + if (Configs.inevitable) + out=new SpecialTransactionalFile(filename,"rw"); + else + out=new TransactionalFile(filename,"rw"); + + root=new TableIndexPage[INITIALPAGEHASH]; + if(out.length()>0){ + for(int i=0;i readStatistics() { + throw new UnsupportedOperationException("Not supported yet."); + } + + public void updateEntry(String id, int rowsize, int location, long position) throws IOException { + throw new UnsupportedOperationException("Not supported yet."); + } + + public List scanIndex(List rows) throws IOException { + throw new UnsupportedOperationException("Not supported yet."); + } + + public List scanIndex() throws IOException { + throw new UnsupportedOperationException("Not supported yet."); + } +} \ No newline at end of file diff --git a/Robust/Transactions/tuplesoup/core/Row.java b/Robust/Transactions/tuplesoup/core/Row.java new file mode 100644 index 00000000..4f4454b5 --- /dev/null +++ b/Robust/Transactions/tuplesoup/core/Row.java @@ -0,0 +1,458 @@ +/* + * Copyright (c) 2007, Solido Systems + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * Neither the name of Solido Systems nor the names of its contributors may be + * used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ +package com.solidosystems.tuplesoup.core; + +import TransactionalIO.interfaces.IOOperations; +import java.io.*; +import java.util.*; + +/** + * Holds a row of data + */ +public class Row { + + private String id; + private int size; + private Hashtable values; + + /** + * Creates a new empty row with the given row id. + */ + public Row(String id) { + this.id = id; + size = -1; + values = new Hashtable(); + } + + /** + * Returns the number of keys in this row. + */ + public int getKeyCount() { + return values.size(); + } + + public Set keySet() { + return values.keySet(); + } + + /** + * Returns the actual size in bytes this row will take when written to a stream. + */ + public int getSize() { + if (size == -1) { + recalcSize(); + } + return size; + } + + /** + * Returns a hashcode for this row. This hashcode will be based purely on the id of the row. + */ + public int hashCode() { + return id.hashCode(); + } + + public boolean equals(Object obj) { + try { + Row r = (Row) obj; + return r.id.equals(id); + } catch (Exception e) { + } + return false; + } + + /** + * Returns the id of this row. + */ + public String getId() { + return id; + } + + /** + * Stores the given value for the given key. + */ + public void put(String key, Value value) { + size = -1; + values.put(key, value); + } + + /** + * Stores the given string wrapped in a value object for the given key. + */ + public void put(String key, String value) { + size = -1; + values.put(key, new Value(value)); + } + + /** + * Stores the given int wrapped in a value object for the given key. + */ + public void put(String key, int value) { + size = -1; + values.put(key, new Value(value)); + } + + /** + * Stores the given long wrapped in a value object for the given key. + */ + public void put(String key, long value) { + size = -1; + values.put(key, new Value(value)); + } + + /** + * Stores the given float wrapped in a value object for the given key. + */ + public void put(String key, float value) { + size = -1; + values.put(key, new Value(value)); + } + + /** + * Stores the given double wrapped in a value object for the given key. + */ + public void put(String key, double value) { + size = -1; + values.put(key, new Value(value)); + } + + /** + * Stores the given boolean wrapped in a value object for the given key. + */ + public void put(String key, boolean value) { + size = -1; + values.put(key, new Value(value)); + } + + /** + * Stores the given Date wrapped in a value object for the given key. + */ + public void put(String key, Date value) { + size = -1; + values.put(key, new Value(value)); + } + + /** + * Returns the value stored for the current key, or a null value (not null) if the key does not exist. + */ + public Value get(String key) { + if (!values.containsKey(key)) { + return new Value(); + } + return values.get(key); + } + + /** + * Returns a string representation of the value stored for the current key. + * If the key does not exist, an empty string will be returned. + * See the documentation for Value to learn how the string value is generated. + */ + public String getString(String key) { + if (!values.containsKey(key)) { + return ""; + } + return values.get(key).getString(); + } + + /** + * Returns an int representation of the value stored for the current key. + * If the key does not exist, 0 will be returned. + * See the documentation for Value to learn how the string value is generated. + */ + public int getInt(String key) { + if (!values.containsKey(key)) { + return 0; + } + return values.get(key).getInt(); + } + + /** + * Returns a long representation of the value stored for the current key. + * If the key does not exist, 0 will be returned. + * See the documentation for Value to learn how the string value is generated. + */ + public long getLong(String key) { + if (!values.containsKey(key)) { + return 0; + } + return values.get(key).getLong(); + } + + /** + * Returns a float representation of the value stored for the current key. + * If the key does not exist, 0 will be returned. + * See the documentation for Value to learn how the string value is generated. + */ + public float getFloat(String key) { + if (!values.containsKey(key)) { + return 0f; + } + return values.get(key).getFloat(); + } + + /** + * Returns a double representation of the value stored for the current key. + * If the key does not exist, 0 will be returned. + * See the documentation for Value to learn how the string value is generated. + */ + public double getDouble(String key) { + if (!values.containsKey(key)) { + return 0d; + } + return values.get(key).getDouble(); + } + + /** + * Returns a boolean representation of the value stored for the current key. + * If the key does not exist, false will be returned. + * See the documentation for Value to learn how the string value is generated. + */ + public boolean getBoolean(String key) { + if (!values.containsKey(key)) { + return false; + } + return values.get(key).getBoolean(); + } + + /** + * Returns a Date representation of the value stored for the current key. + * If the key does not exist, the date initialized with 0 will be returned. + * See the documentation for Value to learn how the string value is generated. + */ + public Date getTimestamp(String key) { + if (!values.containsKey(key)) { + return new Date(0); + } + return values.get(key).getTimestamp(); + } + + /** + * Utility function to calculate the distance between ints, allowing for a single wraparound. + */ + protected static int calcSize(int pre, int post) { + if (post > pre) { + return post - pre; + } + return (Integer.MAX_VALUE - pre) + post; + } + + /** + * Recalculate the size of the row. Be aware that this method will actually write the full row to a buffer to calculate the size. + * Its a slow and memory consuming method to call! + */ + private void recalcSize() { + try { + ByteArrayOutputStream bout = new ByteArrayOutputStream(); + DataOutputStream dout = new DataOutputStream(bout); + writeToStream(dout); + size = bout.size(); + dout.close(); + bout.close(); + } catch (Exception e) { + } + } + + /** + * Writes the contents of this row to the given RandomAccessFile + */ + public void writeToFile(RandomAccessFile out) throws IOException { + long pre = out.getFilePointer(); + + out.writeUTF(id); + + Set keys = values.keySet(); + out.writeInt(keys.size()); + Iterator it = keys.iterator(); + while (it.hasNext()) { + String key = it.next(); + Value value = values.get(key); + out.writeUTF(key); + value.writeToFile(out); + } + long post = out.getFilePointer(); + int size = (int) (post - pre); + this.size = size + 4; + out.writeInt(this.size); + } + + /** + * Writes the contents of this row to the given DataOutputStream. + */ + public void writeToStream(DataOutputStream out) throws IOException { + int pre = out.size(); + out.writeUTF(id); + Set keys = values.keySet(); + out.writeInt(keys.size()); + Iterator it = keys.iterator(); + while (it.hasNext()) { + String key = it.next(); + Value value = values.get(key); + out.writeUTF(key); + value.writeToStream(out); + } + int post = out.size(); + int size = calcSize(pre, post); + this.size = size + 4; + out.writeInt(this.size); + } + + public void writeToStream(RandomAccessFile out) throws IOException { + int pre = (int) out.getFilePointer(); + out.writeUTF(id); + Set keys = values.keySet(); + out.writeInt(keys.size()); + Iterator it = keys.iterator(); + while (it.hasNext()) { + String key = it.next(); + Value value = values.get(key); + out.writeUTF(key); + value.writeToStream(out); + } + int post = (int) out.getFilePointer(); + int size = calcSize(pre, post); + this.size = size + 4; + out.writeInt(this.size); + } + + public void writeToStream(IOOperations out) throws IOException { + int pre = (int) out.getFilePointer(); + out.writeUTF(id); + Set keys = values.keySet(); + out.writeInt(keys.size()); + Iterator it = keys.iterator(); + while (it.hasNext()) { + String key = it.next(); + Value value = values.get(key); + out.writeUTF(key); + value.writeToStream(out); + } + int post = (int) out.getFilePointer(); + int size = calcSize(pre, post); + this.size = size + 4; + out.writeInt(this.size); + } + + /** + * Reads a full row from the given DataInputStream and returns it. + */ + public static Row readFromStream(DataInputStream in) throws IOException { + String id = in.readUTF(); + Row row = new Row(id); + int size = in.readInt(); + for (int i = 0; i < size; i++) { + String key = in.readUTF(); + Value value = Value.readFromStream(in); + row.put(key, value); + } + size = in.readInt(); + row.size = size; + return row; + } + + /** + * Returns a string representing this row formatted as the following example: + * (1732)=>{"name":string:"Kasper J. Jeppesen","age":int:31} + * + * @return a string representation of this row + */ + public String toString() { + StringBuffer buf = new StringBuffer(); + buf.append("(" + id + ")=>{"); + Iterator it = values.keySet().iterator(); + boolean first = true; + while (it.hasNext()) { + if (!first) { + buf.append(","); + } else { + first = false; + } + String key = it.next(); + buf.append("\""); + buf.append(key); + buf.append("\":"); + Value value = values.get(key); + buf.append(value.getTypeName()); + buf.append(":"); + if (value.getType() == Value.STRING) { + buf.append("\""); + // TODO: This string should be escaped properly + buf.append(value.getString()); + buf.append("\""); + } else { + buf.append(value.getString()); + } + } + buf.append("}"); + return buf.toString(); + } + + /** + * Shorthand for calling toBasicXMLString("") + */ + public String toBasicXMLString() { + return toBasicXMLString(""); + } + + /** + * Creates an indentation of the given size and calls toBasicXMLString(String) with the indentation string as parameter. + */ + public String toBasicXMLString(int indentation) { + StringBuffer buf = new StringBuffer(); + for (int i = 0; i < indentation; i++) { + buf.append(" "); + } + return toBasicXMLString(buf.toString()); + } + + /** + * Creates a basic xml representation of the row as shown in the following sample: + * <row id="1"> + * <value name="foo" type="string">Bar</value> + * </row> + */ + public String toBasicXMLString(String indentation) { + StringBuffer buf = new StringBuffer(); + buf.append(indentation); + buf.append("\n"); + Iterator it = values.keySet().iterator(); + while (it.hasNext()) { + String key = it.next(); + Value value = values.get(key); + buf.append(indentation); + buf.append(" "); + buf.append(value.toBasicXMLString(key)); + buf.append("\n"); + } + buf.append(indentation); + buf.append("\n"); + return buf.toString(); + } +} \ No newline at end of file diff --git a/Robust/Transactions/tuplesoup/core/RowMatcher.java b/Robust/Transactions/tuplesoup/core/RowMatcher.java new file mode 100644 index 00000000..1debd7ba --- /dev/null +++ b/Robust/Transactions/tuplesoup/core/RowMatcher.java @@ -0,0 +1,108 @@ +/* + * Copyright (c) 2007, Solido Systems + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * Neither the name of Solido Systems nor the names of its contributors may be + * used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + + package com.solidosystems.tuplesoup.core; + + import java.io.*; + import java.util.*; + +public class RowMatcher{ + public final static int LESSTHAN=0; + public final static int EQUALS=1; + public final static int GREATERTHAN=2; + public final static int STARTSWITH=3; + public final static int ENDSWITH=4; + public final static int CONTAINS=5; + public final static int ISNULL=6; + + public final static int NOT=7; + public final static int OR=8; + public final static int AND=9; + public final static int XOR=10; + + private String key=null; + private Value value=null; + private int type=-1; + + private RowMatcher match1=null; + private RowMatcher match2=null; + + public RowMatcher(String key,int type,Value value){ + this.key=key; + this.type=type; + this.value=value; + } + + public RowMatcher(String key,int type){ + this.key=key; + this.type=type; + } + + public RowMatcher(RowMatcher match1,int type,RowMatcher match2){ + this.match1=match1; + this.type=type; + this.match2=match2; + } + + public RowMatcher(int type,RowMatcher match1){ + this.match1=match1; + this.type=type; + } + + /** + * This method needs to be seriously optimized... especially the XOR method + */ + public boolean matches(Row row){ + if(value!=null){ + Value compare=row.get(key); + switch(type){ + case LESSTHAN : return compare.lessThan(value); + case EQUALS : return compare.equals(value); + case GREATERTHAN: return compare.greaterThan(value); + case STARTSWITH : return compare.startsWith(value); + case ENDSWITH : return compare.endsWith(value); + case CONTAINS : return compare.contains(value); + } + }else if(type==ISNULL){ + Value compare=row.get(key); + return compare.isNull(); + }else if((type==AND)||(type==OR)||(type==XOR)){ + switch(type){ + case AND : return match1.matches(row)&&match2.matches(row); + case OR : return match1.matches(row)||match2.matches(row); + case XOR : return (match1.matches(row)||match2.matches(row))&&(!(match1.matches(row)&&match2.matches(row))); + } + }else if(type==NOT){ + return !match1.matches(row); + } + return false; + } +} \ No newline at end of file diff --git a/Robust/Transactions/tuplesoup/core/Table.java b/Robust/Transactions/tuplesoup/core/Table.java new file mode 100644 index 00000000..74c20742 --- /dev/null +++ b/Robust/Transactions/tuplesoup/core/Table.java @@ -0,0 +1,113 @@ +/* + * Copyright (c) 2007, Solido Systems + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * Neither the name of Solido Systems nor the names of its contributors may be + * used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +package com.solidosystems.tuplesoup.core; + +import java.io.*; +import java.util.*; +import java.nio.channels.*; +import com.solidosystems.tuplesoup.filter.*; + +/** + * The table stores a group of rows. + * Every row must have a unique id within a table. + */ +public interface Table{ + // Index type constants + public static final int MEMORY=0; + public static final int FLAT=1; + public static final int PAGED=2; + + // Row location constants + public static final int FILEA=0; + public static final int FILEB=1; + public static final int DELETE=2; + public static final int INDEX=3; + + + /** + * Set the maximal allowable size of the index cache. + */ + public void setIndexCacheSize(int newsize); + + /** + * Close all open file streams + */ + public void close(); + + /** + * Returns the name of this table + */ + public String getTitle(); + + /** + * Returns the location of this tables datafiles + */ + public String getLocation(); + + /** + * Delete the files created by this table object. + * Be aware that this will delete any data stored in this table! + */ + public void deleteFiles(); + + /** + * Adds a row of data to this table. + */ + public void addRow(Row row) throws IOException; + + + /** + * Returns a tuplestream containing the given list of rows + */ + public TupleStream getRows(List rows) throws IOException; + + /** + * Returns a tuplestream containing the rows matching the given rowmatcher + */ + public TupleStream getRows(RowMatcher matcher) throws IOException; + + /** + * Returns a tuplestream containing those rows in the given list that matches the given RowMatcher + */ + public TupleStream getRows(List rows,RowMatcher matcher) throws IOException; + + /** + * Returns a tuplestream of all rows in this table. + */ + public TupleStream getRows() throws IOException; + + /** + * Returns a single row stored in this table. + * If the row does not exist in the table, null will be returned. + */ + public Row getRow(String id) throws IOException; + } \ No newline at end of file diff --git a/Robust/Transactions/tuplesoup/core/TableIndex.java b/Robust/Transactions/tuplesoup/core/TableIndex.java new file mode 100644 index 00000000..ee5363ee --- /dev/null +++ b/Robust/Transactions/tuplesoup/core/TableIndex.java @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2007, Solido Systems + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * Neither the name of Solido Systems nor the names of its contributors may be + * used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +package com.solidosystems.tuplesoup.core; + +import java.util.*; +import java.io.*; + +public interface TableIndex{ + //public Hashtable readStatistics(); + //public void updateEntry(String id,int rowsize,int location,long position) throws IOException; + public void addEntry(String id,int rowsize,int location,long position) throws IOException; + public TableIndexEntry scanIndex(String id) throws IOException; + public List scanIndex(List rows) throws IOException; + public List scanIndex() throws IOException; + public void close(); +} \ No newline at end of file diff --git a/Robust/Transactions/tuplesoup/core/TableIndexEntry.java b/Robust/Transactions/tuplesoup/core/TableIndexEntry.java new file mode 100644 index 00000000..501faf62 --- /dev/null +++ b/Robust/Transactions/tuplesoup/core/TableIndexEntry.java @@ -0,0 +1,166 @@ +/* + * Copyright (c) 2007, Solido Systems + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * Neither the name of Solido Systems nor the names of its contributors may be + * used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +package com.solidosystems.tuplesoup.core; + +import TransactionalIO.interfaces.IOOperations; +import java.io.*; + +public class TableIndexEntry implements Comparable{ + public String id; + public int location; + public long position; + private int size; + private int rowsize; + + public TableIndexEntry(String id,int rowsize,int location,long position){ + this.id=id; + this.location=location; + this.position=position; + this.rowsize=rowsize; + size=-1; + } + public String getId(){ + return id; + } + + public long getPosition(){ + return position; + } + + public int getLocation(){ + return location; + } + + public int getRowSize(){ + return rowsize; + } + + public int compareTo(TableIndexEntry obj) throws ClassCastException{ + TableIndexEntry ent=(TableIndexEntry)obj; + if(position factory = Thread.makeFactory(TableIndexPageTSInf.class); + TableIndexPageTSInf tableindexpage; + + @atomic public interface TableIndexPageTSInf{ + Long getNext(); + Long getLower(); + Long getLocation(); + int getSize(); + int getOffset(); + int getStarthash(); + int getEndhash(); + boolean getFirst(); + TableIndexPage getNextpage(); + TableIndexPage getLowerpage(); + + + void setLowerpage(TableIndexPage lowerpage); + void setNextpage(TableIndexPage nextpage); + void setFirst(boolean val); + void setEndhash(int val); + void setStarthash(int val); + void setOffset(int offset); + void setNext(Long next); + void setSize(int size); + void setLocation(Long location); + void setLower(Long val); + } + + + public TableIndexPage(PagedIndex index, IOOperations/*RandomAccessFile*/ file) throws IOException{ + this.file=file; + this.index=index; + tableindexpage = factory.create(); + tableindexpage.setFirst(false); + tableindexpage.setLocation(file.getFilePointer()); + tableindexpage.setSize(file.readInt()); + tableindexpage.setNext(file.readLong()); + tableindexpage.setLower(file.readLong()); + tableindexpage.setOffset(file.readInt()); + tableindexpage.setEndhash(file.readInt()); + if(tableindexpage.getOffset()>0) { + tableindexpage.setStarthash(file.readInt()); + } + } + + public static TableIndexPage createNewPage(PagedIndex index, IOOperations file,int size) throws IOException{ + long pre=file.length(); + file.seek(pre); + byte[] dummy = new byte[size+BASEOFFSET]; + file.write(dummy); + //file.setLength(file.length()+size+BASEOFFSET); + file.seek(pre); + file.writeInt(size); + file.writeLong(-1l); + file.writeLong(-1l); + file.writeInt(0); + file.writeInt(-1); + file.seek(pre); + + return new TableIndexPage(index,file); + } + + + + public void setFirst(){ + tableindexpage.setFirst(true); + } + + public long getLocation(){ + return tableindexpage.getLocation(); + } + public long getEndLocation(){ + return tableindexpage.getLocation()+tableindexpage.getSize()+BASEOFFSET; + } + + public String toString(){ + StringBuffer buf=new StringBuffer(); + buf.append("{\n"); + buf.append(" location "+tableindexpage.getLocation()+"\n"); + buf.append(" size "+tableindexpage.getSize()+"\n"); + buf.append(" next "+tableindexpage.getNext()+"\n"); + buf.append(" lower "+tableindexpage.getLower()+"\n"); + buf.append(" offset "+tableindexpage.getOffset()+"\n"); + buf.append(" starthash "+tableindexpage.getStarthash()+"\n"); + buf.append(" endhash "+tableindexpage.getEndhash()+"\n"); + buf.append("}\n"); + return buf.toString(); + } + + private void updateMeta() throws IOException{ + file.seek(tableindexpage.getLocation()); + file.writeInt(tableindexpage.getSize()); + file.writeLong(tableindexpage.getNext()); + file.writeLong(tableindexpage.getLower()); + file.writeInt(tableindexpage.getOffset()); + file.writeInt(tableindexpage.getEndhash()); + } + + public TableIndexEntry scanIndex(String id,int hashcode) throws IOException{ + if(!tableindexpage.getFirst()){ + if(hashcodetableindexpage.getEndhash()){ + if(tableindexpage.getNext()==-1)return null; + if(tableindexpage.getNextpage()==null){ + file.seek(tableindexpage.getNext()); + tableindexpage.setNextpage(new TableIndexPage(index,file)); + } + // index.stat_page_next++; + return tableindexpage.getNextpage().scanIndex(id,hashcode); + } + file.seek(tableindexpage.getLocation()+BASEOFFSET); + long pre=file.getFilePointer(); + while(file.getFilePointer()id.length()*2+4+4+8+1+2)return this; + // Check next + if(tableindexpage.getNext()==-1){ + tableindexpage.setNext(file.length()); + updateMeta(); + return createNewPage(index,file,PagedIndex.PAGESIZE); + } + if(tableindexpage.getNextpage()==null){ + file.seek(tableindexpage.getNext()); + tableindexpage.setNextpage(new TableIndexPage(index,file)); + } + + return tableindexpage.getNextpage().getFirstFreePage(id,hashcode); + } + + public void addEntry(String id,int rowsize,int location,long position) throws IOException{ + if(tableindexpage.getOffset()==0) tableindexpage.setStarthash(id.hashCode()); + file.seek(this.tableindexpage.getLocation()+BASEOFFSET+tableindexpage.getOffset()); + TableIndexEntry entry=new TableIndexEntry(id,rowsize,location,position); + entry.writeData(file); + tableindexpage.setOffset(tableindexpage.getOffset()+entry.getSize()); + if(id.hashCode()>tableindexpage.getEndhash())tableindexpage.setEndhash(id.hashCode()); + updateMeta(); + } +} \ No newline at end of file diff --git a/Robust/Transactions/tuplesoup/core/TupleStream.java b/Robust/Transactions/tuplesoup/core/TupleStream.java new file mode 100644 index 00000000..f6a7e81d --- /dev/null +++ b/Robust/Transactions/tuplesoup/core/TupleStream.java @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2007, Solido Systems + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * Neither the name of Solido Systems nor the names of its contributors may be + * used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +package com.solidosystems.tuplesoup.core; + +import java.io.*; + +public abstract class TupleStream{ + public abstract boolean hasNext() throws IOException; + public abstract Row next() throws IOException; +} \ No newline at end of file diff --git a/Robust/Transactions/tuplesoup/core/TupleStreamMerger.java b/Robust/Transactions/tuplesoup/core/TupleStreamMerger.java new file mode 100644 index 00000000..8d71ce4c --- /dev/null +++ b/Robust/Transactions/tuplesoup/core/TupleStreamMerger.java @@ -0,0 +1,74 @@ +/* + * Copyright (c) 2007, Solido Systems + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * Neither the name of Solido Systems nor the names of its contributors may be + * used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +package com.solidosystems.tuplesoup.core; + +import java.io.*; +import java.util.*; + +public class TupleStreamMerger extends TupleStream{ + private List streams; + private TupleStream current=null; + private Row next=null; + + public TupleStreamMerger(){ + streams=new ArrayList(); + } + + public void addStream(TupleStream stream){ + streams.add(stream); + } + + public boolean hasNext() throws IOException{ + if(next!=null)return true; + if(current==null){ + if(streams.size()>0){ + current=streams.remove(0); + }else return false; + } + if(current.hasNext()){ + next=current.next(); + return true; + }else{ + current=null; + return hasNext(); + } + } + + public Row next() throws IOException{ + if(next==null)hasNext(); + Row tmp=next; + next=null; + return tmp; + } + + +} \ No newline at end of file diff --git a/Robust/Transactions/tuplesoup/core/Value.java b/Robust/Transactions/tuplesoup/core/Value.java new file mode 100644 index 00000000..db86b0a7 --- /dev/null +++ b/Robust/Transactions/tuplesoup/core/Value.java @@ -0,0 +1,726 @@ +/* + * Copyright (c) 2007, Solido Systems + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * Neither the name of Solido Systems nor the names of its contributors may be + * used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + + package com.solidosystems.tuplesoup.core; + +import TransactionalIO.core.TransactionalFile; +import TransactionalIO.interfaces.IOOperations; +import java.util.*; +import java.io.*; + + /** + * The Value class holds a single data value. + * Current size estimate without string data: 8+4+4+8+8 = 32 bytes pr value in mem + */ + public class Value{ + public final static int NULL=0; + public final static int STRING=1; + public final static int INT=2; + public final static int LONG=3; + public final static int FLOAT=4; + public final static int DOUBLE=5; + public final static int BOOLEAN=6; + public final static int TIMESTAMP=7; + public final static int BINARY=8; + + private byte type=NULL; // The type of the value being held + private String str_value=null; + private long int_value=0; + private double float_value=0.0; + private byte[] binary=null; + + /** + * Returns the numerical type id for this value. + */ + public int getType(){ + return type; + } + + /** + * Returns the name this value's type. + */ + public String getTypeName(){ + switch(type){ + case STRING : return "string"; + case INT : return "int"; + case LONG : return "long"; + case FLOAT : return "float"; + case DOUBLE : return "double"; + case BOOLEAN : return "boolean"; + case TIMESTAMP : return "timestamp"; + case BINARY : return "binary"; + } + return "null"; + } + + /** + * An implementation of the hashCode method as defined in java.lang.Object + * + * @return a hash code value for this object + */ + public int hashCode(){ + int hash=0; + switch(type){ + case STRING : hash+=str_value.hashCode(); + case INT : hash+=(int)int_value; + case LONG : hash+=(int)int_value; + case FLOAT : hash+=(int)float_value; + case DOUBLE : hash+=(int)float_value; + case BOOLEAN : hash+=(int)int_value; + case TIMESTAMP : hash+=(int)int_value; + case BINARY : hash+=binary.hashCode(); + } + return hash; + } + + /** + * Returns true only if this Value has specifically been set to null. + * + * @return true if the data being held is null, false otherwise + */ + public boolean isNull(){ + return type==NULL; + } + + /** + * Returns -1, 0 or 1 if this value is smaller, equal or larger than the value given as a parameter. + */ + public int compareTo(Value value){ + if(type==STRING){ + return str_value.compareTo(value.getString()); + } + if(lessThan(value))return -1; + if(greaterThan(value))return 1; + return 0; + } + + /** + * Attempts to compare this Value to the value given as parameter and returns true if this value is less than the value given as a parameter. + * The types used for the comparison will always be based on the type of this Value based on the following rules. + *
    + *
  • If this Value is a numeric type, then the other Value will be asked to deliver the same numeric type for the comparison. + *
  • If this Value is a string, then both values will be asked to deliver a double value for the comparison. + *
  • If this Value is a timestamp, then both values will be asked to deliver a long value for the comparison. + *
  • If this Value is a boolean, false will be returned. + *
+ * + * @param value the value this value should be compared to + * @return true if this value is less than the value given as a parameter, false otherwise + */ + public boolean lessThan(Value value){ + switch(type){ + case STRING : return getDouble() + *
  • If this Value is a numeric type, then the other Value will be asked to deliver the same numeric type for the comparison. + *
  • If this Value is a string, then both values will be asked to deliver a double value for the comparison. + *
  • If this Value is a timestamp, then both values will be asked to deliver a long value for the comparison. + *
  • If this Value is a boolean, false will be returned. + * + * + * @param value the value this value should be compared to + * @return true if this value is greater than the value given as a parameter, false otherwise + */ + public boolean greaterThan(Value value){ + switch(type){ + case STRING : return getDouble()>value.getDouble(); + case INT : return getInt()>value.getInt(); + case LONG : return getLong()>value.getLong(); + case FLOAT : return getFloat()>value.getFloat(); + case DOUBLE : return getDouble()>value.getDouble(); + case TIMESTAMP : return getLong()>value.getLong(); + } + return false; + } + + /** + * Returns true if the string representation of this value starts with the string representation of the value given as parameter. + */ + public boolean startsWith(Value value){ + return getString().startsWith(value.getString()); + } + + /** + * Returns true if the string representation of this value ends with the string representation of the value given as parameter. + */ + public boolean endsWith(Value value){ + return getString().endsWith(value.getString()); + } + + /** + * Returns true if the string representation of this value contains the string representation of the value given as parameter. + */ + public boolean contains(Value value){ + return getString().indexOf(value.getString())>-1; + } + + /** + * Returns true if the contents of this value equals the contents of the value given as parameter. + */ + public boolean equals(Object obj){ + try{ + Value val=(Value)obj; + if(val.type==type){ + switch(type){ + case NULL : return true; + case STRING : return str_value.equals(val.str_value); + case INT : return int_value==val.int_value; + case LONG : return int_value==val.int_value; + case FLOAT : return float_value==val.float_value; + case DOUBLE : return float_value==val.float_value; + case BOOLEAN : return int_value==val.int_value; + case TIMESTAMP : return int_value==val.int_value; + case BINARY : if(binary.length==val.binary.length){ + for(int i=0;i"+str_value+""; + case INT : return ""+int_value+""; + case LONG : return ""+int_value+""; + case FLOAT : return ""+float_value+""; + case DOUBLE : return ""+float_value+""; + case BOOLEAN : if(int_value==1){ + return "TRUE"; + }else{ + return "FALSE"; + } + case TIMESTAMP : return ""+new Date(int_value).toString()+""; + case BINARY : return ""+getString()+""; + } + return ""; + } + + /** + * Returns this value as an xml tag. + * The following string is an example of the int value 1234 <value type="int">1234</value> + */ + public String toBasicXMLString(){ + switch(type){ + case STRING : return ""+str_value+""; + case INT : return ""+int_value+""; + case LONG : return ""+int_value+""; + case FLOAT : return ""+float_value+""; + case DOUBLE : return ""+float_value+""; + case BOOLEAN : if(int_value==1){ + return "TRUE"; + }else{ + return "FALSE"; + } + case TIMESTAMP : return ""+new Date(int_value).toString()+""; + case BINARY : return ""+getString()+""; + } + return ""; + } + + /** + * Returns a string representation of this value + */ + public String getString(){ + switch(type){ + case STRING : return str_value; + case INT : return ""+int_value; + case LONG : return ""+int_value; + case FLOAT : return ""+float_value; + case DOUBLE : return ""+float_value; + case BOOLEAN : if(int_value==1){ + return "TRUE"; + }else{ + return "FALSE"; + } + case TIMESTAMP : return new Date(int_value).toString(); + case BINARY : StringBuffer buf=new StringBuffer(); + for(int i=0;i lst){ + + } + public abstract boolean hasNext(); + public abstract Row next(); +} \ No newline at end of file diff --git a/Robust/Transactions/tuplesoup/filter/JavaSort.java b/Robust/Transactions/tuplesoup/filter/JavaSort.java new file mode 100644 index 00000000..1dfb4a1f --- /dev/null +++ b/Robust/Transactions/tuplesoup/filter/JavaSort.java @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2007, Solido Systems + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * Neither the name of Solido Systems nor the names of its contributors may be + * used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +package com.solidosystems.tuplesoup.filter; + +import com.solidosystems.tuplesoup.core.*; +import java.util.*; +import java.io.*; + +public class JavaSort extends Sort{ + private List sortlst; + private int offset; + private int datasize; + + public JavaSort(){ + } + public void initialize(String filename,TupleStream source,List lst) throws IOException{ + sortlst=new ArrayList(); + offset=0; + while(source.hasNext()){ + sortlst.add(source.next()); + } + Collections.sort(sortlst,new SortComparator(lst)); + datasize=sortlst.size(); + } + public boolean hasNext(){ + if(offset buffers) throws IOException{ + while(buffers.size()>2){ + RowBuffer tmp=new RowBuffer(filename+"."+(bufnum++)); + tmp.setCacheSize(allocBuffer()); + // Grab two and sort to buf + RowBuffer a=buffers.remove(0); + RowBuffer b=buffers.remove(0); + mergeSort(tmp,a,b); + a.close(); + freeBuffer(a); + b.close(); + freeBuffer(b); + buffers.add(tmp); + } + if(buffers.size()==1){ + result.close(); + result=buffers.get(0); + result.prepare(); + return; + } + if(buffers.size()==2){ + RowBuffer a=buffers.get(0); + RowBuffer b=buffers.get(1); + mergeSort(result,a,b); + a.close(); + freeBuffer(a); + b.close(); + freeBuffer(b); + result.prepare(); + return; + } + } + + private int allocBuffer(){ + if(BUFFERCACHE>=BUFFERLIMIT){ + BUFFERCACHE-=BUFFERLIMIT; + return BUFFERLIMIT; + } + int tmp=BUFFERCACHE; + BUFFERCACHE=0; + return tmp; + } + private void freeBuffer(RowBuffer buf){ + BUFFERCACHE+=buf.getCacheSize(); + } + + public void initialize(String filename,TupleStream source,List lst) throws IOException{ + this.filename=filename; + compare=new SortComparator(lst); + bufnum=0; + result=new RowBuffer(filename+".result"); + result.setCacheSize(BUFFERLIMIT); + List buffers=new ArrayList(); + int usage=0; + List sortlst=new ArrayList(); + while(source.hasNext()){ + Row row=source.next(); + if(usage+row.getSize()>MEMLIMIT){ + RowBuffer buf=new RowBuffer(filename+"."+(bufnum++)); + buf.setCacheSize(allocBuffer()); + Collections.sort(sortlst,new SortComparator(lst)); + for(int i=0;i(); + } + sortlst.add(row); + usage+=row.getSize(); + } + RowBuffer buf=new RowBuffer(filename+"."+(bufnum++)); + buf.setCacheSize(allocBuffer()); + Collections.sort(sortlst,new SortComparator(lst)); + for(int i=0;i buffers) throws IOException{ + while(buffers.size()>2){ + RowBuffer tmp=new RowBuffer(filename+"."+(bufnum++)); + tmp.setCacheSize(allocBuffer()); + // Grab two and sort to buf + RowBuffer a=buffers.remove(0); + RowBuffer b=buffers.remove(0); + mergeSort(tmp,a,b); + a.close(); + freeBuffer(a); + b.close(); + freeBuffer(b); + buffers.add(tmp); + } + if(buffers.size()==1){ + result.close(); + result=buffers.get(0); + result.prepare(); + return; + } + if(buffers.size()==2){ + RowBuffer a=buffers.get(0); + RowBuffer b=buffers.get(1); + mergeSort(result,a,b); + a.close(); + freeBuffer(a); + b.close(); + freeBuffer(b); + result.prepare(); + return; + } + } + + private int allocBuffer(){ + if(BUFFERCACHE>=BUFFERLIMIT){ + BUFFERCACHE-=BUFFERLIMIT; + return BUFFERLIMIT; + } + int tmp=BUFFERCACHE; + BUFFERCACHE=0; + return tmp; + } + private void freeBuffer(RowBuffer buf){ + BUFFERCACHE+=buf.getCacheSize(); + } + + public void initialize(String filename,TupleStream source,List lst) throws IOException{ + long prepart=System.currentTimeMillis(); + this.filename=filename; + compare=new SortComparator(lst); + bufnum=0; + result=new RowBuffer(filename+".result"); + result.setCacheSize(BUFFERLIMIT); + List buffers=new ArrayList(); + int usage=0; + + List prebuffers=new ArrayList(); + List prebufferends=new ArrayList(); + for(int i=0;i0){ + RowBuffer tmp=prebuffers.get(i); + tmp.addRow(rowa); + prebufferends.set(i,rowa); + rowa=null; + } + }else{ + prebufferends.add(rowa); + RowBuffer tmp=prebuffers.get(i); + tmp.addRow(rowa); + rowa=null; + } + } + if(rowa!=null){ + overflow.addRow(rowa); + over++; + } + } + for(int i=0;i sortlst=new ArrayList(); + while(source.hasNext()){ + Row row=source.next(); + if(usage+row.getSize()>MEMLIMIT){ + RowBuffer buf=new RowBuffer(filename+"."+(bufnum++)); + buf.setCacheSize(allocBuffer()); + Collections.sort(sortlst,new SortComparator(lst)); + for(int i=0;i(); + } + sortlst.add(row); + usage+=row.getSize(); + } + RowBuffer buf=new RowBuffer(filename+"."+(bufnum++)); + buf.setCacheSize(allocBuffer()); + Collections.sort(sortlst,new SortComparator(lst)); + for(int i=0;i lst){ + + } + public abstract boolean hasNext(); + public abstract Row next(); +} \ No newline at end of file diff --git a/Robust/Transactions/tuplesoup/filter/RadixSort.java b/Robust/Transactions/tuplesoup/filter/RadixSort.java new file mode 100644 index 00000000..b64f2086 --- /dev/null +++ b/Robust/Transactions/tuplesoup/filter/RadixSort.java @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2007, Solido Systems + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * Neither the name of Solido Systems nor the names of its contributors may be + * used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +package com.solidosystems.tuplesoup.filter; + +import com.solidosystems.tuplesoup.core.*; +import java.util.*; + +public abstract class RadixSort extends TupleStream{ + public RadixSort(){ + + } + public void initialize(TupleStream source,List lst){ + + } + public abstract boolean hasNext(); + public abstract Row next(); +} \ No newline at end of file diff --git a/Robust/Transactions/tuplesoup/filter/RowBuffer.java b/Robust/Transactions/tuplesoup/filter/RowBuffer.java new file mode 100644 index 00000000..3635c2b6 --- /dev/null +++ b/Robust/Transactions/tuplesoup/filter/RowBuffer.java @@ -0,0 +1,132 @@ +/* + * Copyright (c) 2007, Solido Systems + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * Neither the name of Solido Systems nor the names of its contributors may be + * used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +package com.solidosystems.tuplesoup.filter; + +import java.util.*; +import java.io.*; +import com.solidosystems.tuplesoup.core.*; + +public class RowBuffer extends TupleStream{ + private int CACHESIZE=32768; + private int cacheusage=0; + + private int mempointer=0; + private boolean diskused=false; + + private List membuffer; + private String diskbuffer; + private DataOutputStream out; + private DataInputStream in; + + private Row next=null; + + public RowBuffer(String filename){ + membuffer=new ArrayList(); + diskbuffer=filename; + out=null; + in=null; + } + + public void setCacheSize(int size){ + CACHESIZE=size; + } + public int getCacheSize(){ + return CACHESIZE; + } + + public void addRow(Row row) throws IOException{ + if(cacheusage+row.getSize()<=CACHESIZE){ + membuffer.add(row); + cacheusage+=row.getSize(); + }else{ + cacheusage=CACHESIZE; + if(out==null)out=new DataOutputStream(new BufferedOutputStream(new FileOutputStream(diskbuffer),2048)); + row.writeToStream(out); + diskused=true; + } + } + + public void prepare() throws IOException{ + if(out!=null){ + out.flush(); + out.close(); + } + mempointer=0; + if(diskused)in=new DataInputStream(new BufferedInputStream(new FileInputStream(diskbuffer),2048)); + readNext(); + } + + public void close(){ + try{ + File ftest=new File(diskbuffer); + if(ftest.exists()){ + if(out!=null)out.close(); + if(in!=null)in.close(); + ftest.delete(); + } + }catch(Exception e){ + + } + } + + private void readNext() throws IOException{ + if(mempointer lst=new ArrayList(); + lst.add(new SortRule(key,direction)); + initialize(filename,source,lst); + } + public void initialize(String filename,TupleStream source,String key,int direction,String key2,int direction2) throws IOException{ + List lst=new ArrayList(); + lst.add(new SortRule(key,direction)); + lst.add(new SortRule(key2,direction2)); + initialize(filename,source,lst); + } + public abstract void initialize(String filename,TupleStream source,List lst) throws IOException; + public abstract boolean hasNext() throws IOException; + public abstract Row next() throws IOException; +} \ No newline at end of file diff --git a/Robust/Transactions/tuplesoup/filter/SortComparator.java b/Robust/Transactions/tuplesoup/filter/SortComparator.java new file mode 100644 index 00000000..33fe9093 --- /dev/null +++ b/Robust/Transactions/tuplesoup/filter/SortComparator.java @@ -0,0 +1,67 @@ +/* + * Copyright (c) 2007, Solido Systems + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * Neither the name of Solido Systems nor the names of its contributors may be + * used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +package com.solidosystems.tuplesoup.filter; + +import com.solidosystems.tuplesoup.core.*; +import java.util.*; + +public class SortComparator implements Comparator{ + private List rules; + + public SortComparator(List rules){ + this.rules=rules; + } + + private int compare(int rulenum,Row rowa,Row rowb){ + if(rules.size()<=rulenum)return 0; + SortRule rule=rules.get(rulenum); + Value a=rowa.get(rule.getKey()); + Value b=rowb.get(rule.getKey()); + int result=a.compareTo(b); + // TODO: add direction switcher here + if(result==0){ + rulenum++; + return compare(rulenum,rowa,rowb); + }else{ + if(rule.getDirection()==SortRule.DESC){ + if(result==-1){ + result=1; + }else if(result==1)result=-1; + } + return result; + } + } + + public int compare(Row rowa,Row rowb){ + return compare(0,rowa,rowb); + } +} \ No newline at end of file diff --git a/Robust/Transactions/tuplesoup/filter/SortRule.java b/Robust/Transactions/tuplesoup/filter/SortRule.java new file mode 100644 index 00000000..206ae8d6 --- /dev/null +++ b/Robust/Transactions/tuplesoup/filter/SortRule.java @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2007, Solido Systems + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * Neither the name of Solido Systems nor the names of its contributors may be + * used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +package com.solidosystems.tuplesoup.filter; + +import com.solidosystems.tuplesoup.core.*; + +public class SortRule{ + public static final int ASC=0; + public static final int DESC=1; + + private String key; + private int direction; + + public SortRule(String key, int direction){ + this.key=key; + this.direction=direction; + } + + public int getDirection(){ + return direction; + } + + public String getKey(){ + return key; + } +} \ No newline at end of file diff --git a/Robust/Transactions/tuplesoup/test/BasicTest.java b/Robust/Transactions/tuplesoup/test/BasicTest.java new file mode 100644 index 00000000..dabbf7b7 --- /dev/null +++ b/Robust/Transactions/tuplesoup/test/BasicTest.java @@ -0,0 +1,117 @@ +/* + * Copyright (c) 2007, Solido Systems + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * Neither the name of Solido Systems nor the names of its contributors may be + * used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +package com.solidosystems.tuplesoup.test; + +import java.util.*; + +public class BasicTest{ + public Listerrors=new ArrayList(); + private int lastlength=0; + + public void err(String str){ + while(lastlength<40){ + System.out.print(" "); + lastlength++; + } + outbr(" ERR"); + outbr(" ! "+str); + errors.add(str); + } + + public void printErrorSummary(){ + outbr(""); + if(errors.size()==0){ + outbr("All tests passed!"); + }else if(errors.size()==1){ + outbr("1 test failed!"); + }else{ + outbr(errors.size()+" tests failed!"); + } + } + + public static void die(String reason){ + System.out.println("ERR"); + System.out.println(" ! "+reason); + System.exit(0); + } + + public void ok(){ + while(lastlength<40){ + System.out.print(" "); + lastlength++; + } + outbr(" OK"); + } + + public void outbr(String str){ + outbr(0,str); + } + + public void out(String str){ + out(0,str); + } + + public void outbr(int level, String str){ + switch(level){ + case 0:System.out.print(""); + break; + case 1:System.out.print(" + "); + break; + case 2:System.out.print(" * "); + break; + case 3:System.out.print(" - "); + break; + case 4:System.out.print(" + "); + break; + } + System.out.println(str); + } + + public void out(int level, String str){ + lastlength=0; + switch(level){ + case 0: System.out.print(""); + break; + case 1: System.out.print(" + "); + lastlength+=3; + break; + case 2: System.out.print(" * "); + lastlength+=5; + break; + case 3: System.out.print(" - "); + lastlength+=7; + break; + } + System.out.print(str); + lastlength+=str.length(); + } +} \ No newline at end of file diff --git a/Robust/Transactions/tuplesoup/test/ParallelPerformanceTest.java b/Robust/Transactions/tuplesoup/test/ParallelPerformanceTest.java new file mode 100644 index 00000000..7fdd015f --- /dev/null +++ b/Robust/Transactions/tuplesoup/test/ParallelPerformanceTest.java @@ -0,0 +1,188 @@ +/* + * Copyright (c) 2007, Solido Systems + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * Neither the name of Solido Systems nor the names of its contributors may be + * used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ +package com.solidosystems.tuplesoup.test; + +import com.solidosystems.tuplesoup.core.*; +import com.solidosystems.tuplesoup.filter.*; + +import dstm2.Configs; +import dstm2.Init; +import java.util.*; +import dstm2.Thread; +import java.io.*; + +public class ParallelPerformanceTest extends BasicTest implements Runnable { + + long writetime; + long readtime; + long randomtime; + + public ParallelPerformanceTest() { + String path = "/scratch/TransactionalIO/tuplesoupfiles/"; + try { + int records = 100; + for (int i = 1; i < 2; i++) { + outbr("Running Parallel DualFileTable Performance test"); + outbr(1, i + " x " + (records / i) + " Large records"); + Table table; + /* outbr(2,"Memory index"); + Table table=new DualFileTable("Performance-test",path,Table.MEMORY); + benchmark(table,i,(records/i)); + table.close(); + table.deleteFiles(); + + outbr(2,"Flat index"); + table=new DualFileTable("Performance-test",path,Table.FLAT); + benchmark(table,i,(records/i)); + table.close(); + table.deleteFiles();*/ + + outbr(2, "Paged index"); + table = new DualFileTable("Performance-test", path, Table.PAGED); + benchmark(table, 2, (records / i)); + table.close(); + table.deleteFiles(); + + /* outbr("Running Parallel HashedTable Performance test"); + outbr(1,i+" x "+(records/i)+" Large records"); + + outbr(2,"Memory index"); + table=new HashedTable("Performance-test",path,Table.MEMORY); + benchmark(table,i,(records/i)); + table.close(); + table.deleteFiles(); + + outbr(2,"Flat index"); + table=new HashedTable("Performance-test",path,Table.FLAT); + benchmark(table,i,(records/i)); + table.close(); + table.deleteFiles(); + + outbr(2,"Paged index"); + table=new HashedTable("Performance-test",path,Table.PAGED); + benchmark(table,i,(records/i)); + table.close(); + table.deleteFiles();*/ + } + + + } catch (Exception e) { + e.printStackTrace(); + } + } + + public static void main(String[] args) { + Init.init(args[0], Boolean.valueOf(args[1])); + new ParallelPerformanceTest(); + } + + public void benchmark(Table table, int threadcount, int records) throws Exception { + writetime = 0; + readtime = 0; + randomtime = 0; + List lst = new ArrayList(); + long starttime = System.currentTimeMillis(); + for (int i = 0; i < threadcount; i++) { + Thread thr = new Thread(new ParallelThread(this, table, i + "", records)); + thr.start(); + lst.add(thr); + } + for (int i = 0; i < threadcount; i++) { + lst.get(i).join(); + } + long stoptime = System.currentTimeMillis(); + System.out.println("\n\nExectuin time: " + (long) (stoptime - starttime)); + long committed = Thread.totalCommitted; + long total = Thread.totalTotal; + if (total > 0) { + System.out.printf("Committed: %d\nTotal: %d\nPercent committed: (%d%%)\n", + committed, + total, + (100 * committed) / total); + } else { + System.out.println("No transactions executed!"); + } + // outbr(3, "Write " + writetime + " ms"); + // outbr(3, "Read " + readtime + " ms"); + // outbr(3, "Random " + randomtime + " ms"); + } + + public void run() { + } + + public long benchmarkLargeWrite(Table table, int records, String id) throws IOException { + long pre = System.currentTimeMillis(); + for (int i = 0; i < records; i++) { + Row row = new Row(id + i); + if (Configs.inevitable) { + row.put("key1", "foobarbazinevitable"); + } else { + row.put("key1", "foobarbaztransactiona"); + } + row.put("key2", 123456); + row.put("key3", 3.141592); + row.put("key4", true); + row.put("key5", new Value(new byte[]{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16})); + table.addRow(row); + } + long post = System.currentTimeMillis(); + return post - pre; + } + + public long benchmarkLargeRead(Table table, int records, String id) throws IOException { + long pre = System.currentTimeMillis(); + TupleStream stream = table.getRows(); + while (stream.hasNext()) { + stream.next(); + } + long post = System.currentTimeMillis(); + return post - pre; + } + + public long benchmarkLargeRandomRead(Table table, int records, String id) throws IOException { + long pre = System.currentTimeMillis(); + for (int i = 0; i < records; i++) { + Row row = table.getRow(id + (int) (Math.random() * records)); + System.out.println(Thread.currentThread() + " " + row); + } + long post = System.currentTimeMillis(); + return post - pre; + } + + public void printStats(Hashtable hash) { + Set keys = hash.keySet(); + Iterator it = keys.iterator(); + while (it.hasNext()) { + String key = it.next(); + outbr(4, key + " " + hash.get(key)); + } + } +} \ No newline at end of file diff --git a/Robust/Transactions/tuplesoup/test/ParallelThread.java b/Robust/Transactions/tuplesoup/test/ParallelThread.java new file mode 100644 index 00000000..16fcea8b --- /dev/null +++ b/Robust/Transactions/tuplesoup/test/ParallelThread.java @@ -0,0 +1,70 @@ +/* + * Copyright (c) 2007, Solido Systems + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * Neither the name of Solido Systems nor the names of its contributors may be + * used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + + package com.solidosystems.tuplesoup.test; + + import com.solidosystems.tuplesoup.core.*; + + + +public class ParallelThread implements Runnable{ + String id; + int records; + ParallelPerformanceTest app; + Table table; + + public ParallelThread(ParallelPerformanceTest app,Table table,String id,int records){ + this.id=id; + this.records=records; + this.app=app; + this.table=table; + } + + public void run(){ + try{ + long time; + time=app.benchmarkLargeWrite(table,records,id); + // synchronized(app){ + ///// app.writetime+=time; + // } + /*/ time=app.benchmarkLargeRead(table,records,id); + synchronized(app){ + app.readtime+=time; + }*/ + time=app.benchmarkLargeRandomRead(table,records,id); + // synchronized(app){ + // app.randomtime+=time; + // } + }catch(Exception e){ + e.printStackTrace(); + } + } +} \ No newline at end of file