Skip to content

Commit

Permalink
Optimize toArray of all ByteIterators except RandomByteIterator
Browse files Browse the repository at this point in the history
Method nextBuf is a clever hack that outperforms Random.nextBytes
but performs poorly for all other ByteIterator implementations

This commit moves it to RandomByteIterator and adds efficient
toArray implementations for other ByteIterator classes.

Also InputStreamByteIterator.reset method that unconditionally
throws UnsupportedOperationException is fixed
  • Loading branch information
isopov committed Mar 19, 2018
1 parent ca25820 commit 0ac118c
Show file tree
Hide file tree
Showing 5 changed files with 59 additions and 21 deletions.
11 changes: 10 additions & 1 deletion core/src/main/java/com/yahoo/ycsb/ByteArrayByteIterator.java
Original file line number Diff line number Diff line change
Expand Up @@ -60,5 +60,14 @@ public long bytesLeft() {
public void reset() {
off = originalOffset;
}


@Override
public byte[] toArray() {
int size = (int) bytesLeft();
byte[] bytes = new byte[size];
System.arraycopy(str, off, bytes, 0, size);
off = len;
return bytes;
}

}
18 changes: 4 additions & 14 deletions core/src/main/java/com/yahoo/ycsb/ByteIterator.java
Original file line number Diff line number Diff line change
Expand Up @@ -56,16 +56,6 @@ public Byte next() {

public abstract byte nextByte();

/** @return byte offset immediately after the last valid byte */
public int nextBuf(byte[] buf, int bufOff) {
int sz = bufOff;
while (sz < buf.length && hasNext()) {
buf[sz] = nextByte();
sz++;
}
return sz;
}

public abstract long bytesLeft();

@Override
Expand All @@ -81,7 +71,7 @@ public void remove() {
public void reset() {
throw new UnsupportedOperationException();
}

/** Consumes remaining contents of this object, and returns them as a string. */
public String toString() {
Charset cset = Charset.forName("UTF-8");
Expand All @@ -95,10 +85,10 @@ public byte[] toArray() {
if (left != (int) left) {
throw new ArrayIndexOutOfBoundsException("Too much data to fit in one array!");
}

byte[] ret = new byte[(int) left];
int off = 0;
while (off < ret.length) {
off = nextBuf(ret, off);
for (int i = 0; i < ret.length; i++) {
ret[i] = nextByte();
}
return ret;
}
Expand Down
20 changes: 18 additions & 2 deletions core/src/main/java/com/yahoo/ycsb/InputStreamByteIterator.java
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@
* A ByteIterator that iterates through an inputstream of bytes.
*/
public class InputStreamByteIterator extends ByteIterator {
private long len;
private final long len;
private InputStream ins;
private long off;
private final boolean resetable;
Expand Down Expand Up @@ -63,6 +63,21 @@ public long bytesLeft() {
return len - off;
}

@Override
public byte[] toArray() {
int size = (int) bytesLeft();
byte[] bytes = new byte[size];
try {
if (ins.read(bytes) < size) {
throw new IllegalStateException("Past EOF!");
}
} catch (IOException e) {
throw new IllegalStateException(e);
}
off = len;
return bytes;
}

@Override
public void reset() {
if (resetable) {
Expand All @@ -72,8 +87,9 @@ public void reset() {
} catch (IOException e) {
throw new IllegalStateException("Failed to reset the input stream", e);
}
} else {
throw new UnsupportedOperationException();
}
throw new UnsupportedOperationException();
}

}
19 changes: 16 additions & 3 deletions core/src/main/java/com/yahoo/ycsb/RandomByteIterator.java
Original file line number Diff line number Diff line change
Expand Up @@ -73,8 +73,7 @@ public byte nextByte() {
return buf[bufOff - 1];
}

@Override
public int nextBuf(byte[] buffer, int bufOffset) {
private int nextBuf(byte[] buffer, int bufOffset) {
int ret;
if (len - off < buffer.length - bufOffset) {
ret = (int) (len - off);
Expand All @@ -98,5 +97,19 @@ public long bytesLeft() {
public void reset() {
off = 0;
}


/** Consumes remaining contents of this object, and returns them as a byte array. */
public byte[] toArray() {
long left = bytesLeft();
if (left != (int) left) {
throw new ArrayIndexOutOfBoundsException("Too much data to fit in one array!");
}
byte[] ret = new byte[(int) left];
int bufOffset = 0;
while (bufOffset < ret.length) {
bufOffset = nextBuf(ret, bufOffset);
}
return ret;
}

}
12 changes: 11 additions & 1 deletion core/src/main/java/com/yahoo/ycsb/StringByteIterator.java
Original file line number Diff line number Diff line change
Expand Up @@ -100,7 +100,17 @@ public long bytesLeft() {
public void reset() {
off = 0;
}


@Override
public byte[] toArray() {
byte[] bytes = new byte[(int) bytesLeft()];
for (int i = 0; i < bytes.length; i++) {
bytes[i] = (byte) str.charAt(off + i);
}
off = str.length();
return bytes;
}

/**
* Specialization of general purpose toString() to avoid unnecessary
* copies.
Expand Down

0 comments on commit 0ac118c

Please sign in to comment.