Skip to content

Commit

Permalink
comment
Browse files Browse the repository at this point in the history
  • Loading branch information
davidkyle committed Aug 3, 2023
1 parent 084fa62 commit 4e3c903
Showing 1 changed file with 2 additions and 1 deletion.
Original file line number Diff line number Diff line change
Expand Up @@ -202,7 +202,7 @@ public void testProcessorMuliptleSpans() throws IOException {
var windowTokens = tokenizationResult.getTokens().get(i);
// size of output
int outputSize = windowTokens.tokenIds().length;
// generate low value -ve logits that will not mark
// generate low value -ve scores that will not mark
// the expected result with a high degree of probability
double[] starts = DoubleStream.generate(() -> -randomDoubleBetween(0.001, 1.0, true)).limit(outputSize).toArray();
double[] ends = DoubleStream.generate(() -> -randomDoubleBetween(0.001, 1.0, true)).limit(outputSize).toArray();
Expand All @@ -226,6 +226,7 @@ public void testProcessorMuliptleSpans() throws IOException {
pyTorchResult
);

// The expected answer is the full text of the span containing the answer
int expectedStart = tokenizationResult.getTokens().get(spanContainingTheAnswer).tokens().get(1).get(0).startOffset();
int lastTokenPosition = tokenizationResult.getTokens().get(spanContainingTheAnswer).tokens().get(1).size() - 1;
int expectedEnd = tokenizationResult.getTokens().get(spanContainingTheAnswer).tokens().get(1).get(lastTokenPosition).endOffset();
Expand Down

0 comments on commit 4e3c903

Please sign in to comment.