diff --git a/src/main/java/org/apache/commons/csv/ExtendedBufferedReader.java b/src/main/java/org/apache/commons/csv/ExtendedBufferedReader.java index 94a163962..0c8b0e8d3 100644 --- a/src/main/java/org/apache/commons/csv/ExtendedBufferedReader.java +++ b/src/main/java/org/apache/commons/csv/ExtendedBufferedReader.java @@ -66,6 +66,9 @@ final class ExtendedBufferedReader extends UnsynchronizedBufferedReader { /** Encoder for calculating the number of bytes for each character read. */ private final CharsetEncoder encoder; + /** Bytes {@link #encoder} emits as a byte-order mark on every {@link CharsetEncoder#encode(CharBuffer)} call (for example {@code UTF-16}). */ + private final int bomLength; + /** * Constructs a new instance using the default buffer size. */ @@ -84,6 +87,7 @@ final class ExtendedBufferedReader extends UnsynchronizedBufferedReader { ExtendedBufferedReader(final Reader reader, final Charset charset, final boolean trackBytes) { super(reader); encoder = charset != null && trackBytes ? charset.newEncoder() : null; + bomLength = encoder != null ? measureBomLength(encoder) : 0; } /** @@ -150,18 +154,35 @@ private int getEncodedCharLength(final int previous, final int current) throws C final char cChar = (char) current; final char lChar = (char) previous; if (!Character.isSurrogate(cChar)) { - return encoder.encode(CharBuffer.wrap(new char[] { cChar })).limit(); + return encoder.encode(CharBuffer.wrap(new char[] { cChar })).limit() - bomLength; } if (Character.isHighSurrogate(cChar)) { // Move on to the next char (low surrogate) return 0; } if (Character.isSurrogatePair(lChar, cChar)) { - return encoder.encode(CharBuffer.wrap(new char[] { lChar, cChar })).limit(); + return encoder.encode(CharBuffer.wrap(new char[] { lChar, cChar })).limit() - bomLength; } throw new CharacterCodingException(); } + /** + * Measures the byte-order mark that {@code encoder} prepends to every {@link CharsetEncoder#encode(CharBuffer)} call. Charsets such as {@code UTF-16} emit + * a BOM each time, which would otherwise be counted once per character. The BOM is the constant prefix shared by encoding one and two characters. + * + * @param encoder The encoder to measure. + * @return The byte-order mark length in bytes, or 0 when the encoder writes none. + */ + private static int measureBomLength(final CharsetEncoder encoder) { + try { + final int one = encoder.encode(CharBuffer.wrap(new char[] { 'a' })).limit(); + final int two = encoder.encode(CharBuffer.wrap(new char[] { 'a', 'a' })).limit(); + return Math.max(0, 2 * one - two); + } catch (final CharacterCodingException e) { + return 0; + } + } + /** * Returns the last character that was read as an integer (0 to 65535). This will be the last character returned by any of the read methods. This will not * include a character read using the {@link #peek()} method. If no character has been read then this will return {@link Constants#UNDEFINED}. If the end of diff --git a/src/test/java/org/apache/commons/csv/CSVParserTest.java b/src/test/java/org/apache/commons/csv/CSVParserTest.java index 6d9bdd9e8..c18ea48c3 100644 --- a/src/test/java/org/apache/commons/csv/CSVParserTest.java +++ b/src/test/java/org/apache/commons/csv/CSVParserTest.java @@ -767,6 +767,29 @@ void testGetBytePositionWithSingleByteCharset() throws IOException { } } + @Test + void testGetBytePositionWithBomEmittingCharset() throws IOException { + // The UTF-16 encoder writes a byte-order mark on every encode call, so the per-character + // byte length must exclude it. Otherwise each code unit is counted as 4 bytes (2 for the + // BOM, 2 for the char) and getBytePosition() grows at twice the true rate. + final String code = "a,b\nc,d\n"; + try (CSVParser parser = CSVParser.builder() + .setReader(new StringReader(code)) + .setFormat(CSVFormat.DEFAULT) + .setCharset(StandardCharsets.UTF_16) + .setTrackBytes(true) + .get()) { + final CSVRecord first = parser.nextRecord(); + final CSVRecord second = parser.nextRecord(); + assertNotNull(first); + assertNotNull(second); + assertNull(parser.nextRecord()); + assertEquals(0, first.getBytePosition()); + // "a,b\n" is 4 UTF-16 code units, 2 bytes each. + assertEquals(8, second.getBytePosition()); + } + } + @Test void testGetHeaderComment_HeaderComment1() throws IOException { try (CSVParser parser = CSVParser.parse(CSV_INPUT_HEADER_COMMENT, FORMAT_AUTO_HEADER)) {