BATCH-1995: Updated to handle delimitors at the beginning and end of …

…a line

dictionary.txt

@@ -97,3 +97,5 @@ michael
 minella
 accessors
 subclassing
+ajax
+javascript

spring-batch-infrastructure/src/main/java/org/springframework/batch/item/file/transform/DelimitedLineTokenizer.java

@@ -144,7 +144,6 @@ protected List<String> doTokenize(String line) {
 		int fieldCount = 0;
 
 		for (int i = 0; i < length; i++) {
-
 			char currentChar = chars[i];
 			boolean isEnd = (i == (length - 1));
 
@@ -154,7 +153,7 @@ protected List<String> doTokenize(String line) {
 				int endPosition = (isEnd ? (length - lastCut) : (i - lastCut));
 
 				if (isEnd && isDelimiter) {
-					endPosition--;
+					endPosition = endPosition - delimiter.length();
 				}
 				else if (!isEnd){
 					endPosition = (endPosition - delimiter.length()) + 1;
@@ -233,7 +232,7 @@ private boolean isQuoted(String value) {
 	private boolean isDelimiter(char[] chars, int i, String token) {
 		boolean result = false;
 
-		if(i >= token.length()) {
+		if(i >= token.length() - 1) {
 			String end = new String(chars, (i-token.length()) + 1, token.length());
 			if(token.equals(end)) {
 				result = true;

spring-batch-infrastructure/src/test/java/org/springframework/batch/item/file/transform/DelimitedLineTokenizerTests.java

@@ -138,6 +138,26 @@ public void testDelimitedLineTokenizerString() {
 		assertEquals("c", line.readString(1));
 	}
 
+	@Test
+	public void testDelimitedLineTokenizerStringBeginningOfLine() {
+		AbstractLineTokenizer tokenizer = new DelimitedLineTokenizer(" | ");
+		FieldSet line = tokenizer.tokenize(" | a | b");
+		assertEquals(3, line.getFieldCount());
+		assertEquals("", line.readString(0));
+		assertEquals("a", line.readString(1));
+		assertEquals("b", line.readString(2));
+	}
+
+	@Test
+	public void testDelimitedLineTokenizerStringEndOfLine() {
+		AbstractLineTokenizer tokenizer = new DelimitedLineTokenizer(" | ");
+		FieldSet line = tokenizer.tokenize("a | b | ");
+		assertEquals(3, line.getFieldCount());
+		assertEquals("a", line.readString(0));
+		assertEquals("b", line.readString(1));
+		assertEquals("", line.readString(2));
+	}
+
 	@Test
 	public void testDelimitedLineTokenizerNewlineToken() {
 		AbstractLineTokenizer tokenizer = new DelimitedLineTokenizer("\n");