Force UTF8 for empty string when flush is invoked without a previous extract call

Author: andsel

logstash-core/src/main/java/org/logstash/common/BufferedTokenizerExt.java

@@ -160,9 +160,20 @@ public IRubyObject flush(final ThreadContext context) {
         headToken = new StringBuilder();
         inputSize = 0;
 
-        // create new RubyString with the last data specified encoding
-        RubyString encodedHeadToken = RubyUtil.RUBY.newString(new ByteList(buffer.toString().getBytes(Charset.forName(encodingName))));
-        encodedHeadToken.force_encoding(context, RubyUtil.RUBY.newString(encodingName));
+        // create new RubyString with the last data specified encoding, if exists
+        RubyString encodedHeadToken;
+        if (encodingName != null) {
+            encodedHeadToken = RubyUtil.RUBY.newString(new ByteList(buffer.toString().getBytes(Charset.forName(encodingName))));
+            encodedHeadToken.force_encoding(context, RubyUtil.RUBY.newString(encodingName));
+        } else {
+            // When used with TCP input it could be that on socket connection the flush method
+            // is invoked while no invocation of extract, leaving the encoding name unassigned.
+            // In such case also the headToken must be empty
+            if (!buffer.toString().isEmpty()) {
+                throw new IllegalStateException("invoked flush with unassigned encoding but not empty head token, this shouldn't happen");
+            }
+            encodedHeadToken = (RubyString) buffer;
+        }
 
         return encodedHeadToken;
     }

logstash-core/src/test/java/org/logstash/common/BufferedTokenizerExtTest.java

@@ -144,4 +144,18 @@ public void shouldNotChangeEncodingOfTokensAfterPartitioningWhenRetrieveLastFlus
         RubyEncoding encoding = (RubyEncoding) lastToken.callMethod(context, "encoding");
         assertEquals("ISO-8859-1", encoding.toString());
     }
+
+    @Test
+    public void givenDirectFlushInvocationUTF8EncodingIsApplied() {
+        RubyString rubyString = RubyString.newString(RUBY, new byte[]{(byte) 0xA3, 0x41}); // £ character, A
+        IRubyObject rubyInput = rubyString.force_encoding(context, RUBY.newString("ISO8859-1"));
+
+        // flush and check that the remaining A is still encoded in ISO8859-1
+        IRubyObject lastToken = sut.flush(context);
+        assertEquals("", lastToken.toString());
+
+        // verify encoding "ISO8859-1" is preserved in the Java to Ruby String conversion
+        RubyEncoding encoding = (RubyEncoding) lastToken.callMethod(context, "encoding");
+        assertEquals("UTF-8", encoding.toString());
+    }
 }