Enable TextLoader to accept new lines in quoted fields

src/Microsoft.ML.Data/DataLoadSave/Text/TextLoader.cs

@@ -501,6 +501,13 @@ public class Options
             [Argument(ArgumentType.AtMostOnce, HelpText = "Use separate parsing threads?", ShortName = "threads", Hide = true)]
             public bool UseThreads = true;
 
+            /// <summary>
+            /// If true, new line characters are acceptable inside a quoted field, and thus one field can have multiple lines of text inside it
+            /// If <see cref="TextLoader.Options.AllowQuoting"/> is false, this option is ignored.
+            /// </summary>
+            [Argument(ArgumentType.AtMostOnce, HelpText = "Escape new line characters inside a quoted field? If AllowQuoting is false, this argument is ignored.", ShortName = "multilines", Hide = true)]
+            public bool ReadMultilines = false;
+
             /// <summary>
             /// File containing a header with feature names. If specified, the header defined in the data file is ignored regardless of <see cref="HasHeader"/>.
             /// </summary>
@@ -694,11 +701,11 @@ public Bindings(TextLoader parent, Column[] cols, IMultiStreamSource headerFile,
                     ch.Assert(0 <= inputSize & inputSize < SrcLim);
                     List<ReadOnlyMemory<char>> lines = null;
                     if (headerFile != null)
-                        Cursor.GetSomeLines(headerFile, 1, ref lines);
+                        Cursor.GetSomeLines(headerFile, 1, parent.ReadMultilines, ref lines);
                     if (needInputSize && inputSize == 0)
-                        Cursor.GetSomeLines(dataSample, 100, ref lines);
+                        Cursor.GetSomeLines(dataSample, 100, parent.ReadMultilines, ref lines);
                     else if (headerFile == null && parent.HasHeader)
-                        Cursor.GetSomeLines(dataSample, 1, ref lines);
+                        Cursor.GetSomeLines(dataSample, 1, parent.ReadMultilines, ref lines);
 
                     if (needInputSize && inputSize == 0)
                     {
@@ -1055,7 +1062,7 @@ private static VersionInfo GetVersionInfo()
                 // verWrittenCur: 0x00010009, // Introduced _flags
                 //verWrittenCur: 0x0001000A, // Added ForceVector in Range
                 //verWrittenCur: 0x0001000B, // Header now retained if used and present
-                verWrittenCur: 0x0001000C, // Removed Min and Contiguous from KeyType
+                verWrittenCur: 0x0001000C, // Removed Min and Contiguous from KeyType, and added ReadMultilines flag to OptionFlags
                 verReadableCur: 0x0001000A,
                 verWeCanReadBack: 0x00010009,
                 loaderSignature: LoaderSignature,
@@ -1073,8 +1080,8 @@ private enum OptionFlags : uint
             HasHeader = 0x02,
             AllowQuoting = 0x04,
             AllowSparse = 0x08,
-
-            All = TrimWhitespace | HasHeader | AllowQuoting | AllowSparse
+            ReadMultilines = 0x10,
+            All = TrimWhitespace | HasHeader | AllowQuoting | AllowSparse | ReadMultilines
         }
 
         // This is reserved to mean the range extends to the end (the segment is variable).
@@ -1095,6 +1102,11 @@ private bool HasHeader
             get { return (_flags & OptionFlags.HasHeader) != 0; }
         }
 
+        private bool ReadMultilines
+        {
+            get { return (_flags & OptionFlags.ReadMultilines) != 0; }
+        }
+
         private readonly IHost _host;
         private const string RegistrationName = "TextLoader";
 
@@ -1147,6 +1159,8 @@ internal TextLoader(IHostEnvironment env, Options options = null, IMultiStreamSo
                 _flags |= OptionFlags.AllowQuoting;
             if (options.AllowSparse)
                 _flags |= OptionFlags.AllowSparse;
+            if (options.AllowQuoting && options.ReadMultilines)
+                _flags |= OptionFlags.ReadMultilines;
 
             // REVIEW: This should be persisted (if it should be maintained).
             _maxRows = options.MaxRows ?? long.MaxValue;

src/Microsoft.ML.Data/DataLoadSave/Text/TextLoaderCursor.cs

@@ -4,6 +4,7 @@
 
 using System;
 using System.Collections.Generic;
+using System.IO;
 using System.Text;
 using System.Threading;
 using System.Threading.Tasks;
@@ -145,7 +146,7 @@ public static DataViewRowCursor Create(TextLoader parent, IMultiStreamSource fil
                 SetupCursor(parent, active, 0, out srcNeeded, out cthd);
                 Contracts.Assert(cthd > 0);
 
-                var reader = new LineReader(files, BatchSize, 100, parent.HasHeader, parent._maxRows, 1);
+                var reader = new LineReader(files, BatchSize, 100, parent.HasHeader, parent.ReadMultilines, parent._maxRows, 1);
                 var stats = new ParseStats(parent._host, 1);
                 return new Cursor(parent, stats, active, reader, srcNeeded, cthd);
             }
@@ -162,7 +163,7 @@ public static DataViewRowCursor[] CreateSet(TextLoader parent, IMultiStreamSourc
                 SetupCursor(parent, active, n, out srcNeeded, out cthd);
                 Contracts.Assert(cthd > 0);
 
-                var reader = new LineReader(files, BatchSize, 100, parent.HasHeader, parent._maxRows, cthd);
+                var reader = new LineReader(files, BatchSize, 100, parent.HasHeader, parent.ReadMultilines, parent._maxRows, cthd);
                 var stats = new ParseStats(parent._host, cthd);
                 if (cthd <= 1)
                     return new DataViewRowCursor[1] { new Cursor(parent, stats, active, reader, srcNeeded, 1) };
@@ -204,7 +205,7 @@ public override ValueGetter<DataViewRowId> GetIdGetter()
                     };
             }
 
-            public static void GetSomeLines(IMultiStreamSource source, int count, ref List<ReadOnlyMemory<char>> lines)
+            public static void GetSomeLines(IMultiStreamSource source, int count, bool readMultilines, ref List<ReadOnlyMemory<char>> lines)
             {
                 Contracts.AssertValue(source);
                 Contracts.Assert(count > 0);
@@ -214,7 +215,7 @@ public static void GetSomeLines(IMultiStreamSource source, int count, ref List<R
                     count = 2;
 
                 LineBatch batch;
-                var reader = new LineReader(source, count, 1, false, count, 1);
+                var reader = new LineReader(source, count, 1, false, readMultilines, count, 1);
                 try
                 {
                     batch = reader.GetBatch();
@@ -401,6 +402,7 @@ private sealed class LineReader
             {
                 private readonly long _limit;
                 private readonly bool _hasHeader;
+                private readonly bool _readMultilines;
                 private readonly int _batchSize;
                 private readonly IMultiStreamSource _files;
 
@@ -410,7 +412,7 @@ private sealed class LineReader
                 private Task _thdRead;
                 private volatile bool _abort;
 
-                public LineReader(IMultiStreamSource files, int batchSize, int bufSize, bool hasHeader, long limit, int cref)
+                public LineReader(IMultiStreamSource files, int batchSize, int bufSize, bool hasHeader, bool readMultilines, long limit, int cref)
                 {
                     // Note that files is allowed to be empty.
                     Contracts.AssertValue(files);
@@ -423,6 +425,7 @@ public LineReader(IMultiStreamSource files, int batchSize, int bufSize, bool has
                     _limit = limit;
                     _hasHeader = hasHeader;
                     _batchSize = batchSize;
+                    _readMultilines = readMultilines;
                     _files = files;
                     _cref = cref;
 
@@ -464,9 +467,67 @@ public LineBatch GetBatch()
                     throw Contracts.ExceptDecode(batch.Exception, "Stream reading encountered exception");
                 }
 
+                private static class MultiLineReader
+                {
+                    // When reading lines that contain quoted fields, the quoted fields can contain
+                    // '\n' so we we'll need to read multiple lines (multilines) to get all the fields
+                    // of a given row.
+                    public static string ReadMultiLine(TextReader sr, StringBuilder sb, long lineNum, bool ignoreHashLine)
+                    {
+                        string line;
+                        line = sr.ReadLine();
+
+                        // if it was an empty line or if we've reached the end of file (i.e. line = null)
+                        if (string.IsNullOrEmpty(line))
+                            return line;
+
+                        // In ML.NET we filter out lines beginning with // and # at the beginning of the file
+                        // Or lines beginning with // elsewhere in the file.
+                        // Thus, we don't care to check if there's a quoted multiline when the line begins with
+                        // these chars.
+                        if (line[0] == '/' && line[1] == '/')
+                            return line;
+                        if (ignoreHashLine && line[0] == '#')
+                            return line;
+
+                        // Get more lines until the number of quote characters is even
+                        // 2 consecutive quotes are considered scaped quotes
+                        long numOfQuotes = GetNumberOfChars(line, '"');
+                        if (numOfQuotes % 2 == 0)
+                            return line;
+
+                        sb.Clear();
+                        sb.Append(line);
+                        while (numOfQuotes % 2 != 0)
+                        {
+                            line = sr.ReadLine();
+
+                            if (line == null) // If we've reached the end of the file
+                                throw new EndOfStreamException($"A quote opened on a field on line {lineNum} was never closed, and we've read to the last line in the file without finding the closing quote");
+
+                            sb.Append("\n");
+                            sb.Append(line);
+                            numOfQuotes += GetNumberOfChars(line, '"');
+                        }
+
+                        return sb.ToString();
+                    }
+
+                    public static int GetNumberOfChars(string line, char ch)
+                    {
+                        int count = 0;
+                        foreach (char c in line)
+                        {
+                            if (c == ch) count++;
+                        }
+                        return count;
+                    }
+                }
+
                 private void ThreadProc()
                 {
                     Contracts.Assert(_batchSize >= 2);
+                    var multilineSB = new StringBuilder();
 
                     try
                     {
@@ -487,7 +548,11 @@ private void ThreadProc()
                                     // REVIEW: Avoid allocating a string for every line. This would probably require
                                     // introducing a CharSpan type (similar to ReadOnlyMemory but based on char[] or StringBuilder)
                                     // and implementing all the necessary conversion functionality on it. See task 3871.
-                                    text = rdr.ReadLine();
+                                    if (_readMultilines)
+                                        text = MultiLineReader.ReadMultiLine(rdr, multilineSB, line, true);
+                                    else
+                                        text = rdr.ReadLine();
+
                                     if (text == null)
                                         goto LNext;
                                     line++;
@@ -514,7 +579,11 @@ private void ThreadProc()
                                     if (_abort)
                                         return;
 
-                                    text = rdr.ReadLine();
+                                    if (_readMultilines)
+                                        text = MultiLineReader.ReadMultiLine(rdr, multilineSB, line, false);
+                                    else
+                                        text = rdr.ReadLine();
+
                                     if (text == null)
                                     {
                                         // We're done with this file. Queue the last partial batch.

src/Microsoft.ML.Data/DataLoadSave/Text/TextLoaderParser.cs

@@ -1161,6 +1161,11 @@ private bool FetchNextField(ref ScanInfo scan, ReadOnlySpan<char> span)
                                 scan.QuotingError = true;
                                 break;
                             }
+
+                            // The logic below allow us to escape quotes (") inside quoted
+                            // fields by using doublo quotes (""). I.e. when the loader
+                            // encounters "" inside a quoted field, it will output only one "
+                            // and continue parsing the rest of the field.
                             if (span[ichCur] == '"')
                             {
                                 if (ichCur > ichRun)

src/Microsoft.ML.Data/DataLoadSave/Text/TextLoaderSaverCatalog.cs

@@ -64,7 +64,7 @@ public static TextLoader CreateTextLoader(this DataOperationsCatalog catalog,
                 HasHeader = hasHeader,
                 AllowQuoting = allowQuoting,
                 TrimWhitespace = trimWhitespace,
-                AllowSparse = allowSparse
+                AllowSparse = allowSparse,
             };
 
             return new TextLoader(CatalogUtils.GetEnvironment(catalog), options: options, dataSample: dataSample);

src/Microsoft.ML.Data/DataLoadSave/Text/TextSaver.cs

@@ -827,7 +827,7 @@ internal static void MapText(ReadOnlySpan<char> span, ref StringBuilder sb, char
                 for (; ichCur < ichLim; ichCur++)
                 {
                     char ch = span[ichCur];
-                    if (ch != '"' && ch != sep && ch != ':')
+                    if (ch != '"' && ch != sep && ch != ':' && ch != '\n')
                         continue;
                     if (!quoted)
                     {

test/BaselineOutput/Common/Command/SavePipeTextLoaderWithMoreOptions-1-out.txt

@@ -0,0 +1,33 @@
+#@ TextLoader{
+#@   header+
+#@   sep=tab
+#@   col=id:R4:0
+#@   col=description:TX:1
+#@   col=animal:TX:2
+#@ }
+id	description	animal
+10	this is a description	dog
+11	this is a quoted description	cat
+12	"this is a multiline
+quoted description"	bird
+13	"this has one""doublequote which should be escaped as a single quote"	dog
+14	"this has ""doublequotes"" inside of it"	cat
+15	"this is a multiline
+quoted description with
+
+""doublequotes"" and
+
+empty new lines and
+
+
+
+escaped quotes inside of ""it""
+
+//and this comment with // shouldn't be ignored
+since it is part of the ""multiline"""	bird
+16	here is text after the empty line	dog
+17	this is a line with an empty animal	""
+0	this is a line with an empty id	bird
+19	""	dog
+20	this is the last row description	cat
+Wrote 11 rows of length 3

test/BaselineOutput/Common/Command/SavePipeTextLoaderWithMoreOptions-out.txt

@@ -0,0 +1,33 @@
+#@ TextLoader{
+#@   header+
+#@   sep=tab
+#@   col=id:R4:0
+#@   col=description:TX:1
+#@   col=animal:TX:2
+#@ }
+id	description	animal
+10	this is a description	dog
+11	this is a quoted description	cat
+12	"this is a multiline
+quoted description"	bird
+13	"this has one""doublequote which should be escaped as a single quote"	dog
+14	"this has ""doublequotes"" inside of it"	cat
+15	"this is a multiline
+quoted description with
+
+""doublequotes"" and
+
+empty new lines and
+
+
+
+escaped quotes inside of ""it""
+
+//and this comment with // shouldn't be ignored
+since it is part of the ""multiline"""	bird
+16	here is text after the empty line	dog
+17	this is a line with an empty animal	""
+0	this is a line with an empty id	bird
+19	""	dog
+20	this is the last row description	cat
+Wrote 11 rows of length 3

test/BaselineOutput/Common/EntryPoints/core_manifest.json

@@ -405,6 +405,18 @@
                 "IsNullable": false,
                 "Default": true
               },
+              {
+                "Name": "ReadMultilines",
+                "Type": "Bool",
+                "Desc": "Escape new line characters inside a quoted field? If AllowQuoting is false, this argument is ignored.",
+                "Aliases": [
+                  "multilines"
+                ],
+                "Required": false,
+                "SortOrder": 150.0,
+                "IsNullable": false,
+                "Default": false
+              },
               {
                 "Name": "HeaderFile",
                 "Type": "String",

test/BaselineOutput/Common/TextLoader/multiline.csv

@@ -0,0 +1,33 @@
+10
+11
+12
+13
+14
+15
+16
+17
+0
+19
+20
+this is a description
+this is a quoted description
+this is a multiline\nquoted description
+this has one"doublequote which should be escaped as a single quote
+this has "doublequotes" inside of it
+this is a multiline\nquoted description with\n\n"doublequotes" and\n\nempty new lines and\n\n\n\nescaped quotes inside of "it"\n\n//and this comment with // shouldn't be ignored\nsince it is part of the "multiline"
+here is text after the empty line
+this is a line with an empty animal
+this is a line with an empty id
+
+this is the last row description
+dog
+cat
+bird
+dog
+cat
+bird
+dog
+
+bird
+dog
+cat