fix failing test and remove unnecessary build

Appveyor keeps failing single tests with a permission error, not sure why

Author: markschl

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "seqtool"
-version = "0.2.3"
+version = "0.2.1"
 authors = ["<[email protected]>"]
 description = "General purpose tool for reading, modifying and writing biological sequences."
 license = "MIT"
@@ -46,4 +46,4 @@ exprtk = ["exprtk_rs"]
 
 [profile.release]
 lto = true
-#panic = "abort"
+panic = "abort" # ~ 10% faster

ci/script.sh

@@ -4,17 +4,13 @@ set -ex
 
 main() {
     cross build --features=exprtk --target $TARGET
-    cross build --features=exprtk --target $TARGET --release
 
     if [ ! -z $DISABLE_TESTS ]; then
         return
     fi
 
     cross test --features=exprtk --target $TARGET
-    cross test --features=exprtk --target $TARGET --release
-
     cross run --features=exprtk --target $TARGET
-    cross run --features=exprtk --target $TARGET --release
 }
 
 # we don't run the "test phase" when doing deploys

scripts/time.sh

@@ -33,7 +33,7 @@ time seqkit fq2fa $f > /dev/null
 # random subsampling
 time s sample -f 0.1 $f > /dev/null
 time seqtk sample $f 0.1 > /dev/null
-time seqkit sample -a 0.1 $f > /dev/null
+time seqkit sample -p 0.1 $f > /dev/null
 
 # counting
 time s count $f

src/cfg.rs

@@ -110,7 +110,6 @@ impl<'a> Config<'a> {
         input::io_readers(&self.input_opts, |in_opts, rdr| {
             vars.new_input(in_opts)?;
             input::run_reader(&in_opts.format, rdr, in_opts.cap, in_opts.max_mem, |rec| {
-                let rec = DefRecord::from_rec(rec);
                 vars.set_record(&rec)?;
                 func(&rec, vars)
             })

src/cmd/replace.rs

@@ -2,6 +2,7 @@ use std::borrow::ToOwned;
 use std::str;
 
 use regex;
+use memchr::Memchr;
 
 use lib::twoway_iter::TwowayIter;
 use error::CliResult;
@@ -10,6 +11,7 @@ use io::{SeqAttr, RecordEditor};
 use cfg;
 use lib::util::replace_iter;
 
+
 static USAGE: &'static str = concat!("
 This command does fast search and replace for patterns in sequences
 or ids/descriptions.
@@ -59,8 +61,13 @@ pub fn run() -> CliResult<()> {
             run_replace(&cfg, attr, replacement, replacer, num_threads)?;
         }
     } else {
-        let replacer = BytesReplacer(pattern.as_bytes().to_owned());
-        run_replace(&cfg, attr, replacement, replacer, num_threads)?;
+        if pattern.len() == 1 {
+            let replacer = SingleByteReplacer(pattern.as_bytes()[0]);
+            run_replace(&cfg, attr, replacement, replacer, num_threads)?;
+        } else {
+            let replacer = BytesReplacer(pattern.as_bytes().to_owned());
+            run_replace(&cfg, attr, replacement, replacer, num_threads)?;
+        }
     }
     Ok(())
 }
@@ -94,6 +101,17 @@ trait Replacer {
     fn replace(&self, text: &[u8], replacement: &[u8], out: &mut Vec<u8>) -> CliResult<()>;
 }
 
+struct SingleByteReplacer(u8);
+
+impl Replacer for SingleByteReplacer {
+    fn replace(&self, text: &[u8], replacement: &[u8], out: &mut Vec<u8>) -> CliResult<()> {
+        let matches = Memchr::new(self.0, text).map(|start| (start, start + 1));
+        replace_iter(text, replacement, out, matches);
+        Ok(())
+    }
+}
+
+
 struct BytesReplacer(Vec<u8>);
 
 impl Replacer for BytesReplacer {
@@ -112,7 +130,7 @@ impl Replacer for BytesRegexReplacer {
             let matches = self.0.find_iter(text).map(|m| (m.start(), m.end()));
             replace_iter(text, replacement, out, matches);
         } else {
-            // slower, requires allocations
+            // requires allocations
             let replaced = self.0.replace_all(text, replacement);
             out.extend_from_slice(&replaced);
         }
@@ -129,7 +147,7 @@ impl Replacer for RegexReplacer {
             let matches = self.0.find_iter(string).map(|m| (m.start(), m.end()));
             replace_iter(text, replacement, out, matches);
         } else {
-            // slower, requires allocations
+            // requires allocations
             let replacement = str::from_utf8(replacement)?;
             let replaced = self.0.replace_all(string, replacement);
             out.extend_from_slice(replaced.as_bytes());

src/io/csv.rs

@@ -1,7 +1,7 @@
 use std::io;
 use std::convert::AsRef;
 use std::collections::HashMap;
-use std::borrow::ToOwned;
+use std::borrow::{Cow,ToOwned};
 
 use csv;
 use lib::util::match_fields;
@@ -168,10 +168,19 @@ impl Record for CsvRecord {
         (self.id_bytes(), self.desc_bytes())
     }
 
+    fn get_header(&self) -> SeqHeader {
+        let (id, desc) = self.id_desc_bytes();
+        SeqHeader::IdDesc(id, desc)
+    }
+
     fn raw_seq(&self) -> &[u8] {
         self.data.get(self.cols.seq_col).unwrap_or(b"")
     }
 
+    fn has_seq_lines(&self) -> bool {
+        false
+    }
+
     fn qual(&self) -> Option<&[u8]> {
         self.cols.qual_col.map(|i| self.data.get(i).unwrap_or(b""))
     }
@@ -180,5 +189,3 @@ impl Record for CsvRecord {
         to.extend_from_slice(self.raw_seq())
     }
 }
-
-// Writer

src/io/fasta.rs

@@ -1,34 +1,76 @@
+
 use std::io;
+use std::borrow::{Cow,ToOwned};
+use std::cell::Cell;
+
+use memchr::memchr;
 
 use error::CliResult;
-use seq_io::fasta;
+use seq_io::fasta::{self, Record as FR};
 use super::*;
 
-// Record
+// Wrapper for FASTA record
 
-impl<'a> Record for fasta::RefRecord<'a> {
-    fn id_bytes(&self) -> &[u8] {
-        <fasta::RefRecord as fasta::Record>::id_bytes(self)
+pub struct FastaRecord<'a> {
+    rec: fasta::RefRecord<'a>,
+    delim: Cell<Option<Option<usize>>>
+}
+
+impl<'a> FastaRecord<'a> {
+    #[inline(always)]
+    pub fn new(inner: fasta::RefRecord<'a>) -> FastaRecord<'a> {
+        FastaRecord {
+            rec: inner,
+            delim: Cell::new(None)
+        }
     }
 
-    fn desc_bytes(&self) -> Option<&[u8]> {
-        <fasta::RefRecord as fasta::Record>::desc_bytes(self)
+    #[inline(always)]
+    fn _get_header(&self) -> (&[u8], Option<&[u8]>) {
+        if let Some(d) = self.delim.get() {
+            if let Some(d) = d {
+                let (id, desc) = self.rec.head().split_at(d);
+                (id, Some(&desc[1..]))
+            } else {
+                (self.rec.head(), None)
+            }
+        } else {
+            self.delim.set(Some(memchr(b' ', self.rec.head())));
+            self._get_header()
+        }
     }
+}
 
+impl<'a> Record for FastaRecord<'a> {
+    fn id_bytes(&self) -> &[u8] {
+        self._get_header().0
+    }
+    fn desc_bytes(&self) -> Option<&[u8]> {
+        self._get_header().1
+    }
     fn id_desc_bytes(&self) -> (&[u8], Option<&[u8]>) {
-        <fasta::RefRecord as fasta::Record>::id_desc_bytes(self)
+        self._get_header()
+    }
+    fn delim(&self) -> Option<Option<usize>> {
+        self.delim.get()
+    }
+    fn set_delim(&self, delim: Option<usize>) {
+        self.delim.set(Some(delim))
+    }
+    fn get_header(&self) -> SeqHeader {
+        SeqHeader::FullHeader(self.rec.head())
     }
-
     fn raw_seq(&self) -> &[u8] {
-        <fasta::RefRecord as fasta::Record>::seq(self)
+        self.rec.seq()
+    }
+    fn has_seq_lines(&self) -> bool {
+        self.rec.num_seq_lines() > 1
     }
-
     fn qual(&self) -> Option<&[u8]> {
         None
     }
-
     fn seq_segments(&self) -> SeqLineIter {
-        SeqLineIter::Fasta(self.seq_lines())
+        SeqLineIter::Fasta(self.rec.seq_lines())
     }
 }
 
@@ -48,13 +90,16 @@ impl<W: io::Write> FastaWriter<W> {
     }
 }
 
+
 impl<W: io::Write> SeqWriter for FastaWriter<W> {
-    fn write(&mut self, id: &[u8], desc: Option<&[u8]>, record: &Record) -> CliResult<()> {
+    fn write(&mut self, record: &Record) -> CliResult<()> {
+        match record.get_header() {
+            SeqHeader::IdDesc(id, desc) => fasta::write_id_desc(&mut self.io_writer, id, desc)?,
+            SeqHeader::FullHeader(h) => fasta::write_head(&mut self.io_writer, h)?,
+        }
         if let Some(wrap) = self.wrap {
-            fasta::write_id_desc(&mut self.io_writer, id, desc)?;
             fasta::write_wrap_seq_iter(&mut self.io_writer, record.seq_segments(), wrap)?;
         } else {
-            fasta::write_id_desc(&mut self.io_writer, id, desc)?;
             fasta::write_seq_iter(&mut self.io_writer, record.seq_segments())?;
         }
         Ok(())

src/io/fastq.rs

@@ -1,37 +1,77 @@
+
 use std::io;
 use error::CliResult;
+use std::borrow::Cow;
+use std::cell::Cell;
+
+use memchr::memchr;
 
-use seq_io::fastq;
+use seq_io::fastq::{self, Record as FR};
 use super::*;
 
-// Record
+// Wrapper for FASTQ record
 
-impl<'a> Record for fastq::RefRecord<'a> {
-    fn id_bytes(&self) -> &[u8] {
-        <fastq::RefRecord as fastq::Record>::id_bytes(self)
+pub struct FastqRecord<'a> {
+    rec: fastq::RefRecord<'a>,
+    delim: Cell<Option<Option<usize>>>
+}
+
+impl<'a> FastqRecord<'a> {
+    #[inline(always)]
+    pub fn new(inner: fastq::RefRecord<'a>) -> FastqRecord<'a> {
+        FastqRecord {
+            rec: inner,
+            delim: Cell::new(None)
+        }
     }
 
-    fn desc_bytes(&self) -> Option<&[u8]> {
-        <fastq::RefRecord as fastq::Record>::desc_bytes(self)
+    #[inline(always)]
+    fn _get_header(&self) -> (&[u8], Option<&[u8]>) {
+        if let Some(d) = self.delim.get() {
+            if let Some(d) = d {
+                let (id, desc) = self.rec.head().split_at(d);
+                (id, Some(&desc[1..]))
+            } else {
+                (self.rec.head(), None)
+            }
+        } else {
+            self.delim.set(Some(memchr(b' ', self.rec.head())));
+            self._get_header()
+        }
     }
+}
 
+impl<'a> Record for FastqRecord<'a> {
+    fn id_bytes(&self) -> &[u8] {
+        self._get_header().0
+    }
+    fn desc_bytes(&self) -> Option<&[u8]> {
+        self._get_header().1
+    }
     fn id_desc_bytes(&self) -> (&[u8], Option<&[u8]>) {
-        <fastq::RefRecord as fastq::Record>::id_desc_bytes(self)
+        self._get_header()
+    }
+    fn delim(&self) -> Option<Option<usize>> {
+        self.delim.get()
+    }
+    fn set_delim(&self, delim: Option<usize>) {
+        self.delim.set(Some(delim))
+    }
+    fn get_header(&self) -> SeqHeader {
+        SeqHeader::FullHeader(self.rec.head())
     }
-
     fn raw_seq(&self) -> &[u8] {
-        <fastq::RefRecord as fastq::Record>::seq(self)
+        self.rec.seq()
     }
-
-    fn qual(&self) -> Option<&[u8]> {
-        Some(<fastq::RefRecord as fastq::Record>::qual(self))
+    fn has_seq_lines(&self) -> bool {
+        false
     }
-
-    fn write_seq(&self, to: &mut Vec<u8>) {
-        to.extend_from_slice(self.raw_seq())
+    fn qual(&self) -> Option<&[u8]> {
+        Some(<fastq::RefRecord as fastq::Record>::qual(&self.rec))
     }
 }
 
+
 // Writer
 
 pub struct FastqWriter<W: io::Write>(W);
@@ -43,13 +83,17 @@ impl<W: io::Write> FastqWriter<W> {
 }
 
 impl<W: io::Write> SeqWriter for FastqWriter<W> {
-    fn write(&mut self, id: &[u8], desc: Option<&[u8]>, record: &Record) -> CliResult<()> {
+    fn write(&mut self, record: &Record) -> CliResult<()> {
         let qual = record.qual().ok_or("Qualities missing!")?;
         // Using .raw_seq() is possible only because FASTA cannot be used as input source
-        // (no quality info). Might become a problem if getting the quality info from other sources
+        // (no quality info). Might change if getting the quality info from other sources
         // (mothur-style .qual files)
         let seq = record.raw_seq();
-        fastq::write_parts(&mut self.0, id, desc, seq, qual)?;
+
+        match record.get_header() {
+            SeqHeader::IdDesc(id, desc) => fastq::write_parts(&mut self.0, id, desc, seq, qual)?,
+            SeqHeader::FullHeader(h) => fastq::write_to(&mut self.0, h, seq, qual)?,
+        }
         Ok(())
     }
 }