Remove magic dependency on Linux

This uses the Rust tree_magic package instead.

Most work was already done in
#600.
This just sets tree_magic to build unconditionally instead of always on
Windows.

This improves the memory usage of docs.rs by over a factor of 5 when
uploading files in some cases. This can be tested locally like so:

Run minio in the background:
```
docker run -p 9000:9000 -e MINIO_ACCESS_KEY=password -e MINIO_SECRET_KEY=password minio/minio server /data
```

Navigate to localhost:9000/ and set both the access key and secret key
to 'password'. Add a new bucket called `rust-docs-rs`.

Now in another terminal, run docs.rs. This assumes you already have the
CRATESFYI_DATABASE_URL set up locally as described in
[developing without docker-compose](https://forge.rust-lang.org/docs-rs/no-docker-compose.html):

```
 # set up a fairly large file
mkdir -p ignored/tmp
yes | head -n $((1024 * 1024 * 50)) > ignored/tmp/100MB

git checkout master
cargo build --release
RUST_LOG=cratesfyi,info S3_ENDPOINT=http://localhost:9000 AWS_ACCESS_KEY_ID=password AWS_SECRET_ACCESS_KEY=password valgrind --tool=massif --massif-out-file=libmagic.massif.out target/release/cratesfyi database add-directory ignored/tmp/
 # this will show ~1.5 GB used, depending on your system
ms_print libmagic.massif.out | less

 # now try without libmagic
git checkout druid
cargo build --release
RUST_LOG=cratesfyi,info S3_ENDPOINT=http://localhost:9000 AWS_ACCESS_KEY_ID=password AWS_SECRET_ACCESS_KEY=password valgrind --tool=massif --massif-out-file=no_libmagic.massif.out target/release/cratesfyi database add-directory ignored/tmp/
 # this will show ~250 MB used, depending on your system
ms_print no_libmagic.massif.out | less
```

Author: jyn514

Cargo.lock

@@ -38,6 +38,7 @@ prometheus = { version = "0.7.0", default-features = false }
 lazy_static = "1.0.0"
 rustwide = "0.5.0"
 tempdir = "0.3"
+mime_guess = "2"
 
 # iron dependencies
 iron = "0.5"
@@ -48,11 +49,9 @@ staticfile = { version = "0.4", features = [ "cache" ] }
 
 [target.'cfg(not(windows))'.dependencies]
 libc = "0.2"
-magic = "0.12"
 
 [target.'cfg(windows)'.dependencies]
 path-slash = "0.1.1"
-mime_guess = "2.0.1"
 
 [dependencies.postgres]
 version = "0.15"

Cargo.toml

@@ -17,8 +17,6 @@ use rusoto_s3::{S3, PutObjectRequest, GetObjectRequest, S3Client};
 use rusoto_core::region::Region;
 use rusoto_credential::DefaultCredentialsProvider;
 use std::ffi::OsStr;
-#[cfg(not(windows))]
-use magic::{Cookie, flags};
 
 const MAX_CONCURRENT_UPLOADS: usize = 1000;
 
@@ -157,13 +155,12 @@ pub fn add_path_into_database<P: AsRef<Path>>(conn: &Connection,
                                               prefix: &str,
                                               path: P)
                                               -> Result<Json> {
-    let trans = conn.transaction()?;
-    #[cfg(not(windows))]
-    let mime_data = load_mime_data()?;
     use std::collections::HashMap;
-    let mut file_paths_and_mimes: HashMap<PathBuf, String> = HashMap::new();
     use futures::future::Future;
 
+    let trans = conn.transaction()?;
+    let mut file_paths_and_mimes: HashMap<PathBuf, String> = HashMap::new();
+
     let mut rt = ::tokio::runtime::Runtime::new().unwrap();
 
     let mut to_upload = get_file_list(&path)?;
@@ -194,10 +191,7 @@ pub fn add_path_into_database<P: AsRef<Path>>(conn: &Connection,
             #[cfg(not(windows))]
             let bucket_path = bucket_path.into_os_string().into_string().unwrap();
 
-            #[cfg(windows)]
             let mime = detect_mime(&content, &file_path)?;
-            #[cfg(not(windows))]
-            let mime = detect_mime(&content, &file_path, &mime_data)?;
 
             if let Some(client) = &client {
                 futures.push(client.put_object(PutObjectRequest {
@@ -260,20 +254,6 @@ pub fn add_path_into_database<P: AsRef<Path>>(conn: &Connection,
     file_list_to_json(file_list_with_mimes)
 }
 
-#[cfg(not(windows))]
-fn load_mime_data() -> Result<Cookie> {
-    let cookie = Cookie::open(flags::MIME_TYPE)?;
-    cookie.load::<&str>(&[])?;
-    Ok(cookie)
-}
-
-#[cfg(not(windows))]
-fn detect_mime(content: &Vec<u8>, file_path: &Path, cookie: &Cookie) -> Result<String> {
-    let mime = cookie.buffer(&content)?;
-    correct_mime(&mime, &file_path)
-}
-
-#[cfg(windows)]
 fn detect_mime(_content: &Vec<u8>, file_path: &Path) -> Result<String> {
     let mime = mime_guess::from_path(file_path).first_raw().map(|m| m).unwrap_or("text/plain");
     correct_mime(&mime, &file_path)
@@ -393,12 +373,7 @@ mod test {
     }
 
     fn check_mime(content: &str, path: &str, expected_mime: &str) {
-        #[cfg(not(windows))]
-        let mime_data = load_mime_data().unwrap();
-        #[cfg(windows)]
         let detected_mime = detect_mime(&content.as_bytes().to_vec(), Path::new(&path));
-        #[cfg(not(windows))]
-        let detected_mime = detect_mime(&content.as_bytes().to_vec(), Path::new(&path), &mime_data);
         let detected_mime = detected_mime.expect("no mime was given");
         assert_eq!(detected_mime, expected_mime);
     }

src/db/file.rs

@@ -39,16 +39,13 @@ extern crate tokio;
 extern crate systemstat;
 extern crate rustwide;
 extern crate tempdir;
+extern crate mime_guess;
 
-#[cfg(not(windows))]
-extern crate magic;
 #[cfg(not(windows))]
 extern crate libc;
 
 #[cfg(windows)]
 extern crate path_slash;
-#[cfg(windows)]
-extern crate mime_guess;
 
 #[cfg(test)]
 extern crate once_cell;