Skip to content

Use different library for mime on windows #600

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Feb 14, 2020
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 15 additions & 10 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

7 changes: 5 additions & 2 deletions Cargo.toml
Original file line number Diff line number Diff line change
@@ -18,11 +18,9 @@ reqwest = "0.9"
semver = "0.9"
slug = "=0.1.1"
env_logger = "0.6"
magic = "0.12"
r2d2 = "0.8"
r2d2_postgres = "0.14"
url = "1.4"
libc = "0.2"
badge = { version = "0", path = "src/web/badge" }
failure = "0.1.3"
comrak = { version = "0.3", default-features = false }
@@ -48,8 +46,13 @@ handlebars-iron = "0.22"
params = "0.6"
staticfile = { version = "0.4", features = [ "cache" ] }

[target.'cfg(not(windows))'.dependencies]
libc = "0.2"
magic = "0.12"

[target.'cfg(windows)'.dependencies]
path-slash = "0.1.1"
mime_guess = "2.0.1"

[dependencies.postgres]
version = "0.15"
97 changes: 71 additions & 26 deletions src/db/file.rs
Original file line number Diff line number Diff line change
@@ -16,7 +16,9 @@ use failure::err_msg;
use rusoto_s3::{S3, PutObjectRequest, GetObjectRequest, S3Client};
use rusoto_core::region::Region;
use rusoto_credential::DefaultCredentialsProvider;

use std::ffi::OsStr;
#[cfg(not(windows))]
use magic::{Cookie, flags};

const MAX_CONCURRENT_UPLOADS: usize = 1000;

@@ -155,12 +157,9 @@ pub fn add_path_into_database<P: AsRef<Path>>(conn: &Connection,
prefix: &str,
path: P)
-> Result<Json> {
use magic::{Cookie, flags};
let cookie = Cookie::open(flags::MIME_TYPE)?;
cookie.load::<&str>(&[])?;

let trans = conn.transaction()?;

#[cfg(not(windows))]
let mime_data = load_mime_data()?;
use std::collections::HashMap;
let mut file_paths_and_mimes: HashMap<PathBuf, String> = HashMap::new();
use futures::future::Future;
@@ -195,25 +194,10 @@ pub fn add_path_into_database<P: AsRef<Path>>(conn: &Connection,
#[cfg(not(windows))]
let bucket_path = bucket_path.into_os_string().into_string().unwrap();

let mime = {
let mime = cookie.buffer(&content)?;
// css's are causing some problem in browsers
// magic will return text/plain for css file types
// convert them to text/css
// do the same for javascript files
if mime == "text/plain" {
let e = file_path.extension().unwrap_or_default();
if e == "css" {
"text/css".to_owned()
} else if e == "js" {
"application/javascript".to_owned()
} else {
mime.to_owned()
}
} else {
mime.to_owned()
}
};
#[cfg(windows)]
let mime = detect_mime(&content, &file_path)?;
#[cfg(not(windows))]
let mime = detect_mime(&content, &file_path, &mime_data)?;

if let Some(client) = &client {
futures.push(client.put_object(PutObjectRequest {
@@ -276,6 +260,42 @@ pub fn add_path_into_database<P: AsRef<Path>>(conn: &Connection,
file_list_to_json(file_list_with_mimes)
}

#[cfg(not(windows))]
fn load_mime_data() -> Result<Cookie> {
let cookie = Cookie::open(flags::MIME_TYPE)?;
cookie.load::<&str>(&[])?;
Ok(cookie)
}

#[cfg(not(windows))]
fn detect_mime(content: &Vec<u8>, file_path: &Path, cookie: &Cookie) -> Result<String> {
let mime = cookie.buffer(&content)?;
correct_mime(&mime, &file_path)
}

#[cfg(windows)]
fn detect_mime(_content: &Vec<u8>, file_path: &Path) -> Result<String> {
let mime = mime_guess::from_path(file_path).first_raw().map(|m| m).unwrap_or("text/plain");
correct_mime(&mime, &file_path)
}

fn correct_mime(mime: &str, file_path: &Path) -> Result<String> {
Ok(match mime {
"text/plain" | "text/troff" | "text/x-markdown" | "text/x-rust" | "text/x-toml" => {
match file_path.extension().and_then(OsStr::to_str) {
Some("md") => "text/markdown",
Some("rs") => "text/rust",
Some("markdown") => "text/markdown",
Some("css") => "text/css",
Some("toml") => "text/toml",
Some("js") => "application/javascript",
Some("json") => "application/json",
_ => mime
}
}
_ => mime
}.to_owned())
}


fn file_list_to_json(file_list: Vec<(String, PathBuf)>) -> Result<Json> {
@@ -343,7 +363,7 @@ pub fn move_to_s3(conn: &Connection, n: usize) -> Result<usize> {
mod test {
extern crate env_logger;
use std::env;
use super::get_file_list;
use super::*;

#[test]
fn test_get_file_list() {
@@ -356,4 +376,29 @@ mod test {
let files = get_file_list(env::current_dir().unwrap().join("Cargo.toml")).unwrap();
assert_eq!(files[0], std::path::Path::new("Cargo.toml"));
}
#[test]
fn test_mime_types() {
check_mime("/ignored", ".gitignore", "text/plain");
check_mime("[package]", "hello.toml","text/toml");
check_mime(".ok { color:red; }", "hello.css","text/css");
check_mime("var x = 1", "hello.js","application/javascript");
check_mime("<html>", "hello.html","text/html");
check_mime("## HELLO", "hello.hello.md","text/markdown");
check_mime("## WORLD", "hello.markdown","text/markdown");
check_mime("{}", "hello.json","application/json");
check_mime("hello world", "hello.txt","text/plain");
check_mime("//! Simple module to ...", "file.rs", "text/rust");
}

fn check_mime(content: &str, path: &str, expected_mime: &str) {
#[cfg(not(windows))]
let mime_data = load_mime_data().unwrap();
#[cfg(windows)]
let detected_mime = detect_mime(&content.as_bytes().to_vec(), Path::new(&path));
#[cfg(not(windows))]
let detected_mime = detect_mime(&content.as_bytes().to_vec(), Path::new(&path), &mime_data);
let detected_mime = detected_mime.expect("no mime was given");
assert_eq!(detected_mime, expected_mime);
}
}

9 changes: 7 additions & 2 deletions src/lib.rs
Original file line number Diff line number Diff line change
@@ -16,7 +16,6 @@ extern crate reqwest;
extern crate time;
extern crate semver;
extern crate slug;
extern crate magic;
extern crate iron;
extern crate router;
extern crate staticfile;
@@ -26,7 +25,6 @@ extern crate r2d2;
extern crate r2d2_postgres;
extern crate url;
extern crate params;
extern crate libc;
extern crate badge;
extern crate crates_index_diff;
extern crate toml;
@@ -42,8 +40,15 @@ extern crate systemstat;
extern crate rustwide;
extern crate tempdir;

#[cfg(not(windows))]
extern crate magic;
#[cfg(not(windows))]
extern crate libc;

#[cfg(windows)]
extern crate path_slash;
#[cfg(windows)]
extern crate mime_guess;

#[cfg(test)]
extern crate once_cell;