Merge pull request #256 from snshn/more-tests-fixes-and-improvements

More tests, fixes, improvements
Y2Z · Jun 2, 2021 · 22a031a · 22a031a
2 parents d7a82a0 + 6e6a60b
commit 22a031a
Show file tree

Hide file tree

Showing 23 changed files with 1,202 additions and 1,174 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "monolith"
-version = "2.4.1"
+version = "2.5.0"
 authors = [
     "Sunshine <[email protected]>",
     "Mahdi Robatipoor <[email protected]>",
@@ -22,20 +22,20 @@ include = [
 license = "CC0-1.0"
 
 [dependencies]
-atty = "0.2" # Used for highlighting network errors
+atty = "0.2.14" # Used for highlighting network errors
 base64 = "0.13.0"
 chrono = "0.4.19" # Used for formatting creation timestamp
 clap = "2.33.3"
 cssparser = "0.28.1"
 html5ever = "0.24.1"
-regex = "1.4.3" # Used for parsing srcset
-sha2 = "0.9.2" # Used for calculating checksums during integrity checks
-url = "2.2.0"
+regex = "1.5.4" # Used for parsing srcset and NOSCRIPT
+sha2 = "0.9.5" # Used for calculating checksums during integrity checks
+url = "2.2.2"
 
 [dependencies.reqwest]
-version = "0.11.0"
+version = "0.11.3"
 default-features = false
 features = ["default-tls", "blocking", "gzip"]
 
 [dev-dependencies]
-assert_cmd = "1.0.2"
+assert_cmd = "1.0.4"
diff --git a/README.md b/README.md
@@ -79,11 +79,11 @@ or
  - `-j`: Exclude JavaScript
  - `-k`: Accept invalid X.509 (TLS) certificates
  - `-M`: Don't add timestamp and URL information
- - `-n`: Extract contents of NOSCRIPT tags
+ - `-n`: Extract contents of NOSCRIPT elements
  - `-o`: Write output to `file`
  - `-s`: Be quiet
  - `-t`: Adjust `network request timeout`
- - `-u`: Provide `custom User-Agent`
+ - `-u`: Provide custom `User-Agent`
  - `-v`: Exclude videos
 
 ---------------------------------------------------
@@ -99,20 +99,16 @@ Please open an issue if something is wrong, that helps make this project better.
 ---------------------------------------------------
 
 ## Related projects
- - `Monolith Chrome Extension`: https://github.com/rhysd/monolith-of-web
- - `Pagesaver`: https://github.com/distributed-mind/pagesaver
- - `Personal WayBack Machine`: https://github.com/popey/pwbm
- - `Hako`: https://github.com/dmpop/hako
- - `Monk`: https://gitlab.com/fisherdarling/monk
+ - Monolith Chrome Extension: https://github.com/rhysd/monolith-of-web
+ - Pagesaver: https://github.com/distributed-mind/pagesaver
+ - Personal WayBack Machine: https://github.com/popey/pwbm
+ - Hako: https://github.com/dmpop/hako
+ - Monk: https://gitlab.com/fisherdarling/monk
 
 ---------------------------------------------------
 
 ## License
 
-<a href="https://creativecommons.org/publicdomain/zero/1.0/">
-    <img src="https://i.creativecommons.org/p/zero/1.0/88x31.png" alt="CC0-1.0" />
-</a>
-<br />
 To the extent possible under law, the author(s) have dedicated all copyright related and neighboring rights to this software to the public domain worldwide.
 This software is distributed without any warranty.
 

diff --git a/src/css.rs b/src/css.rs
@@ -6,7 +6,7 @@ use std::collections::HashMap;
 use url::Url;
 
 use crate::opts::Options;
-use crate::url::{data_to_data_url, resolve_url};
+use crate::url::{create_data_url, resolve_url};
 use crate::utils::retrieve_asset;
 
 const CSS_PROPS_WITH_IMAGE_URLS: &[&str] = &[
@@ -55,14 +55,6 @@ pub fn embed_css(
     .unwrap()
 }
 
-pub fn enquote(input: String, double: bool) -> String {
-    if double {
-        format!("\"{}\"", input.replace("\"", "\\\""))
-    } else {
-        format!("'{}'", input.replace("'", "\\'"))
-    }
-}
-
 pub fn format_ident(ident: &str) -> String {
     let mut res: String = String::new();
     let _ = serialize_identifier(ident, &mut res);
@@ -207,7 +199,7 @@ pub fn process_css<'a>(
                         depth + 1,
                     ) {
                         Ok((import_contents, import_final_url, _import_media_type)) => {
-                            let mut import_data_url = data_to_data_url(
+                            let mut import_data_url = create_data_url(
                                 "text/css",
                                 embed_css(
                                     cache,
@@ -221,15 +213,18 @@ pub fn process_css<'a>(
                                 &import_final_url,
                             );
                             import_data_url.set_fragment(import_full_url.fragment());
-                            result.push_str(enquote(import_data_url.to_string(), false).as_str());
+                            result.push_str(
+                                format_quoted_string(&import_data_url.to_string()).as_str(),
+                            );
                         }
                         Err(_) => {
                             // Keep remote reference if unable to retrieve the asset
                             if import_full_url.scheme() == "http"
                                 || import_full_url.scheme() == "https"
                             {
-                                result
-                                    .push_str(enquote(import_full_url.to_string(), false).as_str());
+                                result.push_str(
+                                    format_quoted_string(&import_full_url.to_string()).as_str(),
+                                );
                             }
                         }
                     }
@@ -241,7 +236,7 @@ pub fn process_css<'a>(
                         }
 
                         if options.no_images && is_image_url_prop(curr_prop.as_str()) {
-                            result.push_str(enquote(str!(empty_image!()), false).as_str());
+                            result.push_str(format_quoted_string(empty_image!()).as_str());
                         } else {
                             let resolved_url: Url = resolve_url(&document_url, value);
                             match retrieve_asset(
@@ -254,17 +249,20 @@ pub fn process_css<'a>(
                             ) {
                                 Ok((data, final_url, media_type)) => {
                                     let mut data_url =
-                                        data_to_data_url(&media_type, &data, &final_url);
+                                        create_data_url(&media_type, &data, &final_url);
                                     data_url.set_fragment(resolved_url.fragment());
-                                    result.push_str(enquote(data_url.to_string(), false).as_str());
+                                    result.push_str(
+                                        format_quoted_string(&data_url.to_string()).as_str(),
+                                    );
                                 }
                                 Err(_) => {
                                     // Keep remote reference if unable to retrieve the asset
                                     if resolved_url.scheme() == "http"
                                         || resolved_url.scheme() == "https"
                                     {
                                         result.push_str(
-                                            enquote(resolved_url.to_string(), false).as_str(),
+                                            format_quoted_string(&resolved_url.to_string())
+                                                .as_str(),
                                         );
                                     }
                                 }
@@ -346,7 +344,7 @@ pub fn process_css<'a>(
                         depth + 1,
                     ) {
                         Ok((css, final_url, _media_type)) => {
-                            let mut data_url = data_to_data_url(
+                            let mut data_url = create_data_url(
                                 "text/css",
                                 embed_css(
                                     cache,
@@ -360,18 +358,19 @@ pub fn process_css<'a>(
                                 &final_url,
                             );
                             data_url.set_fragment(full_url.fragment());
-                            result.push_str(enquote(data_url.to_string(), false).as_str());
+                            result.push_str(format_quoted_string(&data_url.to_string()).as_str());
                         }
                         Err(_) => {
                             // Keep remote reference if unable to retrieve the asset
                             if full_url.scheme() == "http" || full_url.scheme() == "https" {
-                                result.push_str(enquote(full_url.to_string(), false).as_str());
+                                result
+                                    .push_str(format_quoted_string(&full_url.to_string()).as_str());
                             }
                         }
                     }
                 } else {
                     if is_image_url_prop(curr_prop.as_str()) && options.no_images {
-                        result.push_str(enquote(str!(empty_image!()), false).as_str());
+                        result.push_str(format_quoted_string(empty_image!()).as_str());
                     } else {
                         let full_url: Url = resolve_url(&document_url, value);
                         match retrieve_asset(
@@ -383,14 +382,17 @@ pub fn process_css<'a>(
                             depth + 1,
                         ) {
                             Ok((data, final_url, media_type)) => {
-                                let mut data_url = data_to_data_url(&media_type, &data, &final_url);
+                                let mut data_url = create_data_url(&media_type, &data, &final_url);
                                 data_url.set_fragment(full_url.fragment());
-                                result.push_str(enquote(data_url.to_string(), false).as_str());
+                                result
+                                    .push_str(format_quoted_string(&data_url.to_string()).as_str());
                             }
                             Err(_) => {
                                 // Keep remote reference if unable to retrieve the asset
                                 if full_url.scheme() == "http" || full_url.scheme() == "https" {
-                                    result.push_str(enquote(full_url.to_string(), false).as_str());
+                                    result.push_str(
+                                        format_quoted_string(&full_url.to_string()).as_str(),
+                                    );
                                 }
                             }
                         }