refactor: create util folder, move nap_sym to napi/sym, move http_cac…

…he to cache folder (denoland#16857)
DavidDeSimone · Nov 28, 2022 · 2d4c46c · 2d4c46c
1 parent f526513
commit 2d4c46c
Show file tree

Hide file tree

Showing 57 changed files with 940 additions and 920 deletions.
diff --git a/Cargo.toml b/Cargo.toml
@@ -5,7 +5,7 @@ resolver = "2"
 members = [
   "bench_util",
   "cli",
-  "cli/napi_sym",
+  "cli/napi/sym",
   "core",
   "ops",
   "runtime",
@@ -47,7 +47,7 @@ deno_core = { version = "0.161.0", path = "./core" }
 deno_ops = { version = "0.39.0", path = "./ops" }
 serde_v8 = { version = "0.72.0", path = "./serde_v8" }
 deno_runtime = { version = "0.87.0", path = "./runtime" }
-napi_sym = { version = "0.9.0", path = "./cli/napi_sym" }
+napi_sym = { version = "0.9.0", path = "./cli/napi/sym" }
 deno_bench_util = { version = "0.73.0", path = "./bench_util" }
 test_util = { path = "./test_util" }
 

diff --git a/cli/args/config_file.rs b/cli/args/config_file.rs
@@ -3,10 +3,9 @@
 use crate::args::ConfigFlag;
 use crate::args::Flags;
 use crate::args::TaskFlags;
-use crate::fs_util;
-use crate::fs_util::canonicalize_path;
-use crate::fs_util::specifier_parent;
-use crate::fs_util::specifier_to_file_path;
+use crate::util::fs::canonicalize_path;
+use crate::util::path::specifier_parent;
+use crate::util::path::specifier_to_file_path;
 
 use deno_core::anyhow::anyhow;
 use deno_core::anyhow::bail;
@@ -467,7 +466,7 @@ impl ConfigFile {
             ..
           }) = &flags.subcommand
           {
-            let task_cwd = fs_util::canonicalize_path(&PathBuf::from(path))?;
+            let task_cwd = canonicalize_path(&PathBuf::from(path))?;
             if let Some(path) = Self::discover_from(&task_cwd, &mut checked)? {
               return Ok(Some(path));
             }

diff --git a/cli/args/lockfile.rs b/cli/args/lockfile.rs
@@ -20,6 +20,7 @@ use crate::npm::NpmPackageId;
 use crate::npm::NpmPackageReq;
 use crate::npm::NpmResolutionPackage;
 use crate::tools::fmt::format_json;
+use crate::util;
 use crate::Flags;
 
 #[derive(Debug)]
@@ -260,7 +261,7 @@ impl Lockfile {
   /// is not included, insert it.
   fn check_or_insert(&mut self, specifier: &str, code: &str) -> bool {
     if let Some(lockfile_checksum) = self.content.remote.get(specifier) {
-      let compiled_checksum = crate::checksum::gen(&[code.as_bytes()]);
+      let compiled_checksum = util::checksum::gen(&[code.as_bytes()]);
       lockfile_checksum == &compiled_checksum
     } else {
       self.insert(specifier, code);
@@ -269,7 +270,7 @@ impl Lockfile {
   }
 
   fn insert(&mut self, specifier: &str, code: &str) {
-    let checksum = crate::checksum::gen(&[code.as_bytes()]);
+    let checksum = util::checksum::gen(&[code.as_bytes()]);
     self.content.remote.insert(specifier.to_string(), checksum);
     self.has_content_changed = true;
   }
@@ -359,7 +360,7 @@ impl deno_graph::source::Locker for Locker {
   }
 
   fn get_checksum(&self, content: &str) -> String {
-    crate::checksum::gen(&[content.as_bytes()])
+    util::checksum::gen(&[content.as_bytes()])
   }
 
   fn get_filename(&self) -> Option<String> {

diff --git a/cli/args/mod.rs b/cli/args/mod.rs
@@ -22,6 +22,10 @@ pub use config_file::TsConfig;
 pub use config_file::TsConfigForEmit;
 pub use config_file::TsConfigType;
 pub use config_file::TsTypeLib;
+use deno_runtime::deno_tls::rustls;
+use deno_runtime::deno_tls::rustls_native_certs::load_native_certs;
+use deno_runtime::deno_tls::rustls_pemfile;
+use deno_runtime::deno_tls::webpki_roots;
 pub use flags::*;
 pub use lockfile::Lockfile;
 pub use lockfile::LockfileError;
@@ -40,16 +44,130 @@ use deno_runtime::inspector_server::InspectorServer;
 use deno_runtime::permissions::PermissionsOptions;
 use std::collections::BTreeMap;
 use std::env;
+use std::io::BufReader;
 use std::net::SocketAddr;
 use std::path::PathBuf;
 use std::sync::Arc;
 
 use crate::cache::DenoDir;
-use crate::file_fetcher::get_root_cert_store;
-use crate::file_fetcher::CacheSetting;
-use crate::fs_util;
+use crate::util::fs::canonicalize_path_maybe_not_exists;
 use crate::version;
 
+/// Indicates how cached source files should be handled.
+#[derive(Debug, Clone, Eq, PartialEq)]
+pub enum CacheSetting {
+  /// Only the cached files should be used.  Any files not in the cache will
+  /// error.  This is the equivalent of `--cached-only` in the CLI.
+  Only,
+  /// No cached source files should be used, and all files should be reloaded.
+  /// This is the equivalent of `--reload` in the CLI.
+  ReloadAll,
+  /// Only some cached resources should be used.  This is the equivalent of
+  /// `--reload=https://deno.land/std` or
+  /// `--reload=https://deno.land/std,https://deno.land/x/example`.
+  ReloadSome(Vec<String>),
+  /// The usability of a cached value is determined by analyzing the cached
+  /// headers and other metadata associated with a cached response, reloading
+  /// any cached "non-fresh" cached responses.
+  RespectHeaders,
+  /// The cached source files should be used for local modules.  This is the
+  /// default behavior of the CLI.
+  Use,
+}
+
+impl CacheSetting {
+  pub fn should_use_for_npm_package(&self, package_name: &str) -> bool {
+    match self {
+      CacheSetting::ReloadAll => false,
+      CacheSetting::ReloadSome(list) => {
+        if list.iter().any(|i| i == "npm:") {
+          return false;
+        }
+        let specifier = format!("npm:{}", package_name);
+        if list.contains(&specifier) {
+          return false;
+        }
+        true
+      }
+      _ => true,
+    }
+  }
+}
+
+/// Create and populate a root cert store based on the passed options and
+/// environment.
+pub fn get_root_cert_store(
+  maybe_root_path: Option<PathBuf>,
+  maybe_ca_stores: Option<Vec<String>>,
+  maybe_ca_file: Option<String>,
+) -> Result<RootCertStore, AnyError> {
+  let mut root_cert_store = RootCertStore::empty();
+  let ca_stores: Vec<String> = maybe_ca_stores
+    .or_else(|| {
+      let env_ca_store = env::var("DENO_TLS_CA_STORE").ok()?;
+      Some(
+        env_ca_store
+          .split(',')
+          .map(|s| s.trim().to_string())
+          .filter(|s| !s.is_empty())
+          .collect(),
+      )
+    })
+    .unwrap_or_else(|| vec!["mozilla".to_string()]);
+
+  for store in ca_stores.iter() {
+    match store.as_str() {
+      "mozilla" => {
+        root_cert_store.add_server_trust_anchors(
+          webpki_roots::TLS_SERVER_ROOTS.0.iter().map(|ta| {
+            rustls::OwnedTrustAnchor::from_subject_spki_name_constraints(
+              ta.subject,
+              ta.spki,
+              ta.name_constraints,
+            )
+          }),
+        );
+      }
+      "system" => {
+        let roots = load_native_certs().expect("could not load platform certs");
+        for root in roots {
+          root_cert_store
+            .add(&rustls::Certificate(root.0))
+            .expect("Failed to add platform cert to root cert store");
+        }
+      }
+      _ => {
+        return Err(anyhow!("Unknown certificate store \"{}\" specified (allowed: \"system,mozilla\")", store));
+      }
+    }
+  }
+
+  let ca_file = maybe_ca_file.or_else(|| env::var("DENO_CERT").ok());
+  if let Some(ca_file) = ca_file {
+    let ca_file = if let Some(root) = &maybe_root_path {
+      root.join(&ca_file)
+    } else {
+      PathBuf::from(ca_file)
+    };
+    let certfile = std::fs::File::open(&ca_file)?;
+    let mut reader = BufReader::new(certfile);
+
+    match rustls_pemfile::certs(&mut reader) {
+      Ok(certs) => {
+        root_cert_store.add_parsable_certificates(&certs);
+      }
+      Err(e) => {
+        return Err(anyhow!(
+          "Unable to add pem file to certificate store: {}",
+          e
+        ));
+      }
+    }
+  }
+
+  Ok(root_cert_store)
+}
+
 /// Overrides for the options below that when set will
 /// use these values over the values derived from the
 /// CLI flags or config file.
@@ -176,7 +294,7 @@ impl CliOptions {
     } else {
       std::env::current_dir()?.join("node_modules")
     };
-    Ok(Some(fs_util::canonicalize_path_maybe_not_exists(&path)?))
+    Ok(Some(canonicalize_path_maybe_not_exists(&path)?))
   }
 
   pub fn resolve_root_cert_store(&self) -> Result<RootCertStore, AnyError> {

diff --git a/cli/cache/disk_cache.rs b/cli/cache/disk_cache.rs
@@ -1,7 +1,8 @@
 // Copyright 2018-2022 the Deno authors. All rights reserved. MIT license.
 
-use crate::fs_util;
-use crate::http_cache::url_to_filename;
+use super::http_cache::url_to_filename;
+use super::CACHE_PERM;
+use crate::util::fs::atomic_write_file;
 
 use deno_core::url::Host;
 use deno_core::url::Url;
@@ -144,7 +145,7 @@ impl DiskCache {
       Some(parent) => self.ensure_dir_exists(parent),
       None => Ok(()),
     }?;
-    fs_util::atomic_write_file(&path, data, crate::http_cache::CACHE_PERM)
+    atomic_write_file(&path, data, CACHE_PERM)
       .map_err(|e| with_io_context(&e, format!("{:#?}", &path)))
   }
 }

diff --git a/cli/http_cache.rs → cli/cache/http_cache.rs b/cli/http_cache.rs → cli/cache/http_cache.rs
@@ -3,8 +3,8 @@
 //! as defined in RFC 7234 (<https://tools.ietf.org/html/rfc7234>).
 //! Currently it's a very simplified version to fulfill Deno needs
 //! at hand.
-use crate::fs_util;
 use crate::http_util::HeadersMap;
+use crate::util;
 use deno_core::error::generic_error;
 use deno_core::error::AnyError;
 use deno_core::serde::Deserialize;
@@ -19,7 +19,7 @@ use std::path::Path;
 use std::path::PathBuf;
 use std::time::SystemTime;
 
-pub const CACHE_PERM: u32 = 0o644;
+use super::CACHE_PERM;
 
 /// Turn base of url (scheme, hostname, port) into a valid filename.
 /// This method replaces port part with a special string token (because
@@ -68,31 +68,32 @@ pub fn url_to_filename(url: &Url) -> Option<PathBuf> {
   // NOTE: fragment is omitted on purpose - it's not taken into
   // account when caching - it denotes parts of webpage, which
   // in case of static resources doesn't make much sense
-  let hashed_filename = crate::checksum::gen(&[rest_str.as_bytes()]);
+  let hashed_filename = util::checksum::gen(&[rest_str.as_bytes()]);
   cache_filename.push(hashed_filename);
   Some(cache_filename)
 }
 
+/// Cached metadata about a url.
 #[derive(Serialize, Deserialize)]
-pub struct Metadata {
+pub struct CachedUrlMetadata {
   pub headers: HeadersMap,
   pub url: String,
   #[serde(default = "SystemTime::now")]
   pub now: SystemTime,
 }
 
-impl Metadata {
+impl CachedUrlMetadata {
   pub fn write(&self, cache_filename: &Path) -> Result<(), AnyError> {
     let metadata_filename = Self::filename(cache_filename);
     let json = serde_json::to_string_pretty(self)?;
-    fs_util::atomic_write_file(&metadata_filename, json, CACHE_PERM)?;
+    util::fs::atomic_write_file(&metadata_filename, json, CACHE_PERM)?;
     Ok(())
   }
 
-  pub fn read(cache_filename: &Path) -> Result<Metadata, AnyError> {
-    let metadata_filename = Metadata::filename(cache_filename);
+  pub fn read(cache_filename: &Path) -> Result<Self, AnyError> {
+    let metadata_filename = Self::filename(cache_filename);
     let metadata = fs::read_to_string(metadata_filename)?;
-    let metadata: Metadata = serde_json::from_str(&metadata)?;
+    let metadata: Self = serde_json::from_str(&metadata)?;
     Ok(metadata)
   }
 
@@ -149,10 +150,10 @@ impl HttpCache {
       url_to_filename(url)
         .ok_or_else(|| generic_error("Can't convert url to filename."))?,
     );
-    let metadata_filename = Metadata::filename(&cache_filename);
+    let metadata_filename = CachedUrlMetadata::filename(&cache_filename);
     let file = File::open(cache_filename)?;
     let metadata = fs::read_to_string(metadata_filename)?;
-    let metadata: Metadata = serde_json::from_str(&metadata)?;
+    let metadata: CachedUrlMetadata = serde_json::from_str(&metadata)?;
     Ok((file, metadata.headers, metadata.now))
   }
 
@@ -172,9 +173,9 @@ impl HttpCache {
       .expect("Cache filename should have a parent dir");
     self.ensure_dir_exists(parent_filename)?;
     // Cache content
-    fs_util::atomic_write_file(&cache_filename, content, CACHE_PERM)?;
+    util::fs::atomic_write_file(&cache_filename, content, CACHE_PERM)?;
 
-    let metadata = Metadata {
+    let metadata = CachedUrlMetadata {
       now: SystemTime::now(),
       url: url.to_string(),
       headers: headers_map,

diff --git a/cli/cache/mod.rs b/cli/cache/mod.rs
@@ -19,6 +19,7 @@ mod common;
 mod deno_dir;
 mod disk_cache;
 mod emit;
+mod http_cache;
 mod incremental;
 mod node;
 mod parsed_source;
@@ -28,10 +29,15 @@ pub use common::FastInsecureHasher;
 pub use deno_dir::DenoDir;
 pub use disk_cache::DiskCache;
 pub use emit::EmitCache;
+pub use http_cache::CachedUrlMetadata;
+pub use http_cache::HttpCache;
 pub use incremental::IncrementalCache;
 pub use node::NodeAnalysisCache;
 pub use parsed_source::ParsedSourceCache;
 
+/// Permissions used to save a file in the disk caches.
+pub const CACHE_PERM: u32 = 0o644;
+
 /// A "wrapper" for the FileFetcher and DiskCache for the Deno CLI that provides
 /// a concise interface to the DENO_DIR when building module graphs.
 pub struct FetchCacher {