Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

perf(compile): code cache #26528

Merged
merged 25 commits into from
Nov 18, 2024
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
25 commits
Select commit Hold shift + click to select a range
7236756
perf(compile): use less memory
dsherret Oct 22, 2024
1928daf
working now with typescript
dsherret Oct 23, 2024
32913e7
Merge branch 'main' into perf_deno_compile_less_memory
dsherret Oct 23, 2024
788f4ab
Tell v8 that something is a string like before.
dsherret Oct 23, 2024
742ae3f
fix byonm issue
dsherret Oct 23, 2024
1ad5f62
Merge branch 'main' into perf_deno_compile_less_memory
dsherret Oct 23, 2024
fd13339
maybe fix ci
dsherret Oct 24, 2024
1e0dabd
do not store data urls in the binary
dsherret Oct 24, 2024
d7cd10b
switch to le because this is not network
dsherret Oct 24, 2024
c156944
review
dsherret Oct 24, 2024
938c3e0
perf(compile): code cache for initial load
dsherret Oct 24, 2024
0a7c050
use distinct strategies for compile
dsherret Oct 24, 2024
0957e09
use distinct strategies for compile
dsherret Oct 24, 2024
bad329a
tests
dsherret Oct 24, 2024
e9e4ad2
support --no-code-cache
dsherret Oct 24, 2024
8a910fd
lint
dsherret Oct 24, 2024
bc08455
Merge branch 'main' into perf_compile_code_cache
dsherret Oct 24, 2024
815be3a
remove unused use
dsherret Oct 24, 2024
ec0d0c7
fix test
dsherret Oct 24, 2024
1140105
Do not subtract with overflow when deserializing.
dsherret Oct 24, 2024
5b13157
maybe fix test failing because they had the same binary name
dsherret Oct 24, 2024
d360dda
Merge branch 'main' into perf_compile_code_cache
dsherret Nov 18, 2024
488f6dc
update after merge
dsherret Nov 18, 2024
0e38fab
lint
dsherret Nov 18, 2024
b1a784b
Merge branch 'main' into perf_compile_code_cache
dsherret Nov 18, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
fix byonm issue
  • Loading branch information
dsherret committed Oct 23, 2024
commit 742ae3f535595fca45fa1f40a98e0c12ab66583f
13 changes: 7 additions & 6 deletions cli/standalone/binary.rs
Original file line number Diff line number Diff line change
Expand Up @@ -571,7 +571,7 @@ impl<'a> DenoCompileBinaryWriter<'a> {
}
}
InnerCliNpmResolverRef::Byonm(resolver) => {
let npm_vfs_builder = VfsBuilder::new(root_path.clone())?;
let npm_vfs_builder = self.build_npm_vfs(&root_path, cli_options)?;
(
Some(npm_vfs_builder),
Some(NodeModules::Byonm {
Expand All @@ -591,7 +591,6 @@ impl<'a> DenoCompileBinaryWriter<'a> {
}
};
let mut vfs = if let Some(npm_vfs) = maybe_npm_vfs {
// todo: probably need to modify this a bit
npm_vfs
} else {
VfsBuilder::new(root_path.clone())?
Expand Down Expand Up @@ -742,8 +741,8 @@ impl<'a> DenoCompileBinaryWriter<'a> {
} else {
// DO NOT include the user's registry url as it may contain credentials,
// but also don't make this dependent on the registry url
let root_path = npm_resolver.global_cache_root_folder();
let mut builder = VfsBuilder::new(root_path)?;
let global_cache_root_path = npm_resolver.global_cache_root_folder();
let mut builder = VfsBuilder::new(global_cache_root_path)?;
let mut packages =
npm_resolver.all_system_packages(&self.npm_system_info);
packages.sort_by(|a, b| a.id.cmp(&b.id)); // determinism
Expand All @@ -753,12 +752,12 @@ impl<'a> DenoCompileBinaryWriter<'a> {
builder.add_dir_recursive(&folder)?;
}

// Flatten all the registries folders into a single "node_modules/localhost" folder
// Flatten all the registries folders into a single ".deno_compile_node_modules/localhost" folder
// that will be used by denort when loading the npm cache. This avoids us exposing
// the user's private registry information and means we don't have to bother
// serializing all the different registry config into the binary.
builder.with_root_dir(|root_dir| {
root_dir.name = "node_modules".to_string();
root_dir.name = ".deno_compile_node_modules".to_string();
let mut new_entries = Vec::with_capacity(root_dir.entries.len());
let mut localhost_entries = IndexMap::new();
for entry in std::mem::take(&mut root_dir.entries) {
Expand Down Expand Up @@ -793,6 +792,8 @@ impl<'a> DenoCompileBinaryWriter<'a> {
root_dir.entries = new_entries;
});

builder.set_new_root_path(root_path.to_path_buf())?;

Ok(builder)
}
}
Expand Down
2 changes: 1 addition & 1 deletion cli/standalone/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -407,7 +407,7 @@ pub async fn run(data: StandaloneData) -> Result<i32, AnyError> {
let root_dir_url =
Arc::new(ModuleSpecifier::from_directory_path(&root_path).unwrap());
let main_module = root_dir_url.join(&metadata.entrypoint_key).unwrap();
let root_node_modules_path = root_path.join("node_modules");
let root_node_modules_path = root_path.join(".deno_compile_node_modules");
let npm_cache_dir = NpmCacheDir::new(
&RealDenoCacheEnv,
root_node_modules_path.clone(),
Expand Down
9 changes: 6 additions & 3 deletions cli/standalone/serialization.rs
Original file line number Diff line number Diff line change
Expand Up @@ -229,9 +229,10 @@ impl<'a> DenoCompileModuleData<'a> {

fn into_string_unsafe(data: Cow<'static, [u8]>) -> ModuleSourceCode {
match data {
Cow::Borrowed(d) => ModuleSourceCode::String(unsafe {
FastString::from_static(std::str::from_utf8_unchecked(d))
}),
Cow::Borrowed(d) => ModuleSourceCode::String(
// SAFETY: we know this is a valid utf8 string
unsafe { FastString::from_static(std::str::from_utf8_unchecked(d)) },
),
Cow::Owned(d) => ModuleSourceCode::Bytes(d.into_boxed_slice().into()),
}
}
Expand Down Expand Up @@ -440,6 +441,7 @@ fn deserialize_npm_snapshot(
Ok((input, id))
}

#[allow(clippy::needless_lifetimes)] // clippy bug
fn parse_root_package<'a>(
id_to_npm_id: &'a impl Fn(usize) -> Result<NpmPackageId, AnyError>,
) -> impl Fn(&[u8]) -> Result<(&[u8], (PackageReq, NpmPackageId)), AnyError> + 'a
Expand All @@ -452,6 +454,7 @@ fn deserialize_npm_snapshot(
}
}

#[allow(clippy::needless_lifetimes)] // clippy bug
fn parse_package_dep<'a>(
id_to_npm_id: &'a impl Fn(usize) -> Result<NpmPackageId, AnyError>,
) -> impl Fn(&[u8]) -> Result<(&[u8], (String, NpmPackageId)), AnyError> + 'a
Expand Down
19 changes: 19 additions & 0 deletions cli/standalone/virtual_fs.rs
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,25 @@ impl VfsBuilder {
})
}

pub fn set_new_root_path(
&mut self,
root_path: PathBuf,
) -> Result<(), AnyError> {
self.root_path = canonicalize_path(&root_path)?;
self.root_dir = VirtualDirectory {
name: self
.root_path
.file_stem()
.map(|s| s.to_string_lossy().into_owned())
.unwrap_or("root".to_string()),
entries: vec![VfsEntry::Dir(VirtualDirectory {
name: std::mem::take(&mut self.root_dir.name),
entries: std::mem::take(&mut self.root_dir.entries),
})],
};
Ok(())
}

pub fn with_root_dir<R>(
&mut self,
with_root: impl FnOnce(&mut VirtualDirectory) -> R,
Expand Down
2 changes: 1 addition & 1 deletion tests/specs/compile/byonm_main_sub_dir/deno.json
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
{
"unstable": ["byonm"]
"nodeModulesDir": "manual"
}
3 changes: 2 additions & 1 deletion tests/specs/compile/detect_cjs/output.out
Original file line number Diff line number Diff line change
@@ -1 +1,2 @@
error: Module not found: file:///[WILDLINE]/add.js
error: Uncaught SyntaxError: The requested module './add.js' does not provide an export named 'add'
at <anonymous> (file:///[WILDLINE])