Skip to content

no_std for wtf8, sre_engine, compiler-core, literal #7051

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
youknowone merged 2 commits into from
Feb 8, 2026
Merged

no_std for wtf8, sre_engine, compiler-core, literal #7051

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
a96977f
`no_std` for wtf8
youknowone Feb 8, 2026
2c5ece9
`no_std` for sre_engine, compiler-core, literal
youknowone Feb 8, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion crates/compiler-core/src/bytecode.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ use crate::{
varint::{read_varint, read_varint_with_start, write_varint, write_varint_with_start},
{OneIndexed, SourceLocation},
};
use alloc::{collections::BTreeSet, fmt, vec::Vec};
use alloc::{borrow::ToOwned, boxed::Box, collections::BTreeSet, fmt, string::String, vec::Vec};
use bitflags::bitflags;
use core::{hash, mem, ops::Deref};
use itertools::Itertools;
Expand Down Expand Up @@ -804,6 +804,7 @@ impl<C: Constant> fmt::Debug for CodeObject<C> {
#[cfg(test)]
mod tests {
use super::*;
use alloc::{vec, vec::Vec};

#[test]
fn test_exception_table_encode_decode() {
Expand Down
1 change: 1 addition & 0 deletions crates/compiler-core/src/frozen.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
use crate::bytecode::*;
use crate::marshal::{self, Read, ReadBorrowed, Write};
use alloc::vec::Vec;

/// A frozen module. Holds a frozen code object and whether it is part of a package
#[derive(Copy, Clone)]
Expand Down
1 change: 1 addition & 0 deletions crates/compiler-core/src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
#![no_std]
#![doc(html_logo_url = "https://raw.githubusercontent.com/RustPython/RustPython/main/logo.png")]
#![doc(html_root_url = "https://docs.rs/rustpython-compiler-core/")]

Expand Down
1 change: 1 addition & 0 deletions crates/compiler-core/src/marshal.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
use crate::{OneIndexed, SourceLocation, bytecode::*};
use alloc::{boxed::Box, vec::Vec};
use core::convert::Infallible;
use malachite_bigint::{BigInt, Sign};
use num_complex::Complex64;
Expand Down
2 changes: 2 additions & 0 deletions crates/compiler-core/src/varint.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@
//! Uses 6-bit chunks with a continuation bit (0x40) to encode integers.
//! Used for exception tables and line number tables.

use alloc::vec::Vec;

/// Write a variable-length unsigned integer using 6-bit chunks.
/// Returns the number of bytes written.
#[inline]
Expand Down
2 changes: 2 additions & 0 deletions crates/literal/src/complex.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,6 @@
use crate::float;
use alloc::borrow::ToOwned;
use alloc::string::{String, ToString};

/// Convert a complex number to a string.
pub fn to_string(re: f64, im: f64) -> String {
Expand Down
31 changes: 18 additions & 13 deletions crates/literal/src/escape.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
use alloc::string::String;
use rustpython_wtf8::{CodePoint, Wtf8};

#[derive(Debug, PartialEq, Eq, Copy, Clone, Hash, is_macro::Is)]
Expand Down Expand Up @@ -55,9 +56,9 @@ pub unsafe trait Escape {
/// # Safety
///
/// This string must only contain printable characters.
unsafe fn write_source(&self, formatter: &mut impl std::fmt::Write) -> core::fmt::Result;
fn write_body_slow(&self, formatter: &mut impl std::fmt::Write) -> core::fmt::Result;
fn write_body(&self, formatter: &mut impl std::fmt::Write) -> core::fmt::Result {
unsafe fn write_source(&self, formatter: &mut impl core::fmt::Write) -> core::fmt::Result;
fn write_body_slow(&self, formatter: &mut impl core::fmt::Write) -> core::fmt::Result;
fn write_body(&self, formatter: &mut impl core::fmt::Write) -> core::fmt::Result {
if self.changed() {
self.write_body_slow(formatter)
} else {
Expand Down Expand Up @@ -117,7 +118,7 @@ impl<'a> UnicodeEscape<'a> {
pub struct StrRepr<'r, 'a>(&'r UnicodeEscape<'a>);

impl StrRepr<'_, '_> {
pub fn write(&self, formatter: &mut impl std::fmt::Write) -> core::fmt::Result {
pub fn write(&self, formatter: &mut impl core::fmt::Write) -> core::fmt::Result {
let quote = self.0.layout().quote.to_char();
formatter.write_char(quote)?;
self.0.write_body(formatter)?;
Expand Down Expand Up @@ -216,7 +217,7 @@ impl UnicodeEscape<'_> {
fn write_char(
ch: CodePoint,
quote: Quote,
formatter: &mut impl std::fmt::Write,
formatter: &mut impl core::fmt::Write,
) -> core::fmt::Result {
let Some(ch) = ch.to_char() else {
return write!(formatter, "\\u{:04x}", ch.to_u32());
Expand Down Expand Up @@ -260,15 +261,15 @@ unsafe impl Escape for UnicodeEscape<'_> {
&self.layout
}

unsafe fn write_source(&self, formatter: &mut impl std::fmt::Write) -> core::fmt::Result {
unsafe fn write_source(&self, formatter: &mut impl core::fmt::Write) -> core::fmt::Result {
formatter.write_str(unsafe {
// SAFETY: this function must be called only when source is printable characters (i.e. no surrogates)
std::str::from_utf8_unchecked(self.source.as_bytes())
core::str::from_utf8_unchecked(self.source.as_bytes())
})
}

#[cold]
fn write_body_slow(&self, formatter: &mut impl std::fmt::Write) -> core::fmt::Result {
fn write_body_slow(&self, formatter: &mut impl core::fmt::Write) -> core::fmt::Result {
for ch in self.source.code_points() {
Self::write_char(ch, self.layout().quote, formatter)?;
}
Expand Down Expand Up @@ -378,7 +379,11 @@ impl AsciiEscape<'_> {
}
}

fn write_char(ch: u8, quote: Quote, formatter: &mut impl std::fmt::Write) -> core::fmt::Result {
fn write_char(
ch: u8,
quote: Quote,
formatter: &mut impl core::fmt::Write,
) -> core::fmt::Result {
match ch {
b'\t' => formatter.write_str("\\t"),
b'\n' => formatter.write_str("\\n"),
Expand All @@ -404,15 +409,15 @@ unsafe impl Escape for AsciiEscape<'_> {
&self.layout
}

unsafe fn write_source(&self, formatter: &mut impl std::fmt::Write) -> core::fmt::Result {
unsafe fn write_source(&self, formatter: &mut impl core::fmt::Write) -> core::fmt::Result {
formatter.write_str(unsafe {
// SAFETY: this function must be called only when source is printable ascii characters
std::str::from_utf8_unchecked(self.source)
core::str::from_utf8_unchecked(self.source)
})
}

#[cold]
fn write_body_slow(&self, formatter: &mut impl std::fmt::Write) -> core::fmt::Result {
fn write_body_slow(&self, formatter: &mut impl core::fmt::Write) -> core::fmt::Result {
for ch in self.source {
Self::write_char(*ch, self.layout().quote, formatter)?;
}
Expand All @@ -423,7 +428,7 @@ unsafe impl Escape for AsciiEscape<'_> {
pub struct BytesRepr<'r, 'a>(&'r AsciiEscape<'a>);

impl BytesRepr<'_, '_> {
pub fn write(&self, formatter: &mut impl std::fmt::Write) -> core::fmt::Result {
pub fn write(&self, formatter: &mut impl core::fmt::Write) -> core::fmt::Result {
let quote = self.0.layout().quote.to_char();
formatter.write_char('b')?;
formatter.write_char(quote)?;
Expand Down
5 changes: 4 additions & 1 deletion crates/literal/src/float.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,9 @@
use crate::format::Case;
use alloc::borrow::ToOwned;
use alloc::format;
use alloc::string::{String, ToString};
use core::f64;
use num_traits::{Float, Zero};
use std::f64;

pub fn parse_str(literal: &str) -> Option<f64> {
parse_inner(literal.trim().as_bytes())
Expand Down
4 changes: 4 additions & 0 deletions crates/literal/src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,7 @@
#![no_std]

extern crate alloc;

pub mod char;
pub mod complex;
pub mod escape;
Expand Down
1 change: 1 addition & 0 deletions crates/sre_engine/src/engine.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ use crate::string::{
};

use super::{MAXREPEAT, SreAtCode, SreCatCode, SreInfo, SreOpcode, StrDrive, StringCursor};
use alloc::{vec, vec::Vec};
use core::{convert::TryFrom, ptr::null};
use optional::Optioned;

Expand Down
4 changes: 4 additions & 0 deletions crates/sre_engine/src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,7 @@
#![no_std]

extern crate alloc;

pub mod constants;
pub mod engine;
pub mod string;
Expand Down
21 changes: 13 additions & 8 deletions crates/wtf8/src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,8 +33,17 @@
//! [WTF-8]: https://simonsapin.github.io/wtf-8
//! [`OsStr`]: std::ffi::OsStr

#![no_std]
#![allow(clippy::precedence, clippy::match_overlapping_arm)]

extern crate alloc;

use alloc::borrow::{Cow, ToOwned};
use alloc::boxed::Box;
use alloc::collections::TryReserveError;
use alloc::string::String;
use alloc::vec::Vec;
use core::borrow::Borrow;
Comment on lines +36 to +46
Copy link
Contributor

@coderabbitai coderabbitai bot Feb 8, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Potential issue | 🔴 Critical

🧩 Analysis chain

🏁 Script executed:

fd 'Cargo.toml' crates/wtf8 --exec cat {}

Repository: RustPython/RustPython

Length of output: 458

🏁 Script executed:

grep -A 50 '\[workspace\.dependencies\]' Cargo.toml | head -100

Repository: RustPython/RustPython

Length of output: 2625

🏁 Script executed:

# Check the actual lib.rs to see import usage and configuration
head -100 crates/wtf8/src/lib.rs

Repository: RustPython/RustPython

Length of output: 3814

🏁 Script executed:

# Check how itertools and bstr are used in the code
rg -n "itertools::" crates/wtf8/src/ -A 2

Repository: RustPython/RustPython

Length of output: 211

🏁 Script executed:

# Search for actual usage of Either, Itertools, ByteSlice, ByteVec
rg "Either|Itertools|ByteSlice|ByteVec" crates/wtf8/src/ --max-count=20

Repository: RustPython/RustPython

Length of output: 364

🏁 Script executed:

# Check for any cargo features or configuration related to no_std in wtf8
rg "feature\|no_std" crates/wtf8/ -B 2 -A 2

Repository: RustPython/RustPython

Length of output: 47

🏁 Script executed:

# Check if there are any tests or CI configuration that builds with no_std
fd 'Cargo.toml|\.rs' crates/wtf8 --exec grep -l "no_std\|no-std" {}

Repository: RustPython/RustPython

Length of output: 87

🏁 Script executed:

# Let's check if bstr crate documentation or source indicates no_std requirements
# First, let me see if there's a Cargo.lock or any evidence of successful builds
git log --oneline crates/wtf8/Cargo.toml | head -5

Repository: RustPython/RustPython

Length of output: 122

🏁 Script executed:

# Check the commit that added no_std support to see what pattern was used
git show 2c5ece945 --stat | head -30

Repository: RustPython/RustPython

Length of output: 1737

🏁 Script executed:

# Check if wtf8 was included in that commit and see what changes were made
git show 2c5ece945 -- crates/wtf8/ | head -200

Repository: RustPython/RustPython

Length of output: 6676

🏁 Script executed:

# Check if there's any .cargo/config.toml configuration related to building crates
cat .cargo/config.toml | head -50

Repository: RustPython/RustPython

Length of output: 343

🏁 Script executed:

# Check if there are any actual usages of ascii and memchr
rg "ascii::|memchr::" crates/wtf8/src/ -A 2

Repository: RustPython/RustPython

Length of output: 1327

🏁 Script executed:

# Check if any of the dependencies have been explicitly configured for no_std elsewhere
# Let's search for how other no_std crates in the workspace handle dependencies
rg "default-features.*false" Cargo.toml | head -20

Repository: RustPython/RustPython

Length of output: 576

🏁 Script executed:

# Check if there are any tests or build configurations that verify no_std compilation
rg "no.std|no-std" . --type toml | head -20

Repository: RustPython/RustPython

Length of output: 47

🏁 Script executed:

# Let me verify if these specific dependency versions support no_std out of the box
# by checking their documentation or configuration patterns used elsewhere in the repo
python3 << 'EOF'
# Check if other crates in the workspace use similar dependencies
import subprocess
import re

# Search for patterns of how dependencies are configured in Cargo.toml files
result = subprocess.run(['rg', 'ascii|bstr|itertools|memchr', '--type', 'toml', '-B2', '-A2'], 
                       capture_output=True, text=True)
print(result.stdout)
EOF

Repository: RustPython/RustPython

Length of output: 47

🌐 Web query:

itertools 0.14.0 no_std alloc feature requirements

💡 Result:

For itertools = 0.14.0, std is enabled by default (default = ["use_std"]). The crate’s features are: use_std (which implies use_alloc) and use_alloc alone. [1][2]

no_std + alloc (no std)

Use default-features = false and enable use_alloc:

[dependencies]
itertools = { version = "0.14.0", default-features = false, features = ["use_alloc"] }

This avoids use_std (and thus avoids enabling either/use_std) while allowing the parts of itertools that rely on alloc. [1][2]

no_std without alloc (core-only)

Disable default features and don’t enable use_alloc:

[dependencies]
itertools = { version = "0.14.0", default-features = false }

(You can only use the subset of itertools that doesn’t require allocation.) [1][2]

Note: In a #![no_std] binary, alloc-using code also requires that your target provides the alloc crate and that you have a global allocator available at link/runtime. [1]

Sources: [1] Cargo.toml for itertools 0.14.0 on docs.rs, [2] docs.rs feature flags page.

itertools requires explicit no_std configuration — add default-features = false, features = ["use_alloc"] to the workspace dependency.

The crate uses #![no_std] with itertools::Either and itertools::Itertools, but itertools 0.14.0 defaults to use_std. Update the workspace dependency in Cargo.toml:

itertools = { version = "0.14.0", default-features = false, features = ["use_alloc"] }

The other dependencies (ascii, bstr, memchr) support no_std with their default configurations.

🤖 Prompt for AI Agents 
In `@crates/wtf8/src/lib.rs` around lines 36 - 46, The workspace dependency for
itertools must be configured for no_std; update the Cargo.toml entry for
itertools to disable default features and enable its alloc support by setting
itertools = { version = "0.14.0", default-features = false, features =
["use_alloc"] } so the crate using #![no_std] and alloc (see extern crate alloc
and uses like itertools::Either / itertools::Itertools) compiles without pulling
in std.

use core::fmt;
use core::hash::{Hash, Hasher};
use core::iter::FusedIterator;
Expand All @@ -46,10 +55,6 @@ use core_char::MAX_LEN_UTF8;
use core_char::{MAX_LEN_UTF16, encode_utf8_raw, encode_utf16_raw, len_utf8};
use core_str::{next_code_point, next_code_point_reverse};
use itertools::{Either, Itertools};
use std::borrow::{Borrow, Cow};
use std::collections::TryReserveError;
use std::string::String;
use std::vec::Vec;

use bstr::{ByteSlice, ByteVec};

Expand Down Expand Up @@ -665,7 +670,7 @@ impl PartialEq<str> for Wtf8 {
impl fmt::Debug for Wtf8 {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
fn write_str_escaped(f: &mut fmt::Formatter<'_>, s: &str) -> fmt::Result {
use std::fmt::Write;
use core::fmt::Write;
for c in s.chars().flat_map(|c| c.escape_debug()) {
f.write_char(c)?
}
Expand Down Expand Up @@ -765,7 +770,7 @@ impl Wtf8 {
#[inline]
pub fn from_bytes(b: &[u8]) -> Option<&Self> {
let mut rest = b;
while let Err(e) = std::str::from_utf8(rest) {
while let Err(e) = core::str::from_utf8(rest) {
rest = &rest[e.valid_up_to()..];
let _ = Self::decode_surrogate(rest)?;
rest = &rest[3..];
Expand Down Expand Up @@ -899,7 +904,7 @@ impl Wtf8 {
{
self.chunks().flat_map(move |chunk| match chunk {
Wtf8Chunk::Utf8(s) => Either::Left(f(s).map_into()),
Wtf8Chunk::Surrogate(c) => Either::Right(std::iter::once(c)),
Wtf8Chunk::Surrogate(c) => Either::Right(core::iter::once(c)),
})
}

Expand Down Expand Up @@ -1469,7 +1474,7 @@ impl<'a> Iterator for Wtf8Chunks<'a> {
}
None => {
let s =
unsafe { str::from_utf8_unchecked(std::mem::take(&mut self.wtf8).as_bytes()) };
unsafe { str::from_utf8_unchecked(core::mem::take(&mut self.wtf8).as_bytes()) };
(!s.is_empty()).then_some(Wtf8Chunk::Utf8(s))
}
}
Expand Down
Loading