Skip to content

Commit 4efbaef

Browse files
DonJayamanneDonJayamanne
authored
Enable tests on windows and validate by resolving environments from all symlinks (microsoft#67)
1 parent 69c1403 commit 4efbaef

14 files changed

Lines changed: 389 additions & 47 deletions

File tree

‎.github/workflows/pr-check.yml‎

Lines changed: 53 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -28,9 +28,6 @@ jobs:
2828
- os: windows-latest
2929
target: x86_64-pc-windows-msvc
3030
run_cli: "yes"
31-
- os: windows-latest
32-
target: aarch64-pc-windows-msvc
33-
run_cli: "no"
3431
- os: ubuntu-latest
3532
target: x86_64-unknown-linux-musl
3633
run_cli: "yes"
@@ -197,26 +194,35 @@ jobs:
197194
# Some of these tests are very specific and need to be run in isolation.
198195
# E.g. we need to ensure we have a poetry project setup correctly (without .venv created using `pip -m venv .venv`).
199196
# We can try to use the previous `tests` job, but that gets very complicated.
200-
name: Other Tests
197+
name: Env Tests
201198
runs-on: ${{ matrix.os }}
202199
strategy:
203200
fail-fast: false
204201
matrix:
205202
include:
206-
- feature: ci-poetry-global
203+
- feature: ci-poetry-global # Poetry tests with envs stored in standard location
207204
os: ubuntu-latest
208205
target: x86_64-unknown-linux-musl
209-
- feature: ci-poetry-project
206+
- feature: ci-poetry-project # Poetry tests, with poetry envs in project
210207
os: ubuntu-latest
211208
target: x86_64-unknown-linux-musl
212-
- feature: ci-poetry-custom
209+
- feature: ci-poetry-custom # Poetry tests with envs stored in a custom location
213210
os: ubuntu-latest
214211
target: x86_64-unknown-linux-musl
212+
- feature: ci-poetry-global # Poetry tests with envs stored in standard location
213+
os: windows-latest
214+
target: x86_64-pc-windows-msvc
215+
- feature: ci-poetry-project # Poetry tests, with poetry envs in project
216+
os: windows-latest
217+
target: x86_64-pc-windows-msvc
218+
- feature: ci-poetry-custom # Poetry tests with envs stored in a custom location
219+
os: windows-latest
220+
target: x86_64-pc-windows-msvc
215221
steps:
216222
- name: Checkout
217223
uses: actions/checkout@v4
218224

219-
# Setup Poetry
225+
# region Setup Poetry
220226
- name: Set Python 3.x to PATH
221227
if: startsWith( matrix.feature, 'ci-poetry')
222228
uses: actions/setup-python@v5
@@ -225,16 +231,33 @@ jobs:
225231

226232
- name: Set Python 3.12 to PATH
227233
if: startsWith( matrix.feature, 'ci-poetry')
234+
id: setupPython312
228235
uses: actions/setup-python@v5
229236
with:
230237
python-version: "3.12"
231238

232239
- name: Set Python 3.11 to PATH
233240
if: startsWith( matrix.feature, 'ci-poetry')
241+
id: setupPython311
234242
uses: actions/setup-python@v5
235243
with:
236244
python-version: "3.11"
237245

246+
- name: Python 3.12 Path
247+
if: startsWith( matrix.feature, 'ci-poetry') && startsWith( matrix.os, 'windows')
248+
run: echo "PYTHON_3_12_PATH=${{ steps.setupPython312.outputs.python-path }}" >> $GITHUB_ENV
249+
shell: bash
250+
251+
- name: Python 3.12 Path
252+
if: startsWith( matrix.feature, 'ci-poetry') && startsWith( matrix.os, 'windows')
253+
run: echo $PYTHON_3_12_PATH
254+
shell: bash
255+
256+
- name: Python 3.11 Path
257+
if: startsWith( matrix.feature, 'ci-poetry') && startsWith( matrix.os, 'windows')
258+
run: echo "PYTHON_3_11_PATH=${{ steps.setupPython311.outputs.python-path }}" >> $GITHUB_ENV
259+
shell: bash
260+
238261
- name: Install Poetry (envs globally)
239262
if: startsWith( matrix.feature, 'ci-poetry-global')
240263
uses: snok/install-poetry@93ada01c735cc8a383ce0ce2ae205a21c415379b
@@ -260,6 +283,11 @@ jobs:
260283
virtualenvs-path: ~/my-custom-path
261284
installer-parallel: true
262285

286+
- name: Petry exe
287+
if: startsWith( matrix.feature, 'ci-poetry')
288+
run: which poetry
289+
shell: bash
290+
263291
- name: Petry config
264292
if: startsWith( matrix.feature, 'ci-poetry')
265293
run: poetry config --list
@@ -272,13 +300,23 @@ jobs:
272300
shell: bash
273301

274302
- name: Petry virtual env setup 3.12
275-
if: startsWith( matrix.feature, 'ci-poetry')
276-
run: poetry env use python3.12
303+
if: startsWith( matrix.feature, 'ci-poetry') && startsWith( matrix.os, 'ubuntu')
304+
run: poetry env use 3.12
305+
shell: bash
306+
307+
- name: Petry virtual env setup 3.12
308+
if: startsWith( matrix.feature, 'ci-poetry') && startsWith( matrix.os, 'windows')
309+
run: poetry env use $PYTHON_3_12_PATH
277310
shell: bash
278311

279312
- name: Petry virtual env setup 3.11
280-
if: startsWith( matrix.feature, 'ci-poetry')
281-
run: poetry env use python3.11
313+
if: startsWith( matrix.feature, 'ci-poetry') && startsWith( matrix.os, 'ubuntu')
314+
run: poetry env use 3.11
315+
shell: bash
316+
317+
- name: Petry virtual env setup 3.11
318+
if: startsWith( matrix.feature, 'ci-poetry') && startsWith( matrix.os, 'windows')
319+
run: poetry env use $PYTHON_3_11_PATH
282320
shell: bash
283321

284322
- name: Petry list envs
@@ -296,6 +334,8 @@ jobs:
296334
# run: set
297335
# shell: bash
298336

337+
# endregion
338+
299339
# Rust
300340
- name: Rust Tool Chain setup
301341
uses: dtolnay/rust-toolchain@stable
@@ -366,7 +406,7 @@ jobs:
366406
shell: bash
367407

368408
- name: Find Environments
369-
run: cargo run --release --target ${{ matrix.target }} -- find -v
409+
run: cargo run --release --target ${{ matrix.target }}
370410
shell: bash
371411

372412
- name: Run Tests

‎Cargo.lock‎

Lines changed: 1 addition & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎crates/pet-core/src/os_environment.rs‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ use std::{
77
sync::{Arc, Mutex},
88
};
99

10+
use log::trace;
1011
use pet_fs::path::norm_case;
1112

1213
pub trait Environment {
@@ -78,7 +79,7 @@ impl Environment for EnvironmentApi {
7879
let mut paths =
7980
env::split_paths(&self.get_env_var("PATH".to_string()).unwrap_or_default())
8081
.collect::<Vec<PathBuf>>();
81-
82+
trace!("Env PATH: {:?}", paths);
8283
vec![
8384
PathBuf::from("/bin"),
8485
PathBuf::from("/etc"),

‎crates/pet-mac-commandlinetools/src/lib.rs‎

Lines changed: 42 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -48,8 +48,10 @@ impl Locator for MacCmdLineTools {
4848

4949
if !env
5050
.executable
51-
.to_string_lossy()
52-
.starts_with("/Library/Developer/CommandLineTools/usr/bin/python")
51+
.starts_with("/Library/Developer/CommandLineTools/usr/bin")
52+
&& !env.executable.starts_with(
53+
"/Library/Developer/CommandLineTools/Library/Frameworks/Python3.framework/Versions",
54+
)
5355
{
5456
return None;
5557
}
@@ -70,6 +72,33 @@ impl Locator for MacCmdLineTools {
7072
symlinks.push(symlink);
7173
}
7274

75+
// Possible we got the file /Library/Developer/CommandLineTools/Library/Frameworks/Python3.framework/Versions/3.9/bin/python3.9
76+
// We know that /Library/Developer/CommandLineTools/usr/bin/python3 is a symlink to the above.
77+
if env
78+
.executable
79+
.starts_with("/Library/Developer/CommandLineTools/usr/bin")
80+
{
81+
let exe = PathBuf::from("/Library/Developer/CommandLineTools/usr/bin/python3");
82+
if let Some(symlink) = resolve_symlink(&exe) {
83+
if symlinks.contains(&symlink) {
84+
symlinks.push(symlink);
85+
86+
// Rest of the files in this directory are also symlinks to the same exe.
87+
for exe in find_executables(PathBuf::from(
88+
"/Library/Developer/CommandLineTools/usr/bin",
89+
)) {
90+
if !symlinks.contains(&exe) {
91+
if let Some(symlink) = resolve_symlink(&exe) {
92+
if symlinks.contains(&symlink) {
93+
symlinks.push(exe);
94+
}
95+
}
96+
}
97+
}
98+
}
99+
}
100+
}
101+
73102
// We know /usr/bin/python3 can end up pointing to this same Python exe as well
74103
// Hence look for those symlinks as well.
75104
// Unfortunately /usr/bin/python3 is not a real symlink
@@ -106,6 +135,17 @@ impl Locator for MacCmdLineTools {
106135
symlinks.sort();
107136
symlinks.dedup();
108137

138+
// Find other exes that are symlinks to the same exe in /Library/Developer/CommandLineTools/usr/bin
139+
for exe in find_executables("/Library/Developer/CommandLineTools/usr/bin") {
140+
if !symlinks.contains(&exe) {
141+
if let Some(symlink) = resolve_symlink(&exe) {
142+
if symlinks.contains(&symlink) {
143+
symlinks.push(exe);
144+
}
145+
}
146+
}
147+
}
148+
109149
if prefix.is_none() {
110150
// We would have identified the symlinks by now.
111151
// Look for the one with the path `/Library/Developer/CommandLineTools/Library/Frameworks/Python3.framework/Versions/3.9/bin/python3.9`

‎crates/pet-mac-xcode/src/lib.rs‎

Lines changed: 32 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,11 @@ use pet_core::{
77
Locator,
88
};
99
use pet_fs::path::resolve_symlink;
10-
use pet_python_utils::env::{PythonEnv, ResolvedPythonEnv};
1110
use pet_python_utils::version;
11+
use pet_python_utils::{
12+
env::{PythonEnv, ResolvedPythonEnv},
13+
executable::find_executables,
14+
};
1215
use pet_virtualenv::is_virtualenv;
1316
use std::path::PathBuf;
1417

@@ -65,7 +68,34 @@ impl Locator for MacXCode {
6568
// /Applications/Xcode.app/Contents/Developer/Library/Frameworks/Python3.framework/Versions/3.9/bin/python3.9
6669
// Verify this and add that to the list of symlinks as well.
6770
if let Some(symlink) = resolve_symlink(&env.executable) {
68-
symlinks.push(symlink);
71+
symlinks.push(symlink.clone());
72+
73+
// All exes in the bin directory of the symlink are also symlinks (thats generally of the form /Applications/Xcode.app/Contents/Developer/Library/Frameworks/Python3.framework/Versions/3.9/bin/python3.9)
74+
for exe in find_executables(symlink.parent().unwrap()) {
75+
symlinks.push(exe);
76+
}
77+
}
78+
79+
// Possible the env.executable is "/Applications/Xcode.app/Contents/Developer/Library/Frameworks/Python3.framework/Versions/3.9/bin/python3.9"
80+
// The symlink to the above exe is in /Applications/Xcode.app/Contents/Developer/usr/bin/python3
81+
// Lets try to find that, because /usr/bin/python3 could also exist and when we run python, the sys.execuctable points to the file /Applications/Xcode.app/Contents/Developer/usr/bin/python3
82+
// The name of the `Xcode.app` folder can be different on other machines, e.g. on CI it is `Xcode_15.0.1.app`
83+
let xcode_folder_name = exe_str.split('/').nth(2).unwrap_or_default();
84+
85+
let bin = PathBuf::from(format!(
86+
"/Applications/{}/Contents/Developer/usr/bin",
87+
xcode_folder_name
88+
));
89+
let exe = bin.join("python3");
90+
if let Some(symlink) = resolve_symlink(&exe) {
91+
if symlinks.contains(&symlink) {
92+
symlinks.push(exe.clone());
93+
94+
// All exes in this directory are symlinks
95+
for exe in find_executables(bin) {
96+
symlinks.push(exe);
97+
}
98+
}
6999
}
70100

71101
// We know /usr/bin/python3 can end up pointing to this same Python exe as well

‎crates/pet-poetry/src/environment_locations.rs‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ use std::{
1515

1616
use crate::{
1717
config::Config, env_variables::EnvVariables, environment::create_poetry_env,
18-
pyproject_toml::PyProjectToml,
18+
manager::PoetryManager, pyproject_toml::PyProjectToml,
1919
};
2020

2121
lazy_static! {
@@ -26,6 +26,7 @@ lazy_static! {
2626
pub fn list_environments(
2727
env: &EnvVariables,
2828
project_dirs: &[PathBuf],
29+
manager: Option<PoetryManager>,
2930
) -> Option<Vec<PythonEnvironment>> {
3031
if project_dirs.is_empty() {
3132
return None;
@@ -76,7 +77,9 @@ pub fn list_environments(
7677
.unwrap_or_default();
7778
// Look for .venv as well, in case we create the virtual envs in the local project folder.
7879
if name.starts_with(&virtualenv_prefix) || name.starts_with(".venv") {
79-
if let Some(env) = create_poetry_env(&virtual_env, project_dir.clone(), None) {
80+
if let Some(env) =
81+
create_poetry_env(&virtual_env, project_dir.clone(), manager.clone())
82+
{
8083
envs.push(env);
8184
}
8285
}

‎crates/pet-poetry/src/lib.rs‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -105,13 +105,14 @@ impl Poetry {
105105
managers: vec![],
106106
environments: vec![],
107107
};
108-
if let Some(manager) = manager {
108+
if let Some(manager) = &manager {
109109
result.managers.push(manager.to_manager());
110110
}
111111
if let Ok(values) = self.project_dirs.lock() {
112112
let project_dirs = values.clone();
113113
drop(values);
114-
let envs = list_environments(&self.env_vars, &project_dirs.clone()).unwrap_or_default();
114+
let envs = list_environments(&self.env_vars, &project_dirs.clone(), manager)
115+
.unwrap_or_default();
115116
result.environments.extend(envs.clone());
116117
}
117118

‎crates/pet-poetry/src/manager.rs‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
// Copyright (c) Microsoft Corporation.
22
// Licensed under the MIT License.
33

4+
use log::trace;
45
use pet_core::manager::{EnvManager, EnvManagerType};
56
use std::{env, path::PathBuf};
67

@@ -35,8 +36,10 @@ impl PoetryManager {
3536
if let Some(poetry_home) = &env_variables.poetry_home {
3637
if std::env::consts::OS == "windows" {
3738
search_paths.push(poetry_home.join("bin").join("poetry.exe"));
39+
search_paths.push(poetry_home.join("venv").join("bin").join("poetry.exe"));
3840
}
3941
search_paths.push(poetry_home.join("bin").join("poetry"));
42+
search_paths.push(poetry_home.join("venv").join("bin").join("poetry"));
4043
}
4144
if std::env::consts::OS == "windows" {
4245
if let Some(app_data) = env_variables.app_data.clone() {
@@ -71,6 +74,10 @@ impl PoetryManager {
7174
app_data.join("Python").join("scripts").join("poetry"), // https://python-poetry.org/docs/#installing-with-the-official-installer
7275
);
7376
}
77+
search_paths.push(
78+
// Found after installing on Windows via github actions.
79+
home.join(".local").join("bin").join("poetry"),
80+
);
7481
} else if std::env::consts::OS == "macos" {
7582
search_paths.push(
7683
// https://python-poetry.org/docs/#installing-with-the-official-installer
@@ -111,9 +118,16 @@ impl PoetryManager {
111118
if executable.is_file() {
112119
return Some(PoetryManager { executable });
113120
}
121+
if std::env::consts::OS == "windows" {
122+
let executable = each.join("poetry.exe");
123+
if executable.is_file() {
124+
return Some(PoetryManager { executable });
125+
}
126+
}
114127
}
115128
}
116129
}
130+
trace!("Poetry exe not found");
117131
None
118132
}
119133
pub fn to_manager(&self) -> EnvManager {

‎crates/pet-python-utils/src/env.rs‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -97,6 +97,11 @@ impl ResolvedPythonEnv {
9797
match result {
9898
Ok(output) => {
9999
let output = String::from_utf8(output.stdout).unwrap().trim().to_string();
100+
trace!(
101+
"Python Execution for {:?} produced an output {:?}",
102+
executable,
103+
output
104+
);
100105
if let Some((_, output)) = output.split_once(PYTHON_INFO_JSON_SEPARATOR) {
101106
if let Ok(info) = serde_json::from_str::<InterpreterInfo>(output) {
102107
Some(Self {

‎crates/pet-windows-registry/src/lib.rs‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -88,8 +88,10 @@ impl Locator for WindowsRegistry {
8888
if let Some(result) = self.find_with_cache() {
8989
// Find the same env here
9090
for found_env in result.environments {
91-
if env.executable.to_str() == env.executable.to_str() {
92-
return Some(found_env);
91+
if let Some(ref python_executable_path) = found_env.executable {
92+
if python_executable_path == &env.executable {
93+
return Some(found_env);
94+
}
9395
}
9496
}
9597
}

0 commit comments

Comments
 (0)