1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
use std::path::Path;
use tracing::debug;
use crate::commands::pip::loggers::{InstallLogger, ResolveLogger};
use crate::commands::pip::operations::Modifications;
use crate::commands::project::{
EnvironmentSpecification, PlatformState, ProjectError, resolve_environment, sync_environment,
};
use crate::printer::Printer;
use crate::settings::ResolverInstallerSettings;
use uv_cache::{Cache, CacheBucket};
use uv_cache_info::CacheInfo;
use uv_cache_key::{cache_digest, hash_digest};
use uv_client::BaseClientBuilder;
use uv_configuration::{Concurrency, Constraints, TargetTriple};
use uv_distribution_types::{
BuiltDist, Dist, Identifier, Node, Resolution, ResolvedDist, SourceDist,
};
use uv_fs::PythonExt;
use uv_preview::Preview;
use uv_python::{Interpreter, PythonEnvironment, canonicalize_executable};
use uv_types::SourceTreeEditablePolicy;
use uv_workspace::WorkspaceCache;
/// An ephemeral [`PythonEnvironment`] for running an individual command.
#[derive(Debug)]
pub(crate) struct EphemeralEnvironment(PythonEnvironment);
impl From<PythonEnvironment> for EphemeralEnvironment {
fn from(environment: PythonEnvironment) -> Self {
Self(environment)
}
}
impl From<EphemeralEnvironment> for PythonEnvironment {
fn from(environment: EphemeralEnvironment) -> Self {
environment.0
}
}
impl EphemeralEnvironment {
/// Set the ephemeral overlay for a Python environment.
pub(crate) fn set_overlay(&self, contents: impl AsRef<[u8]>) -> Result<(), ProjectError> {
let site_packages = self
.0
.site_packages()
.next()
.ok_or(ProjectError::NoSitePackages)?;
let overlay_path = site_packages.join("_uv_ephemeral_overlay.pth");
fs_err::write(overlay_path, contents)?;
Ok(())
}
/// Enable system site packages for a Python environment.
pub(crate) fn set_system_site_packages(&self) -> Result<(), ProjectError> {
self.0
.set_pyvenv_cfg("include-system-site-packages", "true")?;
Ok(())
}
/// Set the `extends-environment` key in the `pyvenv.cfg` file to the given path.
///
/// Ephemeral environments created by `uv run --with` extend a parent (virtual or system)
/// environment by adding a `.pth` file to the ephemeral environment's `site-packages`
/// directory. The `pth` file contains Python code to dynamically add the parent
/// environment's `site-packages` directory to Python's import search paths in addition to
/// the ephemeral environment's `site-packages` directory. This works well at runtime, but
/// is too dynamic for static analysis tools like ty to understand. As such, we
/// additionally write the `sys.prefix` of the parent environment to the
/// `extends-environment` key of the ephemeral environment's `pyvenv.cfg` file, making it
/// easier for these tools to statically and reliably understand the relationship between
/// the two environments.
pub(crate) fn set_parent_environment(
&self,
parent_environment_sys_prefix: &Path,
) -> Result<(), ProjectError> {
self.0.set_pyvenv_cfg(
"extends-environment",
&parent_environment_sys_prefix.escape_for_python(),
)?;
Ok(())
}
/// Returns the path to the environment's scripts directory.
pub(crate) fn scripts(&self) -> &Path {
self.0.scripts()
}
/// Returns the path to the environment's Python executable.
pub(crate) fn sys_executable(&self) -> &Path {
self.0.interpreter().sys_executable()
}
pub(crate) fn sys_prefix(&self) -> &Path {
self.0.interpreter().sys_prefix()
}
}
/// A [`PythonEnvironment`] stored in the cache.
#[derive(Debug)]
pub(crate) struct CachedEnvironment(PythonEnvironment);
impl From<CachedEnvironment> for PythonEnvironment {
fn from(environment: CachedEnvironment) -> Self {
environment.0
}
}
#[derive(Debug, Clone, Hash)]
struct CachedEnvironmentDist {
dist: ResolvedDist,
hashes: uv_pypi_types::HashDigests,
cache_info: Option<CacheInfo>,
}
impl CachedEnvironment {
/// Get or create an [`CachedEnvironment`] based on a given set of requirements.
pub(crate) async fn from_spec(
spec: EnvironmentSpecification<'_>,
build_constraints: Constraints,
interpreter: &Interpreter,
python_platform: Option<&TargetTriple>,
settings: &ResolverInstallerSettings,
client_builder: &BaseClientBuilder<'_>,
state: &PlatformState,
resolve: Box<dyn ResolveLogger>,
install: Box<dyn InstallLogger>,
installer_metadata: bool,
concurrency: &Concurrency,
cache: &Cache,
workspace_cache: &WorkspaceCache,
printer: Printer,
preview: Preview,
) -> Result<Self, ProjectError> {
let interpreter = Self::base_interpreter(interpreter, cache)?;
// Resolve the requirements with the interpreter.
let resolution = Resolution::from(
resolve_environment(
spec,
&interpreter,
python_platform,
SourceTreeEditablePolicy::Project,
build_constraints.clone(),
&settings.resolver,
client_builder,
state,
resolve,
concurrency,
cache,
workspace_cache,
printer,
preview,
)
.await?,
);
// Hash the resolution by hashing the generated lockfile.
let resolution_hash = {
let mut distributions = resolution
.graph()
.node_weights()
.filter_map(|node| match node {
Node::Dist {
dist,
hashes,
install: true,
} => Some((dist, hashes)),
Node::Dist { install: false, .. } | Node::Root => None,
})
.map(|(dist, hashes)| {
Ok(CachedEnvironmentDist {
dist: dist.clone(),
hashes: hashes.clone(),
cache_info: Self::cache_info(dist).map_err(ProjectError::from)?,
})
})
.collect::<Result<Vec<_>, ProjectError>>()?;
distributions.sort_unstable_by(|left, right| {
left.dist
.distribution_id()
.cmp(&right.dist.distribution_id())
});
hash_digest(&distributions)
};
// Construct a hash for the environment.
//
// Use the canonicalized base interpreter path since that's the interpreter we performed the
// resolution with and the interpreter the environment will be created with.
//
// We cache environments independent of the environment they'd be layered on top of. The
// assumption is such that the environment will _not_ be modified by the user or uv;
// otherwise, we risk cache poisoning. For example, if we were to write a `.pth` file to
// the cached environment, it would be shared across all projects that use the same
// interpreter and the same cached dependencies.
//
// TODO(zanieb): We should include the version of the base interpreter in the hash, so if
// the interpreter at the canonicalized path changes versions we construct a new
// environment.
let interpreter_hash =
cache_digest(&canonicalize_executable(interpreter.sys_executable())?);
// Search in the content-addressed cache.
let cache_entry = cache.entry(CacheBucket::Environments, interpreter_hash, resolution_hash);
if let Ok(root) = cache.resolve_link(cache_entry.path()) {
if let Ok(environment) = PythonEnvironment::from_root(root, cache) {
return Ok(Self(environment));
}
}
// Create the environment in the cache, then relocate it to its content-addressed location.
let temp_dir = cache.venv_dir()?;
let venv = uv_virtualenv::create_venv(
temp_dir.path(),
interpreter,
uv_virtualenv::Prompt::None,
false,
uv_virtualenv::OnExisting::Remove(uv_virtualenv::RemovalReason::TemporaryEnvironment),
true,
false,
false,
)?;
sync_environment(
venv,
&resolution,
Modifications::Exact,
build_constraints,
settings.into(),
client_builder,
state,
install,
installer_metadata,
concurrency,
cache,
printer,
preview,
)
.await?;
// Now that the environment is complete, sync it to its content-addressed location.
let id = cache.persist(temp_dir.keep(), cache_entry.path()).await?;
let root = cache.archive(&id);
Ok(Self(PythonEnvironment::from_root(root, cache)?))
}
/// Return any mutable cache info that should invalidate a cached environment for a given
/// distribution.
fn cache_info(dist: &ResolvedDist) -> Result<Option<CacheInfo>, uv_cache_info::CacheInfoError> {
let path = match dist {
ResolvedDist::Installed { .. } => return Ok(None),
ResolvedDist::Installable { dist, .. } => match dist.as_ref() {
Dist::Built(BuiltDist::Path(wheel)) => wheel.install_path.as_ref(),
Dist::Source(SourceDist::Path(sdist)) => sdist.install_path.as_ref(),
Dist::Source(SourceDist::Directory(directory)) => directory.install_path.as_ref(),
_ => return Ok(None),
},
};
Ok(Some(CacheInfo::from_path(path)?))
}
/// Return the [`Interpreter`] to use for the cached environment, based on a given
/// [`Interpreter`].
///
/// When caching, always use the base interpreter, rather than that of the virtual
/// environment.
fn base_interpreter(
interpreter: &Interpreter,
cache: &Cache,
) -> Result<Interpreter, uv_python::Error> {
let base_python = if cfg!(unix) {
interpreter.find_base_python()?
} else {
interpreter.to_base_python()?
};
if base_python == interpreter.sys_executable() {
debug!(
"Caching via base interpreter: `{}`",
interpreter.sys_executable().display()
);
Ok(interpreter.clone())
} else {
let base_interpreter = Interpreter::query(base_python, cache)?;
debug!(
"Caching via base interpreter: `{}`",
base_interpreter.sys_executable().display()
);
Ok(base_interpreter)
}
}
}