pyo3/types/
string.rs

1#[cfg(not(Py_LIMITED_API))]
2use crate::exceptions::PyUnicodeDecodeError;
3use crate::ffi_ptr_ext::FfiPtrExt;
4use crate::instance::Borrowed;
5use crate::py_result_ext::PyResultExt;
6use crate::types::bytes::PyBytesMethods;
7use crate::types::PyBytes;
8use crate::{ffi, Bound, Py, PyAny, PyResult, Python};
9#[cfg(RustPython)]
10use crate::{
11    sync::PyOnceLock,
12    types::{PyType, PyTypeMethods},
13};
14use alloc::borrow::Cow;
15use core::ffi::CStr;
16use core::{fmt, str};
17
18/// Represents raw data backing a Python `str`.
19///
20/// Python internally stores strings in various representations. This enumeration
21/// represents those variations.
22#[cfg(not(Py_LIMITED_API))]
23#[derive(Clone, Copy, Debug, PartialEq, Eq)]
24pub enum PyStringData<'a> {
25    /// UCS1 representation.
26    Ucs1(&'a [u8]),
27
28    /// UCS2 representation.
29    Ucs2(&'a [u16]),
30
31    /// UCS4 representation.
32    Ucs4(&'a [u32]),
33}
34
35#[cfg(not(Py_LIMITED_API))]
36impl<'a> PyStringData<'a> {
37    /// Obtain the raw bytes backing this instance as a [u8] slice.
38    pub fn as_bytes(&self) -> &[u8] {
39        match self {
40            Self::Ucs1(s) => s,
41            Self::Ucs2(s) => unsafe {
42                core::slice::from_raw_parts(s.as_ptr().cast(), s.len() * self.value_width_bytes())
43            },
44            Self::Ucs4(s) => unsafe {
45                core::slice::from_raw_parts(s.as_ptr().cast(), s.len() * self.value_width_bytes())
46            },
47        }
48    }
49
50    /// Size in bytes of each value/item in the underlying slice.
51    #[inline]
52    pub fn value_width_bytes(&self) -> usize {
53        match self {
54            Self::Ucs1(_) => 1,
55            Self::Ucs2(_) => 2,
56            Self::Ucs4(_) => 4,
57        }
58    }
59
60    /// Convert the raw data to a Rust string.
61    ///
62    /// For UCS-1 / UTF-8, returns a borrow into the original slice. For UCS-2 and UCS-4,
63    /// returns an owned string.
64    ///
65    /// Returns [PyUnicodeDecodeError] if the string data isn't valid in its purported
66    /// storage format. This should only occur for strings that were created via Python
67    /// C APIs that skip input validation (like `PyUnicode_FromKindAndData`) and should
68    /// never occur for strings that were created from Python code.
69    pub fn to_string(self, py: Python<'_>) -> PyResult<Cow<'a, str>> {
70        match self {
71            Self::Ucs1(data) => match str::from_utf8(data) {
72                Ok(s) => Ok(Cow::Borrowed(s)),
73                Err(e) => Err(PyUnicodeDecodeError::new_utf8(py, data, e)?.into()),
74            },
75            Self::Ucs2(data) => match String::from_utf16(data) {
76                Ok(s) => Ok(Cow::Owned(s)),
77                Err(e) => {
78                    let mut message = e.to_string().as_bytes().to_vec();
79                    message.push(0);
80
81                    Err(PyUnicodeDecodeError::new(
82                        py,
83                        c"utf-16",
84                        self.as_bytes(),
85                        0..self.as_bytes().len(),
86                        CStr::from_bytes_with_nul(&message).unwrap(),
87                    )?
88                    .into())
89                }
90            },
91            Self::Ucs4(data) => match data.iter().copied().map(char::from_u32).collect() {
92                Some(s) => Ok(Cow::Owned(s)),
93                None => Err(PyUnicodeDecodeError::new(
94                    py,
95                    c"utf-32",
96                    self.as_bytes(),
97                    0..self.as_bytes().len(),
98                    c"error converting utf-32",
99                )?
100                .into()),
101            },
102        }
103    }
104
105    /// Convert the raw data to a Rust string, possibly with data loss.
106    ///
107    /// Invalid code points will be replaced with `U+FFFD REPLACEMENT CHARACTER`.
108    ///
109    /// Returns a borrow into original data, when possible, or owned data otherwise.
110    ///
111    /// The return value of this function should only disagree with [Self::to_string]
112    /// when that method would error.
113    pub fn to_string_lossy(self) -> Cow<'a, str> {
114        match self {
115            Self::Ucs1(data) => String::from_utf8_lossy(data),
116            Self::Ucs2(data) => Cow::Owned(String::from_utf16_lossy(data)),
117            Self::Ucs4(data) => Cow::Owned(
118                data.iter()
119                    .map(|&c| char::from_u32(c).unwrap_or('\u{FFFD}'))
120                    .collect(),
121            ),
122        }
123    }
124}
125
126/// Represents a Python `string` (a Unicode string object).
127///
128/// Values of this type are accessed via PyO3's smart pointers, e.g. as
129/// [`Py<PyString>`][crate::Py] or [`Bound<'py, PyString>`][Bound].
130///
131/// For APIs available on `str` objects, see the [`PyStringMethods`] trait which is implemented for
132/// [`Bound<'py, PyString>`][Bound].
133///
134/// # Equality
135///
136/// For convenience, [`Bound<'py, PyString>`] implements [`PartialEq<str>`] to allow comparing the
137/// data in the Python string to a Rust UTF-8 string slice.
138///
139/// This is not always the most appropriate way to compare Python strings, as Python string
140/// subclasses may have different equality semantics. In situations where subclasses overriding
141/// equality might be relevant, use [`PyAnyMethods::eq`](crate::types::any::PyAnyMethods::eq), at
142/// cost of the additional overhead of a Python method call.
143///
144/// ```rust
145/// # use pyo3::prelude::*;
146/// use pyo3::types::PyString;
147///
148/// # Python::attach(|py| {
149/// let py_string = PyString::new(py, "foo");
150/// // via PartialEq<str>
151/// assert_eq!(py_string, "foo");
152///
153/// // via Python equality
154/// assert!(py_string.as_any().eq("foo").unwrap());
155/// # });
156/// ```
157#[repr(transparent)]
158pub struct PyString(PyAny);
159
160#[cfg(not(RustPython))]
161pyobject_native_type_core!(PyString, pyobject_native_static_type_object!(ffi::PyUnicode_Type), "builtins", "str", #checkfunction=ffi::PyUnicode_Check);
162
163#[cfg(RustPython)]
164pyobject_native_type_core!(
165    PyString,
166    |py| {
167        static TYPE: PyOnceLock<Py<PyType>> = PyOnceLock::new();
168        TYPE.import(py, "builtins", "str").unwrap().as_type_ptr()
169    },
170    "builtins",
171    "str",
172    #checkfunction=ffi::PyUnicode_Check
173);
174
175impl PyString {
176    /// Creates a new Python string object.
177    ///
178    /// Panics if out of memory.
179    pub fn new<'py>(py: Python<'py>, s: &str) -> Bound<'py, PyString> {
180        let ptr = s.as_ptr().cast();
181        let len = s.len() as ffi::Py_ssize_t;
182        unsafe {
183            ffi::PyUnicode_FromStringAndSize(ptr, len)
184                .assume_owned(py)
185                .cast_into_unchecked()
186        }
187    }
188
189    /// Creates a new Python string object from bytes.
190    ///
191    /// Returns PyMemoryError if out of memory.
192    /// Returns [PyUnicodeDecodeError] if the slice is not a valid UTF-8 string.
193    pub fn from_bytes<'py>(py: Python<'py>, s: &[u8]) -> PyResult<Bound<'py, PyString>> {
194        let ptr = s.as_ptr().cast();
195        let len = s.len() as ffi::Py_ssize_t;
196        unsafe {
197            ffi::PyUnicode_FromStringAndSize(ptr, len)
198                .assume_owned_or_err(py)
199                .cast_into_unchecked()
200        }
201    }
202
203    /// Intern the given string
204    ///
205    /// This will return a reference to the same Python string object if called repeatedly with the same string.
206    ///
207    /// Note that while this is more memory efficient than [`PyString::new`], it unconditionally allocates a
208    /// temporary Python string object and is thereby slower than [`PyString::new`].
209    ///
210    /// Panics if out of memory.
211    pub fn intern<'py>(py: Python<'py>, s: &str) -> Bound<'py, PyString> {
212        let ptr = s.as_ptr().cast();
213        let len = s.len() as ffi::Py_ssize_t;
214        unsafe {
215            let mut ob = ffi::PyUnicode_FromStringAndSize(ptr, len);
216            if !ob.is_null() {
217                ffi::PyUnicode_InternInPlace(&mut ob);
218            }
219            ob.assume_owned(py).cast_into_unchecked()
220        }
221    }
222
223    /// Attempts to create a Python string from a Python [bytes-like object].
224    ///
225    /// The `encoding` and `errors` parameters are optional:
226    /// - If `encoding` is `None`, the default encoding is used (UTF-8).
227    /// - If `errors` is `None`, the default error handling is used ("strict").
228    ///
229    /// See the [Python documentation on codecs] for more information.
230    ///
231    /// [bytes-like object]: (https://docs.python.org/3/glossary.html#term-bytes-like-object).
232    /// [Python documentation on codecs]: https://docs.python.org/3/library/codecs.html#standard-encodings
233    pub fn from_encoded_object<'py>(
234        src: &Bound<'py, PyAny>,
235        encoding: Option<&CStr>,
236        errors: Option<&CStr>,
237    ) -> PyResult<Bound<'py, PyString>> {
238        let encoding = encoding.map_or(core::ptr::null(), CStr::as_ptr);
239        let errors = errors.map_or(core::ptr::null(), CStr::as_ptr);
240        // Safety:
241        // - `src` is a valid Python object
242        // - `encoding` and `errors` are either null or valid C strings. `encoding` and `errors` are
243        //   documented as allowing null.
244        // - `ffi::PyUnicode_FromEncodedObject` returns a new `str` object, or sets an error.
245        unsafe {
246            ffi::PyUnicode_FromEncodedObject(src.as_ptr(), encoding, errors)
247                .assume_owned_or_err(src.py())
248                .cast_into_unchecked()
249        }
250    }
251
252    /// Creates a Python string using a format string.
253    ///
254    /// This function is similar to [`format!`], but it returns a Python string object instead of a Rust string.
255    #[inline]
256    pub fn from_fmt<'py>(
257        py: Python<'py>,
258        args: fmt::Arguments<'_>,
259    ) -> PyResult<Bound<'py, PyString>> {
260        if let Some(static_string) = args.as_str() {
261            return Ok(PyString::new(py, static_string));
262        };
263
264        #[cfg(all(Py_3_14, not(Py_LIMITED_API)))]
265        {
266            use crate::fmt::PyUnicodeWriter;
267            use core::fmt::Write as _;
268
269            let mut writer = PyUnicodeWriter::new(py)?;
270            writer
271                .write_fmt(args)
272                .map_err(|_| writer.take_error().expect("expected error"))?;
273            writer.into_py_string()
274        }
275
276        #[cfg(any(not(Py_3_14), Py_LIMITED_API))]
277        {
278            Ok(PyString::new(py, &format!("{args}")))
279        }
280    }
281}
282
283/// Implementation of functionality for [`PyString`].
284///
285/// These methods are defined for the `Bound<'py, PyString>` smart pointer, so to use method call
286/// syntax these methods are separated into a trait, because stable Rust does not yet support
287/// `arbitrary_self_types`.
288#[doc(alias = "PyString")]
289pub trait PyStringMethods<'py>: crate::sealed::Sealed {
290    /// Gets the Python string as a Rust UTF-8 string slice.
291    ///
292    /// Returns a `UnicodeEncodeError` if the input is not valid unicode
293    /// (containing unpaired surrogates).
294    #[cfg(any(Py_3_10, not(Py_LIMITED_API)))]
295    fn to_str(&self) -> PyResult<&str>;
296
297    /// Converts the `PyString` into a Rust string, avoiding copying when possible.
298    ///
299    /// Returns a `UnicodeEncodeError` if the input is not valid unicode
300    /// (containing unpaired surrogates).
301    fn to_cow(&self) -> PyResult<Cow<'_, str>>;
302
303    /// Converts the `PyString` into a Rust string.
304    ///
305    /// Unpaired surrogates invalid UTF-8 sequences are
306    /// replaced with `U+FFFD REPLACEMENT CHARACTER`.
307    fn to_string_lossy(&self) -> Cow<'_, str>;
308
309    /// Encodes this string as a Python `bytes` object, using UTF-8 encoding.
310    fn encode_utf8(&self) -> PyResult<Bound<'py, PyBytes>>;
311
312    /// Obtains the raw data backing the Python string.
313    ///
314    /// If the Python string object was created through legacy APIs, its internal storage format
315    /// will be canonicalized before data is returned.
316    ///
317    /// # Safety
318    ///
319    /// This function implementation relies on manually decoding a C bitfield. In practice, this
320    /// works well on common little-endian architectures such as x86_64, where the bitfield has a
321    /// common representation (even if it is not part of the C spec). The PyO3 CI tests this API on
322    /// x86_64 platforms.
323    ///
324    /// By using this API, you accept responsibility for testing that PyStringData behaves as
325    /// expected on the targets where you plan to distribute your software.
326    #[cfg(not(any(Py_LIMITED_API, GraalPy, PyPy)))]
327    unsafe fn data(&self) -> PyResult<PyStringData<'_>>;
328}
329
330impl<'py> PyStringMethods<'py> for Bound<'py, PyString> {
331    #[cfg(any(Py_3_10, not(Py_LIMITED_API)))]
332    fn to_str(&self) -> PyResult<&str> {
333        self.as_borrowed().to_str()
334    }
335
336    fn to_cow(&self) -> PyResult<Cow<'_, str>> {
337        self.as_borrowed().to_cow()
338    }
339
340    fn to_string_lossy(&self) -> Cow<'_, str> {
341        self.as_borrowed().to_string_lossy()
342    }
343
344    fn encode_utf8(&self) -> PyResult<Bound<'py, PyBytes>> {
345        unsafe {
346            ffi::PyUnicode_AsUTF8String(self.as_ptr())
347                .assume_owned_or_err(self.py())
348                .cast_into_unchecked::<PyBytes>()
349        }
350    }
351
352    #[cfg(not(any(Py_LIMITED_API, GraalPy, PyPy)))]
353    unsafe fn data(&self) -> PyResult<PyStringData<'_>> {
354        unsafe { self.as_borrowed().data() }
355    }
356}
357
358impl<'a> Borrowed<'a, '_, PyString> {
359    #[cfg(any(Py_3_10, not(Py_LIMITED_API)))]
360    pub(crate) fn to_str(self) -> PyResult<&'a str> {
361        // PyUnicode_AsUTF8AndSize only available on limited API starting with 3.10.
362        let mut size: ffi::Py_ssize_t = 0;
363        let data: *const u8 =
364            unsafe { ffi::PyUnicode_AsUTF8AndSize(self.as_ptr(), &mut size).cast() };
365        if data.is_null() {
366            Err(crate::PyErr::fetch(self.py()))
367        } else {
368            Ok(unsafe {
369                core::str::from_utf8_unchecked(core::slice::from_raw_parts(data, size as usize))
370            })
371        }
372    }
373
374    pub(crate) fn to_cow(self) -> PyResult<Cow<'a, str>> {
375        // TODO: this method can probably be deprecated once Python 3.9 support is dropped,
376        // because all versions then support the more efficient `to_str`.
377        #[cfg(any(Py_3_10, not(Py_LIMITED_API)))]
378        {
379            self.to_str().map(Cow::Borrowed)
380        }
381
382        #[cfg(not(any(Py_3_10, not(Py_LIMITED_API))))]
383        {
384            let bytes = self.encode_utf8()?;
385            Ok(Cow::Owned(
386                unsafe { str::from_utf8_unchecked(bytes.as_bytes()) }.to_owned(),
387            ))
388        }
389    }
390
391    fn to_string_lossy(self) -> Cow<'a, str> {
392        let ptr = self.as_ptr();
393        let py = self.py();
394
395        #[cfg(any(Py_3_10, not(Py_LIMITED_API)))]
396        if let Ok(s) = self.to_str() {
397            return Cow::Borrowed(s);
398        }
399
400        let bytes = unsafe {
401            ffi::PyUnicode_AsEncodedString(ptr, c"utf-8".as_ptr(), c"surrogatepass".as_ptr())
402                .assume_owned(py)
403                .cast_into_unchecked::<PyBytes>()
404        };
405        Cow::Owned(String::from_utf8_lossy(bytes.as_bytes()).into_owned())
406    }
407
408    #[cfg(not(any(Py_LIMITED_API, GraalPy, PyPy)))]
409    unsafe fn data(self) -> PyResult<PyStringData<'a>> {
410        unsafe {
411            let ptr = self.as_ptr();
412
413            #[cfg(not(Py_3_12))]
414            #[allow(deprecated)]
415            {
416                let ready = ffi::PyUnicode_READY(ptr);
417                if ready != 0 {
418                    // Exception was created on failure.
419                    return Err(crate::PyErr::fetch(self.py()));
420                }
421            }
422
423            // The string should be in its canonical form after calling `PyUnicode_READY()`.
424            // And non-canonical form not possible after Python 3.12. So it should be safe
425            // to call these APIs.
426            let length = ffi::PyUnicode_GET_LENGTH(ptr) as usize;
427            let raw_data = ffi::PyUnicode_DATA(ptr);
428            let kind = ffi::PyUnicode_KIND(ptr);
429
430            match kind {
431                ffi::PyUnicode_1BYTE_KIND => Ok(PyStringData::Ucs1(core::slice::from_raw_parts(
432                    raw_data as *const u8,
433                    length,
434                ))),
435                ffi::PyUnicode_2BYTE_KIND => Ok(PyStringData::Ucs2(core::slice::from_raw_parts(
436                    raw_data as *const u16,
437                    length,
438                ))),
439                ffi::PyUnicode_4BYTE_KIND => Ok(PyStringData::Ucs4(core::slice::from_raw_parts(
440                    raw_data as *const u32,
441                    length,
442                ))),
443                _ => unreachable!(),
444            }
445        }
446    }
447}
448
449impl Py<PyString> {
450    /// Gets the Python string as a Rust UTF-8 string slice.
451    ///
452    /// Returns a `UnicodeEncodeError` if the input is not valid unicode
453    /// (containing unpaired surrogates).
454    ///
455    /// Because `str` objects are immutable, the returned slice is independent of
456    /// the GIL lifetime.
457    #[cfg(any(Py_3_10, not(Py_LIMITED_API)))]
458    pub fn to_str<'a>(&'a self, py: Python<'_>) -> PyResult<&'a str> {
459        self.bind_borrowed(py).to_str()
460    }
461
462    /// Converts the `PyString` into a Rust string, avoiding copying when possible.
463    ///
464    /// Returns a `UnicodeEncodeError` if the input is not valid unicode
465    /// (containing unpaired surrogates).
466    ///
467    /// Because `str` objects are immutable, the returned slice is independent of
468    /// the GIL lifetime.
469    pub fn to_cow<'a>(&'a self, py: Python<'_>) -> PyResult<Cow<'a, str>> {
470        self.bind_borrowed(py).to_cow()
471    }
472
473    /// Converts the `PyString` into a Rust string.
474    ///
475    /// Unpaired surrogates invalid UTF-8 sequences are
476    /// replaced with `U+FFFD REPLACEMENT CHARACTER`.
477    ///
478    /// Because `str` objects are immutable, the returned slice is independent of
479    /// the GIL lifetime.
480    pub fn to_string_lossy<'a>(&'a self, py: Python<'_>) -> Cow<'a, str> {
481        self.bind_borrowed(py).to_string_lossy()
482    }
483}
484
485/// Compares whether the data in the Python string is equal to the given UTF8.
486///
487/// In some cases Python equality might be more appropriate; see the note on [`PyString`].
488impl PartialEq<str> for Bound<'_, PyString> {
489    #[inline]
490    fn eq(&self, other: &str) -> bool {
491        self.as_borrowed() == *other
492    }
493}
494
495/// Compares whether the data in the Python string is equal to the given UTF8.
496///
497/// In some cases Python equality might be more appropriate; see the note on [`PyString`].
498impl PartialEq<&'_ str> for Bound<'_, PyString> {
499    #[inline]
500    fn eq(&self, other: &&str) -> bool {
501        self.as_borrowed() == **other
502    }
503}
504
505/// Compares whether the data in the Python string is equal to the given UTF8.
506///
507/// In some cases Python equality might be more appropriate; see the note on [`PyString`].
508impl PartialEq<Bound<'_, PyString>> for str {
509    #[inline]
510    fn eq(&self, other: &Bound<'_, PyString>) -> bool {
511        *self == other.as_borrowed()
512    }
513}
514
515/// Compares whether the data in the Python string is equal to the given UTF8.
516///
517/// In some cases Python equality might be more appropriate; see the note on [`PyString`].
518impl PartialEq<&'_ Bound<'_, PyString>> for str {
519    #[inline]
520    fn eq(&self, other: &&Bound<'_, PyString>) -> bool {
521        *self == other.as_borrowed()
522    }
523}
524
525/// Compares whether the data in the Python string is equal to the given UTF8.
526///
527/// In some cases Python equality might be more appropriate; see the note on [`PyString`].
528impl PartialEq<Bound<'_, PyString>> for &'_ str {
529    #[inline]
530    fn eq(&self, other: &Bound<'_, PyString>) -> bool {
531        **self == other.as_borrowed()
532    }
533}
534
535/// Compares whether the data in the Python string is equal to the given UTF8.
536///
537/// In some cases Python equality might be more appropriate; see the note on [`PyString`].
538impl PartialEq<str> for &'_ Bound<'_, PyString> {
539    #[inline]
540    fn eq(&self, other: &str) -> bool {
541        self.as_borrowed() == other
542    }
543}
544
545/// Compares whether the data in the Python string is equal to the given UTF8.
546///
547/// In some cases Python equality might be more appropriate; see the note on [`PyString`].
548impl PartialEq<str> for Borrowed<'_, '_, PyString> {
549    #[inline]
550    fn eq(&self, other: &str) -> bool {
551        #[cfg(not(Py_3_13))]
552        {
553            self.to_cow().is_ok_and(|s| s == other)
554        }
555
556        #[cfg(Py_3_13)]
557        unsafe {
558            ffi::PyUnicode_EqualToUTF8AndSize(
559                self.as_ptr(),
560                other.as_ptr().cast(),
561                other.len() as _,
562            ) == 1
563        }
564    }
565}
566
567/// Compares whether the data in the Python string is equal to the given UTF8.
568///
569/// In some cases Python equality might be more appropriate; see the note on [`PyString`].
570impl PartialEq<&str> for Borrowed<'_, '_, PyString> {
571    #[inline]
572    fn eq(&self, other: &&str) -> bool {
573        *self == **other
574    }
575}
576
577/// Compares whether the data in the Python string is equal to the given UTF8.
578///
579/// In some cases Python equality might be more appropriate; see the note on [`PyString`].
580impl PartialEq<Borrowed<'_, '_, PyString>> for str {
581    #[inline]
582    fn eq(&self, other: &Borrowed<'_, '_, PyString>) -> bool {
583        other == self
584    }
585}
586
587/// Compares whether the data in the Python string is equal to the given UTF8.
588///
589/// In some cases Python equality might be more appropriate; see the note on [`PyString`].
590impl PartialEq<Borrowed<'_, '_, PyString>> for &'_ str {
591    #[inline]
592    fn eq(&self, other: &Borrowed<'_, '_, PyString>) -> bool {
593        other == self
594    }
595}
596
597#[cfg(test)]
598mod tests {
599    use super::*;
600    use crate::{exceptions::PyLookupError, types::PyAnyMethods as _, IntoPyObject};
601
602    #[test]
603    fn test_to_cow_utf8() {
604        Python::attach(|py| {
605            let s = "ascii 🐈";
606            let py_string = PyString::new(py, s);
607            assert_eq!(s, py_string.to_cow().unwrap());
608        })
609    }
610
611    #[test]
612    fn test_to_cow_surrogate() {
613        Python::attach(|py| {
614            let py_string = py
615                .eval(cr"'\ud800'", None, None)
616                .unwrap()
617                .cast_into::<PyString>()
618                .unwrap();
619            assert!(py_string.to_cow().is_err());
620        })
621    }
622
623    #[test]
624    fn test_to_cow_unicode() {
625        Python::attach(|py| {
626            let s = "哈哈🐈";
627            let py_string = PyString::new(py, s);
628            assert_eq!(s, py_string.to_cow().unwrap());
629        })
630    }
631
632    #[test]
633    fn test_encode_utf8_unicode() {
634        Python::attach(|py| {
635            let s = "哈哈🐈";
636            let obj = PyString::new(py, s);
637            assert_eq!(s.as_bytes(), obj.encode_utf8().unwrap().as_bytes());
638        })
639    }
640
641    #[test]
642    fn test_encode_utf8_surrogate() {
643        Python::attach(|py| {
644            let obj: Py<PyAny> = py.eval(cr"'\ud800'", None, None).unwrap().into();
645            assert!(obj
646                .bind(py)
647                .cast::<PyString>()
648                .unwrap()
649                .encode_utf8()
650                .is_err());
651        })
652    }
653
654    #[test]
655    fn test_to_string_lossy() {
656        Python::attach(|py| {
657            let py_string = py
658                .eval(cr"'🐈 Hello \ud800World'", None, None)
659                .unwrap()
660                .cast_into::<PyString>()
661                .unwrap();
662
663            assert_eq!(py_string.to_string_lossy(), "🐈 Hello ���World");
664        })
665    }
666
667    #[test]
668    fn test_debug_string() {
669        Python::attach(|py| {
670            let s = "Hello\n".into_pyobject(py).unwrap();
671            assert_eq!(format!("{s:?}"), "'Hello\\n'");
672        })
673    }
674
675    #[test]
676    fn test_display_string() {
677        Python::attach(|py| {
678            let s = "Hello\n".into_pyobject(py).unwrap();
679            assert_eq!(format!("{s}"), "Hello\n");
680        })
681    }
682
683    #[test]
684    fn test_string_from_encoded_object() {
685        Python::attach(|py| {
686            let py_bytes = PyBytes::new(py, b"ab\xFFcd");
687
688            // default encoding is utf-8, default error handler is strict
689            let py_string = PyString::from_encoded_object(&py_bytes, None, None).unwrap_err();
690            assert!(py_string
691                .get_type(py)
692                .is(py.get_type::<crate::exceptions::PyUnicodeDecodeError>()));
693
694            // with `ignore` error handler, the invalid byte is dropped
695            let py_string =
696                PyString::from_encoded_object(&py_bytes, None, Some(c"ignore")).unwrap();
697
698            let result = py_string.to_cow().unwrap();
699            assert_eq!(result, "abcd");
700        });
701    }
702
703    #[test]
704    fn test_string_from_encoded_object_with_invalid_encoding_errors() {
705        Python::attach(|py| {
706            let py_bytes = PyBytes::new(py, b"abcd");
707
708            // invalid encoding
709            let err = PyString::from_encoded_object(&py_bytes, Some(c"wat"), None).unwrap_err();
710            assert!(err.is_instance(py, &py.get_type::<PyLookupError>()));
711            assert_eq!(err.to_string(), "LookupError: unknown encoding: wat");
712
713            // invalid error handler
714            let err =
715                PyString::from_encoded_object(&PyBytes::new(py, b"ab\xFFcd"), None, Some(c"wat"))
716                    .unwrap_err();
717            assert!(err.is_instance(py, &py.get_type::<PyLookupError>()));
718            assert_eq!(
719                err.to_string(),
720                "LookupError: unknown error handler name 'wat'"
721            );
722        });
723    }
724
725    #[test]
726    #[cfg(not(any(Py_LIMITED_API, PyPy, GraalPy)))]
727    fn test_string_data_ucs1() {
728        Python::attach(|py| {
729            let s = PyString::new(py, "hello, world");
730            let data = unsafe { s.data().unwrap() };
731
732            assert_eq!(data, PyStringData::Ucs1(b"hello, world"));
733            assert_eq!(data.to_string(py).unwrap(), Cow::Borrowed("hello, world"));
734            assert_eq!(data.to_string_lossy(), Cow::Borrowed("hello, world"));
735        })
736    }
737
738    #[test]
739    #[cfg(not(any(Py_LIMITED_API, PyPy, GraalPy)))]
740    fn test_string_data_ucs1_invalid() {
741        Python::attach(|py| {
742            // 0xfe is not allowed in UTF-8.
743            let buffer = b"f\xfe\0";
744            let ptr = unsafe {
745                crate::ffi::PyUnicode_FromKindAndData(
746                    crate::ffi::PyUnicode_1BYTE_KIND as _,
747                    buffer.as_ptr().cast(),
748                    2,
749                )
750            };
751            assert!(!ptr.is_null());
752            let s = unsafe { ptr.assume_owned(py).cast_into_unchecked::<PyString>() };
753            let data = unsafe { s.data().unwrap() };
754            assert_eq!(data, PyStringData::Ucs1(b"f\xfe"));
755            let err = data.to_string(py).unwrap_err();
756            assert!(err.get_type(py).is(py.get_type::<PyUnicodeDecodeError>()));
757            assert!(err
758                .to_string()
759                .contains("'utf-8' codec can't decode byte 0xfe in position 1"));
760            assert_eq!(data.to_string_lossy(), Cow::Borrowed("f�"));
761        });
762    }
763
764    #[test]
765    #[cfg(not(any(Py_LIMITED_API, PyPy, GraalPy)))]
766    fn test_string_data_ucs2() {
767        Python::attach(|py| {
768            let s = py.eval(c"'foo\\ud800'", None, None).unwrap();
769            let py_string = s.cast::<PyString>().unwrap();
770            let data = unsafe { py_string.data().unwrap() };
771
772            assert_eq!(data, PyStringData::Ucs2(&[102, 111, 111, 0xd800]));
773            assert_eq!(
774                data.to_string_lossy(),
775                Cow::Owned::<str>("foo�".to_string())
776            );
777        })
778    }
779
780    #[test]
781    #[cfg(all(not(any(Py_LIMITED_API, PyPy, GraalPy)), target_endian = "little"))]
782    fn test_string_data_ucs2_invalid() {
783        Python::attach(|py| {
784            // U+FF22 (valid) & U+d800 (never valid)
785            let buffer = b"\x22\xff\x00\xd8\x00\x00";
786            let ptr = unsafe {
787                crate::ffi::PyUnicode_FromKindAndData(
788                    crate::ffi::PyUnicode_2BYTE_KIND as _,
789                    buffer.as_ptr().cast(),
790                    2,
791                )
792            };
793            assert!(!ptr.is_null());
794            let s = unsafe { ptr.assume_owned(py).cast_into_unchecked::<PyString>() };
795            let data = unsafe { s.data().unwrap() };
796            assert_eq!(data, PyStringData::Ucs2(&[0xff22, 0xd800]));
797            let err = data.to_string(py).unwrap_err();
798            assert!(err.get_type(py).is(py.get_type::<PyUnicodeDecodeError>()));
799            assert!(err
800                .to_string()
801                .contains("'utf-16' codec can't decode bytes in position 0-3"));
802            assert_eq!(data.to_string_lossy(), Cow::Owned::<str>("Ｂ�".into()));
803        });
804    }
805
806    #[test]
807    #[cfg(not(any(Py_LIMITED_API, PyPy, GraalPy)))]
808    fn test_string_data_ucs4() {
809        Python::attach(|py| {
810            let s = "哈哈🐈";
811            let py_string = PyString::new(py, s);
812            let data = unsafe { py_string.data().unwrap() };
813
814            assert_eq!(data, PyStringData::Ucs4(&[21704, 21704, 128008]));
815            assert_eq!(data.to_string_lossy(), Cow::Owned::<str>(s.to_string()));
816        })
817    }
818
819    #[test]
820    #[cfg(all(not(any(Py_LIMITED_API, PyPy, GraalPy)), target_endian = "little"))]
821    fn test_string_data_ucs4_invalid() {
822        Python::attach(|py| {
823            // U+20000 (valid) & U+d800 (never valid)
824            let buffer = b"\x00\x00\x02\x00\x00\xd8\x00\x00\x00\x00\x00\x00";
825            let ptr = unsafe {
826                crate::ffi::PyUnicode_FromKindAndData(
827                    crate::ffi::PyUnicode_4BYTE_KIND as _,
828                    buffer.as_ptr().cast(),
829                    2,
830                )
831            };
832            assert!(!ptr.is_null());
833            let s = unsafe { ptr.assume_owned(py).cast_into_unchecked::<PyString>() };
834            let data = unsafe { s.data().unwrap() };
835            assert_eq!(data, PyStringData::Ucs4(&[0x20000, 0xd800]));
836            let err = data.to_string(py).unwrap_err();
837            assert!(err.get_type(py).is(py.get_type::<PyUnicodeDecodeError>()));
838            assert!(err
839                .to_string()
840                .contains("'utf-32' codec can't decode bytes in position 0-7"));
841            assert_eq!(data.to_string_lossy(), Cow::Owned::<str>("𠀀�".into()));
842        });
843    }
844
845    #[test]
846    #[cfg(not(any(Py_LIMITED_API, PyPy, GraalPy)))]
847    fn test_pystring_from_bytes() {
848        Python::attach(|py| {
849            let result = PyString::from_bytes(py, "\u{2122}".as_bytes());
850            assert!(result.is_ok());
851            let result = PyString::from_bytes(py, b"\x80");
852            assert!(result
853                .unwrap_err()
854                .get_type(py)
855                .is(py.get_type::<PyUnicodeDecodeError>()));
856        });
857    }
858
859    #[test]
860    fn test_intern_string() {
861        Python::attach(|py| {
862            let py_string1 = PyString::intern(py, "foo");
863            assert_eq!(py_string1, "foo");
864
865            let py_string2 = PyString::intern(py, "foo");
866            assert_eq!(py_string2, "foo");
867
868            assert_eq!(py_string1.as_ptr(), py_string2.as_ptr());
869
870            let py_string3 = PyString::intern(py, "bar");
871            assert_eq!(py_string3, "bar");
872
873            assert_ne!(py_string1.as_ptr(), py_string3.as_ptr());
874        });
875    }
876
877    #[test]
878    fn test_py_to_str_utf8() {
879        Python::attach(|py| {
880            let s = "ascii 🐈";
881            let py_string = PyString::new(py, s).unbind();
882
883            #[cfg(any(Py_3_10, not(Py_LIMITED_API)))]
884            assert_eq!(s, py_string.to_str(py).unwrap());
885
886            assert_eq!(s, py_string.to_cow(py).unwrap());
887        })
888    }
889
890    #[test]
891    fn test_py_to_str_surrogate() {
892        Python::attach(|py| {
893            let py_string: Py<PyString> = py
894                .eval(cr"'\ud800'", None, None)
895                .unwrap()
896                .extract()
897                .unwrap();
898
899            #[cfg(any(Py_3_10, not(Py_LIMITED_API)))]
900            assert!(py_string.to_str(py).is_err());
901
902            assert!(py_string.to_cow(py).is_err());
903        })
904    }
905
906    #[test]
907    fn test_py_to_string_lossy() {
908        Python::attach(|py| {
909            let py_string: Py<PyString> = py
910                .eval(cr"'🐈 Hello \ud800World'", None, None)
911                .unwrap()
912                .extract()
913                .unwrap();
914            assert_eq!(py_string.to_string_lossy(py), "🐈 Hello ���World");
915        })
916    }
917
918    #[test]
919    fn test_comparisons() {
920        Python::attach(|py| {
921            let s = "hello, world";
922            let py_string = PyString::new(py, s);
923
924            assert_eq!(py_string, "hello, world");
925
926            assert_eq!(py_string, s);
927            assert_eq!(&py_string, s);
928            assert_eq!(s, py_string);
929            assert_eq!(s, &py_string);
930
931            assert_eq!(py_string, *s);
932            assert_eq!(&py_string, *s);
933            assert_eq!(*s, py_string);
934            assert_eq!(*s, &py_string);
935
936            let py_string = py_string.as_borrowed();
937
938            assert_eq!(py_string, s);
939            assert_eq!(&py_string, s);
940            assert_eq!(s, py_string);
941            assert_eq!(s, &py_string);
942
943            assert_eq!(py_string, *s);
944            assert_eq!(*s, py_string);
945        })
946    }
947}
pyo3/types/string.rs

pyo3/types/
string.rs
