diff --git a/Lib/test/test_io.py b/Lib/test/test_io.py index 2cf84d9aae..1e594c4964 100644 --- a/Lib/test/test_io.py +++ b/Lib/test/test_io.py @@ -1818,10 +1818,6 @@ def test_garbage_collection(self): support.gc_collect() self.assertIsNone(wr(), wr) - @unittest.expectedFailure # TODO: RUSTPYTHON - def test_error_through_destructor(self): - return super().test_error_through_destructor() - def test_args_error(self): # Issue #17275 with self.assertRaisesRegex(TypeError, "BufferedReader"): @@ -1844,12 +1840,8 @@ def test_bad_readinto_type(self): self.assertIsInstance(cm.exception.__cause__, TypeError) @unittest.expectedFailure # TODO: RUSTPYTHON - def test_flush_error_on_close(self): - return super().test_flush_error_on_close() - - @unittest.expectedFailure # TODO: RUSTPYTHON - def test_seek_character_device_file(self): - return super().test_seek_character_device_file() + def test_error_through_destructor(self): + return super().test_error_through_destructor() def test_truncate_on_read_only(self): return super().test_truncate_on_read_only() @@ -2166,10 +2158,6 @@ def test_slow_close_from_thread(self): class CBufferedWriterTest(BufferedWriterTest, SizeofTest): tp = io.BufferedWriter - @unittest.expectedFailure # TODO: RUSTPYTHON - def test_error_through_destructor(self): - return super().test_error_through_destructor() - def test_initialization(self): rawio = self.MockRawIO() bufio = self.tp(rawio) @@ -2205,8 +2193,8 @@ def test_args_error(self): self.tp(self.BytesIO(), 1024, 1024, 1024) @unittest.expectedFailure # TODO: RUSTPYTHON - def test_flush_error_on_close(self): - return super().test_flush_error_on_close() + def test_error_through_destructor(self): + return super().test_error_through_destructor() @unittest.skip('TODO: RUSTPYTHON; fallible allocation') def test_constructor(self): @@ -2692,10 +2680,6 @@ def test_interleaved_readline_write(self): class CBufferedRandomTest(BufferedRandomTest, SizeofTest): tp = io.BufferedRandom - @unittest.expectedFailure # TODO: RUSTPYTHON - def test_error_through_destructor(self): - return super().test_error_through_destructor() - @unittest.skipIf(sys.platform == 'win32', 'TODO: RUSTPYTHON; cyclic GC not supported, causes file locking') @unittest.expectedFailure # TODO: RUSTPYTHON def test_garbage_collection(self): @@ -2708,16 +2692,8 @@ def test_args_error(self): self.tp(self.BytesIO(), 1024, 1024, 1024) @unittest.expectedFailure # TODO: RUSTPYTHON - def test_flush_error_on_close(self): - return super().test_flush_error_on_close() - - @unittest.expectedFailure # TODO: RUSTPYTHON - def test_seek_character_device_file(self): - return super().test_seek_character_device_file() - - @unittest.expectedFailure # TODO: RUSTPYTHON; f.read1(1) returns b'a' - def test_read1_after_write(self): - return super().test_read1_after_write() + def test_error_through_destructor(self): + return super().test_error_through_destructor() @unittest.skip('TODO: RUSTPYTHON; fallible allocation') def test_constructor(self): @@ -3395,7 +3371,24 @@ def test_multibyte_seek_and_tell(self): self.assertEqual(f.tell(), p1) f.close() - @unittest.expectedFailure # TODO: RUSTPYTHON + def test_tell_after_readline_with_cr(self): + # Test for gh-141314: TextIOWrapper.tell() assertion failure + # when dealing with standalone carriage returns + data = b'line1\r' + with self.open(os_helper.TESTFN, "wb") as f: + f.write(data) + + with self.open(os_helper.TESTFN, "r") as f: + # Read line that ends with \r + line = f.readline() + self.assertEqual(line, "line1\n") + # This should not cause an assertion failure + pos = f.tell() + # Verify we can seek back to this position + f.seek(pos) + remaining = f.read() + self.assertEqual(remaining, "") + def test_seek_with_encoder_state(self): f = self.open(os_helper.TESTFN, "w", encoding="euc_jis_2004") f.write("\u00e6\u0300") @@ -4130,10 +4123,6 @@ class CTextIOWrapperTest(TextIOWrapperTest): io = io shutdown_error = "LookupError: unknown encoding: ascii" - @unittest.expectedFailure # TODO: RUSTPYTHON - def test_error_through_destructor(self): - return super().test_error_through_destructor() - @unittest.expectedFailure # TODO: RUSTPYTHON def test_initialization(self): r = self.BytesIO(b"\xc3\xa9\n\n") @@ -4291,6 +4280,10 @@ def test_reconfigure_write_fromascii(self): def test_reconfigure_write_through(self): return super().test_reconfigure_write_through() + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_error_through_destructor(self): + return super().test_error_through_destructor() + @unittest.expectedFailure # TODO: RUSTPYTHON def test_repr(self): return super().test_repr() @@ -4306,6 +4299,11 @@ def test_recursive_repr(self): def test_pickling_subclass(self): return super().test_pickling_subclass() + # TODO: RUSTPYTHON; euc_jis_2004 encoding not supported + @unittest.expectedFailure + def test_seek_with_encoder_state(self): + return super().test_seek_with_encoder_state() + class PyTextIOWrapperTest(TextIOWrapperTest): io = pyio @@ -4319,6 +4317,11 @@ def test_constructor(self): def test_newlines(self): return super().test_newlines() + # TODO: RUSTPYTHON; euc_jis_2004 encoding not supported + @unittest.expectedFailure + def test_seek_with_encoder_state(self): + return super().test_seek_with_encoder_state() + class IncrementalNewlineDecoderTest(unittest.TestCase): @@ -4836,22 +4839,6 @@ class CMiscIOTest(MiscIOTest): name_of_module = "io", "_io" extra_exported = "BlockingIOError", - @unittest.expectedFailure # TODO: RUSTPYTHON; BufferedWriter seeks on non-seekable pipe - def test_nonblock_pipe_write_bigbuf(self): - return super().test_nonblock_pipe_write_bigbuf() - - @unittest.expectedFailure # TODO: RUSTPYTHON; BufferedWriter seeks on non-seekable pipe - def test_nonblock_pipe_write_smallbuf(self): - return super().test_nonblock_pipe_write_smallbuf() - - @unittest.expectedFailure # TODO: RUSTPYTHON - def test_warn_on_dealloc(self): - return super().test_warn_on_dealloc() - - @unittest.expectedFailure # TODO: RUSTPYTHON - def test_warn_on_dealloc_fd(self): - return super().test_warn_on_dealloc_fd() - def test_readinto_buffer_overflow(self): # Issue #18025 class BadReader(self.io.BufferedIOBase): @@ -4916,6 +4903,16 @@ def test_daemon_threads_shutdown_stderr_deadlock(self): def test_check_encoding_errors(self): return super().test_check_encoding_errors() + # TODO: RUSTPYTHON; ResourceWarning not triggered by _io.FileIO + @unittest.expectedFailure + def test_warn_on_dealloc(self): + return super().test_warn_on_dealloc() + + # TODO: RUSTPYTHON; ResourceWarning not triggered by _io.FileIO + @unittest.expectedFailure + def test_warn_on_dealloc_fd(self): + return super().test_warn_on_dealloc_fd() + class PyMiscIOTest(MiscIOTest): io = pyio diff --git a/crates/derive-impl/src/pystructseq.rs b/crates/derive-impl/src/pystructseq.rs index 2ebb05075e..32b603fe47 100644 --- a/crates/derive-impl/src/pystructseq.rs +++ b/crates/derive-impl/src/pystructseq.rs @@ -316,8 +316,44 @@ pub(crate) fn impl_pystruct_sequence_data( // Generate try_from_elements trait override only when try_from_object=true let try_from_elements_trait_override = if try_from_object { - let visible_field_idents: Vec<_> = visible_fields.iter().map(|f| &f.ident).collect(); - let skipped_field_idents: Vec<_> = skipped_fields.iter().map(|f| &f.ident).collect(); + let visible_field_inits: Vec<_> = visible_fields + .iter() + .map(|f| { + let ident = &f.ident; + let cfg_attrs = &f.cfg_attrs; + if cfg_attrs.is_empty() { + quote! { #ident: iter.next().unwrap().clone().try_into_value(vm)?, } + } else { + quote! { + #(#cfg_attrs)* + #ident: iter.next().unwrap().clone().try_into_value(vm)?, + } + } + }) + .collect(); + let skipped_field_inits: Vec<_> = skipped_fields + .iter() + .map(|f| { + let ident = &f.ident; + let cfg_attrs = &f.cfg_attrs; + if cfg_attrs.is_empty() { + quote! { + #ident: match iter.next() { + Some(v) => v.clone().try_into_value(vm)?, + None => vm.ctx.none(), + }, + } + } else { + quote! { + #(#cfg_attrs)* + #ident: match iter.next() { + Some(v) => v.clone().try_into_value(vm)?, + None => vm.ctx.none(), + }, + } + } + }) + .collect(); quote! { fn try_from_elements( elements: Vec<::rustpython_vm::PyObjectRef>, @@ -325,11 +361,8 @@ pub(crate) fn impl_pystruct_sequence_data( ) -> ::rustpython_vm::PyResult { let mut iter = elements.into_iter(); Ok(Self { - #(#visible_field_idents: iter.next().unwrap().clone().try_into_value(vm)?,)* - #(#skipped_field_idents: match iter.next() { - Some(v) => v.clone().try_into_value(vm)?, - None => vm.ctx.none(), - },)* + #(#visible_field_inits)* + #(#skipped_field_inits)* }) } } diff --git a/crates/vm/src/stdlib/io.rs b/crates/vm/src/stdlib/io.rs index 0167df2bed..89dc8bca92 100644 --- a/crates/vm/src/stdlib/io.rs +++ b/crates/vm/src/stdlib/io.rs @@ -191,6 +191,7 @@ mod _io { borrow::Cow, io::{self, Cursor, SeekFrom, prelude::*}, ops::Range, + sync::atomic::{AtomicBool, Ordering}, }; #[allow(clippy::let_and_return)] @@ -935,7 +936,7 @@ mod _io { let rewind = self.raw_offset() + (self.pos - self.write_pos); if rewind != 0 { self.raw_seek(-rewind, 1, vm)?; - self.raw_pos = -rewind; + self.raw_pos -= rewind; } while self.write_pos < self.write_end { @@ -999,7 +1000,10 @@ mod _io { }; if offset >= -self.pos && offset <= available { self.pos += offset; - return Ok(current - available + offset); + // GH-95782: character devices may report raw position 0 + // even after reading, which would make this negative + let result = current - available + offset; + return Ok(if result < 0 { 0 } else { result }); } } } @@ -1111,9 +1115,51 @@ mod _io { } } - // TODO: something something check if error is BlockingIOError? - let _ = self.flush(vm); + // if BlockingIOError, shift buffer + // and try to buffer the new data; otherwise propagate the error + match self.flush(vm) { + Ok(()) => {} + Err(e) if e.fast_isinstance(vm.ctx.exceptions.blocking_io_error) => { + if self.readable() { + self.reset_read(); + } + // Shift buffer and adjust positions + let shift = self.write_pos; + if shift > 0 { + self.buffer + .copy_within(shift as usize..self.write_end as usize, 0); + self.write_end -= shift; + self.raw_pos -= shift; + self.pos -= shift; + self.write_pos = 0; + } + let avail = self.buffer.len() - self.write_end as usize; + if buf_len <= avail { + // Everything can be buffered + let buf = obj.borrow_buf(); + self.buffer[self.write_end as usize..][..buf_len].copy_from_slice(&buf); + self.write_end += buf_len as Offset; + self.pos += buf_len as Offset; + return Ok(buf_len); + } + // Buffer as much as possible and return BlockingIOError + let buf = obj.borrow_buf(); + self.buffer[self.write_end as usize..][..avail].copy_from_slice(&buf[..avail]); + self.write_end += avail as Offset; + self.pos += avail as Offset; + return Err(vm.invoke_exception( + vm.ctx.exceptions.blocking_io_error.to_owned(), + vec![ + vm.new_pyobj(EAGAIN), + vm.new_pyobj("write could not complete without blocking"), + vm.new_pyobj(avail), + ], + )?); + } + Err(e) => return Err(e), + } + // Only reach here if flush succeeded let offset = self.raw_offset(); if offset != 0 { self.raw_seek(-offset, 1, vm)?; @@ -1556,6 +1602,7 @@ mod _io { const SEEKABLE: bool = false; fn data(&self) -> &PyThreadMutex; + fn closing(&self) -> &AtomicBool; fn lock(&self, vm: &VirtualMachine) -> PyResult> { self.data() @@ -1694,19 +1741,25 @@ mod _io { Ok(self.lock(vm)?.raw.clone()) } + /// Get raw stream without holding the lock (for calling Python code safely) + fn get_raw_unlocked(&self, vm: &VirtualMachine) -> PyResult { + let data = self.lock(vm)?; + Ok(data.check_init(vm)?.to_owned()) + } + #[pygetset] fn closed(&self, vm: &VirtualMachine) -> PyResult { - self.lock(vm)?.check_init(vm)?.get_attr("closed", vm) + self.get_raw_unlocked(vm)?.get_attr("closed", vm) } #[pygetset] fn name(&self, vm: &VirtualMachine) -> PyResult { - self.lock(vm)?.check_init(vm)?.get_attr("name", vm) + self.get_raw_unlocked(vm)?.get_attr("name", vm) } #[pygetset] fn mode(&self, vm: &VirtualMachine) -> PyResult { - self.lock(vm)?.check_init(vm)?.get_attr("mode", vm) + self.get_raw_unlocked(vm)?.get_attr("mode", vm) } #[pymethod] @@ -1750,17 +1803,19 @@ mod _io { #[pymethod] fn close(zelf: PyRef, vm: &VirtualMachine) -> PyResult { - { + // Don't hold the lock while calling Python code to avoid reentrant lock issues + let raw = { let data = zelf.lock(vm)?; let raw = data.check_init(vm)?; if file_closed(raw, vm)? { return Ok(vm.ctx.none()); } - } + raw.to_owned() + }; + // Set closing flag so that concurrent write() calls will fail + zelf.closing().store(true, Ordering::Release); let flush_res = vm.call_method(zelf.as_object(), "flush", ()).map(drop); - let data = zelf.lock(vm)?; - let raw = data.raw.as_ref().unwrap(); - let close_res = vm.call_method(raw, "close", ()); + let close_res = vm.call_method(&raw, "close", ()); exception_chain(flush_res, close_res) } @@ -1774,10 +1829,9 @@ mod _io { Self::WRITABLE } - // TODO: this should be the default for an equivalent of _PyObject_GetState #[pymethod] - fn __reduce__(zelf: PyObjectRef, vm: &VirtualMachine) -> PyResult { - Err(vm.new_type_error(format!("cannot pickle '{}' object", zelf.class().name()))) + fn __getstate__(zelf: PyObjectRef, vm: &VirtualMachine) -> PyResult { + Err(vm.new_type_error(format!("cannot pickle '{}' instances", zelf.class().name()))) } } @@ -1830,6 +1884,10 @@ mod _io { let n = std::cmp::min(have as usize, n); return Ok(data.read_fast(n).unwrap()); } + // Flush write buffer before reading + if data.writable() { + data.flush_rewind(vm)?; + } let mut v = vec![0; n]; data.reset_read(); let r = data @@ -1855,6 +1913,19 @@ mod _io { ensure_unclosed(raw, "readinto of closed file", vm)?; data.readinto_generic(buf.into(), true, vm) } + + #[pymethod] + fn flush(&self, vm: &VirtualMachine) -> PyResult<()> { + // For read-only buffers, flush just calls raw.flush() + // Don't hold the lock while calling Python code to avoid reentrant lock issues + let raw = { + let data = self.reader().lock(vm)?; + data.check_init(vm)?.to_owned() + }; + ensure_unclosed(&raw, "flush of closed file", vm)?; + vm.call_method(&raw, "flush", ())?; + Ok(()) + } } fn exception_chain(e1: PyResult<()>, e2: PyResult) -> PyResult { @@ -1874,6 +1945,7 @@ mod _io { struct BufferedReader { _base: _BufferedIOBase, data: PyThreadMutex, + closing: AtomicBool, } impl BufferedMixin for BufferedReader { @@ -1884,6 +1956,10 @@ mod _io { fn data(&self) -> &PyThreadMutex { &self.data } + + fn closing(&self) -> &AtomicBool { + &self.closing + } } impl BufferedReadable for BufferedReader { @@ -1922,6 +1998,27 @@ mod _io { #[pymethod] fn write(&self, obj: ArgBytesLike, vm: &VirtualMachine) -> PyResult { + // Check if close() is in progress (Issue #31976) + // If closing, wait for close() to complete by spinning until raw is closed. + // Note: This spin-wait has no timeout because close() is expected to always + // complete (flush + fd close). + if self.writer().closing().load(Ordering::Acquire) { + loop { + let raw = { + let data = self.writer().lock(vm)?; + match &data.raw { + Some(raw) => raw.to_owned(), + None => break, // detached + } + }; + if file_closed(&raw, vm)? { + break; + } + // Yield to other threads + std::thread::yield_now(); + } + return Err(vm.new_value_error("write to closed file".to_owned())); + } let mut data = self.writer().lock(vm)?; let raw = data.check_init(vm)?; ensure_unclosed(raw, "write to closed file", vm)?; @@ -1944,6 +2041,7 @@ mod _io { struct BufferedWriter { _base: _BufferedIOBase, data: PyThreadMutex, + closing: AtomicBool, } impl BufferedMixin for BufferedWriter { @@ -1954,6 +2052,10 @@ mod _io { fn data(&self) -> &PyThreadMutex { &self.data } + + fn closing(&self) -> &AtomicBool { + &self.closing + } } impl BufferedWritable for BufferedWriter { @@ -1990,6 +2092,7 @@ mod _io { struct BufferedRandom { _base: _BufferedIOBase, data: PyThreadMutex, + closing: AtomicBool, } impl BufferedMixin for BufferedRandom { @@ -2001,6 +2104,10 @@ mod _io { fn data(&self) -> &PyThreadMutex { &self.data } + + fn closing(&self) -> &AtomicBool { + &self.closing + } } impl BufferedReadable for BufferedRandom { @@ -3347,8 +3454,8 @@ mod _io { } #[pymethod] - fn __reduce__(zelf: PyObjectRef, vm: &VirtualMachine) -> PyResult { - Err(vm.new_type_error(format!("cannot pickle '{}' object", zelf.class().name()))) + fn __getstate__(zelf: PyObjectRef, vm: &VirtualMachine) -> PyResult { + Err(vm.new_type_error(format!("cannot pickle '{}' instances", zelf.class().name()))) } } @@ -4908,7 +5015,7 @@ mod fileio { zelf: &Py, read_byte: OptionalSize, vm: &VirtualMachine, - ) -> PyResult> { + ) -> PyResult>> { if !zelf.mode.load().contains(Mode::READABLE) { return Err(new_unsupported_operation( vm, @@ -4926,6 +5033,10 @@ mod fileio { vm.check_signals()?; continue; } + // Non-blocking mode: return None if EAGAIN + Err(e) if e.raw_os_error() == Some(libc::EAGAIN) => { + return Ok(None); + } Err(e) => return Err(Self::io_error(zelf, e, vm)), } }; @@ -4941,17 +5052,28 @@ mod fileio { vm.check_signals()?; continue; } + // Non-blocking mode: return None if EAGAIN (only if no data read yet) + Err(e) if e.raw_os_error() == Some(libc::EAGAIN) => { + if bytes.is_empty() { + return Ok(None); + } + break; + } Err(e) => return Err(Self::io_error(zelf, e, vm)), } } bytes }; - Ok(bytes) + Ok(Some(bytes)) } #[pymethod] - fn readinto(zelf: &Py, obj: ArgMemoryBuffer, vm: &VirtualMachine) -> PyResult { + fn readinto( + zelf: &Py, + obj: ArgMemoryBuffer, + vm: &VirtualMachine, + ) -> PyResult> { if !zelf.mode.load().contains(Mode::READABLE) { return Err(new_unsupported_operation( vm, @@ -4971,15 +5093,23 @@ mod fileio { vm.check_signals()?; continue; } + // Non-blocking mode: return None if EAGAIN + Err(e) if e.raw_os_error() == Some(libc::EAGAIN) => { + return Ok(None); + } Err(e) => return Err(Self::io_error(zelf, e, vm)), } }; - Ok(ret) + Ok(Some(ret)) } #[pymethod] - fn write(zelf: &Py, obj: ArgBytesLike, vm: &VirtualMachine) -> PyResult { + fn write( + zelf: &Py, + obj: ArgBytesLike, + vm: &VirtualMachine, + ) -> PyResult> { if !zelf.mode.load().contains(Mode::WRITABLE) { return Err(new_unsupported_operation( vm, @@ -4989,12 +5119,15 @@ mod fileio { let mut handle = zelf.get_fd(vm)?; - let len = obj - .with_ref(|b| handle.write(b)) - .map_err(|err| Self::io_error(zelf, err, vm))?; + let len = match obj.with_ref(|b| handle.write(b)) { + Ok(n) => n, + // Non-blocking mode: return None if EAGAIN + Err(e) if e.raw_os_error() == Some(libc::EAGAIN) => return Ok(None), + Err(e) => return Err(Self::io_error(zelf, e, vm)), + }; //return number of bytes written - Ok(len) + Ok(Some(len)) } #[pymethod] @@ -5060,8 +5193,8 @@ mod fileio { } #[pymethod] - fn __reduce__(zelf: PyObjectRef, vm: &VirtualMachine) -> PyResult { - Err(vm.new_type_error(format!("cannot pickle '{}' object", zelf.class().name()))) + fn __getstate__(zelf: PyObjectRef, vm: &VirtualMachine) -> PyResult { + Err(vm.new_type_error(format!("cannot pickle '{}' instances", zelf.class().name()))) } }