Vendor things

This commit is contained in:
John Doty 2024-03-08 11:03:01 -08:00
parent 5deceec006
commit 977e3c17e5
19434 changed files with 10682014 additions and 0 deletions

View file

@ -0,0 +1,255 @@
use std::ffi::c_void;
use std::fmt;
use std::fs::File;
use std::io;
use std::mem::size_of;
use std::os::windows::io::AsRawHandle;
use windows_sys::Win32::Foundation::{
RtlNtStatusToDosError, HANDLE, NTSTATUS, STATUS_NOT_FOUND, STATUS_PENDING, STATUS_SUCCESS,
};
use windows_sys::Win32::System::WindowsProgramming::{
NtDeviceIoControlFile, IO_STATUS_BLOCK, IO_STATUS_BLOCK_0,
};
const IOCTL_AFD_POLL: u32 = 0x00012024;
#[link(name = "ntdll")]
extern "system" {
/// See <https://processhacker.sourceforge.io/doc/ntioapi_8h.html#a0d4d550cad4d62d75b76961e25f6550c>
///
/// This is an undocumented API and as such not part of <https://github.com/microsoft/win32metadata>
/// from which `windows-sys` is generated, and also unlikely to be added, so
/// we manually declare it here
fn NtCancelIoFileEx(
FileHandle: HANDLE,
IoRequestToCancel: *mut IO_STATUS_BLOCK,
IoStatusBlock: *mut IO_STATUS_BLOCK,
) -> NTSTATUS;
}
/// Winsock2 AFD driver instance.
///
/// All operations are unsafe due to IO_STATUS_BLOCK parameter are being used by Afd driver during STATUS_PENDING before I/O Completion Port returns its result.
#[derive(Debug)]
pub struct Afd {
fd: File,
}
#[repr(C)]
#[derive(Debug)]
pub struct AfdPollHandleInfo {
pub handle: HANDLE,
pub events: u32,
pub status: NTSTATUS,
}
unsafe impl Send for AfdPollHandleInfo {}
#[repr(C)]
pub struct AfdPollInfo {
pub timeout: i64,
// Can have only value 1.
pub number_of_handles: u32,
pub exclusive: u32,
pub handles: [AfdPollHandleInfo; 1],
}
impl fmt::Debug for AfdPollInfo {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("AfdPollInfo").finish()
}
}
impl Afd {
/// Poll `Afd` instance with `AfdPollInfo`.
///
/// # Unsafety
///
/// This function is unsafe due to memory of `IO_STATUS_BLOCK` still being used by `Afd` instance while `Ok(false)` (`STATUS_PENDING`).
/// `iosb` needs to be untouched after the call while operation is in effective at ALL TIME except for `cancel` method.
/// So be careful not to `poll` twice while polling.
/// User should deallocate there overlapped value when error to prevent memory leak.
pub unsafe fn poll(
&self,
info: &mut AfdPollInfo,
iosb: *mut IO_STATUS_BLOCK,
overlapped: *mut c_void,
) -> io::Result<bool> {
let info_ptr = info as *mut _ as *mut c_void;
(*iosb).Anonymous.Status = STATUS_PENDING;
let status = NtDeviceIoControlFile(
self.fd.as_raw_handle() as HANDLE,
0,
None,
overlapped,
iosb,
IOCTL_AFD_POLL,
info_ptr,
size_of::<AfdPollInfo>() as u32,
info_ptr,
size_of::<AfdPollInfo>() as u32,
);
match status {
STATUS_SUCCESS => Ok(true),
STATUS_PENDING => Ok(false),
_ => Err(io::Error::from_raw_os_error(
RtlNtStatusToDosError(status) as i32
)),
}
}
/// Cancel previous polled request of `Afd`.
///
/// iosb needs to be used by `poll` first for valid `cancel`.
///
/// # Unsafety
///
/// This function is unsafe due to memory of `IO_STATUS_BLOCK` still being used by `Afd` instance while `Ok(false)` (`STATUS_PENDING`).
/// Use it only with request is still being polled so that you have valid `IO_STATUS_BLOCK` to use.
/// User should NOT deallocate there overlapped value after the `cancel` to prevent double free.
pub unsafe fn cancel(&self, iosb: *mut IO_STATUS_BLOCK) -> io::Result<()> {
if (*iosb).Anonymous.Status != STATUS_PENDING {
return Ok(());
}
let mut cancel_iosb = IO_STATUS_BLOCK {
Anonymous: IO_STATUS_BLOCK_0 { Status: 0 },
Information: 0,
};
let status = NtCancelIoFileEx(self.fd.as_raw_handle() as HANDLE, iosb, &mut cancel_iosb);
if status == STATUS_SUCCESS || status == STATUS_NOT_FOUND {
return Ok(());
}
Err(io::Error::from_raw_os_error(
RtlNtStatusToDosError(status) as i32
))
}
}
cfg_io_source! {
use std::mem::zeroed;
use std::os::windows::io::{FromRawHandle, RawHandle};
use std::ptr::null_mut;
use std::sync::atomic::{AtomicUsize, Ordering};
use super::iocp::CompletionPort;
use windows_sys::Win32::{
Foundation::{UNICODE_STRING, INVALID_HANDLE_VALUE},
System::WindowsProgramming::{
OBJECT_ATTRIBUTES, FILE_SKIP_SET_EVENT_ON_HANDLE,
},
Storage::FileSystem::{FILE_OPEN, NtCreateFile, SetFileCompletionNotificationModes, SYNCHRONIZE, FILE_SHARE_READ, FILE_SHARE_WRITE},
};
const AFD_HELPER_ATTRIBUTES: OBJECT_ATTRIBUTES = OBJECT_ATTRIBUTES {
Length: size_of::<OBJECT_ATTRIBUTES>() as u32,
RootDirectory: 0,
ObjectName: &AFD_OBJ_NAME as *const _ as *mut _,
Attributes: 0,
SecurityDescriptor: null_mut(),
SecurityQualityOfService: null_mut(),
};
const AFD_OBJ_NAME: UNICODE_STRING = UNICODE_STRING {
Length: (AFD_HELPER_NAME.len() * size_of::<u16>()) as u16,
MaximumLength: (AFD_HELPER_NAME.len() * size_of::<u16>()) as u16,
Buffer: AFD_HELPER_NAME.as_ptr() as *mut _,
};
const AFD_HELPER_NAME: &[u16] = &[
'\\' as _,
'D' as _,
'e' as _,
'v' as _,
'i' as _,
'c' as _,
'e' as _,
'\\' as _,
'A' as _,
'f' as _,
'd' as _,
'\\' as _,
'M' as _,
'i' as _,
'o' as _
];
static NEXT_TOKEN: AtomicUsize = AtomicUsize::new(0);
impl AfdPollInfo {
pub fn zeroed() -> AfdPollInfo {
unsafe { zeroed() }
}
}
impl Afd {
/// Create new Afd instance.
pub(crate) fn new(cp: &CompletionPort) -> io::Result<Afd> {
let mut afd_helper_handle: HANDLE = INVALID_HANDLE_VALUE;
let mut iosb = IO_STATUS_BLOCK {
Anonymous: IO_STATUS_BLOCK_0 { Status: 0 },
Information: 0,
};
unsafe {
let status = NtCreateFile(
&mut afd_helper_handle as *mut _,
SYNCHRONIZE,
&AFD_HELPER_ATTRIBUTES as *const _ as *mut _,
&mut iosb,
null_mut(),
0,
FILE_SHARE_READ | FILE_SHARE_WRITE,
FILE_OPEN,
0,
null_mut(),
0,
);
if status != STATUS_SUCCESS {
let raw_err = io::Error::from_raw_os_error(
RtlNtStatusToDosError(status) as i32
);
let msg = format!("Failed to open \\Device\\Afd\\Mio: {}", raw_err);
return Err(io::Error::new(raw_err.kind(), msg));
}
let fd = File::from_raw_handle(afd_helper_handle as RawHandle);
// Increment by 2 to reserve space for other types of handles.
// Non-AFD types (currently only NamedPipe), use odd numbered
// tokens. This allows the selector to differentiate between them
// and dispatch events accordingly.
let token = NEXT_TOKEN.fetch_add(2, Ordering::Relaxed) + 2;
let afd = Afd { fd };
cp.add_handle(token, &afd.fd)?;
match SetFileCompletionNotificationModes(
afd_helper_handle,
FILE_SKIP_SET_EVENT_ON_HANDLE as u8 // This is just 2, so fits in u8
) {
0 => Err(io::Error::last_os_error()),
_ => Ok(afd),
}
}
}
}
}
pub const POLL_RECEIVE: u32 = 0b0_0000_0001;
pub const POLL_RECEIVE_EXPEDITED: u32 = 0b0_0000_0010;
pub const POLL_SEND: u32 = 0b0_0000_0100;
pub const POLL_DISCONNECT: u32 = 0b0_0000_1000;
pub const POLL_ABORT: u32 = 0b0_0001_0000;
pub const POLL_LOCAL_CLOSE: u32 = 0b0_0010_0000;
// Not used as it indicated in each event where a connection is connected, not
// just the first time a connection is established.
// Also see https://github.com/piscisaureus/wepoll/commit/8b7b340610f88af3d83f40fb728e7b850b090ece.
pub const POLL_CONNECT: u32 = 0b0_0100_0000;
pub const POLL_ACCEPT: u32 = 0b0_1000_0000;
pub const POLL_CONNECT_FAIL: u32 = 0b1_0000_0000;
pub const KNOWN_EVENTS: u32 = POLL_RECEIVE
| POLL_RECEIVE_EXPEDITED
| POLL_SEND
| POLL_DISCONNECT
| POLL_ABORT
| POLL_LOCAL_CLOSE
| POLL_ACCEPT
| POLL_CONNECT_FAIL;

View file

@ -0,0 +1,169 @@
use std::fmt;
use super::afd;
use super::iocp::CompletionStatus;
use crate::Token;
#[derive(Clone)]
pub struct Event {
pub flags: u32,
pub data: u64,
}
pub fn token(event: &Event) -> Token {
Token(event.data as usize)
}
impl Event {
pub(super) fn new(token: Token) -> Event {
Event {
flags: 0,
data: usize::from(token) as u64,
}
}
pub(super) fn set_readable(&mut self) {
self.flags |= afd::POLL_RECEIVE
}
#[cfg(feature = "os-ext")]
pub(super) fn set_writable(&mut self) {
self.flags |= afd::POLL_SEND;
}
pub(super) fn from_completion_status(status: &CompletionStatus) -> Event {
Event {
flags: status.bytes_transferred(),
data: status.token() as u64,
}
}
pub(super) fn to_completion_status(&self) -> CompletionStatus {
CompletionStatus::new(self.flags, self.data as usize, std::ptr::null_mut())
}
#[cfg(feature = "os-ext")]
pub(super) fn to_completion_status_with_overlapped(
&self,
overlapped: *mut super::Overlapped,
) -> CompletionStatus {
CompletionStatus::new(self.flags, self.data as usize, overlapped)
}
}
pub(crate) const READABLE_FLAGS: u32 = afd::POLL_RECEIVE
| afd::POLL_DISCONNECT
| afd::POLL_ACCEPT
| afd::POLL_ABORT
| afd::POLL_CONNECT_FAIL;
pub(crate) const WRITABLE_FLAGS: u32 = afd::POLL_SEND | afd::POLL_ABORT | afd::POLL_CONNECT_FAIL;
pub(crate) const ERROR_FLAGS: u32 = afd::POLL_CONNECT_FAIL;
pub(crate) const READ_CLOSED_FLAGS: u32 =
afd::POLL_DISCONNECT | afd::POLL_ABORT | afd::POLL_CONNECT_FAIL;
pub(crate) const WRITE_CLOSED_FLAGS: u32 = afd::POLL_ABORT | afd::POLL_CONNECT_FAIL;
pub fn is_readable(event: &Event) -> bool {
event.flags & READABLE_FLAGS != 0
}
pub fn is_writable(event: &Event) -> bool {
event.flags & WRITABLE_FLAGS != 0
}
pub fn is_error(event: &Event) -> bool {
event.flags & ERROR_FLAGS != 0
}
pub fn is_read_closed(event: &Event) -> bool {
event.flags & READ_CLOSED_FLAGS != 0
}
pub fn is_write_closed(event: &Event) -> bool {
event.flags & WRITE_CLOSED_FLAGS != 0
}
pub fn is_priority(event: &Event) -> bool {
event.flags & afd::POLL_RECEIVE_EXPEDITED != 0
}
pub fn is_aio(_: &Event) -> bool {
// Not supported.
false
}
pub fn is_lio(_: &Event) -> bool {
// Not supported.
false
}
pub fn debug_details(f: &mut fmt::Formatter<'_>, event: &Event) -> fmt::Result {
#[allow(clippy::trivially_copy_pass_by_ref)]
fn check_flags(got: &u32, want: &u32) -> bool {
(got & want) != 0
}
debug_detail!(
FlagsDetails(u32),
check_flags,
afd::POLL_RECEIVE,
afd::POLL_RECEIVE_EXPEDITED,
afd::POLL_SEND,
afd::POLL_DISCONNECT,
afd::POLL_ABORT,
afd::POLL_LOCAL_CLOSE,
afd::POLL_CONNECT,
afd::POLL_ACCEPT,
afd::POLL_CONNECT_FAIL,
);
f.debug_struct("event")
.field("flags", &FlagsDetails(event.flags))
.field("data", &event.data)
.finish()
}
pub struct Events {
/// Raw I/O event completions are filled in here by the call to `get_many`
/// on the completion port above. These are then processed to run callbacks
/// which figure out what to do after the event is done.
pub statuses: Box<[CompletionStatus]>,
/// Literal events returned by `get` to the upwards `EventLoop`. This file
/// doesn't really modify this (except for the waker), instead almost all
/// events are filled in by the `ReadinessQueue` from the `poll` module.
pub events: Vec<Event>,
}
impl Events {
pub fn with_capacity(cap: usize) -> Events {
// Note that it's possible for the output `events` to grow beyond the
// capacity as it can also include deferred events, but that's certainly
// not the end of the world!
Events {
statuses: vec![CompletionStatus::zero(); cap].into_boxed_slice(),
events: Vec::with_capacity(cap),
}
}
pub fn is_empty(&self) -> bool {
self.events.is_empty()
}
pub fn capacity(&self) -> usize {
self.events.capacity()
}
pub fn len(&self) -> usize {
self.events.len()
}
pub fn get(&self, idx: usize) -> Option<&Event> {
self.events.get(idx)
}
pub fn clear(&mut self) {
self.events.clear();
for status in self.statuses.iter_mut() {
*status = CompletionStatus::zero();
}
}
}

View file

@ -0,0 +1,30 @@
use std::os::windows::io::RawHandle;
use windows_sys::Win32::Foundation::{CloseHandle, HANDLE};
/// Wrapper around a Windows HANDLE so that we close it upon drop in all scenarios
#[derive(Debug)]
pub struct Handle(HANDLE);
impl Handle {
#[inline]
pub fn new(handle: HANDLE) -> Self {
Self(handle)
}
pub fn raw(&self) -> HANDLE {
self.0
}
pub fn into_raw(self) -> RawHandle {
let ret = self.0;
// This is super important so that drop is not called!
std::mem::forget(self);
ret as RawHandle
}
}
impl Drop for Handle {
fn drop(&mut self) {
unsafe { CloseHandle(self.0) };
}
}

View file

@ -0,0 +1,40 @@
use std::fmt;
use std::ops::{Deref, DerefMut};
use windows_sys::Win32::System::WindowsProgramming::IO_STATUS_BLOCK;
pub struct IoStatusBlock(IO_STATUS_BLOCK);
cfg_io_source! {
use windows_sys::Win32::System::WindowsProgramming::{IO_STATUS_BLOCK_0};
impl IoStatusBlock {
pub fn zeroed() -> Self {
Self(IO_STATUS_BLOCK {
Anonymous: IO_STATUS_BLOCK_0 { Status: 0 },
Information: 0,
})
}
}
}
unsafe impl Send for IoStatusBlock {}
impl Deref for IoStatusBlock {
type Target = IO_STATUS_BLOCK;
fn deref(&self) -> &Self::Target {
&self.0
}
}
impl DerefMut for IoStatusBlock {
fn deref_mut(&mut self) -> &mut Self::Target {
&mut self.0
}
}
impl fmt::Debug for IoStatusBlock {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("IoStatusBlock").finish()
}
}

View file

@ -0,0 +1,273 @@
//! Bindings to IOCP, I/O Completion Ports
use super::{Handle, Overlapped};
use std::cmp;
use std::fmt;
use std::io;
use std::mem;
use std::os::windows::io::*;
use std::time::Duration;
use windows_sys::Win32::Foundation::{HANDLE, INVALID_HANDLE_VALUE};
use windows_sys::Win32::System::IO::{
CreateIoCompletionPort, GetQueuedCompletionStatusEx, PostQueuedCompletionStatus, OVERLAPPED,
OVERLAPPED_ENTRY,
};
/// A handle to an Windows I/O Completion Port.
#[derive(Debug)]
pub(crate) struct CompletionPort {
handle: Handle,
}
/// A status message received from an I/O completion port.
///
/// These statuses can be created via the `new` or `empty` constructors and then
/// provided to a completion port, or they are read out of a completion port.
/// The fields of each status are read through its accessor methods.
#[derive(Clone, Copy)]
#[repr(transparent)]
pub struct CompletionStatus(OVERLAPPED_ENTRY);
impl fmt::Debug for CompletionStatus {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "CompletionStatus(OVERLAPPED_ENTRY)")
}
}
unsafe impl Send for CompletionStatus {}
unsafe impl Sync for CompletionStatus {}
impl CompletionPort {
/// Creates a new I/O completion port with the specified concurrency value.
///
/// The number of threads given corresponds to the level of concurrency
/// allowed for threads associated with this port. Consult the Windows
/// documentation for more information about this value.
pub fn new(threads: u32) -> io::Result<CompletionPort> {
let ret = unsafe { CreateIoCompletionPort(INVALID_HANDLE_VALUE, 0, 0, threads) };
if ret == 0 {
Err(io::Error::last_os_error())
} else {
Ok(CompletionPort {
handle: Handle::new(ret),
})
}
}
/// Associates a new `HANDLE` to this I/O completion port.
///
/// This function will associate the given handle to this port with the
/// given `token` to be returned in status messages whenever it receives a
/// notification.
///
/// Any object which is convertible to a `HANDLE` via the `AsRawHandle`
/// trait can be provided to this function, such as `std::fs::File` and
/// friends.
#[cfg(any(feature = "net", feature = "os-ext"))]
pub fn add_handle<T: AsRawHandle + ?Sized>(&self, token: usize, t: &T) -> io::Result<()> {
let ret = unsafe {
CreateIoCompletionPort(t.as_raw_handle() as HANDLE, self.handle.raw(), token, 0)
};
if ret == 0 {
Err(io::Error::last_os_error())
} else {
Ok(())
}
}
/// Dequeues a number of completion statuses from this I/O completion port.
///
/// This function is the same as `get` except that it may return more than
/// one status. A buffer of "zero" statuses is provided (the contents are
/// not read) and then on success this function will return a sub-slice of
/// statuses which represent those which were dequeued from this port. This
/// function does not wait to fill up the entire list of statuses provided.
///
/// Like with `get`, a timeout may be specified for this operation.
pub fn get_many<'a>(
&self,
list: &'a mut [CompletionStatus],
timeout: Option<Duration>,
) -> io::Result<&'a mut [CompletionStatus]> {
debug_assert_eq!(
mem::size_of::<CompletionStatus>(),
mem::size_of::<OVERLAPPED_ENTRY>()
);
let mut removed = 0;
let timeout = duration_millis(timeout);
let len = cmp::min(list.len(), <u32>::max_value() as usize) as u32;
let ret = unsafe {
GetQueuedCompletionStatusEx(
self.handle.raw(),
list.as_ptr() as *mut _,
len,
&mut removed,
timeout,
0,
)
};
if ret == 0 {
Err(io::Error::last_os_error())
} else {
Ok(&mut list[..removed as usize])
}
}
/// Posts a new completion status onto this I/O completion port.
///
/// This function will post the given status, with custom parameters, to the
/// port. Threads blocked in `get` or `get_many` will eventually receive
/// this status.
pub fn post(&self, status: CompletionStatus) -> io::Result<()> {
let ret = unsafe {
PostQueuedCompletionStatus(
self.handle.raw(),
status.0.dwNumberOfBytesTransferred,
status.0.lpCompletionKey,
status.0.lpOverlapped,
)
};
if ret == 0 {
Err(io::Error::last_os_error())
} else {
Ok(())
}
}
}
impl AsRawHandle for CompletionPort {
fn as_raw_handle(&self) -> RawHandle {
self.handle.raw() as RawHandle
}
}
impl FromRawHandle for CompletionPort {
unsafe fn from_raw_handle(handle: RawHandle) -> CompletionPort {
CompletionPort {
handle: Handle::new(handle as HANDLE),
}
}
}
impl IntoRawHandle for CompletionPort {
fn into_raw_handle(self) -> RawHandle {
self.handle.into_raw()
}
}
impl CompletionStatus {
/// Creates a new completion status with the provided parameters.
///
/// This function is useful when creating a status to send to a port with
/// the `post` method. The parameters are opaquely passed through and not
/// interpreted by the system at all.
pub(crate) fn new(bytes: u32, token: usize, overlapped: *mut Overlapped) -> Self {
CompletionStatus(OVERLAPPED_ENTRY {
dwNumberOfBytesTransferred: bytes,
lpCompletionKey: token,
lpOverlapped: overlapped as *mut _,
Internal: 0,
})
}
/// Creates a new borrowed completion status from the borrowed
/// `OVERLAPPED_ENTRY` argument provided.
///
/// This method will wrap the `OVERLAPPED_ENTRY` in a `CompletionStatus`,
/// returning the wrapped structure.
#[cfg(feature = "os-ext")]
pub fn from_entry(entry: &OVERLAPPED_ENTRY) -> &Self {
// Safety: CompletionStatus is repr(transparent) w/ OVERLAPPED_ENTRY, so
// a reference to one is guaranteed to be layout compatible with the
// reference to another.
unsafe { &*(entry as *const _ as *const _) }
}
/// Creates a new "zero" completion status.
///
/// This function is useful when creating a stack buffer or vector of
/// completion statuses to be passed to the `get_many` function.
pub fn zero() -> Self {
Self::new(0, 0, std::ptr::null_mut())
}
/// Returns the number of bytes that were transferred for the I/O operation
/// associated with this completion status.
pub fn bytes_transferred(&self) -> u32 {
self.0.dwNumberOfBytesTransferred
}
/// Returns the completion key value associated with the file handle whose
/// I/O operation has completed.
///
/// A completion key is a per-handle key that is specified when it is added
/// to an I/O completion port via `add_handle` or `add_socket`.
pub fn token(&self) -> usize {
self.0.lpCompletionKey as usize
}
/// Returns a pointer to the `Overlapped` structure that was specified when
/// the I/O operation was started.
pub fn overlapped(&self) -> *mut OVERLAPPED {
self.0.lpOverlapped
}
/// Returns a pointer to the internal `OVERLAPPED_ENTRY` object.
pub fn entry(&self) -> &OVERLAPPED_ENTRY {
&self.0
}
}
#[inline]
fn duration_millis(dur: Option<Duration>) -> u32 {
if let Some(dur) = dur {
// `Duration::as_millis` truncates, so round up. This avoids
// turning sub-millisecond timeouts into a zero timeout, unless
// the caller explicitly requests that by specifying a zero
// timeout.
let dur_ms = dur
.checked_add(Duration::from_nanos(999_999))
.unwrap_or(dur)
.as_millis();
cmp::min(dur_ms, u32::MAX as u128) as u32
} else {
u32::MAX
}
}
#[cfg(test)]
mod tests {
use super::{CompletionPort, CompletionStatus};
#[test]
fn is_send_sync() {
fn is_send_sync<T: Send + Sync>() {}
is_send_sync::<CompletionPort>();
}
#[test]
fn get_many() {
let c = CompletionPort::new(1).unwrap();
c.post(CompletionStatus::new(1, 2, 3 as *mut _)).unwrap();
c.post(CompletionStatus::new(4, 5, 6 as *mut _)).unwrap();
let mut s = vec![CompletionStatus::zero(); 4];
{
let s = c.get_many(&mut s, None).unwrap();
assert_eq!(s.len(), 2);
assert_eq!(s[0].bytes_transferred(), 1);
assert_eq!(s[0].token(), 2);
assert_eq!(s[0].overlapped(), 3 as *mut _);
assert_eq!(s[1].bytes_transferred(), 4);
assert_eq!(s[1].token(), 5);
assert_eq!(s[1].overlapped(), 6 as *mut _);
}
assert_eq!(s[2].bytes_transferred(), 0);
assert_eq!(s[2].token(), 0);
assert_eq!(s[2].overlapped(), 0 as *mut _);
}
}

View file

@ -0,0 +1,154 @@
mod afd;
pub mod event;
pub use event::{Event, Events};
mod handle;
use handle::Handle;
mod io_status_block;
mod iocp;
mod overlapped;
use overlapped::Overlapped;
mod selector;
pub use selector::Selector;
// Macros must be defined before the modules that use them
cfg_net! {
/// Helper macro to execute a system call that returns an `io::Result`.
//
// Macro must be defined before any modules that uses them.
macro_rules! syscall {
($fn: ident ( $($arg: expr),* $(,)* ), $err_test: path, $err_value: expr) => {{
let res = unsafe { $fn($($arg, )*) };
if $err_test(&res, &$err_value) {
Err(io::Error::last_os_error())
} else {
Ok(res)
}
}};
}
mod net;
pub(crate) mod tcp;
pub(crate) mod udp;
pub use selector::{SelectorInner, SockState};
}
cfg_os_ext! {
pub(crate) mod named_pipe;
}
mod waker;
pub(crate) use waker::Waker;
cfg_io_source! {
use std::io;
use std::os::windows::io::RawSocket;
use std::pin::Pin;
use std::sync::{Arc, Mutex};
use crate::{Interest, Registry, Token};
struct InternalState {
selector: Arc<SelectorInner>,
token: Token,
interests: Interest,
sock_state: Pin<Arc<Mutex<SockState>>>,
}
impl Drop for InternalState {
fn drop(&mut self) {
let mut sock_state = self.sock_state.lock().unwrap();
sock_state.mark_delete();
}
}
pub struct IoSourceState {
// This is `None` if the socket has not yet been registered.
//
// We box the internal state to not increase the size on the stack as the
// type might move around a lot.
inner: Option<Box<InternalState>>,
}
impl IoSourceState {
pub fn new() -> IoSourceState {
IoSourceState { inner: None }
}
pub fn do_io<T, F, R>(&self, f: F, io: &T) -> io::Result<R>
where
F: FnOnce(&T) -> io::Result<R>,
{
let result = f(io);
if let Err(ref e) = result {
if e.kind() == io::ErrorKind::WouldBlock {
self.inner.as_ref().map_or(Ok(()), |state| {
state
.selector
.reregister(state.sock_state.clone(), state.token, state.interests)
})?;
}
}
result
}
pub fn register(
&mut self,
registry: &Registry,
token: Token,
interests: Interest,
socket: RawSocket,
) -> io::Result<()> {
if self.inner.is_some() {
Err(io::ErrorKind::AlreadyExists.into())
} else {
registry
.selector()
.register(socket, token, interests)
.map(|state| {
self.inner = Some(Box::new(state));
})
}
}
pub fn reregister(
&mut self,
registry: &Registry,
token: Token,
interests: Interest,
) -> io::Result<()> {
match self.inner.as_mut() {
Some(state) => {
registry
.selector()
.reregister(state.sock_state.clone(), token, interests)
.map(|()| {
state.token = token;
state.interests = interests;
})
}
None => Err(io::ErrorKind::NotFound.into()),
}
}
pub fn deregister(&mut self) -> io::Result<()> {
match self.inner.as_mut() {
Some(state) => {
{
let mut sock_state = state.sock_state.lock().unwrap();
sock_state.mark_delete();
}
self.inner = None;
Ok(())
}
None => Err(io::ErrorKind::NotFound.into()),
}
}
}
}

File diff suppressed because it is too large Load diff

View file

@ -0,0 +1,111 @@
use std::io;
use std::mem;
use std::net::SocketAddr;
use std::sync::Once;
use windows_sys::Win32::Networking::WinSock::{
closesocket, ioctlsocket, socket, AF_INET, AF_INET6, FIONBIO, IN6_ADDR, IN6_ADDR_0,
INVALID_SOCKET, IN_ADDR, IN_ADDR_0, SOCKADDR, SOCKADDR_IN, SOCKADDR_IN6, SOCKADDR_IN6_0,
SOCKET,
};
/// Initialise the network stack for Windows.
fn init() {
static INIT: Once = Once::new();
INIT.call_once(|| {
// Let standard library call `WSAStartup` for us, we can't do it
// ourselves because otherwise using any type in `std::net` would panic
// when it tries to call `WSAStartup` a second time.
drop(std::net::UdpSocket::bind("127.0.0.1:0"));
});
}
/// Create a new non-blocking socket.
pub(crate) fn new_ip_socket(addr: SocketAddr, socket_type: i32) -> io::Result<SOCKET> {
let domain = match addr {
SocketAddr::V4(..) => AF_INET,
SocketAddr::V6(..) => AF_INET6,
};
new_socket(domain.into(), socket_type)
}
pub(crate) fn new_socket(domain: u32, socket_type: i32) -> io::Result<SOCKET> {
init();
let socket = syscall!(
socket(domain as i32, socket_type, 0),
PartialEq::eq,
INVALID_SOCKET
)?;
if let Err(err) = syscall!(ioctlsocket(socket, FIONBIO, &mut 1), PartialEq::ne, 0) {
let _ = unsafe { closesocket(socket) };
return Err(err);
}
Ok(socket as SOCKET)
}
/// A type with the same memory layout as `SOCKADDR`. Used in converting Rust level
/// SocketAddr* types into their system representation. The benefit of this specific
/// type over using `SOCKADDR_STORAGE` is that this type is exactly as large as it
/// needs to be and not a lot larger. And it can be initialized cleaner from Rust.
#[repr(C)]
pub(crate) union SocketAddrCRepr {
v4: SOCKADDR_IN,
v6: SOCKADDR_IN6,
}
impl SocketAddrCRepr {
pub(crate) fn as_ptr(&self) -> *const SOCKADDR {
self as *const _ as *const SOCKADDR
}
}
pub(crate) fn socket_addr(addr: &SocketAddr) -> (SocketAddrCRepr, i32) {
match addr {
SocketAddr::V4(ref addr) => {
// `s_addr` is stored as BE on all machine and the array is in BE order.
// So the native endian conversion method is used so that it's never swapped.
let sin_addr = unsafe {
let mut s_un = mem::zeroed::<IN_ADDR_0>();
s_un.S_addr = u32::from_ne_bytes(addr.ip().octets());
IN_ADDR { S_un: s_un }
};
let sockaddr_in = SOCKADDR_IN {
sin_family: AF_INET as u16, // 1
sin_port: addr.port().to_be(),
sin_addr,
sin_zero: [0; 8],
};
let sockaddr = SocketAddrCRepr { v4: sockaddr_in };
(sockaddr, mem::size_of::<SOCKADDR_IN>() as i32)
}
SocketAddr::V6(ref addr) => {
let sin6_addr = unsafe {
let mut u = mem::zeroed::<IN6_ADDR_0>();
u.Byte = addr.ip().octets();
IN6_ADDR { u }
};
let u = unsafe {
let mut u = mem::zeroed::<SOCKADDR_IN6_0>();
u.sin6_scope_id = addr.scope_id();
u
};
let sockaddr_in6 = SOCKADDR_IN6 {
sin6_family: AF_INET6 as u16, // 23
sin6_port: addr.port().to_be(),
sin6_addr,
sin6_flowinfo: addr.flowinfo(),
Anonymous: u,
};
let sockaddr = SocketAddrCRepr { v6: sockaddr_in6 };
(sockaddr, mem::size_of::<SOCKADDR_IN6>() as i32)
}
}
}

View file

@ -0,0 +1,35 @@
use crate::sys::windows::Event;
use std::cell::UnsafeCell;
use std::fmt;
use windows_sys::Win32::System::IO::{OVERLAPPED, OVERLAPPED_ENTRY};
#[repr(C)]
pub(crate) struct Overlapped {
inner: UnsafeCell<OVERLAPPED>,
pub(crate) callback: fn(&OVERLAPPED_ENTRY, Option<&mut Vec<Event>>),
}
#[cfg(feature = "os-ext")]
impl Overlapped {
pub(crate) fn new(cb: fn(&OVERLAPPED_ENTRY, Option<&mut Vec<Event>>)) -> Overlapped {
Overlapped {
inner: UnsafeCell::new(unsafe { std::mem::zeroed() }),
callback: cb,
}
}
pub(crate) fn as_ptr(&self) -> *const OVERLAPPED {
self.inner.get()
}
}
impl fmt::Debug for Overlapped {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("Overlapped").finish()
}
}
unsafe impl Send for Overlapped {}
unsafe impl Sync for Overlapped {}

View file

@ -0,0 +1,741 @@
use super::afd::{self, Afd, AfdPollInfo};
use super::io_status_block::IoStatusBlock;
use super::Event;
use crate::sys::Events;
cfg_net! {
use crate::sys::event::{
ERROR_FLAGS, READABLE_FLAGS, READ_CLOSED_FLAGS, WRITABLE_FLAGS, WRITE_CLOSED_FLAGS,
};
use crate::Interest;
}
use super::iocp::{CompletionPort, CompletionStatus};
use std::collections::VecDeque;
use std::ffi::c_void;
use std::io;
use std::marker::PhantomPinned;
use std::os::windows::io::RawSocket;
use std::pin::Pin;
#[cfg(debug_assertions)]
use std::sync::atomic::AtomicUsize;
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::{Arc, Mutex};
use std::time::Duration;
use windows_sys::Win32::Foundation::{
ERROR_INVALID_HANDLE, ERROR_IO_PENDING, HANDLE, STATUS_CANCELLED, WAIT_TIMEOUT,
};
use windows_sys::Win32::System::IO::OVERLAPPED;
#[derive(Debug)]
struct AfdGroup {
#[cfg_attr(not(feature = "net"), allow(dead_code))]
cp: Arc<CompletionPort>,
afd_group: Mutex<Vec<Arc<Afd>>>,
}
impl AfdGroup {
pub fn new(cp: Arc<CompletionPort>) -> AfdGroup {
AfdGroup {
afd_group: Mutex::new(Vec::new()),
cp,
}
}
pub fn release_unused_afd(&self) {
let mut afd_group = self.afd_group.lock().unwrap();
afd_group.retain(|g| Arc::strong_count(g) > 1);
}
}
cfg_io_source! {
const POLL_GROUP__MAX_GROUP_SIZE: usize = 32;
impl AfdGroup {
pub fn acquire(&self) -> io::Result<Arc<Afd>> {
let mut afd_group = self.afd_group.lock().unwrap();
if afd_group.len() == 0 {
self._alloc_afd_group(&mut afd_group)?;
} else {
// + 1 reference in Vec
if Arc::strong_count(afd_group.last().unwrap()) > POLL_GROUP__MAX_GROUP_SIZE {
self._alloc_afd_group(&mut afd_group)?;
}
}
match afd_group.last() {
Some(arc) => Ok(arc.clone()),
None => unreachable!(
"Cannot acquire afd, {:#?}, afd_group: {:#?}",
self, afd_group
),
}
}
fn _alloc_afd_group(&self, afd_group: &mut Vec<Arc<Afd>>) -> io::Result<()> {
let afd = Afd::new(&self.cp)?;
let arc = Arc::new(afd);
afd_group.push(arc);
Ok(())
}
}
}
#[derive(Debug)]
enum SockPollStatus {
Idle,
Pending,
Cancelled,
}
#[derive(Debug)]
pub struct SockState {
iosb: IoStatusBlock,
poll_info: AfdPollInfo,
afd: Arc<Afd>,
base_socket: RawSocket,
user_evts: u32,
pending_evts: u32,
user_data: u64,
poll_status: SockPollStatus,
delete_pending: bool,
// last raw os error
error: Option<i32>,
_pinned: PhantomPinned,
}
impl SockState {
fn update(&mut self, self_arc: &Pin<Arc<Mutex<SockState>>>) -> io::Result<()> {
assert!(!self.delete_pending);
// make sure to reset previous error before a new update
self.error = None;
if let SockPollStatus::Pending = self.poll_status {
if (self.user_evts & afd::KNOWN_EVENTS & !self.pending_evts) == 0 {
/* All the events the user is interested in are already being monitored by
* the pending poll operation. It might spuriously complete because of an
* event that we're no longer interested in; when that happens we'll submit
* a new poll operation with the updated event mask. */
} else {
/* A poll operation is already pending, but it's not monitoring for all the
* events that the user is interested in. Therefore, cancel the pending
* poll operation; when we receive it's completion package, a new poll
* operation will be submitted with the correct event mask. */
if let Err(e) = self.cancel() {
self.error = e.raw_os_error();
return Err(e);
}
return Ok(());
}
} else if let SockPollStatus::Cancelled = self.poll_status {
/* The poll operation has already been cancelled, we're still waiting for
* it to return. For now, there's nothing that needs to be done. */
} else if let SockPollStatus::Idle = self.poll_status {
/* No poll operation is pending; start one. */
self.poll_info.exclusive = 0;
self.poll_info.number_of_handles = 1;
self.poll_info.timeout = i64::MAX;
self.poll_info.handles[0].handle = self.base_socket as HANDLE;
self.poll_info.handles[0].status = 0;
self.poll_info.handles[0].events = self.user_evts | afd::POLL_LOCAL_CLOSE;
// Increase the ref count as the memory will be used by the kernel.
let overlapped_ptr = into_overlapped(self_arc.clone());
let result = unsafe {
self.afd
.poll(&mut self.poll_info, &mut *self.iosb, overlapped_ptr)
};
if let Err(e) = result {
let code = e.raw_os_error().unwrap();
if code == ERROR_IO_PENDING as i32 {
/* Overlapped poll operation in progress; this is expected. */
} else {
// Since the operation failed it means the kernel won't be
// using the memory any more.
drop(from_overlapped(overlapped_ptr as *mut _));
if code == ERROR_INVALID_HANDLE as i32 {
/* Socket closed; it'll be dropped. */
self.mark_delete();
return Ok(());
} else {
self.error = e.raw_os_error();
return Err(e);
}
}
}
self.poll_status = SockPollStatus::Pending;
self.pending_evts = self.user_evts;
} else {
unreachable!("Invalid poll status during update, {:#?}", self)
}
Ok(())
}
fn cancel(&mut self) -> io::Result<()> {
match self.poll_status {
SockPollStatus::Pending => {}
_ => unreachable!("Invalid poll status during cancel, {:#?}", self),
};
unsafe {
self.afd.cancel(&mut *self.iosb)?;
}
self.poll_status = SockPollStatus::Cancelled;
self.pending_evts = 0;
Ok(())
}
// This is the function called from the overlapped using as Arc<Mutex<SockState>>. Watch out for reference counting.
fn feed_event(&mut self) -> Option<Event> {
self.poll_status = SockPollStatus::Idle;
self.pending_evts = 0;
let mut afd_events = 0;
// We use the status info in IO_STATUS_BLOCK to determine the socket poll status. It is unsafe to use a pointer of IO_STATUS_BLOCK.
unsafe {
if self.delete_pending {
return None;
} else if self.iosb.Anonymous.Status == STATUS_CANCELLED {
/* The poll request was cancelled by CancelIoEx. */
} else if self.iosb.Anonymous.Status < 0 {
/* The overlapped request itself failed in an unexpected way. */
afd_events = afd::POLL_CONNECT_FAIL;
} else if self.poll_info.number_of_handles < 1 {
/* This poll operation succeeded but didn't report any socket events. */
} else if self.poll_info.handles[0].events & afd::POLL_LOCAL_CLOSE != 0 {
/* The poll operation reported that the socket was closed. */
self.mark_delete();
return None;
} else {
afd_events = self.poll_info.handles[0].events;
}
}
afd_events &= self.user_evts;
if afd_events == 0 {
return None;
}
// In mio, we have to simulate Edge-triggered behavior to match API usage.
// The strategy here is to intercept all read/write from user that could cause WouldBlock usage,
// then reregister the socket to reset the interests.
self.user_evts &= !afd_events;
Some(Event {
data: self.user_data,
flags: afd_events,
})
}
pub fn is_pending_deletion(&self) -> bool {
self.delete_pending
}
pub fn mark_delete(&mut self) {
if !self.delete_pending {
if let SockPollStatus::Pending = self.poll_status {
drop(self.cancel());
}
self.delete_pending = true;
}
}
fn has_error(&self) -> bool {
self.error.is_some()
}
}
cfg_io_source! {
impl SockState {
fn new(raw_socket: RawSocket, afd: Arc<Afd>) -> io::Result<SockState> {
Ok(SockState {
iosb: IoStatusBlock::zeroed(),
poll_info: AfdPollInfo::zeroed(),
afd,
base_socket: get_base_socket(raw_socket)?,
user_evts: 0,
pending_evts: 0,
user_data: 0,
poll_status: SockPollStatus::Idle,
delete_pending: false,
error: None,
_pinned: PhantomPinned,
})
}
/// True if need to be added on update queue, false otherwise.
fn set_event(&mut self, ev: Event) -> bool {
/* afd::POLL_CONNECT_FAIL and afd::POLL_ABORT are always reported, even when not requested by the caller. */
let events = ev.flags | afd::POLL_CONNECT_FAIL | afd::POLL_ABORT;
self.user_evts = events;
self.user_data = ev.data;
(events & !self.pending_evts) != 0
}
}
}
impl Drop for SockState {
fn drop(&mut self) {
self.mark_delete();
}
}
/// Converts the pointer to a `SockState` into a raw pointer.
/// To revert see `from_overlapped`.
fn into_overlapped(sock_state: Pin<Arc<Mutex<SockState>>>) -> *mut c_void {
let overlapped_ptr: *const Mutex<SockState> =
unsafe { Arc::into_raw(Pin::into_inner_unchecked(sock_state)) };
overlapped_ptr as *mut _
}
/// Convert a raw overlapped pointer into a reference to `SockState`.
/// Reverts `into_overlapped`.
fn from_overlapped(ptr: *mut OVERLAPPED) -> Pin<Arc<Mutex<SockState>>> {
let sock_ptr: *const Mutex<SockState> = ptr as *const _;
unsafe { Pin::new_unchecked(Arc::from_raw(sock_ptr)) }
}
/// Each Selector has a globally unique(ish) ID associated with it. This ID
/// gets tracked by `TcpStream`, `TcpListener`, etc... when they are first
/// registered with the `Selector`. If a type that is previously associated with
/// a `Selector` attempts to register itself with a different `Selector`, the
/// operation will return with an error. This matches windows behavior.
#[cfg(debug_assertions)]
static NEXT_ID: AtomicUsize = AtomicUsize::new(0);
/// Windows implementation of `sys::Selector`
///
/// Edge-triggered event notification is simulated by resetting internal event flag of each socket state `SockState`
/// and setting all events back by intercepting all requests that could cause `io::ErrorKind::WouldBlock` happening.
///
/// This selector is currently only support socket due to `Afd` driver is winsock2 specific.
#[derive(Debug)]
pub struct Selector {
#[cfg(debug_assertions)]
id: usize,
pub(super) inner: Arc<SelectorInner>,
}
impl Selector {
pub fn new() -> io::Result<Selector> {
SelectorInner::new().map(|inner| {
#[cfg(debug_assertions)]
let id = NEXT_ID.fetch_add(1, Ordering::Relaxed) + 1;
Selector {
#[cfg(debug_assertions)]
id,
inner: Arc::new(inner),
}
})
}
pub fn try_clone(&self) -> io::Result<Selector> {
Ok(Selector {
#[cfg(debug_assertions)]
id: self.id,
inner: Arc::clone(&self.inner),
})
}
/// # Safety
///
/// This requires a mutable reference to self because only a single thread
/// can poll IOCP at a time.
pub fn select(&mut self, events: &mut Events, timeout: Option<Duration>) -> io::Result<()> {
self.inner.select(events, timeout)
}
pub(super) fn clone_port(&self) -> Arc<CompletionPort> {
self.inner.cp.clone()
}
#[cfg(feature = "os-ext")]
pub(super) fn same_port(&self, other: &Arc<CompletionPort>) -> bool {
Arc::ptr_eq(&self.inner.cp, other)
}
}
cfg_io_source! {
use super::InternalState;
use crate::Token;
impl Selector {
pub(super) fn register(
&self,
socket: RawSocket,
token: Token,
interests: Interest,
) -> io::Result<InternalState> {
SelectorInner::register(&self.inner, socket, token, interests)
}
pub(super) fn reregister(
&self,
state: Pin<Arc<Mutex<SockState>>>,
token: Token,
interests: Interest,
) -> io::Result<()> {
self.inner.reregister(state, token, interests)
}
#[cfg(debug_assertions)]
pub fn id(&self) -> usize {
self.id
}
}
}
#[derive(Debug)]
pub struct SelectorInner {
pub(super) cp: Arc<CompletionPort>,
update_queue: Mutex<VecDeque<Pin<Arc<Mutex<SockState>>>>>,
afd_group: AfdGroup,
is_polling: AtomicBool,
}
// We have ensured thread safety by introducing lock manually.
unsafe impl Sync for SelectorInner {}
impl SelectorInner {
pub fn new() -> io::Result<SelectorInner> {
CompletionPort::new(0).map(|cp| {
let cp = Arc::new(cp);
let cp_afd = Arc::clone(&cp);
SelectorInner {
cp,
update_queue: Mutex::new(VecDeque::new()),
afd_group: AfdGroup::new(cp_afd),
is_polling: AtomicBool::new(false),
}
})
}
/// # Safety
///
/// May only be calling via `Selector::select`.
pub fn select(&self, events: &mut Events, timeout: Option<Duration>) -> io::Result<()> {
events.clear();
if timeout.is_none() {
loop {
let len = self.select2(&mut events.statuses, &mut events.events, None)?;
if len == 0 {
continue;
}
break Ok(());
}
} else {
self.select2(&mut events.statuses, &mut events.events, timeout)?;
Ok(())
}
}
pub fn select2(
&self,
statuses: &mut [CompletionStatus],
events: &mut Vec<Event>,
timeout: Option<Duration>,
) -> io::Result<usize> {
assert!(!self.is_polling.swap(true, Ordering::AcqRel));
unsafe { self.update_sockets_events() }?;
let result = self.cp.get_many(statuses, timeout);
self.is_polling.store(false, Ordering::Relaxed);
match result {
Ok(iocp_events) => Ok(unsafe { self.feed_events(events, iocp_events) }),
Err(ref e) if e.raw_os_error() == Some(WAIT_TIMEOUT as i32) => Ok(0),
Err(e) => Err(e),
}
}
unsafe fn update_sockets_events(&self) -> io::Result<()> {
let mut update_queue = self.update_queue.lock().unwrap();
for sock in update_queue.iter_mut() {
let mut sock_internal = sock.lock().unwrap();
if !sock_internal.is_pending_deletion() {
sock_internal.update(sock)?;
}
}
// remove all sock which do not have error, they have afd op pending
update_queue.retain(|sock| sock.lock().unwrap().has_error());
self.afd_group.release_unused_afd();
Ok(())
}
// It returns processed count of iocp_events rather than the events itself.
unsafe fn feed_events(
&self,
events: &mut Vec<Event>,
iocp_events: &[CompletionStatus],
) -> usize {
let mut n = 0;
let mut update_queue = self.update_queue.lock().unwrap();
for iocp_event in iocp_events.iter() {
if iocp_event.overlapped().is_null() {
events.push(Event::from_completion_status(iocp_event));
n += 1;
continue;
} else if iocp_event.token() % 2 == 1 {
// Handle is a named pipe. This could be extended to be any non-AFD event.
let callback = (*(iocp_event.overlapped() as *mut super::Overlapped)).callback;
let len = events.len();
callback(iocp_event.entry(), Some(events));
n += events.len() - len;
continue;
}
let sock_state = from_overlapped(iocp_event.overlapped());
let mut sock_guard = sock_state.lock().unwrap();
if let Some(e) = sock_guard.feed_event() {
events.push(e);
n += 1;
}
if !sock_guard.is_pending_deletion() {
update_queue.push_back(sock_state.clone());
}
}
self.afd_group.release_unused_afd();
n
}
}
cfg_io_source! {
use std::mem::size_of;
use std::ptr::null_mut;
use windows_sys::Win32::Networking::WinSock::{
WSAGetLastError, WSAIoctl, SIO_BASE_HANDLE, SIO_BSP_HANDLE,
SIO_BSP_HANDLE_POLL, SIO_BSP_HANDLE_SELECT, SOCKET_ERROR,
};
impl SelectorInner {
fn register(
this: &Arc<Self>,
socket: RawSocket,
token: Token,
interests: Interest,
) -> io::Result<InternalState> {
let flags = interests_to_afd_flags(interests);
let sock = {
let sock = this._alloc_sock_for_rawsocket(socket)?;
let event = Event {
flags,
data: token.0 as u64,
};
sock.lock().unwrap().set_event(event);
sock
};
let state = InternalState {
selector: this.clone(),
token,
interests,
sock_state: sock.clone(),
};
this.queue_state(sock);
unsafe { this.update_sockets_events_if_polling()? };
Ok(state)
}
// Directly accessed in `IoSourceState::do_io`.
pub(super) fn reregister(
&self,
state: Pin<Arc<Mutex<SockState>>>,
token: Token,
interests: Interest,
) -> io::Result<()> {
{
let event = Event {
flags: interests_to_afd_flags(interests),
data: token.0 as u64,
};
state.lock().unwrap().set_event(event);
}
// FIXME: a sock which has_error true should not be re-added to
// the update queue because it's already there.
self.queue_state(state);
unsafe { self.update_sockets_events_if_polling() }
}
/// This function is called by register() and reregister() to start an
/// IOCTL_AFD_POLL operation corresponding to the registered events, but
/// only if necessary.
///
/// Since it is not possible to modify or synchronously cancel an AFD_POLL
/// operation, and there can be only one active AFD_POLL operation per
/// (socket, completion port) pair at any time, it is expensive to change
/// a socket's event registration after it has been submitted to the kernel.
///
/// Therefore, if no other threads are polling when interest in a socket
/// event is (re)registered, the socket is added to the 'update queue', but
/// the actual syscall to start the IOCTL_AFD_POLL operation is deferred
/// until just before the GetQueuedCompletionStatusEx() syscall is made.
///
/// However, when another thread is already blocked on
/// GetQueuedCompletionStatusEx() we tell the kernel about the registered
/// socket event(s) immediately.
unsafe fn update_sockets_events_if_polling(&self) -> io::Result<()> {
if self.is_polling.load(Ordering::Acquire) {
self.update_sockets_events()
} else {
Ok(())
}
}
fn queue_state(&self, sock_state: Pin<Arc<Mutex<SockState>>>) {
let mut update_queue = self.update_queue.lock().unwrap();
update_queue.push_back(sock_state);
}
fn _alloc_sock_for_rawsocket(
&self,
raw_socket: RawSocket,
) -> io::Result<Pin<Arc<Mutex<SockState>>>> {
let afd = self.afd_group.acquire()?;
Ok(Arc::pin(Mutex::new(SockState::new(raw_socket, afd)?)))
}
}
fn try_get_base_socket(raw_socket: RawSocket, ioctl: u32) -> Result<RawSocket, i32> {
let mut base_socket: RawSocket = 0;
let mut bytes: u32 = 0;
unsafe {
if WSAIoctl(
raw_socket as usize,
ioctl,
null_mut(),
0,
&mut base_socket as *mut _ as *mut c_void,
size_of::<RawSocket>() as u32,
&mut bytes,
null_mut(),
None,
) != SOCKET_ERROR
{
Ok(base_socket)
} else {
Err(WSAGetLastError())
}
}
}
fn get_base_socket(raw_socket: RawSocket) -> io::Result<RawSocket> {
let res = try_get_base_socket(raw_socket, SIO_BASE_HANDLE);
if let Ok(base_socket) = res {
return Ok(base_socket);
}
// The `SIO_BASE_HANDLE` should not be intercepted by LSPs, therefore
// it should not fail as long as `raw_socket` is a valid socket. See
// https://docs.microsoft.com/en-us/windows/win32/winsock/winsock-ioctls.
// However, at least one known LSP deliberately breaks it, so we try
// some alternative IOCTLs, starting with the most appropriate one.
for &ioctl in &[
SIO_BSP_HANDLE_SELECT,
SIO_BSP_HANDLE_POLL,
SIO_BSP_HANDLE,
] {
if let Ok(base_socket) = try_get_base_socket(raw_socket, ioctl) {
// Since we know now that we're dealing with an LSP (otherwise
// SIO_BASE_HANDLE would't have failed), only return any result
// when it is different from the original `raw_socket`.
if base_socket != raw_socket {
return Ok(base_socket);
}
}
}
// If the alternative IOCTLs also failed, return the original error.
let os_error = res.unwrap_err();
let err = io::Error::from_raw_os_error(os_error);
Err(err)
}
}
impl Drop for SelectorInner {
fn drop(&mut self) {
loop {
let events_num: usize;
let mut statuses: [CompletionStatus; 1024] = [CompletionStatus::zero(); 1024];
let result = self
.cp
.get_many(&mut statuses, Some(std::time::Duration::from_millis(0)));
match result {
Ok(iocp_events) => {
events_num = iocp_events.iter().len();
for iocp_event in iocp_events.iter() {
if iocp_event.overlapped().is_null() {
// Custom event
} else if iocp_event.token() % 2 == 1 {
// Named pipe, dispatch the event so it can release resources
let callback = unsafe {
(*(iocp_event.overlapped() as *mut super::Overlapped)).callback
};
callback(iocp_event.entry(), None);
} else {
// drain sock state to release memory of Arc reference
let _sock_state = from_overlapped(iocp_event.overlapped());
}
}
}
Err(_) => {
break;
}
}
if events_num == 0 {
// continue looping until all completion statuses have been drained
break;
}
}
self.afd_group.release_unused_afd();
}
}
cfg_net! {
fn interests_to_afd_flags(interests: Interest) -> u32 {
let mut flags = 0;
if interests.is_readable() {
flags |= READABLE_FLAGS | READ_CLOSED_FLAGS | ERROR_FLAGS;
}
if interests.is_writable() {
flags |= WRITABLE_FLAGS | WRITE_CLOSED_FLAGS | ERROR_FLAGS;
}
flags
}
}

View file

@ -0,0 +1,66 @@
use std::io;
use std::net::{self, SocketAddr};
use std::os::windows::io::AsRawSocket;
use windows_sys::Win32::Networking::WinSock::{self, SOCKET, SOCKET_ERROR, SOCK_STREAM};
use crate::sys::windows::net::{new_ip_socket, socket_addr};
pub(crate) fn new_for_addr(address: SocketAddr) -> io::Result<SOCKET> {
new_ip_socket(address, SOCK_STREAM)
}
pub(crate) fn bind(socket: &net::TcpListener, addr: SocketAddr) -> io::Result<()> {
use WinSock::bind;
let (raw_addr, raw_addr_length) = socket_addr(&addr);
syscall!(
bind(
socket.as_raw_socket() as _,
raw_addr.as_ptr(),
raw_addr_length
),
PartialEq::eq,
SOCKET_ERROR
)?;
Ok(())
}
pub(crate) fn connect(socket: &net::TcpStream, addr: SocketAddr) -> io::Result<()> {
use WinSock::connect;
let (raw_addr, raw_addr_length) = socket_addr(&addr);
let res = syscall!(
connect(
socket.as_raw_socket() as _,
raw_addr.as_ptr(),
raw_addr_length
),
PartialEq::eq,
SOCKET_ERROR
);
match res {
Err(err) if err.kind() != io::ErrorKind::WouldBlock => Err(err),
_ => Ok(()),
}
}
pub(crate) fn listen(socket: &net::TcpListener, backlog: u32) -> io::Result<()> {
use std::convert::TryInto;
use WinSock::listen;
let backlog = backlog.try_into().unwrap_or(i32::max_value());
syscall!(
listen(socket.as_raw_socket() as _, backlog),
PartialEq::eq,
SOCKET_ERROR
)?;
Ok(())
}
pub(crate) fn accept(listener: &net::TcpListener) -> io::Result<(net::TcpStream, SocketAddr)> {
// The non-blocking state of `listener` is inherited. See
// https://docs.microsoft.com/en-us/windows/win32/api/winsock2/nf-winsock2-accept#remarks.
listener.accept()
}

View file

@ -0,0 +1,46 @@
use std::io;
use std::mem::{self, MaybeUninit};
use std::net::{self, SocketAddr};
use std::os::windows::io::{AsRawSocket, FromRawSocket};
use std::os::windows::raw::SOCKET as StdSocket; // windows-sys uses usize, stdlib uses u32/u64.
use crate::sys::windows::net::{new_ip_socket, socket_addr};
use windows_sys::Win32::Networking::WinSock::{
bind as win_bind, getsockopt, IPPROTO_IPV6, IPV6_V6ONLY, SOCKET_ERROR, SOCK_DGRAM,
};
pub fn bind(addr: SocketAddr) -> io::Result<net::UdpSocket> {
let raw_socket = new_ip_socket(addr, SOCK_DGRAM)?;
let socket = unsafe { net::UdpSocket::from_raw_socket(raw_socket as StdSocket) };
let (raw_addr, raw_addr_length) = socket_addr(&addr);
syscall!(
win_bind(raw_socket, raw_addr.as_ptr(), raw_addr_length),
PartialEq::eq,
SOCKET_ERROR
)?;
Ok(socket)
}
pub(crate) fn only_v6(socket: &net::UdpSocket) -> io::Result<bool> {
let mut optval: MaybeUninit<i32> = MaybeUninit::uninit();
let mut optlen = mem::size_of::<i32>() as i32;
syscall!(
getsockopt(
socket.as_raw_socket() as usize,
IPPROTO_IPV6 as i32,
IPV6_V6ONLY as i32,
optval.as_mut_ptr().cast(),
&mut optlen,
),
PartialEq::eq,
SOCKET_ERROR
)?;
debug_assert_eq!(optlen as usize, mem::size_of::<i32>());
// Safety: `getsockopt` initialised `optval` for us.
let optval = unsafe { optval.assume_init() };
Ok(optval != 0)
}

View file

@ -0,0 +1,29 @@
use crate::sys::windows::Event;
use crate::sys::windows::Selector;
use crate::Token;
use super::iocp::CompletionPort;
use std::io;
use std::sync::Arc;
#[derive(Debug)]
pub struct Waker {
token: Token,
port: Arc<CompletionPort>,
}
impl Waker {
pub fn new(selector: &Selector, token: Token) -> io::Result<Waker> {
Ok(Waker {
token,
port: selector.clone_port(),
})
}
pub fn wake(&self) -> io::Result<()> {
let mut ev = Event::new(self.token);
ev.set_readable();
self.port.post(ev.to_completion_status())
}
}