io_int.h

/*-
 * Copyright (c) 2026, The XTC Project
 * Use of this source code is governed by the ISC License.
 *
 * src/inc/io_int.h
 *      Internal struct definition for the L1 backend implementations.
 */
 
#ifndef XTC_IO_INT_H
#define XTC_IO_INT_H
 
#include "xtc_io.h"
#include <stdatomic.h>
 
#if defined(XTC_IO_BACKEND_EPOLL)
/* nothing to predefine; epoll keeps tags via epoll_data_t */
#elif defined(XTC_IO_BACKEND_KQUEUE)
/* kqueue's EV_ADD is idempotent and EV_DELETE silently ignores absent
 * filters, so we keep an authoritative side-table of currently-
 * registered fds to enforce M2's semantic contract. */
#elif defined(XTC_IO_BACKEND_SOLARIS)
/* illumos event ports are one-shot per association, so we maintain
 * the same side-table as kqueue and re-arm on each delivery. */
struct __xtc_solaris_reg {
        int       fd;
        uint32_t  interest;
        void     *tag;
};
#elif defined(XTC_IO_BACKEND_IOCP)
#include <pthread.h>   /* MSVC: the compat shim; MinGW: winpthreads */
/*
 * Windows IOCP backend (round 2): native completion port plus the
 * AFD poll fast path that turns a connected/listening socket's
 * readiness into an IOCP completion.
 *
 * Each socket registration owns one AFD poll OVERLAPPED that is
 * armed (submitted via NtDeviceIoControlFile(IOCTL_AFD_POLL)) into
 * the kernel and re-armed after every completion -- level-triggered
 * emulation matching the epoll/kqueue contract.
 *
 * OVERLAPPED OWNERSHIP RULE (the classic IOCP correctness invariant,
 * enforced throughout io_iocp.c):
 *   The OVERLAPPED (ov) and the AFD_POLL_INFO (poll_info) embedded in
 *   a registration belong to the KERNEL from the moment the AFD poll
 *   is armed (NtDeviceIoControlFile returns STATUS_PENDING) until the
 *   matching completion is dequeued from the port by
 *   GetQueuedCompletionStatusEx.  While armed (pending == 1) neither
 *   buffer may be freed or reused.  Deregistering an armed socket
 *   issues NtCancelIoFileEx and marks the registration dead; the
 *   storage is only released once the (possibly canceled) completion
 *   is reaped.  This is why the struct embeds ov/poll_info by value
 *   and the registration node is freed lazily, never inline with
 *   xtc_io_del_fd while a poll is in flight.
 *
 * The OVERLAPPED is the FIRST member so a completion's
 * OVERLAPPED_ENTRY.lpOverlapped pointer can be cast straight back to
 * the owning registration.  Each registration is a separately
 * heap-allocated node with a STABLE address (held in a pointer array,
 * never an inline array), because the kernel-owned OVERLAPPED carries
 * a back-pointer to its node for the whole time the poll is armed --
 * a realloc or swap-remove of an inline array would dangle it.
 */
struct __xtc_iocp_reg {
        struct __xtc_iocp_overlapped *ovp; /* OVERLAPPED + AFD_POLL_INFO (heap) */
        int       fd;               /* the SOCKET as an int (Winsock handle) */
        void     *base;             /* base socket HANDLE (SIO_BASE_HANDLE) */
        uint32_t  interest;
        void     *tag;
        int       pending;          /* 1 while an AFD poll is armed in the kernel */
        int       dead;             /* deregistered; awaiting terminal completion */
        int64_t   armed_at_ns;      /* __os_clock_mono() when this arm was issued;
                                     * drives the bounded re-poll workaround for
                                     * the AFD async-completion bug (see io_iocp.c
                                     * __xtc_iocp_repoll_sweep) */
};
/* A file AIO (pread/pwrite) in flight on the IOCP backend.  The
 * file HANDLE is associated with the completion port, so the
 * overlapped ReadFile/WriteFile completion is dequeued by
 * GetQueuedCompletionStatusEx like any socket event -- no hEvent and
 * no WaitForMultipleObjects.  fsync has no async form on Windows
 * (FlushFileBuffers is synchronous) and is offloaded.  The OVERLAPPED
 * is the FIRST member so the completion's lpOverlapped recovers the
 * node; ownership follows the same kernel-owns-while-pending rule as
 * the socket poll OVERLAPPED above. */
struct __xtc_iocp_aio {
        void *ov;      /* OVERLAPPED * (heap; first field; owned here) */
        void *aio;     /* xtc_aio_t * awaiting completion */
        void *fh;      /* HANDLE: the file, for GetOverlappedResult */
};
#elif defined(XTC_IO_BACKEND_AIX)
/* AIX pollset_* backend.  Like solaris/kqueue, we maintain a
 * side-table for duplicate detection and to map fd -> user tag
 * (pollset itself doesn't carry udata). */
struct __xtc_aix_reg {
        int       fd;
        uint32_t  interest;
        void     *tag;
};
#elif defined(XTC_IO_BACKEND_URING)
#include <liburing.h>
#include <pthread.h>
/*
 * Per-fd state for the io_uring backend.  The user_data passed to
 * each POLL_ADD points at one of these.  io->tab maps fd -> the LIVE
 * node; a retired node whose poll is still in the kernel waits on
 * io->zombies for its terminal CQE.
 *
 *   armed    a poll for this node is in the kernel (CQEs may arrive).
 *            Cleared by the terminal CQE (no IORING_CQE_F_MORE).
 *   dead     retired by del/mod; no longer in io->tab.
 *   pending  on io->pending: needs an SQE the SQ had no room for (a
 *            re-arm for a live node, a cancel for a dead armed one).
 *   linger   retired by del while its one-shot poll was still armed, and
 *            NOT cancelled: it waits on io->lingers (lnext/lprev) for a
 *            re-registration of the same socket to adopt it, or for the
 *            end of the next poll to cancel it.  ino/dev identify the
 *            socket (a closed fd's number can be reused).
 *
 * A node is on io->zombies exactly while dead && armed && !linger, and
 * is freed once dead && !armed && !pending.
 */
struct __xtc_uring_fd {
        int        fd;
        uint32_t   interest;
        void      *tag;
        int        is_wakeup;     /* 1 for the internal wakeup pipe */
        int        dead;
        int        armed;
        int        pending;
        int        linger;
        uint64_t   ino;            /* socket identity (0 = not a socket) */
        uint64_t   dev;
        struct __xtc_uring_fd *next;   /* io->zombies linkage */
        struct __xtc_uring_fd *prev;   /* io->zombies linkage */
        struct __xtc_uring_fd *pnext;  /* io->pending linkage */
        struct __xtc_uring_fd *lnext;  /* io->lingers linkage */
        struct __xtc_uring_fd *lprev;
};
#elif defined(XTC_IO_BACKEND_POLL)
#include <poll.h>
#include <pthread.h>
#elif defined(XTC_IO_BACKEND_SELECT)
#include <sys/select.h>
#include <pthread.h>
#elif defined(XTC_IO_BACKEND_SIM)
/* Deterministic-simulation backend (DST): no kernel poller.  Readiness
 * and file-AIO completions come from a scripted in-process event store
 * driven against the virtual clock; the wakeup is an in-process flag.
 * See src/io/io_sim.c. */
#else
# error "M2 build expects XTC_IO_BACKEND_{POLL,EPOLL,URING,KQUEUE,IOCP,SOLARIS,AIX,SELECT,SIM} to be defined"
#endif
 
struct xtc_io {
        int wakeup_rfd;
        int wakeup_wfd;
 
#if defined(XTC_IO_BACKEND_EPOLL)
        int epfd;
#elif defined(XTC_IO_BACKEND_KQUEUE)
        int epfd;                          /* kqueue fd */
        int *reg_fds;                      /* registered fd list */
        int  n_reg;
        int  cap_reg;
#elif defined(XTC_IO_BACKEND_SOLARIS)
        int epfd;                          /* event-port fd */
        struct __xtc_solaris_reg *reg_fds;
        int  n_reg;
        int  cap_reg;
#elif defined(XTC_IO_BACKEND_IOCP)
        void                  *iocp;       /* HANDLE: the completion port */
        void                  *afd;        /* HANDLE: \Device\Afd, port-associated */
        _Atomic int            wakeup_pending; /* 1 = a wakeup completion is queued */
        struct __xtc_iocp_reg **reg_iocp;   /* live registration nodes (stable) */
        int  n_reg;
        int  cap_reg;
        struct __xtc_iocp_reg **dead_iocp;  /* deregistered, awaiting completion */
        int  n_dead;
        int  cap_dead;
        struct __xtc_iocp_aio  *aio_pend;   /* file AIOs in flight */
        int  n_aio;
        int  cap_aio;
        /*
         * Cross-loop deferred unregister, as io_uring/poll/select have:
         * reg_iocp/dead_iocp are a USERSPACE registry owned by this io's
         * poll thread.  A migrated fiber's xtc_proc_wait_fd cleanup used to
         * call xtc_io_del_fd on this io from ANOTHER thread while the owner
         * was mid-poll -- racing the swap-remove and the dead-list free:
         * heap corruption (/m5/exec/Blk3_cross_loop_del_fd under MSVC and
         * MinGW, 1.55.0 and 1.56.0).  A foreign thread now queues the fd
         * under del_lock and nudges the owner, which drains the queue at the
         * top of xtc_io_poll and before a re-registration.
         */
        int             *pending_del;
        int              n_pending_del;
        int              cap_pending_del;
        _Atomic int      has_pending_del;
        pthread_mutex_t  del_lock;
        pthread_t        owner_tid;
        _Atomic int      owner_set;
#elif defined(XTC_IO_BACKEND_AIX)
        int   ps;                          /* pollset_t */
        void *reg_aix;                     /* struct __xtc_aix_reg * */
        int   n_reg;
        int   cap_reg;
#elif defined(XTC_IO_BACKEND_URING)
        struct io_uring  ring;
        struct __xtc_uring_fd **tab;     /* fd -> live node (NULL = none) */
        int              tab_cap;        /* entries in tab */
        struct __xtc_uring_fd *zombies;  /* deleted fds awaiting terminal CQE */
        struct __xtc_uring_fd *pending;  /* nodes awaiting an SQE (SQ was full) */
        int              n_zombies;      /* length of zombies (diagnostics) */
        struct __xtc_uring_fd *lingers;  /* deleted, still armed, adoptable */
        struct __xtc_uring_fd **ltab;    /* fd -> lingering node; sized as tab */
        unsigned char   *idle;           /* fd -> 1: last del found it armed */
        int              n_lingers;
        unsigned long    n_adopted;      /* re-registrations served by a linger */
        unsigned long    n_cancels;      /* POLL_REMOVEs submitted (tests) */
        /*
         * Cross-loop deferred unregister queue.  io->tab / io->zombies and
         * the SQ ring are single-producer, owned by this io's loop thread.
         * A migratable fiber that parked on this loop but was woken via a
         * non-fd path (timeout / xtc_proc_wake / mailbox) and then work-
         * stolen resumes on ANOTHER thread with its park_fd still live, and
         * its xtc_proc_wait_fd cleanup would otherwise call xtc_io_del_fd on
         * THIS io from the wrong thread -- racing the fd list AND the SQ ring
         * (the native-path concurrent-commit collapse, TSan-caught
         * 2026-08-30).  Instead the foreign thread posts the fd here under
         * del_lock and nudges this loop; the owning thread drains it at the
         * top of xtc_io_poll and performs the real unregister on its own
         * ring.  del_lock guards ONLY this small queue, never the hot fds
         * list or the ring. */
        int             *pending_del;    /* fds awaiting owner-thread unregister */
        int              n_pending_del;
        int              cap_pending_del;
        _Atomic int      has_pending_del;  /* fast, lock-free "is queue non-empty?" */
        pthread_mutex_t  del_lock;        /* guards ONLY pending_del (not fds/ring) */
        /*
         * The exec-relative id of the loop that owns this io, purely as a
         * LABEL for XTC_TAIL_REAP (which is emitted inside xtc_io_poll, where
         * the reap outcome is visible but the loop is not).  -1 until the
         * owning loop publishes it; never used for control flow.
         */
        int              tail_loop_id;
        /*
         * The owner: the thread that polls this io and the only one that
         * may submit to its ring.  A per-thread token, never reused (not a
         * pthread_t: glibc reuses a dead thread's descriptor, so a second
         * xtc_exec_run's worker can compare EQUAL to the first run's).  0 =
         * no owner yet.  owner_active: the owner declared itself
         * (__xtc_io_set_owner, at the start of a loop run) and has not left
         * (__xtc_io_owner_leave, at its end).  A thread touching an io that
         * has no active owner takes it over (__own); on a SINGLE_ISSUER ring
         * that rebuilds the ring for it (__ring_rebind), and a loop run
         * upgrades a COOP ring to SINGLE_ISSUER the same way.
         * owner_tid/owner_set mirror the owner for the debugger (xtc-rings).
         */
        _Atomic uint64_t owner_tok;
        _Atomic int      owner_active;    /* owner declared (set_owner), not left */
        pthread_t        owner_tid;       /* the thread that polls this io */
        _Atomic int      owner_set;       /* 1 once owner_tid is recorded */
        /*
         * Ring mode.  ring_flags: the IORING_SETUP_* the ring was built with
         * (rebuilds reuse them).  ring_single: SINGLE_ISSUER (the DEFER
         * tier), so only the creating thread may submit or reap and an owner
         * change rebuilds the ring.  n_aio: file-AIO SQEs in the kernel; a
         * rebuild needs 0 (see __xtc_io_owner_leave).
         * carry: events a leaving owner reaped while it waited for its AIO,
         * returned by the next poll before anything else.  ring_dead: a
         * rebuild found no memory for the new ring; the io is unusable.
         */
        unsigned         ring_flags;
        int              ring_single;
        int              ring_try_defer;  /* a loop run may upgrade to DEFER */
        int              n_aio;
        int              n_aio_queued;    /* of n_aio: prepared, not yet submitted */
        int              ring_dead;
        xtc_io_event_t  *carry;
        int              n_carry;
        int              cap_carry;
        unsigned         n_rebinds;       /* ring rebuilds (tests) */
        unsigned long    n_enter_submit;  /* enters that carried SQEs (tests) */
        unsigned long    n_enter_wait;    /* enters that only waited (tests) */
        /*
         * L2 ring-pointer preempt (INSPIRED BY Glommio's need_preempt():
         * reactor.rs / sys/uring.rs preempt_pointers).  A dedicated tiny
         * ring carrying ONLY a rearmed TIMEOUT SQE, so
         * io_uring_cq_ready(&preempt_ring) -- two relaxed/acquire loads of
         * that ring's own head/tail, no syscall, no signal -- means "the
         * preempt interval elapsed".  A separate ring (not io->ring) is
         * what makes the pointer test isolate the timeout: I/O CQEs on the
         * main ring advance its head/tail unpredictably, but this ring sees
         * only the timeout.  preempt_armed gates it (0 = off, the ring is
         * not initialised). */
        struct io_uring  preempt_ring;
        int              preempt_armed;
        struct __kernel_timespec preempt_ts;   /* the rearm interval */
#elif defined(XTC_IO_BACKEND_POLL)
        struct pollfd *pfds;
        void         **tags;
        int            n;
        int            cap;
        int           *slot_of;   /* fd -> slot + 1 (0 = not registered) */
        int            slot_cap;  /* entries in slot_of */
        /*
         * Cross-loop deferred unregister, as io_uring has: the fd arrays are
         * a USERSPACE registry owned by this io's poll thread.  A migrated
         * fiber's xtc_proc_wait_fd cleanup used to xtc_io_del_fd this io from
         * ANOTHER thread while the owner was mid-poll -- racing the array
         * compaction, so a registration could survive its fiber and its fd's
         * close (select then failed the whole poll with EBADF and the worker
         * thread exited, stranding every fiber on its loop).  A foreign
         * thread now queues the fd under del_lock and nudges the owner, which
         * drains the queue at the top of xtc_io_poll.
         */
        int             *pending_del;
        int              n_pending_del;
        int              cap_pending_del;
        _Atomic int      has_pending_del;
        pthread_mutex_t  del_lock;
        pthread_t        owner_tid;
        _Atomic int      owner_set;
#elif defined(XTC_IO_BACKEND_SELECT)
        /* Parallel fd[], interest[], tag[] arrays.  fd_set is built
         * each poll() call from these.  Capped at FD_SETSIZE. */
        int           *fds;
        uint32_t      *interests;
        void         **tags;
        int            n;
        int            cap;
        int           *slot_of;   /* fd -> slot + 1 (0 = not registered) */
        int            slot_cap;  /* entries in slot_of */
        /*
         * Cross-loop deferred unregister, as io_uring has: the fd arrays are
         * a USERSPACE registry owned by this io's poll thread.  A migrated
         * fiber's xtc_proc_wait_fd cleanup used to xtc_io_del_fd this io from
         * ANOTHER thread while the owner was mid-poll -- racing the array
         * compaction, so a registration could survive its fiber and its fd's
         * close (select then failed the whole poll with EBADF and the worker
         * thread exited, stranding every fiber on its loop).  A foreign
         * thread now queues the fd under del_lock and nudges the owner, which
         * drains the queue at the top of xtc_io_poll.
         */
        int             *pending_del;
        int              n_pending_del;
        int              cap_pending_del;
        _Atomic int      has_pending_del;
        pthread_mutex_t  del_lock;
        pthread_t        owner_tid;
        _Atomic int      owner_set;
#elif defined(XTC_IO_BACKEND_SIM)
        /* Registered fds (tag map) for readiness simulation, a scripted
         * event queue ordered by virtual-time due, and an in-process
         * wakeup flag.  Defined in io_sim.c; opaque here. */
        struct __xtc_sim_io *sim;
#endif
};
 
/*
 * Cross-loop deferred fd-unregister.  Each backend .c provides one:
 * io_uring queues the fd and drains it on the owning thread (its fds
 * list + SQ ring are single-owner); the other backends passthrough to
 * xtc_io_del_fd (kernel-synchronized or their own registry).  Internal;
 * the sole caller is xtc_proc_wait_fd's post-migration cleanup.
 */
int __xtc_io_defer_del_fd(xtc_io_t *io, int fd);
 
/*
 * Label an io with the exec-relative id of the loop that owns it, for
 * XTC_TAIL_REAP.  No-op on backends that do not reap CQEs.  Label only --
 * never used for control flow.
 */
void __xtc_io_set_tail_loop_id(xtc_io_t *io, int id);
 
/*
 * Registration accounting (io_uring; XTC_E_NOSYS elsewhere): live
 * registrations, retired nodes awaiting a terminal CQE, nodes queued for
 * an SQE.  Tests assert no node leaks.  Any out pointer may be NULL.
 */
int __xtc_io_reg_stats(xtc_io_t *io, int *live, int *zombies, int *pending);
 
/*
 * io_uring (XTC_E_NOSYS elsewhere): the IORING_SETUP_* flags the main ring
 * has now (after any fallback for an older kernel), how many SQEs the
 * kernel has consumed from the current ring, and how many times the ring
 * was rebuilt (a loop run upgrading it to DEFER, a new owner thread).
 * Tests.  Out pointers may be NULL.
 */
int __xtc_io_uring_info(xtc_io_t *io, unsigned *flags, unsigned *sq_consumed,
    unsigned *rebinds);
 
/*
 * The calling thread stops running this io's loop (an executor worker
 * exiting, xtc_loop_run returning).  On a SINGLE_ISSUER io_uring ring it
 * reaps every file-AIO completion still in the kernel, because only this
 * thread can, and keeps them for the next owner's first poll.  No-op on
 * other backends and when the caller is not the owner.
 */
void __xtc_io_owner_leave(xtc_io_t *io);
 
/*
 * io_uring (XTC_E_NOSYS elsewhere): POLL_REMOVEs submitted, and
 * re-registrations served by adopting a lingering poll, since xtc_io_init.
 * Tests.  Out pointers may be NULL.
 */
int __xtc_io_uring_poll_stats(xtc_io_t *io, unsigned long *cancels,
    unsigned long *adopted);
 
/*
 * io_uring (XTC_E_NOSYS elsewhere): io_uring_enter calls since xtc_io_init
 * that handed SQEs to the kernel, those that only waited for a completion
 * (a poll's wait; not the reap loop's task-work flush), and the SQEs
 * prepared but not yet submitted (see __submit_wait).  Tests.  Out
 * pointers may be NULL.
 */
int __xtc_io_uring_enter_stats(xtc_io_t *io, unsigned long *submits,
    unsigned long *waits, unsigned *queued);
 
/*
 * An xtc_aio_t is read and written in place for its whole flight, so it
 * must cover the 2.0 layout (through `res`).  A constant, not sizeof: a
 * field a later minor appends is read under XTC_FIELD_OK, and a 2.0
 * caller's struct stays valid.
 */
#define __XTC_AIO_SIZE_2_0 (offsetof(xtc_aio_t, res) + sizeof(int32_t))
#define __XTC_AIO_OK(a) ((a) != NULL && (a)->size >= __XTC_AIO_SIZE_2_0)
 
#endif /* XTC_IO_INT_H */