xtc_io.h

/*-
 * Copyright (c) 2026, The XTC Project
 * @stability stable
 * Use of this source code is governed by the ISC License.
 *
 * src/inc/xtc_io.h
 *      The L1 event-notification engine: register interest in fd
 *      readiness, poll for ready events, wake the poller from another
 *      thread.
 *
 *      Exactly one backend is compiled in per binary (configure-time).
 *      M2 ships poll(2) and epoll; later milestones add io_uring,
 *      kqueue, IOCP, Solaris event ports, and AIX pollset.
 *
 *      See M2_CLAIMS.md.
 */
 
#ifndef XTC_IO_H
#define XTC_IO_H
 
#include "xtc_export.h"
 
#include <stddef.h>
#include <stdint.h>
 
typedef struct xtc_io xtc_io_t;
 
/*
 * Event flag bits.  Stable across minor versions; new flags appear
 * at higher bit positions only (PLAN.md (S)18).
 */
#define XTC_IO_READABLE  0x01u
#define XTC_IO_WRITABLE  0x02u
#define XTC_IO_HUP       0x04u
#define XTC_IO_ERR       0x08u
#define XTC_IO_WAKEUP    0x10u   /* set on the synthetic event delivered
                                    by xtc_io_wakeup; tag is NULL. */
#define XTC_IO_AIO       0x20u   /* set on an async file-I/O completion;
                                    tag is the xtc_aio_t's tag. */
 
typedef struct xtc_io_event {
        void     *tag;        /* the value passed at registration; NULL on wakeups */
        uint32_t  flags;      /* XTC_IO_* bitset */
        /*
         * The fd this readiness is FOR, or -1 when the event is not about a
         * registered fd (a wakeup, or an AIO completion whose identity is
         * the tag).
         *
         * Every backend already knows this at the point it fills the event
         * (kqueue's kevent.ident, epoll's own bookkeeping, the uring fd
         * record, ...) and it used to be DISCARDED here -- which forced the
         * dispatcher to GUESS, by unregistering `task->park_fd` from the
         * loop that happened to be polling.  Both halves of that guess are
         * wrong for a migratable fiber: it registered on the loop it was
         * RUNNING on, which after a work-steal is not the dispatching loop,
         * and by then park_fd may already name a different registration.
         *
         * The consequence was not cosmetic.  A stale entry left in a peer
         * loop's registry means SEVERAL loops watch the same open file;
         * whichever polls first dispatches the readiness to ITS OWN tag,
         * waking a task that is not the parker.  Measured on FreeBSD: one fd
         * number registered in ALL TWELVE kqueues of a 12-loop executor.
         * Carrying the fd lets dispatch unregister exactly the registration
         * that fired, so no guess is needed.
         */
        int       fd;
} xtc_io_event_t;
 
/*
 * Async file I/O.  A native completion-based file operation: io_uring
 * submits IORING_OP_READ/WRITE/FSYNC; kqueue submits POSIX AIO with
 * SIGEV_KEVENT and reaps EVFILT_AIO; IOCP uses overlapped reads/writes
 * on the port -- in each case the completion wakes the tagged task.  On
 * a readiness-only backend (epoll, poll, select, event ports, AIX)
 * xtc_io_aio_submit returns XTC_E_NOSYS and the caller offloads the op
 * to the blocking pool instead, because a regular file is not pollable.
 * See xtc_aio(3).
 */
enum {
        XTC_AIO_PREAD     = 0,
        XTC_AIO_PWRITE    = 1,
        XTC_AIO_FSYNC     = 2,    /* full sync: data + metadata */
        XTC_AIO_FDATASYNC = 3,    /* data only (fsync on macOS, which lacks it) */
        XTC_AIO_PREADV    = 4,    /* vectored read (iov/iovcnt) */
        XTC_AIO_PWRITEV   = 5     /* vectored write (iov/iovcnt) */
};
typedef struct xtc_aio {
        size_t    size;       /* sizeof(xtc_aio_t): XTC_AIO_INIT.  The library
                               * keeps this pointer until completion and reads
                               * and writes the struct IN PLACE, so it refuses
                               * (XTC_E_INVAL) a size that does not cover every
                               * field below; a field appended in a later minor
                               * is read only when .size covers it. */
        int       fd;
        int       op;         /* XTC_AIO_* */
        void     *buf;        /* read/write buffer (scalar ops; NULL for FSYNC/V) */
        /* Vectored ops (PREADV / PWRITEV): iov points at the caller's
         * iovec array (struct iovec *, kept opaque here to avoid a
         * <sys/uio.h> dependency in this header) and iovcnt is its count.
         * Zero/NULL for the scalar and sync ops. */
        void     *iov;        /* const struct iovec * */
        int64_t   off;        /* file offset (ignored for FSYNC) */
        void     *tag;        /* woken on completion (an xtc_task_t *) */
        uint32_t  len;        /* byte count (scalar ops) */
        int       iovcnt;
        /*
         * 0 until the completion is reaped.  CONCURRENCY CONTRACT: on a
         * native completion backend (io_uring / IOCP / kqueue-AIO / Solaris
         * ports) the REAPING thread stores this and the PARKED FIBER reads it
         * in its wake-recheck loop, so it is a cross-thread flag.
         *
         * It is deliberately a PLAIN int, not `_Atomic int`, and the library
         * accesses it through the __os_atomic_{load,store}_i32 helpers in
         * src/inc/os_atomic.h -- which exist precisely for this shape (atomic
         * primitives applied to the ADDRESS of plain storage, giving the same
         * ordering guarantees as an _Atomic-qualified type while leaving
         * ordinary load/store usable on the single-threaded pre-publication
         * init path, where xtc_io_aio_submit clears it).
         *
         * Why not `_Atomic int`: layout is identical either way (gcc and
         * clang, x86-64), but `_Atomic` is not valid C++, and this header has
         * no extern "C" guard of its own (xtc.h has one; xtc_io.h is also
         * includable directly).  g++ -std=c++17 compiles this header today
         * and would fail outright with `'_Atomic' does not name a type`.
         * Consumers allocate xtc_aio_t on the stack, so that is a source
         * break for every C++ embedder, in exchange for nothing the helper
         * accesses do not already provide.  Do not "upgrade" it.
         */
        int       done;
        int32_t   res;        /* bytes transferred, or -errno */
} xtc_aio_t;
 
#define XTC_AIO_INIT { .size = sizeof(xtc_aio_t) }
 
/*
 * PUBLIC: int          xtc_io_init __P((xtc_io_t **));
 * PUBLIC: int          xtc_io_fini __P((xtc_io_t *));
 * PUBLIC: const char  *xtc_io_backend_name __P((void));
 * PUBLIC: int          xtc_io_reg_fd __P((xtc_io_t *, int, uint32_t, void *));
 * PUBLIC: int          xtc_io_mod_fd __P((xtc_io_t *, int, uint32_t, void *));
 * PUBLIC: int          xtc_io_del_fd __P((xtc_io_t *, int));
 * PUBLIC: int          xtc_io_poll __P((xtc_io_t *, xtc_io_event_t *, int, int64_t, int *));
 * PUBLIC: int          xtc_io_aio_submit __P((xtc_io_t *, xtc_aio_t *));
 * PUBLIC: int          xtc_io_wakeup __P((xtc_io_t *));
 *
 * PUBLIC: void         xtc_io_set_iowq_max_workers __P((unsigned, unsigned));
 */
 
/* Lifecycle. */
XTC_API int          xtc_io_init(xtc_io_t **out);
XTC_API int          xtc_io_fini(xtc_io_t *io);
XTC_API const char  *xtc_io_backend_name(void);
 
/* Registration. */
XTC_API int          xtc_io_reg_fd(xtc_io_t *io, int fd, uint32_t interest, void *tag);
XTC_API int          xtc_io_mod_fd(xtc_io_t *io, int fd, uint32_t interest, void *tag);
XTC_API int          xtc_io_del_fd(xtc_io_t *io, int fd);
 
/*
 * xtc_io_poll --
 *      Wait for events.  On return *n_out holds the number of events
 *      written (0..max).
 *
 *      timeout_ns:
 *              == 0  -> non-blocking
 *              >  0  -> wait at most that many nanoseconds
 *              <  0  -> wait indefinitely (until an fd becomes ready or
 *                       xtc_io_wakeup is called)
 */
XTC_API int          xtc_io_poll(xtc_io_t *io, xtc_io_event_t *events, int max,
                                 int64_t timeout_ns, int *n_out);
 
/* Submit an async file op.  XTC_OK if queued (the caller parks until
 * the completion event with tag a->tag arrives, then reads a->res);
 * XTC_E_NOSYS if the backend has no native file completion (caller
 * should offload instead). */
XTC_API int          xtc_io_aio_submit(xtc_io_t *io, xtc_aio_t *a);
 
/*
 * xtc_io_wakeup --
 *      From any thread, cause the next (or in-flight) xtc_io_poll on
 *      this io to return.  Safe to call concurrently from many threads;
 *      multiple wakeups before the next poll coalesce into one event.
 */
XTC_API int          xtc_io_wakeup(xtc_io_t *io);
 
/*
 * Cap the per-ring io_uring io-wq kernel worker pool (Linux io_uring
 * backend only; a no-op on other backends).  See the implementation
 * comment in io_uring.c: prevents N carriers on an M-core box from
 * accumulating N*M idle io-wq kernel threads.  Call before the
 * executor/loop creates its rings.
 */
XTC_API void         xtc_io_set_iowq_max_workers(unsigned bound,
                                                 unsigned unbound);
 
#endif /* XTC_IO_H */