#IPC — Inter-Process Communication (404 ipc.h)

#define FIO_IPC
#include FIO_INCLUDE_FILE

IPC is the messaging backbone of the facil.io worker/master model. Workers fork from a master process. The master keeps listeners, manages state, and handles anything that needs a single authoritative view of the world. Workers handle client connections. IPC bridges the two: a worker sends a message to the master (or to every process in the cluster), the master executes it and optionally streams replies back.

All messages are encrypted in transit with ChaCha20-Poly1305 AEAD, keyed from the process secret. Local IPC runs over a Unix socket (or auto-generated TCP socket). Multi-machine RPC extends the same protocol over TCP with optional UDP peer discovery.

See [./400 io-overview.md](./400 io-overview.md) for where IPC sits in the full IO stack.

#Worker/Master Model

master process
 ├── owns IPC listener socket
 ├── executes messages sent by workers
 ├── forwards cluster/broadcast messages to other machines
 └── worker process × N
      ├── each has its own IO reactor loop
      ├── connects back to master via IPC socket on startup
      └── sends fio_ipc_* messages to reach master or peers

The master is always the hub. Workers can only talk through the master — there is no direct worker-to-worker channel. On shutdown the master sends a local broadcast instructing all workers to stop.

#Configuration Macros

Define before including the header to override defaults.

Macro	Default	Meaning
`FIO_IPC_URL_MAX_LENGTH`	`1024`	Max IPC socket URL length
`FIO_IPC_MAX_LENGTH`	`128 MiB`	Max message payload size

#Types

#`fio_ipc_s` — the message

typedef struct fio_ipc_s {
  fio_io_s *from;               /* originating IO (set by receiver; not transmitted) */
  /* -- wire format starts here -- */
  uint32_t len;                 /* data[] byte count            (AAD - authenticated) */
  uint16_t flags;               /* user-settable flags           (AAD - authenticated) */
  uint16_t routing_flags;       /* internal routing              (AAD - authenticated) */
  uint64_t timestamp;           /* millisecond tick — part of nonce (unencrypted)    */
  uint64_t id;                  /* random 64-bit value — part of nonce (unencrypted) */
  union {
    void (*call)(struct fio_ipc_s *); /* function to call (local IPC; encrypted)  */
    uint32_t opcode;                  /* registered op-code (cluster RPC; encrypted)*/
  };
  void (*on_reply)(struct fio_ipc_s *); /* run on caller when reply arrives (encrypted) */
  void (*on_done)(struct fio_ipc_s *);  /* run on caller when last reply arrives (encrypted) */
  void *udata;                          /* caller-side opaque pointer (encrypted) */
  char data[];                          /* payload + 16-byte Poly1305 MAC at tail (encrypted) */
} fio_ipc_s;

Messages are reference-counted. Everything from call onward (including the payload) is encrypted on the wire. from, timestamp, and id are unencrypted and unauthenticated (they form the nonce). len, flags, and routing_flags are unencrypted but authenticated via AEAD.

#`fio_ipc_args_s` — construction arguments

typedef struct {
  void (*call)(fio_ipc_s *);     /* function to run on target (required for local IPC) */
  void (*on_reply)(fio_ipc_s *); /* called on caller when a reply arrives */
  void (*on_done)(fio_ipc_s *);  /* called on caller when final reply arrives */
  fio_io_s *exclude;             /* IO to exclude from delivery (or FIO_IPC_EXCLUDE_SELF) */
  uint64_t timestamp;            /* force timestamp; 0 = use reactor tick */
  uint64_t id;                   /* force message ID; 0 = random */
  uint32_t opcode;               /* registered op-code; replaces call if non-zero */
  uint16_t flags;                /* user-settable flags */
  bool cluster;                  /* deliver to remote machines in cluster */
  bool workers;                  /* deliver to master + all local workers */
  void *udata;                   /* opaque pointer for reply callbacks */
  fio_buf_info_s *data;          /* payload (use FIO_IPC_DATA macro) */
} fio_ipc_args_s;

#`fio_ipc_opcode_s` — op-code registration

typedef struct fio_ipc_opcode_s {
  uint32_t opcode;                      /* unique non-zero value */
  void (*call)(struct fio_ipc_s *);     /* handler */
  void (*on_reply)(struct fio_ipc_s *); /* reply callback */
  void (*on_done)(struct fio_ipc_s *);  /* final reply callback */
  void *udata;                          /* provided to handler via ipc->udata */
} fio_ipc_opcode_s;

#`fio_ipc_reply_args_s` — reply arguments

typedef struct {
  fio_ipc_s *ipc;       /* original request (required) */
  fio_buf_info_s *data; /* reply payload */
  uint64_t timestamp;   /* 0 = current tick */
  uint64_t id;          /* 0 = use original request ID */
  uint16_t flags;       /* 0 = use original request flags */
  uint8_t done;         /* set to 1 on last reply (triggers on_done on caller) */
  uint8_t flags_set;    /* set to 1 to allow flags=0 override */
} fio_ipc_reply_args_s;

#Helper Macro: `FIO_IPC_DATA`

#define FIO_IPC_DATA(...)  \
  (fio_buf_info_s[]){ __VA_ARGS__, { .len = ((size_t)-1) } }

Composes multiple buffers into a single message without intermediate allocation. Iteration stops at the first entry with len == (size_t)-1.

uint32_t seq = 42;
const char *text = "hello";

fio_ipc_call(.call = my_handler,
             .data = FIO_IPC_DATA(
                 FIO_BUF_INFO2(&seq, sizeof(seq)),
                 FIO_BUF_INFO1((char *)text)));
/* seq and text are copied immediately — stack safe */

#Delivery Macros

All four macros call fio_ipc_new(...) then fio_ipc_send(...). They accept named fields from fio_ipc_args_s.

Macro	Delivered to
`fio_ipc_call(...)`	master only
`fio_ipc_local(...)`	master + all local workers
`fio_ipc_cluster(...)`	master process on every machine (local and remote)
`fio_ipc_broadcast(...)`	master + workers on every machine (local and remote)

/* worker → master */
#define fio_ipc_call(...)      fio_ipc_send(fio_ipc_new(__VA_ARGS__))
/* all local processes */
#define fio_ipc_local(...)     fio_ipc_send(fio_ipc_new(.workers = 1, __VA_ARGS__))
/* remote masters only */
#define fio_ipc_cluster(...)   fio_ipc_send(fio_ipc_new(.cluster = 1, __VA_ARGS__))
/* everywhere */
#define fio_ipc_broadcast(...) fio_ipc_send(fio_ipc_new(.workers = 1, .cluster = 1, __VA_ARGS__))

fio_ipc_cluster and fio_ipc_broadcast require an opcode (not a function pointer) because function pointers are meaningless across machine boundaries.

Use .exclude = FIO_IPC_EXCLUDE_SELF to skip the calling process when broadcasting locally.

FIO_IPC_EXCLUDE_SELF

#define FIO_IPC_EXCLUDE_SELF ((fio_io_s *)((char *)-1LL))

Pass as .exclude in fio_ipc_args_s, or set ipc->from = FIO_IPC_EXCLUDE_SELF before calling fio_ipc_send.

#Quick Examples

#Worker calls master, master replies

/* runs on master */
void on_master(fio_ipc_s *msg) {
  printf("master got: %.*s\n", (int)msg->len, msg->data);
  fio_ipc_reply(msg,
                .data = FIO_IPC_DATA(FIO_BUF_INFO1((char *)"pong")),
                .done = 1);
}

/* runs on calling worker when reply arrives */
void on_reply(fio_ipc_s *msg) {
  printf("worker got reply: %.*s\n", (int)msg->len, msg->data);
}

/* somewhere in a worker callback */
fio_ipc_call(.call    = on_master,
             .on_done = on_reply,
             .data    = FIO_IPC_DATA(FIO_BUF_INFO1((char *)"ping")));

#Local broadcast (skip self)

void on_notify(fio_ipc_s *msg) {
  printf("[%d] notified: %.*s\n", fio_io_pid(), (int)msg->len, msg->data);
}

fio_ipc_local(.call    = on_notify,
              .exclude = FIO_IPC_EXCLUDE_SELF,
              .data    = FIO_IPC_DATA(FIO_BUF_INFO1((char *)"config-reload")));

#Cluster-wide op-code broadcast

#define OP_CACHE_BUST 1

void on_cache_bust(fio_ipc_s *msg) {
  cache_invalidate(msg->data, msg->len);
}

/* register before fio_io_start */
fio_ipc_opcode_register(.opcode = OP_CACHE_BUST, .call = on_cache_bust);
fio_ipc_cluster_listen(9000); /* requires SECRET env var */
fio_io_start(4);

/* later, from any process */
fio_ipc_broadcast(.opcode = OP_CACHE_BUST,
                  .data   = FIO_IPC_DATA(FIO_BUF_INFO1((char *)"key123")));

#Reply: `fio_ipc_reply`

SFUNC void fio_ipc_reply(fio_ipc_reply_args_s args);
/* shadowing macro — first arg is the original request */
#define fio_ipc_reply(r, ...) \
  fio_ipc_reply((fio_ipc_reply_args_s){ .ipc = (r), __VA_ARGS__ })

Send one or more replies from the master back to the caller. Set .done = 1 on the final reply to trigger the caller's on_done callback. Replies inherit call, on_reply, on_done, and udata from the original request.

Streaming replies:

void stream_handler(fio_ipc_s *msg) {
  for (int i = 0; i < 5; ++i) {
    char chunk[32];
    int n = snprintf(chunk, sizeof(chunk), "chunk%d", i);
    fio_ipc_reply(msg,
                  .data = FIO_IPC_DATA(FIO_BUF_INFO2(chunk, (size_t)n)),
                  .done = (i == 4));
  }
}

#Op-Code Registration

Op-codes let you perform cluster-wide RPC without transmitting function pointers (which differ per binary). Register before fio_io_start.

SFUNC int fio_ipc_opcode_register(fio_ipc_opcode_s opcode);
/* named-arg macro */
#define fio_ipc_opcode_register(...) \
  fio_ipc_opcode_register((fio_ipc_opcode_s){ __VA_ARGS__ })

/** Returns registered op-code or NULL. */
SFUNC const fio_ipc_opcode_s *fio_ipc_opcode(uint32_t opcode);

Op-codes must be non-zero uint32_t values.
Values >= 0xFF000000 are reserved for internal use.
Pass call = NULL to unregister.
Must be called before fio_io_start() (not thread-safe after that).
Returns 0 on success, -1 on error.

fio_ipc_opcode_register(.opcode = OP_SYNC, .call = on_sync, .udata = ctx);

/* look up */
const fio_ipc_opcode_s *op = fio_ipc_opcode(OP_SYNC);

/* remove */
fio_ipc_opcode_register(.opcode = OP_SYNC, .call = NULL);

#IPC URL

The IPC socket URL is auto-generated at startup (unix://fio_tmp_<rand>.sock in $TMPDIR). Override before fio_io_start on the master:

/** Returns current IPC socket URL. */
SFUNC const char *fio_ipc_url(void);

/**
 * Sets IPC socket URL. Master only, before IO reactor starts.
 * Pass NULL to regenerate. 'X' / '#' characters in the URL are replaced with
 * random hex digits. Returns 0 on success, -1 on error.
 */
SFUNC int fio_ipc_url_set(const char *url);

fio_ipc_url_set("unix:///var/run/myapp-XXXX.sock"); /* X replaced randomly */
fio_ipc_url_set(NULL);  /* auto-generate */
printf("%s\n", fio_ipc_url());

#Multi-Machine Cluster (RPC)

/**
 * Listen for cluster peers on port. Auto-discovers peers via UDP broadcast.
 * Requires a shared SECRET environment variable (non-random secret).
 * Returns listener handle or NULL if disabled.
 */
SFUNC fio_io_listener_s *fio_ipc_cluster_listen(uint16_t port);

/** Manually connect to a cluster peer (usually unnecessary). */
SFUNC void fio_ipc_cluster_connect(const char *url);

/** Returns last port passed to fio_ipc_cluster_listen / connect, or 0. */
SFUNC uint16_t fio_ipc_cluster_port(void);

Call fio_ipc_cluster_listen from the master before or after fio_io_start. It opens a TCP listener and a UDP broadcast socket on the same port for automatic peer discovery.
Uses the environment's shared secret for encryption (ChaCha20-Poly1305, no forward secrecy). Set SECRET=<shared-key> across all instances.
All instances receive all cluster / broadcast messages.
fio_ipc_cluster_connect lets you manually reach peers on other subnets where UDP broadcast doesn't reach.

/* Typical setup */
fio_ipc_opcode_register(.opcode = OP_SYNC, .call = on_sync);
fio_ipc_cluster_listen(9000); /* returns NULL/no-op if SECRET is unset or random */
fio_io_start(4);

#Core Lifetime API

These are used internally by the delivery macros and are available for advanced use (e.g., constructing messages for local execution, or async message handling).

/** Create a message without sending it. */
SFUNC fio_ipc_s *fio_ipc_new(fio_ipc_args_s args);
#define fio_ipc_new(...) fio_ipc_new((fio_ipc_args_s){ __VA_ARGS__ })

/** Encrypt, route, and free the message. Takes ownership. */
SFUNC void fio_ipc_send(fio_ipc_s *ipc);

/** Encrypt and send directly to a specific IO. Takes ownership. */
SFUNC void fio_ipc_send_to(fio_io_s *to, fio_ipc_s *ipc);

/** Increment reference count. */
SFUNC fio_ipc_s *fio_ipc_dup(fio_ipc_s *msg);

/** Decrement reference count; destroy when zero. */
SFUNC void fio_ipc_free(fio_ipc_s *msg);

/** Release the msg->from IO reference (call if storing msg beyond callback). */
SFUNC void fio_ipc_detach(fio_ipc_s *msg);

/** Set a one-shot callback invoked when sending is complete.
    The callback receives ownership of the decrypted message and must free it. */
SFUNC void fio_ipc_after_send(fio_ipc_s *ipc,
                               void (*fn)(fio_ipc_s *, void *),
                               void *udata);

Typical async pattern:

void my_handler(fio_ipc_s *msg) {
  fio_ipc_s *ref = fio_ipc_dup(msg); /* keep alive past callback return */
  fio_ipc_detach(ref);               /* release the from-IO ref if unneeded */
  do_async_work(ref);                /* stored elsewhere */
}

void async_done(fio_ipc_s *msg) {
  process(msg->data, msg->len);
  fio_ipc_free(msg);
}

#Encryption

/** Encrypt a message in-place (idempotent). Called automatically by fio_ipc_send. */
SFUNC void fio_ipc_encrypt(fio_ipc_s *m);

/** Decrypt a message in-place. Returns 0 on success, non-zero on MAC failure. */
FIO_IFUNC int fio_ipc_decrypt(fio_ipc_s *m);

ChaCha20-Poly1305 AEAD. Key derived from the shared process secret plus the message's id and timestamp fields (which form the nonce). len, flags, and routing_flags are authenticated but unencrypted. Everything else (call/opcode, on_reply, on_done, udata, data) is encrypted.

Decryption failure logs a security warning and the connection is closed. Only processes sharing the same secret can communicate.

#Routing Flags

routing_flags is internal — do not edit manually. Available for inspection:

#define FIO_IPC_FLAG_ENCRYPTED  ((uint16_t)1 << 0) /* message is encrypted */
#define FIO_IPC_FLAG_DONE       ((uint16_t)1 << 1) /* final reply */
#define FIO_IPC_FLAG_OPCODE     ((uint16_t)1 << 2) /* route via op-code */
#define FIO_IPC_FLAG_WORKERS    ((uint16_t)1 << 3) /* deliver to workers too */
#define FIO_IPC_FLAG_CLUSTER    ((uint16_t)1 << 4) /* deliver to remote machines */
#define FIO_IPC_FLAG_REPLY      ((uint16_t)1 << 5) /* this is a reply */
#define FIO_IPC_FLAG_PING       ((uint16_t)1 << 6) /* internal keepalive */

/** Test a flag: returns flag value if set, 0 otherwise. */
#define FIO_IPC_FLAG_TEST(msg, flag) (((msg)->routing_flags & (flag)) == (flag))

#Keepalive

IPC and cluster connections use an automatic ping/pong protocol. On the first timeout the system sends a ping; if no pong arrives before the next timeout the connection is closed. Ping frames are never delivered to user callbacks.

Default timeouts (set in the protocol initializer):

Connection type	Timeout
Local IPC (master ↔ worker)	~6 minutes
Cluster RPC (machine ↔ machine)	~55 seconds

#Memory Management

Messages are reference-counted (fio_ipc_dup / fio_ipc_free).
Callbacks receive a message pointer valid for the duration of the callback. The system frees it automatically after the callback returns.
Call fio_ipc_dup to keep a message alive beyond the callback, then fio_ipc_free when done.
Data passed via FIO_IPC_DATA is copied into the message immediately — source buffers may be stack-allocated or freed right after the call.

#Thread Safety

fio_ipc_call, fio_ipc_local, fio_ipc_cluster, fio_ipc_broadcast, and fio_ipc_reply are safe to call from any thread.
fio_ipc_dup / fio_ipc_free are thread-safe.
fio_ipc_opcode_register is not thread-safe after fio_io_start.
fio_ipc_url_set must be called on the master before fio_io_start.