class-wcmq-reaper.php 9.4 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275
  1. <?php
  2. /**
  3. * Recovers stranded queue rows.
  4. *
  5. * Enforces the invariant that makes the whole design durable:
  6. *
  7. * Every row in `pending` has a scheduled Action Scheduler action.
  8. *
  9. * That invariant, not the enqueue rollback, is what prevents mail loss —
  10. * rollback cannot run after a PHP fatal.
  11. *
  12. * @package studiou-wc-mail-queue
  13. * @copyright 2026 QUADARAX
  14. * @author Dalibor Votruba <dvotruba@quadarax.com>
  15. * @license GPL-2.0-or-later
  16. */
  17. defined( 'ABSPATH' ) || exit;
  18. /**
  19. * Class Studiou_WCMQ_Reaper
  20. */
  21. class Studiou_WCMQ_Reaper {
  22. const HOOK = 'studiou_wcmq_reap';
  23. const INTERVAL = 300;
  24. const ORPHAN_GRACE = 300;
  25. const BATCH = 200;
  26. const ADMIN_THROTTLE = 'studiou_wcmq_reap_throttle';
  27. /**
  28. * Guard against running more than once per request.
  29. *
  30. * @var bool
  31. */
  32. private static $ran_this_request = false;
  33. /**
  34. * Register hooks.
  35. */
  36. public static function init() {
  37. add_action( self::HOOK, array( __CLASS__, 'run' ) );
  38. add_action( 'admin_init', array( __CLASS__, 'maybe_run_from_admin' ) );
  39. }
  40. /**
  41. * Schedule the recurring sweep, exactly once.
  42. *
  43. * Called from `init`, which runs on every request — so a bare
  44. * as_schedule_recurring_action() here would register a fresh action each time.
  45. */
  46. public static function schedule() {
  47. if ( ! function_exists( 'as_next_scheduled_action' ) || ! function_exists( 'as_schedule_recurring_action' ) ) {
  48. return;
  49. }
  50. if ( as_next_scheduled_action( self::HOOK, array(), Studiou_WCMQ_Queue::GROUP ) ) {
  51. return;
  52. }
  53. as_schedule_recurring_action(
  54. time() + self::INTERVAL,
  55. self::INTERVAL,
  56. self::HOOK,
  57. array(),
  58. Studiou_WCMQ_Queue::GROUP,
  59. true
  60. );
  61. }
  62. /**
  63. * Heal from wp-admin, throttled.
  64. *
  65. * This is the trigger that still works when WP-Cron is broken — which is a
  66. * plausible root cause of the wedge in the first place.
  67. */
  68. public static function maybe_run_from_admin() {
  69. if ( get_transient( self::ADMIN_THROTTLE ) ) {
  70. return;
  71. }
  72. set_transient( self::ADMIN_THROTTLE, 1, 60 );
  73. self::run();
  74. }
  75. /**
  76. * Sweep both classes of stranded row.
  77. */
  78. public static function run() {
  79. if ( self::$ran_this_request ) {
  80. return;
  81. }
  82. self::$ran_this_request = true;
  83. if ( ! function_exists( 'as_has_scheduled_action' ) || ! function_exists( 'as_schedule_single_action' ) ) {
  84. return;
  85. }
  86. global $wpdb;
  87. $table = Studiou_WCMQ_DB::queue_table();
  88. $now = time();
  89. $timeout = (int) Studiou_WCMQ_Settings::get( 'claim_timeout' );
  90. // (a) `sending` rows whose worker died. Without this, one PHP fatal wedges
  91. // the queue forever: retention sweeps only sent/failed, the state machine
  92. // never reaches `disabled`, and the slot stays occupied.
  93. // phpcs:ignore WordPress.DB.PreparedSQL.InterpolatedNotPrepared, WordPress.DB.DirectDatabaseQuery
  94. $stale = $wpdb->get_results(
  95. $wpdb->prepare(
  96. "SELECT id, reclaims FROM {$table}
  97. WHERE state = %s AND claimed_at IS NOT NULL AND claimed_at < %d
  98. LIMIT %d",
  99. Studiou_WCMQ_DB::STATE_SENDING,
  100. $now - $timeout,
  101. self::BATCH
  102. )
  103. );
  104. if ( is_array( $stale ) ) {
  105. foreach ( $stale as $row ) {
  106. self::recover_crashed( (int) $row->id, (int) $row->reclaims );
  107. }
  108. }
  109. // (b) `pending` rows with no action — the §2.1 invariant violated. Arises
  110. // when a fatal lands between the INSERT and as_schedule_single_action(),
  111. // or when a retry's re-schedule failed. More likely than (a), because the
  112. // enqueue window is on the hot path of every bulk.
  113. // phpcs:ignore WordPress.DB.PreparedSQL.InterpolatedNotPrepared, WordPress.DB.DirectDatabaseQuery
  114. $orphans = $wpdb->get_results(
  115. $wpdb->prepare(
  116. "SELECT id FROM {$table}
  117. WHERE state = %s AND scheduled_at < %d
  118. LIMIT %d",
  119. Studiou_WCMQ_DB::STATE_PENDING,
  120. $now - self::ORPHAN_GRACE,
  121. self::BATCH
  122. )
  123. );
  124. if ( is_array( $orphans ) ) {
  125. foreach ( $orphans as $row ) {
  126. // The cast is load-bearing. Action Scheduler matches args by
  127. // hashing json_encode($args); the action was scheduled with an int
  128. // so its hash is of "[123]", while $wpdb hands back the string
  129. // "123" whose hash is of "[\"123\"]". Without the cast this returns
  130. // false for a perfectly healthy row and we duplicate its action.
  131. $row_id = (int) $row->id;
  132. if ( as_has_scheduled_action( Studiou_WCMQ_Queue::HOOK, array( $row_id ), Studiou_WCMQ_Queue::GROUP ) ) {
  133. continue;
  134. }
  135. self::repair_orphan( $row_id );
  136. }
  137. }
  138. }
  139. /**
  140. * A `pending` row lost its Action Scheduler action. Give it a new one.
  141. *
  142. * This is NOT a crashed worker and must not touch `reclaims`. The row was
  143. * never claimed, never attempted, and nothing about it is suspect — the
  144. * action went missing (a fatal between INSERT and schedule, a failed retry
  145. * re-schedule, an admin pruning the AS tables). Charging it to the crash
  146. * budget would eventually fail perfectly deliverable mail with
  147. * "worker repeatedly crashed", which is both wrong and unactionable.
  148. *
  149. * There is no runaway risk: the re-slot moves scheduled_at into the future,
  150. * so the next sweep will not see the row until the grace window elapses again.
  151. *
  152. * @param int $row_id Row id.
  153. */
  154. private static function repair_orphan( $row_id ) {
  155. if ( ! self::reslot_and_schedule( $row_id, array(), Studiou_WCMQ_DB::STATE_PENDING ) ) {
  156. return;
  157. }
  158. Studiou_WCMQ_DB::log( 'warning', 'Pending row had no scheduled action; rescheduled.', $row_id );
  159. }
  160. /**
  161. * A `sending` row whose worker died. Charge it to the crash budget.
  162. *
  163. * @param int $row_id Row id.
  164. * @param int $reclaims Reclaims so far.
  165. */
  166. private static function recover_crashed( $row_id, $reclaims ) {
  167. $max_reclaims = (int) Studiou_WCMQ_Settings::get( 'max_reclaims' );
  168. $reclaims = $reclaims + 1;
  169. // `reclaims`, not `attempts`. A worker killed by max_execution_time
  170. // strands whichever row happened to be running when the cumulative batch
  171. // budget expired — that is arbitrary, not a property of the row. Counting
  172. // crashes as send attempts permanently fails healthy mail that was never
  173. // once actually attempted.
  174. if ( $reclaims >= $max_reclaims ) {
  175. // Guarded on `sending`: if another sweep in a parallel process already
  176. // handled this row, we must not double-count or re-fail it.
  177. $affected = Studiou_WCMQ_DB::update_row_if_state(
  178. $row_id,
  179. array(
  180. 'state' => Studiou_WCMQ_DB::STATE_FAILED,
  181. 'claimed_at' => null,
  182. 'reclaims' => $reclaims,
  183. 'last_error' => 'worker repeatedly crashed while handling this mail',
  184. ),
  185. Studiou_WCMQ_DB::STATE_SENDING
  186. );
  187. if ( 1 === (int) $affected ) {
  188. // Terminal state — the order must carry a note, or it records
  189. // nothing at all about a mail the customer never received.
  190. $row = Studiou_WCMQ_DB::get_row( $row_id );
  191. if ( $row ) {
  192. Studiou_WCMQ_Notifier::annotate( $row, false, 'worker repeatedly crashed while handling this mail' );
  193. }
  194. Studiou_WCMQ_DB::log( 'error', sprintf( 'Row failed after %d crash recoveries.', $reclaims ), $row_id );
  195. }
  196. return;
  197. }
  198. if ( ! self::reslot_and_schedule( $row_id, array( 'reclaims' => $reclaims ), Studiou_WCMQ_DB::STATE_SENDING ) ) {
  199. return;
  200. }
  201. Studiou_WCMQ_DB::log(
  202. 'warning',
  203. sprintf( 'Recovered row from a dead worker; rescheduled. Crash recovery %d/%d.', $reclaims, $max_reclaims ),
  204. $row_id
  205. );
  206. }
  207. /**
  208. * Return a row to `pending` on a fresh FUTURE slot and schedule it.
  209. *
  210. * Re-slotting is not optional. A row restored at its original scheduled_at
  211. * now lies in the past, so Action Scheduler fires it on the next pass — and
  212. * thirty recovered rows would all go out at once. That is the burst this
  213. * plugin exists to prevent, delivered by its own recovery path.
  214. *
  215. * @param int $row_id Row id.
  216. * @param array $extra Extra columns to set (e.g. reclaims).
  217. * @param string $expected_state State the row must currently be in.
  218. * @return bool True when the row is pending with a live action.
  219. */
  220. private static function reslot_and_schedule( $row_id, array $extra, $expected_state ) {
  221. list( $slot, $affected ) = Studiou_WCMQ_Queue::with_slot_lock(
  222. time(),
  223. function ( $slot ) use ( $row_id, $extra, $expected_state ) {
  224. $data = array_merge(
  225. $extra,
  226. array(
  227. 'state' => Studiou_WCMQ_DB::STATE_PENDING,
  228. 'claimed_at' => null,
  229. 'scheduled_at' => $slot,
  230. )
  231. );
  232. return Studiou_WCMQ_DB::update_row_if_state( $row_id, $data, $expected_state );
  233. }
  234. );
  235. // If the UPDATE changed no row — a DB error, or the row moved out from
  236. // under us — do NOT schedule. Scheduling anyway is how the reaper loops
  237. // forever: the row stays `sending` with a stale claimed_at and its
  238. // reclaims never persists, so the next sweep computes reclaims=0+1 again,
  239. // schedules another dead action, and never trips max_reclaims. Leave the
  240. // row for the next sweep, which will re-read its true reclaims from the DB.
  241. if ( 1 !== (int) $affected ) {
  242. Studiou_WCMQ_DB::log( 'error', 'Reaper could not move row to pending; not scheduling. Next sweep will retry.', $row_id );
  243. return false;
  244. }
  245. // Guarded exactly as enqueue and retry are. An unguarded call here would
  246. // re-create the orphan `pending` row the reaper exists to repair.
  247. $action_id = as_schedule_single_action( $slot, Studiou_WCMQ_Queue::HOOK, array( (int) $row_id ), Studiou_WCMQ_Queue::GROUP );
  248. if ( ! $action_id ) {
  249. Studiou_WCMQ_DB::log( 'error', 'Reaper could not reschedule; next sweep will retry.', $row_id );
  250. return false;
  251. }
  252. return true;
  253. }
  254. }