1use std::cell::{Cell, RefCell};
8use std::collections::{BTreeMap, HashMap, HashSet, VecDeque};
9use std::panic::{catch_unwind, resume_unwind, AssertUnwindSafe};
10use std::rc::Rc;
11
12use super::bridge_protobuf::{
13 CanonicalWireEdgeFrame, CanonicalWireEdgeKind, WireBridgeProtobufDataBody,
14};
15use crate::ctx::{Ctx, DepTerminal, WaveData};
16use crate::graph::{Graph, GraphNodeOpts, TopologyGroup, TopologyGroupOptions};
17use crate::identity::{canonical_tuple_key, compound_tuple_key};
18use crate::node::{Core, Node, NodeOpts};
19use crate::protocol::{AnyValue, Message};
20use crate::resilience::RetryPolicy;
21use crate::versioning::NodeVersion;
22
23#[derive(Debug, Clone, Copy, PartialEq, Eq)]
24pub enum WireBridgeEnvelopeType {
26 Start,
28 Data,
30 Ack,
32 Nack,
34 Status,
36 Error,
38 Close,
40}
41
42#[derive(Debug, Clone, PartialEq, Eq)]
43pub struct WireBridgeMetadata {
45 pub seq: u64,
47 pub cursor: u64,
49 pub idempotency_key: String,
51 pub attempt: u32,
53 pub max_attempts: u32,
55 pub timestamp_ms: Option<u64>,
57 pub ack_for_seq: Option<u64>,
59 pub request_id: Option<String>,
61}
62
63#[derive(Debug, Clone, PartialEq, Eq)]
64pub enum WireBridgePayload<T> {
66 Data(T),
68 Error(String),
70 Status(String),
72 Close {
74 reason: Option<String>,
76 },
77}
78
79#[derive(Debug, Clone, PartialEq, Eq)]
80pub struct WireBridgeEnvelope<T> {
82 pub session_id: String,
84 pub envelope_type: WireBridgeEnvelopeType,
86 pub payload: Option<WireBridgePayload<T>>,
88 pub metadata: WireBridgeMetadata,
90}
91
92#[derive(Debug, Clone, PartialEq, Eq)]
93pub enum WireBridgeEnvelopeError {
95 EmptySessionId,
97 ZeroSeq,
99 EmptyIdempotencyKey,
101 ZeroAttempt,
103 MaxAttemptsBeforeAttempt,
105 ZeroAckForSeq,
107 MissingAckForSeq,
109 MissingPayload,
111 UnexpectedPayload,
113 PayloadTypeMismatch,
115}
116
117impl std::fmt::Display for WireBridgeEnvelopeError {
118 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
119 f.write_str(match self {
120 Self::EmptySessionId => "wireBridgeEnvelope: session_id must be non-empty",
121 Self::ZeroSeq => "wireBridgeEnvelope: seq must be positive",
122 Self::EmptyIdempotencyKey => "wireBridgeEnvelope: idempotency_key must be non-empty",
123 Self::ZeroAttempt => "wireBridgeEnvelope: attempt must be positive",
124 Self::MaxAttemptsBeforeAttempt => "wireBridgeEnvelope: max_attempts must be >= attempt",
125 Self::ZeroAckForSeq => "wireBridgeEnvelope: ack_for_seq must be positive",
126 Self::MissingAckForSeq => "wireBridgeEnvelope: ack/nack requires ack_for_seq",
127 Self::MissingPayload => "wireBridgeEnvelope: envelope type requires a payload",
128 Self::UnexpectedPayload => "wireBridgeEnvelope: envelope type must not carry a payload",
129 Self::PayloadTypeMismatch => {
130 "wireBridgeEnvelope: payload kind does not match envelope type"
131 }
132 })
133 }
134}
135
136impl std::error::Error for WireBridgeEnvelopeError {}
137
138#[derive(Debug, Clone)]
139pub struct WireBridgeEnvelopeInput<T> {
141 pub session_id: String,
143 pub envelope_type: WireBridgeEnvelopeType,
145 pub seq: u64,
147 pub cursor: u64,
149 pub payload: Option<WireBridgePayload<T>>,
151 pub idempotency_key: Option<String>,
153 pub attempt: u32,
155 pub max_attempts: u32,
157 pub timestamp_ms: Option<u64>,
159 pub ack_for_seq: Option<u64>,
161 pub request_id: Option<String>,
163}
164
165pub fn wire_bridge_idempotency_key(session_id: &str, seq: u64) -> String {
167 canonical_tuple_key(&[session_id, &seq.to_string()])
168}
169
170pub fn wire_bridge_envelope<T>(
172 input: WireBridgeEnvelopeInput<T>,
173) -> Result<WireBridgeEnvelope<T>, WireBridgeEnvelopeError> {
174 if input.session_id.is_empty() {
175 return Err(WireBridgeEnvelopeError::EmptySessionId);
176 }
177 if input.seq == 0 {
178 return Err(WireBridgeEnvelopeError::ZeroSeq);
179 }
180 if input.attempt == 0 {
181 return Err(WireBridgeEnvelopeError::ZeroAttempt);
182 }
183 if input.max_attempts < input.attempt {
184 return Err(WireBridgeEnvelopeError::MaxAttemptsBeforeAttempt);
185 }
186 if matches!(
187 input.envelope_type,
188 WireBridgeEnvelopeType::Ack | WireBridgeEnvelopeType::Nack
189 ) && input.ack_for_seq.is_none()
190 {
191 return Err(WireBridgeEnvelopeError::MissingAckForSeq);
192 }
193 if input.ack_for_seq == Some(0) {
194 return Err(WireBridgeEnvelopeError::ZeroAckForSeq);
195 }
196 validate_payload_for_type(input.envelope_type, &input.payload)?;
197 let idempotency_key = input
198 .idempotency_key
199 .unwrap_or_else(|| wire_bridge_idempotency_key(&input.session_id, input.seq));
200 if idempotency_key.is_empty() {
201 return Err(WireBridgeEnvelopeError::EmptyIdempotencyKey);
202 }
203 Ok(WireBridgeEnvelope {
204 session_id: input.session_id,
205 envelope_type: input.envelope_type,
206 payload: input.payload,
207 metadata: WireBridgeMetadata {
208 seq: input.seq,
209 cursor: input.cursor,
210 idempotency_key,
211 attempt: input.attempt,
212 max_attempts: input.max_attempts,
213 timestamp_ms: input.timestamp_ms,
214 ack_for_seq: input.ack_for_seq,
215 request_id: input.request_id,
216 },
217 })
218}
219
220#[derive(Debug, Clone, PartialEq, Eq)]
221pub enum WireBridgeCommand<T> {
223 Start {
225 idempotency_key: Option<String>,
227 request_id: Option<String>,
229 },
230 Send {
232 payload: T,
234 idempotency_key: Option<String>,
236 request_id: Option<String>,
238 },
239 Ack {
241 ack_for_seq: u64,
243 idempotency_key: Option<String>,
245 request_id: Option<String>,
247 },
248 Nack {
250 ack_for_seq: u64,
252 error: String,
254 idempotency_key: Option<String>,
256 request_id: Option<String>,
258 },
259 Close {
261 reason: Option<String>,
263 idempotency_key: Option<String>,
265 },
266 AckTimeout {
268 seq: u64,
270 attempt: u32,
272 observed_at_ms: Option<u64>,
274 },
275}
276
277#[derive(Debug, Clone, PartialEq, Eq)]
278pub enum WireBridgeReceipt {
280 Ack,
282 Nack,
284}
285
286#[derive(Debug, Clone, PartialEq, Eq)]
287pub enum WireBridgeEvent<TOutbound, TInbound> {
289 Outbound {
291 envelope: WireBridgeEnvelope<TOutbound>,
293 },
294 Inbound {
296 envelope: WireBridgeEnvelope<TInbound>,
298 },
299 Ack {
301 ack_for_seq: u64,
303 envelope: WireBridgeEnvelope<TInbound>,
305 outbound: WireBridgeEnvelope<TOutbound>,
307 },
308 Nack {
310 ack_for_seq: u64,
312 envelope: WireBridgeEnvelope<TInbound>,
314 outbound: WireBridgeEnvelope<TOutbound>,
316 error: String,
318 },
319 Timeout {
321 seq: u64,
323 attempt: u32,
325 },
326 Retry {
328 seq: u64,
330 attempt: u32,
332 delay_ms: u64,
334 error: String,
336 },
337 Exhausted {
339 seq: u64,
341 attempt: u32,
343 error: String,
345 },
346 Cursor {
348 cursor: u64,
350 },
351 Duplicate {
353 seq: u64,
355 cursor: u64,
357 },
358 OutOfOrder {
360 seq: u64,
362 expected: u64,
364 },
365 SessionMismatch {
367 expected: String,
369 actual: String,
371 },
372 LateReceipt {
374 receipt: WireBridgeReceipt,
376 ack_for_seq: u64,
378 },
379 Invalid {
381 error: String,
383 },
384}
385
386#[derive(Debug, Clone, PartialEq, Eq)]
387pub struct WireBridgeAck<TInbound> {
389 pub ack_for_seq: u64,
391 pub envelope: WireBridgeEnvelope<TInbound>,
393}
394
395#[derive(Debug, Clone, PartialEq, Eq)]
396pub struct WireBridgeNack<TInbound> {
398 pub ack_for_seq: u64,
400 pub envelope: WireBridgeEnvelope<TInbound>,
402 pub error: String,
404}
405
406#[derive(Debug, Clone, PartialEq, Eq)]
407pub struct WireBridgeAttempt {
409 pub seq: u64,
411 pub attempt: u32,
413 pub max_attempts: u32,
415}
416
417#[derive(Debug, Clone, Copy, PartialEq, Eq)]
418pub enum WireBridgeStatusState {
420 Idle,
422 Started,
424 Open,
426 Waiting,
428 Closed,
430 Errored,
432 Exhausted,
434}
435
436#[derive(Debug, Clone, PartialEq, Eq)]
437pub struct WireBridgeStatus {
439 pub session_id: String,
441 pub state: WireBridgeStatusState,
443 pub cursor: u64,
445 pub next_seq: u64,
447 pub pending: u64,
449 pub attempts: u64,
451 pub acked: u64,
453 pub nacked: u64,
455 pub errors: u64,
457 pub last_seq: Option<u64>,
459 pub last_delay_ms: Option<u64>,
461}
462
463#[derive(Clone)]
464pub struct WireBridgeOptions {
466 pub name: Option<String>,
468 pub session_id: String,
470 pub retry: RetryPolicy,
472 pub now_ms: Option<Rc<dyn Fn() -> u64>>,
474}
475
476impl WireBridgeOptions {
477 pub fn new(session_id: impl Into<String>) -> Self {
479 Self {
480 name: None,
481 session_id: session_id.into(),
482 retry: RetryPolicy::default(),
483 now_ms: None,
484 }
485 }
486
487 pub fn named(session_id: impl Into<String>, name: impl Into<String>) -> Self {
489 Self {
490 name: Some(name.into()),
491 ..Self::new(session_id)
492 }
493 }
494}
495
496pub struct WireBridgeBundle<TOutbound: Clone + 'static, TInbound: Clone + 'static> {
498 pub command: Node<WireBridgeCommand<TOutbound>>,
500 pub outbound: Node<WireBridgeEnvelope<TOutbound>>,
502 pub inbound: WireBridgeInbound<TInbound>,
504 pub events: Node<WireBridgeEvent<TOutbound, TInbound>>,
506 pub acks: Node<WireBridgeAck<TInbound>>,
508 pub nacks: Node<WireBridgeNack<TInbound>>,
510 pub status: Node<WireBridgeStatus>,
512 pub errors: Node<String>,
514 pub cursor: Node<u64>,
516 pub attempts: Node<WireBridgeAttempt>,
518 command_sources: Rc<RefCell<Vec<Core>>>,
519 inbound_sources: Rc<RefCell<Vec<Core>>>,
520}
521
522impl<TOutbound: Clone + 'static, TInbound: Clone + 'static> WireBridgeBundle<TOutbound, TInbound> {
523 pub fn start(&self) {
525 self.command.set(WireBridgeCommand::Start {
526 idempotency_key: None,
527 request_id: None,
528 });
529 }
530
531 pub fn send(
533 &self,
534 payload: TOutbound,
535 idempotency_key: Option<String>,
536 request_id: Option<String>,
537 ) {
538 self.command.set(WireBridgeCommand::Send {
539 payload,
540 idempotency_key,
541 request_id,
542 });
543 }
544
545 pub fn ack(
547 &self,
548 ack_for_seq: u64,
549 idempotency_key: Option<String>,
550 request_id: Option<String>,
551 ) {
552 self.command.set(WireBridgeCommand::Ack {
553 ack_for_seq,
554 idempotency_key,
555 request_id,
556 });
557 }
558
559 pub fn nack(
561 &self,
562 ack_for_seq: u64,
563 error: impl Into<String>,
564 idempotency_key: Option<String>,
565 request_id: Option<String>,
566 ) {
567 self.command.set(WireBridgeCommand::Nack {
568 ack_for_seq,
569 error: error.into(),
570 idempotency_key,
571 request_id,
572 });
573 }
574
575 pub fn close(&self, reason: Option<String>, idempotency_key: Option<String>) {
577 self.command.set(WireBridgeCommand::Close {
578 reason,
579 idempotency_key,
580 });
581 }
582
583 #[doc(hidden)]
584 pub fn attach_command_source_for_native(&self, source: Core) {
585 attach_wire_bridge_command_source(self, source);
586 }
587
588 #[doc(hidden)]
589 pub fn detach_command_source_for_native(&self, source: Core) {
590 detach_wire_bridge_command_source(&self.command, &self.command_sources, source);
591 }
592
593 #[doc(hidden)]
594 pub fn attach_inbound_source_for_native(&self, source: Core) {
595 attach_wire_bridge_inbound_source(self, source);
596 }
597
598 #[doc(hidden)]
599 pub fn detach_inbound_source_for_native(&self, source: Core) {
600 detach_wire_bridge_inbound_source(&self.inbound, &self.inbound_sources, source);
601 }
602}
603
604#[derive(Debug, Clone, PartialEq, Eq)]
605pub struct RemoteCallRequest<T> {
607 pub operation: String,
609 pub request_id: String,
611 pub payload: T,
613}
614
615impl<T> RemoteCallRequest<T> {
616 pub fn new(operation: impl Into<String>, request_id: impl Into<String>, payload: T) -> Self {
618 Self {
619 operation: operation.into(),
620 request_id: request_id.into(),
621 payload,
622 }
623 }
624}
625
626#[derive(Debug, Clone, PartialEq, Eq)]
627pub enum RemoteCallResponse<T> {
629 Result {
631 operation: String,
633 request_id: String,
635 payload: T,
637 },
638 Error {
640 operation: String,
642 request_id: String,
644 error: String,
646 },
647 Status {
649 operation: String,
651 request_id: String,
653 status: String,
655 },
656}
657
658#[derive(Debug, Clone, PartialEq, Eq)]
659pub struct RemoteCallResult<T> {
661 pub operation: String,
663 pub request_id: String,
665 pub payload: T,
667}
668
669#[derive(Debug, Clone, PartialEq, Eq)]
670pub struct RemoteCallError {
672 pub operation: Option<String>,
674 pub request_id: Option<String>,
676 pub error: String,
678}
679
680#[derive(Debug, Clone, Copy, PartialEq, Eq)]
681pub enum RemoteCallStatusState {
683 Idle,
685 Requested,
687 Responded,
689 Errored,
691 TimedOut,
693 BridgeErrored,
695}
696
697#[derive(Debug, Clone, PartialEq, Eq)]
698pub struct RemoteCallStatus {
700 pub state: RemoteCallStatusState,
702 pub operation: Option<String>,
704 pub request_id: Option<String>,
706 pub pending: usize,
708 pub completed: u64,
710 pub errors: u64,
712 pub timeouts: u64,
714}
715
716#[derive(Debug, Clone, PartialEq, Eq)]
717pub struct RemoteCallTimeout {
719 pub operation: Option<String>,
721 pub request_id: String,
723 pub error: String,
725}
726
727#[derive(Clone)]
728pub struct RemoteCallOptions {
730 pub name: String,
732}
733
734impl Default for RemoteCallOptions {
735 fn default() -> Self {
736 Self {
737 name: "remoteCall".to_owned(),
738 }
739 }
740}
741
742impl RemoteCallOptions {
743 pub fn named(name: impl Into<String>) -> Self {
745 Self { name: name.into() }
746 }
747}
748
749pub struct RemoteCallBundle<TRequest: Clone + 'static, TResponse: Clone + 'static> {
751 bridge_command: Node<WireBridgeCommand<RemoteCallRequest<TRequest>>>,
752 pub responses: Node<RemoteCallResponse<TResponse>>,
754 pub results: Node<RemoteCallResult<TResponse>>,
756 pub status: Node<RemoteCallStatus>,
758 pub errors: Node<RemoteCallError>,
760 pub timeouts: Node<RemoteCallTimeout>,
762}
763
764impl<TRequest: Clone + 'static, TResponse: Clone + 'static> RemoteCallBundle<TRequest, TResponse> {
765 pub fn call(
767 &self,
768 operation: impl Into<String>,
769 request_id: impl Into<String>,
770 payload: TRequest,
771 ) -> RemoteCallRequest<TRequest> {
772 self.call_with_options(operation, request_id, payload, None)
773 }
774
775 pub fn call_with_options(
777 &self,
778 operation: impl Into<String>,
779 request_id: impl Into<String>,
780 payload: TRequest,
781 idempotency_key: Option<String>,
782 ) -> RemoteCallRequest<TRequest> {
783 let request = RemoteCallRequest::new(operation, request_id, payload);
784 assert!(
785 !request.operation.is_empty(),
786 "remote_call: operation must be non-empty"
787 );
788 assert!(
789 !request.request_id.is_empty(),
790 "remote_call: request_id must be non-empty"
791 );
792 self.bridge_command.set(WireBridgeCommand::Send {
793 payload: request.clone(),
794 idempotency_key,
795 request_id: Some(request.request_id.clone()),
796 });
797 request
798 }
799
800 pub fn timeout(
802 &self,
803 request_id: impl Into<String>,
804 operation: Option<String>,
805 error: impl Into<String>,
806 ) -> RemoteCallTimeout {
807 let timeout = RemoteCallTimeout {
808 operation,
809 request_id: request_id.into(),
810 error: error.into(),
811 };
812 assert!(
813 !timeout.request_id.is_empty(),
814 "remote_call: timeout request_id must be non-empty"
815 );
816 assert!(
817 !timeout.error.is_empty(),
818 "remote_call: timeout error must be non-empty"
819 );
820 self.timeouts.set(timeout.clone());
821 timeout
822 }
823}
824
825#[derive(Debug, Clone, PartialEq, Eq)]
826pub enum RemoteResponderEvent<TRequest, TResponse> {
828 Request {
830 request: RemoteCallRequest<TRequest>,
832 seq: u64,
834 },
835 Response {
837 request_id: String,
839 operation: String,
841 command: WireBridgeCommand<RemoteCallResponse<TResponse>>,
843 },
844 Rejected {
846 request_id: Option<String>,
848 operation: Option<String>,
850 error: String,
852 command: Option<WireBridgeCommand<RemoteCallResponse<TResponse>>>,
854 },
855 Invalid {
857 error: String,
859 },
860}
861
862#[derive(Debug, Clone, Copy, PartialEq, Eq)]
863pub enum RemoteResponderStatusState {
865 Idle,
867 Responded,
869 Rejected,
871 Errored,
873}
874
875#[derive(Debug, Clone, PartialEq, Eq)]
876pub struct RemoteResponderStatus {
878 pub state: RemoteResponderStatusState,
880 pub operation: Option<String>,
882 pub request_id: Option<String>,
884 pub handled: u64,
886 pub rejected: u64,
888 pub errors: u64,
890}
891
892pub type RemoteResponderHandler<TRequest, TResponse> =
894 Rc<dyn Fn(&RemoteCallRequest<TRequest>) -> Result<TResponse, String>>;
895
896#[derive(Clone)]
897pub struct RemoteResponderHandlerDefinition<TRequest, TResponse> {
899 pub operation: String,
901 pub handle: RemoteResponderHandler<TRequest, TResponse>,
903}
904
905pub fn remote_responder_handler<TRequest, TResponse>(
907 operation: impl Into<String>,
908 handle: impl Fn(&RemoteCallRequest<TRequest>) -> Result<TResponse, String> + 'static,
909) -> RemoteResponderHandlerDefinition<TRequest, TResponse> {
910 let operation = operation.into();
911 assert!(
912 !operation.is_empty(),
913 "remote_responder_handler: operation must be non-empty"
914 );
915 RemoteResponderHandlerDefinition {
916 operation,
917 handle: Rc::new(handle),
918 }
919}
920
921#[derive(Clone)]
922pub struct RemoteResponderOptions<TRequest, TResponse> {
924 pub name: String,
926 pub handlers: Vec<RemoteResponderHandlerDefinition<TRequest, TResponse>>,
928 pub reject_unknown: bool,
930}
931
932impl<TRequest, TResponse> Default for RemoteResponderOptions<TRequest, TResponse> {
933 fn default() -> Self {
934 Self {
935 name: "remoteResponder".to_owned(),
936 handlers: Vec::new(),
937 reject_unknown: false,
938 }
939 }
940}
941
942impl<TRequest, TResponse> RemoteResponderOptions<TRequest, TResponse> {
943 pub fn named(name: impl Into<String>) -> Self {
945 Self {
946 name: name.into(),
947 ..Self::default()
948 }
949 }
950
951 pub fn with_handlers(
953 mut self,
954 handlers: Vec<RemoteResponderHandlerDefinition<TRequest, TResponse>>,
955 ) -> Self {
956 self.handlers = handlers;
957 self
958 }
959
960 pub fn with_reject_unknown(mut self, reject: bool) -> Self {
962 self.reject_unknown = reject;
963 self
964 }
965}
966
967pub struct RemoteResponderBundle<TRequest: Clone + 'static, TResponse: Clone + 'static> {
969 pub events: Node<RemoteResponderEvent<TRequest, TResponse>>,
971 pub response_commands: Node<WireBridgeCommand<RemoteCallResponse<TResponse>>>,
973 pub requests: Node<RemoteCallRequest<TRequest>>,
975 pub status: Node<RemoteResponderStatus>,
977 pub errors: Node<RemoteCallError>,
979 graph: Graph,
980 bridge_command: Node<WireBridgeCommand<RemoteCallResponse<TResponse>>>,
981 command_sources: Rc<RefCell<Vec<Core>>>,
982 released: Cell<bool>,
983}
984
985impl<TRequest: Clone + 'static, TResponse: Clone + 'static>
986 RemoteResponderBundle<TRequest, TResponse>
987{
988 pub fn release(&self) {
990 if self.released.get() {
991 return;
992 }
993 detach_wire_bridge_command_source(
994 &self.bridge_command,
995 &self.command_sources,
996 self.response_commands.erased(),
997 );
998 let release = catch_unwind(AssertUnwindSafe(|| {
999 self.graph.release_nodes(
1000 &[
1001 self.events.erased(),
1002 self.response_commands.erased(),
1003 self.requests.erased(),
1004 self.status.erased(),
1005 self.errors.erased(),
1006 ],
1007 "remote_responder release",
1008 );
1009 }));
1010 if let Err(panic) = release {
1011 attach_wire_bridge_command_source_parts(
1012 &self.bridge_command,
1013 &self.command_sources,
1014 self.response_commands.erased(),
1015 );
1016 resume_unwind(panic);
1017 }
1018 self.released.set(true);
1019 }
1020}
1021
1022#[derive(Debug, Clone, PartialEq, Eq)]
1023pub enum WireBridgeIngress<T> {
1025 Envelope(WireBridgeEnvelope<T>),
1027 Invalid(String),
1029}
1030
1031#[derive(Clone)]
1032pub struct WireBridgeInbound<T: Clone + 'static> {
1034 node: Node<WireBridgeIngress<T>>,
1035 session_id: String,
1036}
1037
1038impl<T: Clone + 'static> WireBridgeInbound<T> {
1039 pub fn session_id(&self) -> &str {
1041 &self.session_id
1042 }
1043
1044 pub fn down(&self, msgs: Vec<Message<AnyValue>>) {
1046 for msg in msgs {
1047 self.node.down(vec![self.guard_msg(msg)]);
1048 }
1049 }
1050
1051 fn guard_msg(&self, msg: Message<AnyValue>) -> Message<AnyValue> {
1052 match msg {
1053 Message::Data(value) => match value.downcast::<WireBridgeEnvelope<T>>() {
1054 Ok(envelope) => data_msg(WireBridgeIngress::Envelope((*envelope).clone())),
1055 Err(value) => match value.downcast::<WireBridgeIngress<T>>() {
1056 Ok(ingress) => data_msg((*ingress).clone()),
1057 Err(_) => data_msg(WireBridgeIngress::<T>::Invalid(
1058 "wireBridge: inbound DATA must carry a wire bridge envelope".to_owned(),
1059 )),
1060 },
1061 },
1062 Message::Error(error) => data_msg(WireBridgeIngress::<T>::Invalid(
1063 format!(
1064 "{}: inbound protocol ERROR {error} is local misuse; remote errors must arrive as DATA envelope facts",
1065 self.session_id
1066 ),
1067 )),
1068 Message::Complete => data_msg(WireBridgeIngress::<T>::Invalid(
1069 format!(
1070 "{}: inbound protocol COMPLETE is local misuse; remote completion must arrive as a DATA envelope fact",
1071 self.session_id
1072 ),
1073 )),
1074 other => data_msg(WireBridgeIngress::<T>::Invalid(format!(
1075 "{}: inbound port accepts DATA envelope facts only; {other:?} is local protocol traffic",
1076 self.session_id
1077 ))),
1078 }
1079 }
1080
1081 pub fn set(&self, envelope: WireBridgeEnvelope<T>) {
1083 self.down(vec![Message::Data(Rc::new(envelope))]);
1084 }
1085
1086 pub fn subscribe(&self, sink: impl Fn(&Message<AnyValue>) + 'static) -> Box<dyn FnOnce()> {
1088 self.node.subscribe(sink)
1089 }
1090
1091 pub fn erased(&self) -> crate::node::Core {
1093 self.node.erased()
1094 }
1095}
1096
1097fn data_msg<T: 'static>(value: T) -> Message<AnyValue> {
1098 Message::Data(Rc::new(value) as AnyValue)
1099}
1100
1101struct PendingEnvelope<T> {
1102 envelope: WireBridgeEnvelope<T>,
1103 timeout_reported_attempt: Option<u32>,
1104 retry_due_at_ms: Option<u64>,
1105}
1106
1107struct BridgeState<T> {
1108 active: bool,
1109 cleanup_installed: bool,
1110 next_seq: u64,
1111 cursor: u64,
1112 remote_cursor: u64,
1113 pending: BTreeMap<u64, PendingEnvelope<T>>,
1114}
1115
1116pub fn wire_bridge<TOutbound, TInbound>(
1118 graph: &Graph,
1119 opts: WireBridgeOptions,
1120) -> WireBridgeBundle<TOutbound, TInbound>
1121where
1122 TOutbound: Clone + 'static,
1123 TInbound: Clone + 'static,
1124{
1125 assert!(
1126 !opts.session_id.is_empty(),
1127 "wire_bridge: session_id must be non-empty"
1128 );
1129 let name = opts.name.clone().unwrap_or_else(|| "wireBridge".to_owned());
1130 let command_sources = Rc::new(RefCell::new(Vec::new()));
1131 let inbound_sources = Rc::new(RefCell::new(Vec::new()));
1132 let command = graph.state_empty_opts::<WireBridgeCommand<TOutbound>>({
1133 let mut opts = GraphNodeOpts::named(format!("{name}/command"));
1134 opts.node.partial = true;
1135 opts.node.complete_when_deps_complete = false;
1136 opts.node.error_when_deps_error = false;
1137 opts
1138 });
1139 let inbound_node = graph.state_empty_opts::<WireBridgeIngress<TInbound>>(GraphNodeOpts::named(
1140 format!("{name}/inbound"),
1141 ));
1142 let inbound = WireBridgeInbound {
1143 node: inbound_node.clone(),
1144 session_id: opts.session_id.clone(),
1145 };
1146 let events =
1147 wire_bridge_events_node(graph, &command, &inbound_node, name.clone(), opts.clone());
1148 let outbound = project_outbound(graph, &events, &name);
1149 let acks = project_acks(graph, &events, &name);
1150 let nacks = project_nacks(graph, &events, &name);
1151 let status = project_status(graph, &events, &name, opts.session_id.clone());
1152 let errors = project_errors(graph, &events, &name);
1153 let cursor = project_cursor(graph, &events, &name);
1154 let attempts = project_attempts(graph, &events, &name);
1155 WireBridgeBundle {
1156 command,
1157 outbound,
1158 inbound,
1159 events,
1160 acks,
1161 nacks,
1162 status,
1163 errors,
1164 cursor,
1165 attempts,
1166 command_sources,
1167 inbound_sources,
1168 }
1169}
1170
1171#[derive(Clone)]
1172pub struct WireEdgeGroupEdge {
1174 pub edge_id: String,
1176 pub outbound: Option<Node<Vec<u8>>>,
1178}
1179
1180impl WireEdgeGroupEdge {
1181 #[must_use]
1182 pub fn inbound(edge_id: impl Into<String>) -> Self {
1184 Self {
1185 edge_id: edge_id.into(),
1186 outbound: None,
1187 }
1188 }
1189
1190 #[must_use]
1191 pub fn outbound(edge_id: impl Into<String>, outbound: Node<Vec<u8>>) -> Self {
1193 Self {
1194 edge_id: edge_id.into(),
1195 outbound: Some(outbound),
1196 }
1197 }
1198}
1199
1200#[derive(Clone)]
1201pub struct WireEdgeGroupOptions {
1203 pub name: Option<String>,
1205 pub edges: Vec<WireEdgeGroupEdge>,
1207}
1208
1209impl WireEdgeGroupOptions {
1210 #[must_use]
1211 pub fn new(edges: Vec<WireEdgeGroupEdge>) -> Self {
1213 Self { name: None, edges }
1214 }
1215
1216 #[must_use]
1217 pub fn named(name: impl Into<String>, edges: Vec<WireEdgeGroupEdge>) -> Self {
1219 Self {
1220 name: Some(name.into()),
1221 edges,
1222 }
1223 }
1224}
1225
1226#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1227pub enum WireEdgeGroupIssueCode {
1229 MissingSnapshot,
1231 UnknownEdge,
1233 DuplicateDirty,
1235 DuplicateData,
1237 DataBeforeDirty,
1239 CompetingCause,
1241 MalformedFrame,
1243 IncompleteCause,
1245}
1246
1247impl WireEdgeGroupIssueCode {
1248 #[must_use]
1249 pub fn as_str(self) -> &'static str {
1251 match self {
1252 Self::MissingSnapshot => "wire-edge-group-missing-snapshot",
1253 Self::UnknownEdge => "wire-edge-group-unknown-edge",
1254 Self::DuplicateDirty => "wire-edge-group-duplicate-dirty",
1255 Self::DuplicateData => "wire-edge-group-duplicate-data",
1256 Self::DataBeforeDirty => "wire-edge-group-data-before-dirty",
1257 Self::CompetingCause => "wire-edge-group-competing-cause",
1258 Self::MalformedFrame => "wire-edge-group-malformed-frame",
1259 Self::IncompleteCause => "wire-edge-group-incomplete-cause",
1260 }
1261 }
1262}
1263
1264#[derive(Debug, Clone, PartialEq, Eq)]
1265pub struct WireEdgeGroupIssue {
1267 pub code: WireEdgeGroupIssueCode,
1269 pub message: String,
1271 pub edge_id: Option<String>,
1273 pub cause_id: Option<String>,
1275 pub active_cause_id: Option<String>,
1277}
1278
1279#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1280pub enum WireEdgeGroupStatusState {
1282 Idle,
1284 Collecting,
1286 Released,
1288 Issues,
1290}
1291
1292#[derive(Debug, Clone, PartialEq, Eq)]
1293pub struct WireEdgeGroupStatus {
1295 pub state: WireEdgeGroupStatusState,
1297 pub expected_edges: Vec<String>,
1299 pub active_cause_id: Option<String>,
1301 pub dirty: usize,
1303 pub data: usize,
1305 pub released: u64,
1307 pub issues: u64,
1309 pub last_issue: Option<WireEdgeGroupIssue>,
1311}
1312
1313pub struct WireEdgeGroupBundle {
1315 pub inbound: BTreeMap<String, Node<Vec<u8>>>,
1317 pub status: Node<WireEdgeGroupStatus>,
1319 pub issues: Node<WireEdgeGroupIssue>,
1321 graph: Graph,
1322 topology: TopologyGroup,
1323 gate: Node<WireEdgeGroupGate>,
1324 bridge_command: Node<WireBridgeCommand<WireBridgeProtobufDataBody>>,
1325 command_sources: Rc<RefCell<Vec<Core>>>,
1326 commands: Node<WireBridgeCommand<WireBridgeProtobufDataBody>>,
1327 gate_retain: RefCell<Option<Box<dyn FnOnce()>>>,
1328 released: Cell<bool>,
1329}
1330
1331impl WireEdgeGroupBundle {
1332 pub fn release(&self) {
1334 if self.released.get() {
1335 return;
1336 }
1337 detach_wire_bridge_command_source(
1338 &self.bridge_command,
1339 &self.command_sources,
1340 self.commands.erased(),
1341 );
1342 let had_gate_retain = self.gate_retain.borrow().is_some();
1343 let mut gate_retain = self.gate_retain.borrow_mut().take();
1344 let release = catch_unwind(AssertUnwindSafe(|| {
1345 if let Some(release) = gate_retain.take() {
1346 release();
1347 }
1348 self.topology.release_with_reason("wire_edge_group release");
1349 }));
1350 if let Err(panic) = release {
1351 attach_wire_bridge_command_source_parts(
1352 &self.bridge_command,
1353 &self.command_sources,
1354 self.commands.erased(),
1355 );
1356 if had_gate_retain {
1357 self.gate_retain.replace(Some(
1358 self.graph
1359 .retain(&self.gate, "wire_edge_group release rollback"),
1360 ));
1361 }
1362 resume_unwind(panic);
1363 }
1364 self.released.set(true);
1365 }
1366}
1367
1368pub fn wire_edge_group(
1370 graph: &Graph,
1371 bridge: &WireBridgeBundle<WireBridgeProtobufDataBody, WireBridgeProtobufDataBody>,
1372 opts: WireEdgeGroupOptions,
1373) -> WireEdgeGroupBundle {
1374 let name = opts.name.unwrap_or_else(|| "wireEdgeGroup".to_owned());
1375 let edges = normalize_wire_edge_group_edges(opts.edges);
1376 validate_wire_edge_group_outbound_versions(&edges);
1377 let expected = edges
1378 .iter()
1379 .map(|edge| edge.edge_id.clone())
1380 .collect::<Vec<_>>();
1381 let outbound = edges
1382 .iter()
1383 .enumerate()
1384 .filter_map(|(edge_index, edge)| {
1385 edge.outbound
1386 .as_ref()
1387 .map(|node| (edge_index, node.erased()))
1388 })
1389 .collect::<Vec<_>>();
1390 let mut deps = outbound
1391 .iter()
1392 .map(|(_, core)| core.clone())
1393 .collect::<Vec<_>>();
1394 deps.push(bridge.inbound.erased());
1395 let topology =
1396 graph.topology_group_opts(TopologyGroupOptions::named(format!("{name}.wireEdgeGroup")));
1397 let events = topology.node_opts::<WireEdgeGroupEvent, _>(
1398 deps,
1399 wire_edge_group_events_fn(
1400 name.clone(),
1401 edges.clone(),
1402 bridge.inbound.session_id().to_owned(),
1403 outbound
1404 .iter()
1405 .map(|(edge_index, _)| *edge_index)
1406 .collect::<Vec<_>>(),
1407 ),
1408 graph_node_opts(format!("{name}/events"), "wireEdgeGroupEvents"),
1409 );
1410 let release = topology.state_empty_opts::<WireEdgeGroupReleaseCohort>(graph_node_opts(
1411 format!("{name}/release"),
1412 "wireEdgeGroupReleaseCohort",
1413 ));
1414 let gate = topology.node_opts::<WireEdgeGroupGate, _>(
1415 vec![events.erased()],
1416 wire_edge_group_gate_fn(name.clone(), expected.clone(), release.clone()),
1417 graph_node_opts(format!("{name}/gate"), "wireEdgeGroupGate"),
1418 );
1419 let commands = topology.node_opts::<WireBridgeCommand<WireBridgeProtobufDataBody>, _>(
1420 vec![events.erased()],
1421 |ctx| {
1422 for event in ctx.batch::<WireEdgeGroupEvent>(0) {
1423 if let WireEdgeGroupEvent::Outbound { command } = event.as_ref() {
1424 ctx.emit(command.clone());
1425 }
1426 }
1427 },
1428 graph_node_opts(format!("{name}/commands"), "wireEdgeGroupCommands"),
1429 );
1430 let issues = topology.node_opts::<WireEdgeGroupIssue, _>(
1431 vec![events.erased(), gate.erased()],
1432 |ctx| {
1433 for event in ctx.batch::<WireEdgeGroupEvent>(0) {
1434 if let WireEdgeGroupEvent::Issue { issue } = event.as_ref() {
1435 ctx.emit(issue.clone());
1436 }
1437 }
1438 for event in ctx.batch::<WireEdgeGroupGate>(1) {
1439 if let WireEdgeGroupGate::Issue { issue } = event.as_ref() {
1440 ctx.emit(issue.clone());
1441 }
1442 }
1443 },
1444 graph_node_opts(format!("{name}/issues"), "wireEdgeGroupIssues"),
1445 );
1446 let status = topology.node_opts::<WireEdgeGroupStatus, _>(
1447 vec![events.erased(), gate.erased()],
1448 wire_edge_group_status_fn(expected.clone()),
1449 graph_node_opts(format!("{name}/status"), "wireEdgeGroupStatus"),
1450 );
1451 let inbound = expected
1452 .iter()
1453 .map(|edge_id| {
1454 let edge_id_for_node = edge_id.clone();
1455 let node = topology.node_opts::<Vec<u8>, _>(
1456 vec![release.erased()],
1457 move |ctx| {
1458 for cohort in ctx.batch::<WireEdgeGroupReleaseCohort>(0) {
1459 let _ = &cohort.cause_id;
1460 if let Some(value) = cohort.values.get(&edge_id_for_node) {
1461 ctx.emit(value.clone());
1462 }
1463 }
1464 },
1465 graph_node_opts(
1466 format!("{name}/inbound/{edge_id}"),
1467 "wireEdgeGroupInboundEdge",
1468 ),
1469 );
1470 (edge_id.clone(), node)
1471 })
1472 .collect::<BTreeMap<_, _>>();
1473 let attach = catch_unwind(AssertUnwindSafe(|| {
1474 attach_wire_bridge_command_source(bridge, commands.erased());
1475 }));
1476 if let Err(panic) = attach {
1477 topology.release_with_reason("wire_edge_group failed command wiring");
1478 resume_unwind(panic);
1479 }
1480 let gate_retain = RefCell::new(if outbound.is_empty() {
1481 Some(graph.retain(&gate, &format!("{name}.wireEdgeGroup.gate")))
1482 } else {
1483 None
1484 });
1485 WireEdgeGroupBundle {
1486 inbound,
1487 status,
1488 issues,
1489 graph: graph.clone(),
1490 topology,
1491 gate,
1492 bridge_command: bridge.command.clone(),
1493 command_sources: bridge.command_sources.clone(),
1494 commands,
1495 gate_retain,
1496 released: Cell::new(false),
1497 }
1498}
1499
1500#[derive(Clone)]
1501enum WireEdgeGroupEvent {
1502 Outbound {
1503 command: WireBridgeCommand<WireBridgeProtobufDataBody>,
1504 },
1505 Frame {
1506 frame: CanonicalWireEdgeFrame,
1507 },
1508 Issue {
1509 issue: WireEdgeGroupIssue,
1510 },
1511 BridgeEnd,
1512}
1513
1514#[derive(Clone)]
1515enum WireEdgeGroupGate {
1516 Issue {
1517 issue: WireEdgeGroupIssue,
1518 },
1519 Progress {
1520 cause_id: String,
1521 dirty: usize,
1522 data: usize,
1523 },
1524 Release {
1525 cause_id: String,
1526 count: usize,
1527 },
1528}
1529
1530#[derive(Clone)]
1531struct WireEdgeGroupReleaseCohort {
1532 cause_id: String,
1533 values: BTreeMap<String, Vec<u8>>,
1534}
1535
1536#[derive(Clone, Default)]
1537struct WireEdgeGroupOutState {
1538 next_cause: u64,
1539 emitted_once: bool,
1540 pending: BTreeMap<String, WireEdgeGroupPendingOutbound>,
1541 last_emitted_versions: BTreeMap<String, Option<NodeVersion>>,
1542}
1543
1544#[derive(Clone)]
1545struct WireEdgeGroupPendingOutbound {
1546 value: Vec<u8>,
1547 version: Option<NodeVersion>,
1548}
1549
1550const WIRE_EDGE_GROUP_CAUSE_TOMBSTONE_LIMIT: usize = 1024;
1551
1552#[derive(Clone)]
1553struct WireEdgeGroupCauseTombstones {
1554 seen: HashSet<String>,
1555 order: VecDeque<String>,
1556 limit: usize,
1557}
1558
1559impl Default for WireEdgeGroupCauseTombstones {
1560 fn default() -> Self {
1561 Self {
1562 seen: HashSet::new(),
1563 order: VecDeque::new(),
1564 limit: WIRE_EDGE_GROUP_CAUSE_TOMBSTONE_LIMIT,
1565 }
1566 }
1567}
1568
1569impl WireEdgeGroupCauseTombstones {
1570 fn insert(&mut self, cause_id: String) {
1571 if !self.seen.insert(cause_id.clone()) {
1572 return;
1573 }
1574 self.order.push_back(cause_id);
1575 while self.order.len() > self.limit {
1576 if let Some(evicted) = self.order.pop_front() {
1577 self.seen.remove(&evicted);
1578 }
1579 }
1580 }
1581
1582 fn contains(&self, cause_id: &str) -> bool {
1583 self.seen.contains(cause_id)
1584 }
1585}
1586
1587#[derive(Clone, Default)]
1588struct WireEdgeGroupGateState {
1589 active_cause_id: Option<String>,
1590 dirty: HashSet<String>,
1591 data: BTreeMap<String, Vec<u8>>,
1592 failed: WireEdgeGroupCauseTombstones,
1593 released: WireEdgeGroupCauseTombstones,
1594}
1595
1596fn graph_node_opts(name: impl Into<String>, factory: impl Into<String>) -> GraphNodeOpts {
1597 let mut opts = GraphNodeOpts::named(name);
1598 opts.node.factory = Some(factory.into());
1599 opts.node.partial = true;
1600 opts.node.complete_when_deps_complete = false;
1601 opts.node.error_when_deps_error = false;
1602 opts
1603}
1604
1605fn normalize_wire_edge_group_edges(edges: Vec<WireEdgeGroupEdge>) -> Vec<WireEdgeGroupEdge> {
1606 assert!(
1607 !edges.is_empty(),
1608 "wire_edge_group: edges must be non-empty"
1609 );
1610 let mut seen = HashSet::new();
1611 let mut has_inbound = false;
1612 let mut has_outbound = false;
1613 for edge in &edges {
1614 assert!(
1615 !edge.edge_id.is_empty(),
1616 "wire_edge_group: edge_id must be non-empty"
1617 );
1618 assert!(
1619 seen.insert(edge.edge_id.clone()),
1620 "wire_edge_group: duplicate edge_id '{}'",
1621 edge.edge_id
1622 );
1623 if edge.outbound.is_some() {
1624 has_outbound = true;
1625 } else {
1626 has_inbound = true;
1627 }
1628 }
1629 assert!(
1630 !(has_inbound && has_outbound),
1631 "wire_edge_group: inbound and outbound edges must be declared in separate groups"
1632 );
1633 edges
1634}
1635
1636fn validate_wire_edge_group_outbound_versions(edges: &[WireEdgeGroupEdge]) {
1637 for edge in edges {
1638 if edge
1639 .outbound
1640 .as_ref()
1641 .is_some_and(|node| node.version().is_none())
1642 {
1643 panic!(
1644 "wire_edge_group: outbound edge '{}' requires node runtime versioning for D561 fresh-source admission",
1645 edge.edge_id
1646 );
1647 }
1648 }
1649}
1650
1651fn wire_edge_group_issue(
1652 code: WireEdgeGroupIssueCode,
1653 message: impl Into<String>,
1654 edge_id: Option<String>,
1655 cause_id: Option<String>,
1656 active_cause_id: Option<String>,
1657) -> WireEdgeGroupIssue {
1658 WireEdgeGroupIssue {
1659 code,
1660 message: message.into(),
1661 edge_id,
1662 cause_id,
1663 active_cause_id,
1664 }
1665}
1666
1667fn wire_edge_group_send(
1668 frame: CanonicalWireEdgeFrame,
1669) -> WireBridgeCommand<WireBridgeProtobufDataBody> {
1670 WireBridgeCommand::Send {
1671 payload: WireBridgeProtobufDataBody::WireEdge(frame),
1672 idempotency_key: None,
1673 request_id: None,
1674 }
1675}
1676
1677fn wire_edge_group_events_fn(
1678 name: String,
1679 edges: Vec<WireEdgeGroupEdge>,
1680 session_id: String,
1681 outbound_indexes: Vec<usize>,
1682) -> impl Fn(&Ctx) + 'static {
1683 let inbound_index = outbound_indexes.len();
1684 move |ctx| {
1685 let state = wire_edge_group_out_state(ctx);
1686 let mut triggered = false;
1687 for (dep_index, edge_index) in outbound_indexes.iter().enumerate() {
1688 let edge = &edges[*edge_index];
1689 if let Some(waves) = ctx.wave_data().get(dep_index) {
1690 for wave in waves.iter() {
1691 for item in wave {
1692 match item {
1693 WaveData::Data(value) => match value.clone().downcast::<Vec<u8>>() {
1694 Ok(value) => {
1695 let version = edge.outbound.as_ref().and_then(Node::version);
1696 if wire_edge_group_should_admit_outbound_data(
1697 &state,
1698 &edge.edge_id,
1699 version.as_ref(),
1700 ) {
1701 state.borrow_mut().pending.insert(
1702 edge.edge_id.clone(),
1703 WireEdgeGroupPendingOutbound {
1704 value: (*value).clone(),
1705 version,
1706 },
1707 );
1708 triggered = true;
1709 }
1710 }
1711 Err(_) => {
1712 ctx.emit(WireEdgeGroupEvent::Issue {
1713 issue: wire_edge_group_issue(
1714 WireEdgeGroupIssueCode::MalformedFrame,
1715 format!(
1716 "{name}: outbound edge {} must emit Vec<u8> bytes",
1717 edge.edge_id
1718 ),
1719 Some(edge.edge_id.clone()),
1720 None,
1721 None,
1722 ),
1723 });
1724 state.borrow_mut().pending.remove(&edge.edge_id);
1725 }
1726 },
1727 WaveData::Sentinel => {
1728 state.borrow_mut().pending.remove(&edge.edge_id);
1729 triggered = true;
1730 }
1731 }
1732 }
1733 }
1734 }
1735 }
1736 if triggered {
1737 emit_wire_edge_group_outbound(ctx, &name, &edges, &state);
1738 }
1739 for ingress in ctx.batch::<WireBridgeIngress<WireBridgeProtobufDataBody>>(inbound_index) {
1740 if let Some(event) = wire_edge_group_frame_event(&name, &session_id, ingress.as_ref()) {
1741 ctx.emit(event);
1742 }
1743 }
1744 }
1745}
1746
1747fn wire_edge_group_out_state(ctx: &Ctx) -> Rc<RefCell<WireEdgeGroupOutState>> {
1748 if let Some(state) = ctx.state_get::<RefCell<WireEdgeGroupOutState>>() {
1749 return state;
1750 }
1751 ctx.state_set(RefCell::new(WireEdgeGroupOutState {
1752 next_cause: 1,
1753 emitted_once: false,
1754 pending: BTreeMap::new(),
1755 last_emitted_versions: BTreeMap::new(),
1756 }));
1757 ctx.state_get::<RefCell<WireEdgeGroupOutState>>()
1758 .expect("wire-edge out state was just installed")
1759}
1760
1761fn wire_edge_group_should_admit_outbound_data(
1762 state: &Rc<RefCell<WireEdgeGroupOutState>>,
1763 edge_id: &str,
1764 version: Option<&NodeVersion>,
1765) -> bool {
1766 let state = state.borrow();
1767 if !state.emitted_once {
1768 return true;
1769 }
1770 let Some(version) = version else {
1771 return false;
1772 };
1773 if state
1774 .pending
1775 .get(edge_id)
1776 .is_some_and(|pending| pending.version.as_ref() == Some(version))
1777 {
1778 return false;
1779 }
1780 state
1781 .last_emitted_versions
1782 .get(edge_id)
1783 .is_none_or(|last| last.as_ref() != Some(version))
1784}
1785
1786#[cfg(test)]
1787mod wire_edge_group_outbound_admission_tests {
1788 use super::*;
1789
1790 fn version(counter: u64) -> NodeVersion {
1791 NodeVersion::V0 { counter }
1792 }
1793
1794 #[test]
1795 fn pending_version_replay_is_not_re_admitted() {
1796 let state = Rc::new(RefCell::new(WireEdgeGroupOutState {
1797 next_cause: 2,
1798 emitted_once: true,
1799 pending: BTreeMap::from([(
1800 "a".to_owned(),
1801 WireEdgeGroupPendingOutbound {
1802 value: vec![2],
1803 version: Some(version(2)),
1804 },
1805 )]),
1806 last_emitted_versions: BTreeMap::from([("a".to_owned(), Some(version(1)))]),
1807 }));
1808
1809 assert!(
1810 !wire_edge_group_should_admit_outbound_data(&state, "a", Some(&version(2))),
1811 "D561: replay of the already-pending source occurrence is not fresh"
1812 );
1813 assert!(
1814 wire_edge_group_should_admit_outbound_data(&state, "a", Some(&version(3))),
1815 "D561: a later source occurrence remains eligible before cohort completion"
1816 );
1817 }
1818}
1819
1820fn emit_wire_edge_group_outbound(
1821 ctx: &Ctx,
1822 name: &str,
1823 edges: &[WireEdgeGroupEdge],
1824 state: &Rc<RefCell<WireEdgeGroupOutState>>,
1825) {
1826 let missing = {
1827 let state = state.borrow();
1828 edges
1829 .iter()
1830 .filter(|edge| !state.pending.contains_key(&edge.edge_id))
1831 .map(|edge| edge.edge_id.clone())
1832 .collect::<Vec<_>>()
1833 };
1834 if !missing.is_empty() {
1835 for edge_id in missing {
1836 ctx.emit(WireEdgeGroupEvent::Issue {
1837 issue: wire_edge_group_issue(
1838 WireEdgeGroupIssueCode::MissingSnapshot,
1839 format!("{name}: missing outbound snapshot for edge {edge_id}"),
1840 Some(edge_id),
1841 None,
1842 None,
1843 ),
1844 });
1845 }
1846 return;
1847 }
1848 let cause_id = {
1849 let mut state = state.borrow_mut();
1850 let cause_id = compound_tuple_key(
1851 "wire-edge-group-cause",
1852 &[name, &state.next_cause.to_string()],
1853 );
1854 state.next_cause = state.next_cause.saturating_add(1);
1855 cause_id
1856 };
1857 for edge in edges {
1858 ctx.emit(WireEdgeGroupEvent::Outbound {
1859 command: wire_edge_group_send(CanonicalWireEdgeFrame {
1860 kind: CanonicalWireEdgeKind::Dirty,
1861 edge_id: edge.edge_id.clone(),
1862 cause_id: cause_id.clone(),
1863 value: None,
1864 }),
1865 });
1866 }
1867 let pending = state.borrow().pending.clone();
1868 for edge in edges {
1869 if let Some(value) = pending.get(&edge.edge_id) {
1870 ctx.emit(WireEdgeGroupEvent::Outbound {
1871 command: wire_edge_group_send(CanonicalWireEdgeFrame {
1872 kind: CanonicalWireEdgeKind::Data,
1873 edge_id: edge.edge_id.clone(),
1874 cause_id: cause_id.clone(),
1875 value: Some(value.value.clone()),
1876 }),
1877 });
1878 }
1879 }
1880 let mut state = state.borrow_mut();
1881 state.last_emitted_versions.clear();
1882 for edge in edges {
1883 if let Some(pending) = pending.get(&edge.edge_id) {
1884 state
1885 .last_emitted_versions
1886 .insert(edge.edge_id.clone(), pending.version.clone());
1887 }
1888 }
1889 state.pending.clear();
1890 state.emitted_once = true;
1891}
1892
1893fn wire_edge_group_frame_event(
1894 name: &str,
1895 session_id: &str,
1896 ingress: &WireBridgeIngress<WireBridgeProtobufDataBody>,
1897) -> Option<WireEdgeGroupEvent> {
1898 let envelope = match ingress {
1899 WireBridgeIngress::Envelope(envelope) => envelope,
1900 WireBridgeIngress::Invalid(error) => {
1901 return Some(WireEdgeGroupEvent::Issue {
1902 issue: wire_edge_group_issue(
1903 WireEdgeGroupIssueCode::MalformedFrame,
1904 format!("{name}: bridge invalid wire-edge ingress: {error}"),
1905 None,
1906 None,
1907 None,
1908 ),
1909 });
1910 }
1911 };
1912 if let Err(error) = validate_inbound_envelope(envelope) {
1913 return Some(WireEdgeGroupEvent::Issue {
1914 issue: wire_edge_group_issue(
1915 WireEdgeGroupIssueCode::MalformedFrame,
1916 format!("{name}: bridge invalid wire-edge envelope: {error}"),
1917 None,
1918 None,
1919 None,
1920 ),
1921 });
1922 }
1923 if envelope.session_id != session_id {
1924 return Some(WireEdgeGroupEvent::Issue {
1925 issue: wire_edge_group_issue(
1926 WireEdgeGroupIssueCode::MalformedFrame,
1927 format!(
1928 "{name}: bridge session {} did not match expected {session_id}",
1929 envelope.session_id
1930 ),
1931 None,
1932 None,
1933 None,
1934 ),
1935 });
1936 }
1937 match envelope.envelope_type {
1938 WireBridgeEnvelopeType::Close | WireBridgeEnvelopeType::Error => {
1939 Some(WireEdgeGroupEvent::BridgeEnd)
1940 }
1941 WireBridgeEnvelopeType::Data => match &envelope.payload {
1942 Some(WireBridgePayload::Data(WireBridgeProtobufDataBody::WireEdge(frame))) => {
1943 validate_wire_edge_group_frame(name, frame).map_or_else(
1944 || {
1945 Some(WireEdgeGroupEvent::Frame {
1946 frame: frame.clone(),
1947 })
1948 },
1949 |issue| Some(WireEdgeGroupEvent::Issue { issue }),
1950 )
1951 }
1952 Some(WireBridgePayload::Data(WireBridgeProtobufDataBody::Value(_))) => None,
1953 _ => Some(WireEdgeGroupEvent::Issue {
1954 issue: wire_edge_group_issue(
1955 WireEdgeGroupIssueCode::MalformedFrame,
1956 format!("{name}: wire-edge payload must be a wire_edge frame"),
1957 None,
1958 None,
1959 None,
1960 ),
1961 }),
1962 },
1963 WireBridgeEnvelopeType::Start
1964 | WireBridgeEnvelopeType::Ack
1965 | WireBridgeEnvelopeType::Nack
1966 | WireBridgeEnvelopeType::Status => None,
1967 }
1968}
1969
1970fn validate_wire_edge_group_frame(
1971 name: &str,
1972 frame: &CanonicalWireEdgeFrame,
1973) -> Option<WireEdgeGroupIssue> {
1974 if frame.edge_id.is_empty() {
1975 return Some(wire_edge_group_issue(
1976 WireEdgeGroupIssueCode::MalformedFrame,
1977 format!("{name}: wire-edge frame edge_id must be non-empty"),
1978 None,
1979 Some(frame.cause_id.clone()),
1980 None,
1981 ));
1982 }
1983 if frame.cause_id.is_empty() {
1984 return Some(wire_edge_group_issue(
1985 WireEdgeGroupIssueCode::MalformedFrame,
1986 format!("{name}: wire-edge frame cause_id must be non-empty"),
1987 Some(frame.edge_id.clone()),
1988 None,
1989 None,
1990 ));
1991 }
1992 match frame.kind {
1993 CanonicalWireEdgeKind::Dirty if frame.value.is_some() => Some(wire_edge_group_issue(
1994 WireEdgeGroupIssueCode::MalformedFrame,
1995 format!("{name}: DIRTY wire-edge frame must not carry value bytes"),
1996 Some(frame.edge_id.clone()),
1997 Some(frame.cause_id.clone()),
1998 None,
1999 )),
2000 CanonicalWireEdgeKind::Data if frame.value.is_none() => Some(wire_edge_group_issue(
2001 WireEdgeGroupIssueCode::MalformedFrame,
2002 format!("{name}: DATA wire-edge frame requires value bytes"),
2003 Some(frame.edge_id.clone()),
2004 Some(frame.cause_id.clone()),
2005 None,
2006 )),
2007 CanonicalWireEdgeKind::Dirty | CanonicalWireEdgeKind::Data => None,
2008 }
2009}
2010
2011fn wire_edge_group_gate_fn(
2012 name: String,
2013 expected_ids: Vec<String>,
2014 release: Node<WireEdgeGroupReleaseCohort>,
2015) -> impl Fn(&Ctx) + 'static {
2016 let expected = expected_ids.iter().cloned().collect::<HashSet<_>>();
2017 move |ctx| {
2018 let state = wire_edge_group_gate_state(ctx);
2019 for event in ctx.batch::<WireEdgeGroupEvent>(0) {
2020 match event.as_ref() {
2021 WireEdgeGroupEvent::Issue { issue } => {
2022 ctx.emit(WireEdgeGroupGate::Issue {
2023 issue: issue.clone(),
2024 });
2025 fail_wire_edge_group_issue_cause(ctx, &name, &state, issue);
2026 }
2027 WireEdgeGroupEvent::Outbound { .. } => {}
2028 WireEdgeGroupEvent::BridgeEnd => {
2029 let active = state.borrow().active_cause_id.clone();
2030 if let Some(cause_id) = active {
2031 ctx.emit(WireEdgeGroupGate::Issue {
2032 issue: wire_edge_group_issue(
2033 WireEdgeGroupIssueCode::IncompleteCause,
2034 format!(
2035 "{name}: cause {cause_id} ended before all expected edge frames arrived"
2036 ),
2037 None,
2038 Some(cause_id.clone()),
2039 None,
2040 ),
2041 });
2042 wire_edge_group_gate_fail(&state, Some(cause_id));
2043 }
2044 }
2045 WireEdgeGroupEvent::Frame { frame } => {
2046 reduce_wire_edge_group_frame(
2047 ctx,
2048 &name,
2049 &expected,
2050 &expected_ids,
2051 &state,
2052 &release,
2053 frame,
2054 );
2055 }
2056 }
2057 }
2058 }
2059}
2060
2061fn wire_edge_group_gate_state(ctx: &Ctx) -> Rc<RefCell<WireEdgeGroupGateState>> {
2062 if let Some(state) = ctx.state_get::<RefCell<WireEdgeGroupGateState>>() {
2063 return state;
2064 }
2065 ctx.state_set(RefCell::new(WireEdgeGroupGateState::default()));
2066 ctx.state_get::<RefCell<WireEdgeGroupGateState>>()
2067 .expect("wire-edge gate state was just installed")
2068}
2069
2070fn wire_edge_group_gate_reset(state: &Rc<RefCell<WireEdgeGroupGateState>>) {
2071 let mut state = state.borrow_mut();
2072 state.active_cause_id = None;
2073 state.dirty.clear();
2074 state.data.clear();
2075}
2076
2077fn wire_edge_group_gate_fail(
2078 state: &Rc<RefCell<WireEdgeGroupGateState>>,
2079 cause_id: Option<String>,
2080) {
2081 if let Some(cause_id) = cause_id {
2082 state.borrow_mut().failed.insert(cause_id);
2083 }
2084 wire_edge_group_gate_reset(state);
2085}
2086
2087fn fail_wire_edge_group_issue_cause(
2088 ctx: &Ctx,
2089 name: &str,
2090 state: &Rc<RefCell<WireEdgeGroupGateState>>,
2091 issue: &WireEdgeGroupIssue,
2092) {
2093 let Some(cause_id) = issue.cause_id.clone() else {
2094 return;
2095 };
2096 let active = state.borrow().active_cause_id.clone();
2097 if let Some(active_cause_id) = active {
2098 if active_cause_id != cause_id {
2099 emit_wire_edge_group_competing_cause(
2100 ctx,
2101 name,
2102 state,
2103 issue.edge_id.clone(),
2104 cause_id,
2105 active_cause_id,
2106 );
2107 return;
2108 }
2109 wire_edge_group_gate_fail(state, Some(cause_id));
2110 } else {
2111 state.borrow_mut().failed.insert(cause_id);
2112 }
2113}
2114
2115fn emit_wire_edge_group_competing_cause(
2116 ctx: &Ctx,
2117 name: &str,
2118 state: &Rc<RefCell<WireEdgeGroupGateState>>,
2119 edge_id: Option<String>,
2120 cause_id: String,
2121 active_cause_id: String,
2122) {
2123 ctx.emit(WireEdgeGroupGate::Issue {
2124 issue: wire_edge_group_issue(
2125 WireEdgeGroupIssueCode::CompetingCause,
2126 format!("{name}: competing cause {cause_id} arrived while {active_cause_id} is active"),
2127 edge_id,
2128 Some(cause_id.clone()),
2129 Some(active_cause_id.clone()),
2130 ),
2131 });
2132 ctx.emit(WireEdgeGroupGate::Issue {
2133 issue: wire_edge_group_issue(
2134 WireEdgeGroupIssueCode::IncompleteCause,
2135 format!("{name}: active cause {active_cause_id} is incomplete"),
2136 None,
2137 Some(active_cause_id.clone()),
2138 None,
2139 ),
2140 });
2141 wire_edge_group_gate_fail(state, Some(active_cause_id));
2142 state.borrow_mut().failed.insert(cause_id);
2143}
2144
2145fn wire_edge_group_progress(
2146 ctx: &Ctx,
2147 cause_id: String,
2148 state: &Rc<RefCell<WireEdgeGroupGateState>>,
2149) {
2150 let state = state.borrow();
2151 ctx.emit(WireEdgeGroupGate::Progress {
2152 cause_id,
2153 dirty: state.dirty.len(),
2154 data: state.data.len(),
2155 });
2156}
2157
2158fn emit_replayed_wire_edge_group_cause(ctx: &Ctx, name: &str, frame: &CanonicalWireEdgeFrame) {
2159 let code = match frame.kind {
2160 CanonicalWireEdgeKind::Dirty => WireEdgeGroupIssueCode::DuplicateDirty,
2161 CanonicalWireEdgeKind::Data => WireEdgeGroupIssueCode::DuplicateData,
2162 };
2163 ctx.emit(WireEdgeGroupGate::Issue {
2164 issue: wire_edge_group_issue(
2165 code,
2166 format!("{name}: cause {} was already released", frame.cause_id),
2167 Some(frame.edge_id.clone()),
2168 Some(frame.cause_id.clone()),
2169 None,
2170 ),
2171 });
2172}
2173
2174fn reduce_wire_edge_group_frame(
2175 ctx: &Ctx,
2176 name: &str,
2177 expected: &HashSet<String>,
2178 expected_ids: &[String],
2179 state: &Rc<RefCell<WireEdgeGroupGateState>>,
2180 release: &Node<WireEdgeGroupReleaseCohort>,
2181 frame: &CanonicalWireEdgeFrame,
2182) {
2183 if state.borrow().failed.contains(&frame.cause_id) {
2184 ctx.emit(WireEdgeGroupGate::Issue {
2185 issue: wire_edge_group_issue(
2186 WireEdgeGroupIssueCode::IncompleteCause,
2187 format!(
2188 "{}: cause {} was already failed closed",
2189 name, frame.cause_id
2190 ),
2191 Some(frame.edge_id.clone()),
2192 Some(frame.cause_id.clone()),
2193 None,
2194 ),
2195 });
2196 return;
2197 }
2198 if state.borrow().released.contains(&frame.cause_id) {
2199 emit_replayed_wire_edge_group_cause(ctx, name, frame);
2200 return;
2201 }
2202 let active = state.borrow().active_cause_id.clone();
2203 if active.as_deref() != Some(frame.cause_id.as_str()) {
2204 if let Some(active_cause_id) = active.clone() {
2205 emit_wire_edge_group_competing_cause(
2206 ctx,
2207 name,
2208 state,
2209 Some(frame.edge_id.clone()),
2210 frame.cause_id.clone(),
2211 active_cause_id,
2212 );
2213 return;
2214 }
2215 }
2216 if !expected.contains(&frame.edge_id) {
2217 ctx.emit(WireEdgeGroupGate::Issue {
2218 issue: wire_edge_group_issue(
2219 WireEdgeGroupIssueCode::UnknownEdge,
2220 format!("{name}: unknown edge {}", frame.edge_id),
2221 Some(frame.edge_id.clone()),
2222 Some(frame.cause_id.clone()),
2223 None,
2224 ),
2225 });
2226 wire_edge_group_gate_fail(state, Some(frame.cause_id.clone()));
2227 return;
2228 }
2229 match frame.kind {
2230 CanonicalWireEdgeKind::Dirty => {
2231 if active.is_none() {
2232 state.borrow_mut().active_cause_id = Some(frame.cause_id.clone());
2233 }
2234 if !state.borrow_mut().dirty.insert(frame.edge_id.clone()) {
2235 ctx.emit(WireEdgeGroupGate::Issue {
2236 issue: wire_edge_group_issue(
2237 WireEdgeGroupIssueCode::DuplicateDirty,
2238 format!("{name}: duplicate DIRTY for edge {}", frame.edge_id),
2239 Some(frame.edge_id.clone()),
2240 Some(frame.cause_id.clone()),
2241 None,
2242 ),
2243 });
2244 wire_edge_group_gate_fail(state, Some(frame.cause_id.clone()));
2245 return;
2246 }
2247 wire_edge_group_progress(ctx, frame.cause_id.clone(), state);
2248 }
2249 CanonicalWireEdgeKind::Data => {
2250 if state.borrow().active_cause_id.is_none() {
2251 ctx.emit(WireEdgeGroupGate::Issue {
2252 issue: wire_edge_group_issue(
2253 WireEdgeGroupIssueCode::DataBeforeDirty,
2254 format!(
2255 "{name}: DATA for edge {} arrived without an active cause",
2256 frame.edge_id
2257 ),
2258 Some(frame.edge_id.clone()),
2259 Some(frame.cause_id.clone()),
2260 None,
2261 ),
2262 });
2263 wire_edge_group_gate_fail(state, Some(frame.cause_id.clone()));
2264 return;
2265 }
2266 if !state.borrow().dirty.contains(&frame.edge_id) {
2267 ctx.emit(WireEdgeGroupGate::Issue {
2268 issue: wire_edge_group_issue(
2269 WireEdgeGroupIssueCode::DataBeforeDirty,
2270 format!(
2271 "{name}: DATA for edge {} arrived before DIRTY",
2272 frame.edge_id
2273 ),
2274 Some(frame.edge_id.clone()),
2275 Some(frame.cause_id.clone()),
2276 None,
2277 ),
2278 });
2279 wire_edge_group_gate_fail(state, Some(frame.cause_id.clone()));
2280 return;
2281 }
2282 if state.borrow().data.contains_key(&frame.edge_id) {
2283 ctx.emit(WireEdgeGroupGate::Issue {
2284 issue: wire_edge_group_issue(
2285 WireEdgeGroupIssueCode::DuplicateData,
2286 format!("{name}: duplicate DATA for edge {}", frame.edge_id),
2287 Some(frame.edge_id.clone()),
2288 Some(frame.cause_id.clone()),
2289 None,
2290 ),
2291 });
2292 wire_edge_group_gate_fail(state, Some(frame.cause_id.clone()));
2293 return;
2294 }
2295 let value = frame.value.clone().unwrap_or_default();
2296 let ready = {
2297 let state = state.borrow();
2298 state.dirty.len() == expected_ids.len()
2299 && state.data.len().saturating_add(1) == expected_ids.len()
2300 };
2301 if ready {
2302 let mut data = state.borrow().data.clone();
2303 data.insert(frame.edge_id.clone(), value);
2304 let values = expected_ids
2305 .iter()
2306 .filter_map(|edge_id| {
2307 data.get(edge_id)
2308 .map(|value| (edge_id.clone(), value.clone()))
2309 })
2310 .collect::<BTreeMap<_, _>>();
2311 ctx.emit(WireEdgeGroupGate::Release {
2312 cause_id: frame.cause_id.clone(),
2313 count: values.len(),
2314 });
2315 release.set(WireEdgeGroupReleaseCohort {
2316 cause_id: frame.cause_id.clone(),
2317 values,
2318 });
2319 state.borrow_mut().released.insert(frame.cause_id.clone());
2320 wire_edge_group_gate_reset(state);
2321 } else {
2322 state.borrow_mut().data.insert(frame.edge_id.clone(), value);
2323 wire_edge_group_progress(ctx, frame.cause_id.clone(), state);
2324 }
2325 }
2326 }
2327}
2328
2329fn wire_edge_group_status_fn(expected_ids: Vec<String>) -> impl Fn(&Ctx) + 'static {
2330 move |ctx| {
2331 let mut status = ctx.state_get::<WireEdgeGroupStatus>().map_or_else(
2332 || WireEdgeGroupStatus {
2333 state: WireEdgeGroupStatusState::Idle,
2334 expected_edges: expected_ids.clone(),
2335 active_cause_id: None,
2336 dirty: 0,
2337 data: 0,
2338 released: 0,
2339 issues: 0,
2340 last_issue: None,
2341 },
2342 |status| (*status).clone(),
2343 );
2344 for event in ctx.batch::<WireEdgeGroupEvent>(0) {
2345 if let WireEdgeGroupEvent::Issue { issue } = event.as_ref() {
2346 status.state = WireEdgeGroupStatusState::Issues;
2347 status.issues = status.issues.saturating_add(1);
2348 status.last_issue = Some(issue.clone());
2349 }
2350 }
2351 for event in ctx.batch::<WireEdgeGroupGate>(1) {
2352 match event.as_ref() {
2353 WireEdgeGroupGate::Issue { issue } => {
2354 status.state = WireEdgeGroupStatusState::Issues;
2355 status.active_cause_id = None;
2356 status.dirty = 0;
2357 status.data = 0;
2358 status.issues = status.issues.saturating_add(1);
2359 status.last_issue = Some(issue.clone());
2360 }
2361 WireEdgeGroupGate::Progress {
2362 cause_id,
2363 dirty,
2364 data,
2365 } => {
2366 status.state = WireEdgeGroupStatusState::Collecting;
2367 status.active_cause_id = Some(cause_id.clone());
2368 status.dirty = *dirty;
2369 status.data = *data;
2370 }
2371 WireEdgeGroupGate::Release { cause_id, count } => {
2372 status.state = WireEdgeGroupStatusState::Released;
2373 status.active_cause_id = None;
2374 status.dirty = 0;
2375 status.data = 0;
2376 status.released = status
2377 .released
2378 .saturating_add(u64::try_from(*count).unwrap_or(u64::MAX));
2379 let _ = cause_id;
2380 }
2381 }
2382 }
2383 ctx.state_set(status.clone());
2384 ctx.emit(status);
2385 }
2386}
2387
2388pub fn remote_call<TRequest, TResponse>(
2390 graph: &Graph,
2391 bridge: &WireBridgeBundle<RemoteCallRequest<TRequest>, RemoteCallResponse<TResponse>>,
2392) -> RemoteCallBundle<TRequest, TResponse>
2393where
2394 TRequest: Clone + 'static,
2395 TResponse: Clone + 'static,
2396{
2397 remote_call_with_options(graph, bridge, RemoteCallOptions::default())
2398}
2399
2400pub fn remote_call_with_options<TRequest, TResponse>(
2402 graph: &Graph,
2403 bridge: &WireBridgeBundle<RemoteCallRequest<TRequest>, RemoteCallResponse<TResponse>>,
2404 opts: RemoteCallOptions,
2405) -> RemoteCallBundle<TRequest, TResponse>
2406where
2407 TRequest: Clone + 'static,
2408 TResponse: Clone + 'static,
2409{
2410 let name = opts.name;
2411 let timeouts = graph
2412 .state_empty_opts::<RemoteCallTimeout>(GraphNodeOpts::named(format!("{name}/timeouts")));
2413 let responses =
2414 remote_call_responses_node(graph, &bridge.command, &bridge.events, &timeouts, &name);
2415 let results = remote_call_results_node(graph, &responses, &name);
2416 let status = remote_call_status_node(
2417 graph,
2418 &bridge.command,
2419 &bridge.events,
2420 &responses,
2421 &results,
2422 &timeouts,
2423 &name,
2424 );
2425 let errors = remote_call_errors_node(graph, &responses, &timeouts, &bridge.events, &name);
2426 RemoteCallBundle {
2427 bridge_command: bridge.command.clone(),
2428 responses,
2429 results,
2430 status,
2431 errors,
2432 timeouts,
2433 }
2434}
2435
2436pub fn remote_responder<TRequest, TResponse>(
2438 graph: &Graph,
2439 bridge: &WireBridgeBundle<RemoteCallResponse<TResponse>, RemoteCallRequest<TRequest>>,
2440 opts: RemoteResponderOptions<TRequest, TResponse>,
2441) -> RemoteResponderBundle<TRequest, TResponse>
2442where
2443 TRequest: Clone + 'static,
2444 TResponse: Clone + 'static,
2445{
2446 let name = opts.name;
2447 let reject_unknown = opts.reject_unknown;
2448 let handlers = Rc::new(normalize_remote_handlers(opts.handlers));
2449 let events = remote_responder_events_node(graph, bridge, &name, handlers, reject_unknown);
2450 let response_commands = remote_responder_response_commands_node(graph, &events, &name);
2451 let requests = remote_responder_requests_node(graph, &events, &name);
2452 let status = remote_responder_status_node(graph, &events, &name);
2453 let errors = remote_responder_errors_node(graph, &events, &name);
2454 let attach = catch_unwind(AssertUnwindSafe(|| {
2455 attach_wire_bridge_command_source(bridge, response_commands.erased());
2456 }));
2457 if let Err(panic) = attach {
2458 graph.release_nodes(
2459 &[
2460 events.erased(),
2461 response_commands.erased(),
2462 requests.erased(),
2463 status.erased(),
2464 errors.erased(),
2465 ],
2466 "remote_responder failed response command wiring",
2467 );
2468 resume_unwind(panic);
2469 }
2470 RemoteResponderBundle {
2471 events,
2472 response_commands,
2473 requests,
2474 status,
2475 errors,
2476 graph: graph.clone(),
2477 bridge_command: bridge.command.clone(),
2478 command_sources: bridge.command_sources.clone(),
2479 released: Cell::new(false),
2480 }
2481}
2482
2483fn normalize_remote_handlers<TRequest, TResponse>(
2484 handlers: Vec<RemoteResponderHandlerDefinition<TRequest, TResponse>>,
2485) -> HashMap<String, RemoteResponderHandler<TRequest, TResponse>> {
2486 let mut out = HashMap::new();
2487 for handler in handlers {
2488 assert!(
2489 out.insert(handler.operation.clone(), handler.handle)
2490 .is_none(),
2491 "remote_responder: duplicate operation '{}'",
2492 handler.operation
2493 );
2494 }
2495 out
2496}
2497
2498fn remote_call_responses_node<TRequest, TResponse>(
2499 graph: &Graph,
2500 commands: &Node<WireBridgeCommand<RemoteCallRequest<TRequest>>>,
2501 events: &Node<WireBridgeEvent<RemoteCallRequest<TRequest>, RemoteCallResponse<TResponse>>>,
2502 timeouts: &Node<RemoteCallTimeout>,
2503 name: &str,
2504) -> Node<RemoteCallResponse<TResponse>>
2505where
2506 TRequest: Clone + 'static,
2507 TResponse: Clone + 'static,
2508{
2509 graph.node_opts::<RemoteCallResponse<TResponse>, _>(
2510 vec![commands.erased(), events.erased(), timeouts.erased()],
2511 |ctx| {
2512 let state = remote_call_responses_state(ctx);
2513 let mut ready = Vec::new();
2514 {
2515 let mut state = state.borrow_mut();
2516 let preview_requests = ctx
2517 .batch::<WireBridgeCommand<RemoteCallRequest<TRequest>>>(0)
2518 .into_iter()
2519 .filter_map(|command| pending_request_from_command(command.as_ref()))
2520 .collect::<Vec<_>>();
2521 let has_invalid_bridge_event = ctx.batch::<
2522 WireBridgeEvent<RemoteCallRequest<TRequest>, RemoteCallResponse<TResponse>>,
2523 >(1)
2524 .into_iter()
2525 .any(|event| {
2526 matches!(
2527 event.as_ref(),
2528 WireBridgeEvent::Invalid { .. }
2529 | WireBridgeEvent::SessionMismatch { .. }
2530 | WireBridgeEvent::OutOfOrder { .. }
2531 | WireBridgeEvent::LateReceipt { .. }
2532 )
2533 });
2534 if !has_invalid_bridge_event {
2535 for request in preview_requests.iter().cloned() {
2536 let _ = state.pending.insert(request);
2537 }
2538 }
2539 for event in ctx.batch::<
2540 WireBridgeEvent<RemoteCallRequest<TRequest>, RemoteCallResponse<TResponse>>,
2541 >(1) {
2542 if let WireBridgeEvent::Outbound { envelope } = event.as_ref() {
2543 if let Some(request) = pending_request_from_envelope(envelope) {
2544 let _ = state.pending.insert(request);
2545 }
2546 }
2547 }
2548 for event in ctx.batch::<
2549 WireBridgeEvent<RemoteCallRequest<TRequest>, RemoteCallResponse<TResponse>>,
2550 >(1) {
2551 match event.as_ref() {
2552 WireBridgeEvent::Invalid { .. } => {
2553 state.pending.remove_all_unbound();
2554 }
2555 WireBridgeEvent::Inbound { envelope } => {
2556 if let Some(WireBridgePayload::Data(response)) = &envelope.payload {
2557 let request_id = remote_call_response_request_id(response);
2558 if let Some(request) =
2559 state.pending.get_by_request_id(request_id)
2560 {
2561 if !remote_call_response_matches_pending(response, request) {
2562 continue;
2563 }
2564 if remote_call_response_is_terminal(response) {
2565 state.pending.remove_by_request_id(request_id);
2566 }
2567 ready.push(response.clone());
2568 }
2569 }
2570 }
2571 WireBridgeEvent::Nack { outbound, .. } => {
2572 let request = state
2573 .pending
2574 .remove_by_seq(outbound.metadata.seq)
2575 .or_else(|| pending_request_from_envelope(outbound));
2576 if let Some(request) = request {
2577 let _ = request;
2578 }
2579 }
2580 WireBridgeEvent::Exhausted { seq, .. } => {
2581 state.pending.remove_by_seq(*seq);
2582 }
2583 _ => {}
2584 }
2585 }
2586 for timeout in ctx.batch::<RemoteCallTimeout>(2) {
2587 state.pending.remove_by_request_id(&timeout.request_id);
2588 }
2589 }
2590 for response in ready {
2591 ctx.emit(response);
2592 }
2593 },
2594 no_terminal_graph_opts(format!("{name}/responses")),
2595 )
2596}
2597
2598#[derive(Clone, Default)]
2599struct RemoteCallResponsesState {
2600 pending: RemoteCallPendingState,
2601}
2602
2603fn remote_call_responses_state(ctx: &Ctx) -> Rc<RefCell<RemoteCallResponsesState>> {
2604 if let Some(state) = ctx.state_get::<RefCell<RemoteCallResponsesState>>() {
2605 return state;
2606 }
2607 ctx.state_set(RefCell::new(RemoteCallResponsesState::default()));
2608 ctx.state_get::<RefCell<RemoteCallResponsesState>>()
2609 .expect("remote call responses state was just installed")
2610}
2611
2612#[derive(Clone)]
2613struct RemoteCallPendingRequest {
2614 operation: String,
2615 request_id: String,
2616 seq: Option<u64>,
2617}
2618
2619#[derive(Clone, Default)]
2620struct RemoteCallPendingState {
2621 request_ids: HashSet<String>,
2622 by_request_id: HashMap<String, RemoteCallPendingRequest>,
2623 by_seq: HashMap<u64, RemoteCallPendingRequest>,
2624}
2625
2626impl RemoteCallPendingState {
2627 fn get_by_request_id(&self, request_id: &str) -> Option<&RemoteCallPendingRequest> {
2628 self.by_request_id.get(request_id)
2629 }
2630
2631 fn insert(&mut self, request: RemoteCallPendingRequest) -> bool {
2632 if let Some(existing) = self.by_request_id.get_mut(&request.request_id) {
2633 if existing.operation == request.operation && existing.seq.is_none() {
2634 if let Some(seq) = request.seq {
2635 if self.by_seq.contains_key(&seq) {
2636 return false;
2637 }
2638 existing.seq = Some(seq);
2639 self.by_seq.insert(seq, existing.clone());
2640 return true;
2641 }
2642 }
2643 return false;
2644 }
2645 if request
2646 .seq
2647 .is_some_and(|seq| self.by_seq.contains_key(&seq))
2648 {
2649 return false;
2650 }
2651 self.request_ids.insert(request.request_id.clone());
2652 self.by_request_id
2653 .insert(request.request_id.clone(), request.clone());
2654 if let Some(seq) = request.seq {
2655 self.by_seq.insert(seq, request);
2656 }
2657 true
2658 }
2659
2660 fn remove_by_request_id(&mut self, request_id: &str) -> Option<RemoteCallPendingRequest> {
2661 if !self.request_ids.remove(request_id) {
2662 return None;
2663 }
2664 let request = self.by_request_id.remove(request_id)?;
2665 if let Some(seq) = request.seq {
2666 self.by_seq.remove(&seq);
2667 }
2668 Some(request)
2669 }
2670
2671 fn remove_by_seq(&mut self, seq: u64) -> Option<RemoteCallPendingRequest> {
2672 let request = self.by_seq.remove(&seq)?;
2673 self.request_ids.remove(&request.request_id);
2674 self.by_request_id.remove(&request.request_id);
2675 Some(request)
2676 }
2677
2678 fn remove_all_unbound(&mut self) {
2679 let request_ids = self
2680 .by_request_id
2681 .values()
2682 .filter_map(|request| request.seq.is_none().then_some(request.request_id.clone()))
2683 .collect::<Vec<_>>();
2684 for request_id in request_ids {
2685 self.request_ids.remove(&request_id);
2686 self.by_request_id.remove(&request_id);
2687 }
2688 }
2689
2690 fn len(&self) -> usize {
2691 self.request_ids.len()
2692 }
2693}
2694
2695fn pending_request_from_envelope<T>(
2696 envelope: &WireBridgeEnvelope<RemoteCallRequest<T>>,
2697) -> Option<RemoteCallPendingRequest> {
2698 if let Some(WireBridgePayload::Data(request)) = &envelope.payload {
2699 Some(RemoteCallPendingRequest {
2700 operation: request.operation.clone(),
2701 request_id: request.request_id.clone(),
2702 seq: Some(envelope.metadata.seq),
2703 })
2704 } else {
2705 None
2706 }
2707}
2708
2709fn pending_request_from_command<T>(
2710 command: &WireBridgeCommand<RemoteCallRequest<T>>,
2711) -> Option<RemoteCallPendingRequest> {
2712 if let WireBridgeCommand::Send { payload, .. } = command {
2713 Some(RemoteCallPendingRequest {
2714 operation: payload.operation.clone(),
2715 request_id: payload.request_id.clone(),
2716 seq: None,
2717 })
2718 } else {
2719 None
2720 }
2721}
2722
2723fn remote_call_response_request_id<T>(response: &RemoteCallResponse<T>) -> &str {
2724 match response {
2725 RemoteCallResponse::Result { request_id, .. }
2726 | RemoteCallResponse::Error { request_id, .. }
2727 | RemoteCallResponse::Status { request_id, .. } => request_id,
2728 }
2729}
2730
2731fn remote_call_response_is_terminal<T>(response: &RemoteCallResponse<T>) -> bool {
2732 matches!(
2733 response,
2734 RemoteCallResponse::Result { .. } | RemoteCallResponse::Error { .. }
2735 )
2736}
2737
2738fn remote_call_response_operation<T>(response: &RemoteCallResponse<T>) -> &str {
2739 match response {
2740 RemoteCallResponse::Result { operation, .. }
2741 | RemoteCallResponse::Error { operation, .. }
2742 | RemoteCallResponse::Status { operation, .. } => operation,
2743 }
2744}
2745
2746fn remote_call_response_matches_pending<T>(
2747 response: &RemoteCallResponse<T>,
2748 request: &RemoteCallPendingRequest,
2749) -> bool {
2750 remote_call_response_operation(response) == request.operation
2751}
2752
2753fn remote_call_response_key<T>(response: &RemoteCallResponse<T>) -> String {
2754 format!(
2755 "{}\u{0}{}",
2756 remote_call_response_operation(response),
2757 remote_call_response_request_id(response)
2758 )
2759}
2760
2761fn remote_call_request_key(request: &RemoteCallPendingRequest) -> String {
2762 format!("{}\u{0}{}", request.operation, request.request_id)
2763}
2764
2765#[derive(Clone, Default)]
2766struct RemoteResponderCursor {
2767 cursor: u64,
2768 remote_cursor: u64,
2769}
2770
2771enum RemoteResponderInbound<T> {
2772 Request {
2773 request: RemoteCallRequest<T>,
2774 seq: u64,
2775 },
2776 Consumed,
2777 Invalid {
2778 error: String,
2779 },
2780}
2781
2782fn reduce_remote_responder_ingress<T>(
2783 cursor: &mut RemoteResponderCursor,
2784 ingress: WireBridgeIngress<RemoteCallRequest<T>>,
2785 session_id: &str,
2786) -> RemoteResponderInbound<T> {
2787 let envelope = match ingress {
2788 WireBridgeIngress::Envelope(envelope) => envelope,
2789 WireBridgeIngress::Invalid(error) => return RemoteResponderInbound::Invalid { error },
2790 };
2791 if let Err(error) = validate_inbound_envelope(&envelope) {
2792 return RemoteResponderInbound::Invalid {
2793 error: error.to_string(),
2794 };
2795 }
2796 if envelope.session_id != session_id {
2797 return RemoteResponderInbound::Invalid {
2798 error: format!(
2799 "{session_id}: remoteResponder session mismatch: {}",
2800 envelope.session_id
2801 ),
2802 };
2803 }
2804 let seq = envelope.metadata.seq;
2805 let expected = cursor.cursor.saturating_add(1);
2806 if seq <= cursor.cursor {
2807 return RemoteResponderInbound::Invalid {
2808 error: format!(
2809 "remoteResponder: duplicate request seq {seq} at cursor {}",
2810 cursor.cursor
2811 ),
2812 };
2813 }
2814 if seq > expected {
2815 return RemoteResponderInbound::Invalid {
2816 error: format!("remoteResponder: out-of-order request seq {seq}, expected {expected}"),
2817 };
2818 }
2819 if envelope.metadata.cursor < cursor.remote_cursor {
2820 return RemoteResponderInbound::Invalid {
2821 error: format!(
2822 "{session_id}: remoteResponder inbound cursor {} regressed below {}",
2823 envelope.metadata.cursor, cursor.remote_cursor
2824 ),
2825 };
2826 }
2827 cursor.cursor = seq;
2828 cursor.remote_cursor = envelope.metadata.cursor;
2829 if envelope.envelope_type != WireBridgeEnvelopeType::Data {
2830 return RemoteResponderInbound::Consumed;
2831 }
2832 match envelope.payload {
2833 Some(WireBridgePayload::Data(request)) => RemoteResponderInbound::Request { request, seq },
2834 _ => RemoteResponderInbound::Invalid {
2835 error: "remoteResponder: request envelope must carry request DATA".to_owned(),
2836 },
2837 }
2838}
2839
2840fn remote_call_results_node<TResponse>(
2841 graph: &Graph,
2842 responses: &Node<RemoteCallResponse<TResponse>>,
2843 name: &str,
2844) -> Node<RemoteCallResult<TResponse>>
2845where
2846 TResponse: Clone + 'static,
2847{
2848 graph.node_opts::<RemoteCallResult<TResponse>, _>(
2849 vec![responses.erased()],
2850 |ctx| {
2851 for response in ctx.batch::<RemoteCallResponse<TResponse>>(0) {
2852 if let RemoteCallResponse::Result {
2853 operation,
2854 request_id,
2855 payload,
2856 } = response.as_ref()
2857 {
2858 ctx.emit(RemoteCallResult {
2859 operation: operation.clone(),
2860 request_id: request_id.clone(),
2861 payload: payload.clone(),
2862 });
2863 }
2864 }
2865 },
2866 no_terminal_graph_opts(format!("{name}/results")),
2867 )
2868}
2869
2870fn remote_call_status_node<TRequest, TResponse>(
2871 graph: &Graph,
2872 commands: &Node<WireBridgeCommand<RemoteCallRequest<TRequest>>>,
2873 events: &Node<WireBridgeEvent<RemoteCallRequest<TRequest>, RemoteCallResponse<TResponse>>>,
2874 responses: &Node<RemoteCallResponse<TResponse>>,
2875 results: &Node<RemoteCallResult<TResponse>>,
2876 timeouts: &Node<RemoteCallTimeout>,
2877 name: &str,
2878) -> Node<RemoteCallStatus>
2879where
2880 TRequest: Clone + 'static,
2881 TResponse: Clone + 'static,
2882{
2883 graph.node_opts::<RemoteCallStatus, _>(
2884 vec![
2885 commands.erased(),
2886 events.erased(),
2887 responses.erased(),
2888 results.erased(),
2889 timeouts.erased(),
2890 ],
2891 |ctx| {
2892 let mut reducer = ctx
2893 .state_get::<RemoteCallStatusReducer>()
2894 .map_or_else(RemoteCallStatusReducer::default, |reducer| {
2895 (*reducer).clone()
2896 });
2897 for command in ctx.batch::<WireBridgeCommand<RemoteCallRequest<TRequest>>>(0) {
2898 if let Some(request) = pending_request_from_command(command.as_ref()) {
2899 reducer.status.state = RemoteCallStatusState::Requested;
2900 reducer.status.operation = Some(request.operation.clone());
2901 reducer.status.request_id = Some(request.request_id.clone());
2902 if !reducer
2903 .terminal_before_request
2904 .contains(&remote_call_request_key(&request))
2905 {
2906 let _ = reducer.pending.insert(request);
2907 }
2908 }
2909 }
2910 for event in ctx.batch::<
2911 WireBridgeEvent<RemoteCallRequest<TRequest>, RemoteCallResponse<TResponse>>,
2912 >(1) {
2913 if let WireBridgeEvent::Outbound { envelope } = event.as_ref() {
2914 if let Some(request) = pending_request_from_envelope(envelope) {
2915 reducer.status.state = RemoteCallStatusState::Requested;
2916 reducer.status.operation = Some(request.operation.clone());
2917 reducer.status.request_id = Some(request.request_id.clone());
2918 if reducer
2919 .terminal_before_request
2920 .remove(&remote_call_request_key(&request))
2921 {
2922 continue;
2923 }
2924 if !reducer.pending.insert(request) {
2925 reducer.status.state = RemoteCallStatusState::Errored;
2926 reducer.status.errors = reducer.status.errors.saturating_add(1);
2927 }
2928 }
2929 }
2930 }
2931 for event in ctx.batch::<
2932 WireBridgeEvent<RemoteCallRequest<TRequest>, RemoteCallResponse<TResponse>>,
2933 >(1) {
2934 match event.as_ref() {
2935 WireBridgeEvent::Invalid { .. } => {
2936 reducer.pending.remove_all_unbound();
2937 reducer.status.state = RemoteCallStatusState::BridgeErrored;
2938 reducer.status.errors = reducer.status.errors.saturating_add(1);
2939 }
2940 WireBridgeEvent::SessionMismatch { .. }
2941 | WireBridgeEvent::OutOfOrder { .. }
2942 | WireBridgeEvent::LateReceipt { .. } => {
2943 reducer.status.state = RemoteCallStatusState::BridgeErrored;
2944 reducer.status.errors = reducer.status.errors.saturating_add(1);
2945 }
2946 WireBridgeEvent::Nack {
2947 outbound, error, ..
2948 } => {
2949 let request = reducer
2950 .pending
2951 .remove_by_seq(outbound.metadata.seq)
2952 .or_else(|| pending_request_from_envelope(outbound));
2953 reducer.status.state = RemoteCallStatusState::BridgeErrored;
2954 if let Some(request) = request {
2955 reducer.status.operation = Some(request.operation);
2956 reducer.status.request_id = Some(request.request_id);
2957 }
2958 if !error.is_empty() {
2959 reducer.status.errors = reducer.status.errors.saturating_add(1);
2960 }
2961 }
2962 WireBridgeEvent::Exhausted { seq, error, .. } => {
2963 let request = reducer.pending.remove_by_seq(*seq);
2964 reducer.status.state = RemoteCallStatusState::BridgeErrored;
2965 if let Some(request) = request {
2966 reducer.status.operation = Some(request.operation);
2967 reducer.status.request_id = Some(request.request_id);
2968 }
2969 if !error.is_empty() {
2970 reducer.status.errors = reducer.status.errors.saturating_add(1);
2971 }
2972 }
2973 WireBridgeEvent::Inbound { envelope } => {
2974 if let Some(WireBridgePayload::Data(response)) = &envelope.payload {
2975 let request_id = remote_call_response_request_id(response);
2976 if let Some(request) = reducer.pending.get_by_request_id(request_id) {
2977 if !remote_call_response_matches_pending(response, request) {
2978 reducer.status.state = RemoteCallStatusState::Errored;
2979 reducer.status.operation = Some(request.operation.clone());
2980 reducer.status.request_id = Some(request.request_id.clone());
2981 reducer.status.errors =
2982 reducer.status.errors.saturating_add(1);
2983 continue;
2984 }
2985 } else {
2986 reducer.status.state = RemoteCallStatusState::Errored;
2987 reducer.status.operation =
2988 Some(remote_call_response_operation(response).to_owned());
2989 reducer.status.request_id = Some(request_id.to_owned());
2990 reducer.status.errors = reducer.status.errors.saturating_add(1);
2991 continue;
2992 }
2993 match response {
2994 RemoteCallResponse::Result {
2995 ..
2996 } => {
2997 }
2999 RemoteCallResponse::Error {
3000 ..
3001 } => {
3002 }
3004 RemoteCallResponse::Status {
3005 operation,
3006 request_id,
3007 ..
3008 } => {
3009 reducer.status.operation = Some(operation.clone());
3010 reducer.status.request_id = Some(request_id.clone());
3011 }
3012 }
3013 }
3014 }
3015 _ => {}
3016 }
3017 }
3018 for response in ctx.batch::<RemoteCallResponse<TResponse>>(2) {
3019 match response.as_ref() {
3020 RemoteCallResponse::Result { .. } => {}
3021 RemoteCallResponse::Error {
3022 operation,
3023 request_id,
3024 ..
3025 } => {
3026 let removed = reducer.pending.remove_by_request_id(request_id);
3027 if removed.as_ref().is_none_or(|request| request.seq.is_none()) {
3028 reducer
3029 .terminal_before_request
3030 .insert(format!("{}\u{0}{}", operation, request_id));
3031 }
3032 reducer.status.state = RemoteCallStatusState::Errored;
3033 reducer.status.operation = Some(operation.clone());
3034 reducer.status.request_id = Some(request_id.clone());
3035 reducer.status.errors = reducer.status.errors.saturating_add(1);
3036 }
3037 RemoteCallResponse::Status {
3038 operation,
3039 request_id,
3040 ..
3041 } => {
3042 if reducer.pending.get_by_request_id(request_id).is_some() {
3043 reducer.status.operation = Some(operation.clone());
3044 reducer.status.request_id = Some(request_id.clone());
3045 }
3046 }
3047 }
3048 }
3049 for result in ctx.batch::<RemoteCallResult<TResponse>>(3) {
3050 let removed = reducer.pending.remove_by_request_id(&result.request_id);
3051 if removed.as_ref().is_none_or(|request| request.seq.is_none()) {
3052 reducer
3053 .terminal_before_request
3054 .insert(format!("{}\u{0}{}", result.operation, result.request_id));
3055 }
3056 reducer.status.state = RemoteCallStatusState::Responded;
3057 reducer.status.operation = Some(result.operation.clone());
3058 reducer.status.request_id = Some(result.request_id.clone());
3059 reducer.status.completed = reducer.status.completed.saturating_add(1);
3060 }
3061 for timeout in ctx.batch::<RemoteCallTimeout>(4) {
3062 reducer.status.state = RemoteCallStatusState::TimedOut;
3063 reducer.status.operation = timeout.operation.clone();
3064 reducer.status.request_id = Some(timeout.request_id.clone());
3065 reducer.pending.remove_by_request_id(&timeout.request_id);
3066 reducer.status.errors = reducer.status.errors.saturating_add(1);
3067 reducer.status.timeouts = reducer.status.timeouts.saturating_add(1);
3068 }
3069 reducer.status.pending = reducer.pending.len();
3070 ctx.state_set(reducer.clone());
3071 ctx.emit(reducer.status);
3072 },
3073 no_terminal_graph_opts(format!("{name}/status")),
3074 )
3075}
3076
3077#[derive(Clone)]
3078struct RemoteCallStatusReducer {
3079 status: RemoteCallStatus,
3080 pending: RemoteCallPendingState,
3081 terminal_before_request: HashSet<String>,
3082}
3083
3084impl Default for RemoteCallStatusReducer {
3085 fn default() -> Self {
3086 Self {
3087 status: initial_remote_call_status(),
3088 pending: RemoteCallPendingState::default(),
3089 terminal_before_request: HashSet::new(),
3090 }
3091 }
3092}
3093
3094fn initial_remote_call_status() -> RemoteCallStatus {
3095 RemoteCallStatus {
3096 state: RemoteCallStatusState::Idle,
3097 operation: None,
3098 request_id: None,
3099 pending: 0,
3100 completed: 0,
3101 errors: 0,
3102 timeouts: 0,
3103 }
3104}
3105
3106fn remote_call_errors_node<TRequest, TResponse>(
3107 graph: &Graph,
3108 responses: &Node<RemoteCallResponse<TResponse>>,
3109 timeouts: &Node<RemoteCallTimeout>,
3110 events: &Node<WireBridgeEvent<RemoteCallRequest<TRequest>, RemoteCallResponse<TResponse>>>,
3111 name: &str,
3112) -> Node<RemoteCallError>
3113where
3114 TRequest: Clone + 'static,
3115 TResponse: Clone + 'static,
3116{
3117 graph.node_opts::<RemoteCallError, _>(
3118 vec![responses.erased(), timeouts.erased(), events.erased()],
3119 |ctx| {
3120 let mut pending = ctx
3121 .state_get::<RemoteCallPendingState>()
3122 .map_or_else(RemoteCallPendingState::default, |pending| {
3123 (*pending).clone()
3124 });
3125 for event in ctx.batch::<
3126 WireBridgeEvent<RemoteCallRequest<TRequest>, RemoteCallResponse<TResponse>>,
3127 >(2) {
3128 if let WireBridgeEvent::Outbound { envelope } = event.as_ref() {
3129 if let Some(request) = pending_request_from_envelope(envelope) {
3130 if !pending.insert(request.clone()) {
3131 ctx.emit(RemoteCallError {
3132 operation: Some(request.operation),
3133 request_id: Some(request.request_id.clone()),
3134 error: format!(
3135 "remote_call: duplicate in-flight request_id '{}'",
3136 request.request_id
3137 ),
3138 });
3139 }
3140 }
3141 }
3142 }
3143 let mut accepted_response_counts = HashMap::new();
3144 for response in ctx.batch::<RemoteCallResponse<TResponse>>(0) {
3145 *accepted_response_counts
3146 .entry(remote_call_response_key(response.as_ref()))
3147 .or_insert(0usize) += 1;
3148 match response.as_ref() {
3149 RemoteCallResponse::Error {
3150 operation,
3151 request_id,
3152 error,
3153 } => {
3154 pending.remove_by_request_id(request_id);
3155 ctx.emit(RemoteCallError {
3156 operation: Some(operation.clone()),
3157 request_id: Some(request_id.clone()),
3158 error: error.clone(),
3159 });
3160 }
3161 RemoteCallResponse::Result { request_id, .. } => {
3162 pending.remove_by_request_id(request_id);
3163 }
3164 RemoteCallResponse::Status { .. } => {}
3165 }
3166 }
3167 for timeout in ctx.batch::<RemoteCallTimeout>(1) {
3168 pending.remove_by_request_id(&timeout.request_id);
3169 ctx.emit(RemoteCallError {
3170 operation: timeout.operation.clone(),
3171 request_id: Some(timeout.request_id.clone()),
3172 error: timeout.error.clone(),
3173 });
3174 }
3175 for event in ctx.batch::<
3176 WireBridgeEvent<RemoteCallRequest<TRequest>, RemoteCallResponse<TResponse>>,
3177 >(2) {
3178 match event.as_ref() {
3179 WireBridgeEvent::Nack {
3180 outbound, error, ..
3181 } => {
3182 let request = pending
3183 .remove_by_seq(outbound.metadata.seq)
3184 .or_else(|| pending_request_from_envelope(outbound));
3185 ctx.emit(RemoteCallError {
3186 operation: request.as_ref().map(|request| request.operation.clone()),
3187 request_id: request.as_ref().map(|request| request.request_id.clone()),
3188 error: error.clone(),
3189 });
3190 }
3191 WireBridgeEvent::Exhausted { seq, error, .. } => {
3192 let request = pending.remove_by_seq(*seq);
3193 ctx.emit(RemoteCallError {
3194 operation: request.as_ref().map(|request| request.operation.clone()),
3195 request_id: request.as_ref().map(|request| request.request_id.clone()),
3196 error: error.clone(),
3197 });
3198 }
3199 WireBridgeEvent::Invalid { error } => ctx.emit(RemoteCallError {
3200 operation: None,
3201 request_id: None,
3202 error: error.clone(),
3203 }),
3204 WireBridgeEvent::Inbound { envelope } => {
3205 if let Some(WireBridgePayload::Data(response)) = &envelope.payload {
3206 let response_key = remote_call_response_key(response);
3207 if let Some(count) = accepted_response_counts.get_mut(&response_key) {
3208 if *count > 0 {
3209 *count -= 1;
3210 continue;
3211 }
3212 }
3213 let request_id = remote_call_response_request_id(response);
3214 if let Some(request) = pending.get_by_request_id(request_id) {
3215 if !remote_call_response_matches_pending(response, request) {
3216 ctx.emit(RemoteCallError {
3217 operation: Some(request.operation.clone()),
3218 request_id: Some(request.request_id.clone()),
3219 error: format!(
3220 "remote_call: response operation '{}' did not match pending operation '{}'",
3221 remote_call_response_operation(response),
3222 request.operation
3223 ),
3224 });
3225 }
3226 } else {
3227 ctx.emit(RemoteCallError {
3228 operation: Some(remote_call_response_operation(response).to_owned()),
3229 request_id: Some(request_id.to_owned()),
3230 error:
3231 "remote_call: orphan response for unknown or completed request"
3232 .to_owned(),
3233 });
3234 }
3235 }
3236 }
3237 WireBridgeEvent::SessionMismatch { .. }
3238 | WireBridgeEvent::OutOfOrder { .. }
3239 | WireBridgeEvent::LateReceipt { .. } => ctx.emit(RemoteCallError {
3240 operation: None,
3241 request_id: None,
3242 error: remote_call_bridge_error_message(event.as_ref()),
3243 }),
3244 _ => {}
3245 }
3246 }
3247 ctx.state_set(pending);
3248 },
3249 no_terminal_graph_opts(format!("{name}/errors")),
3250 )
3251}
3252
3253fn remote_call_bridge_error_message<TRequest, TResponse>(
3254 event: &WireBridgeEvent<RemoteCallRequest<TRequest>, RemoteCallResponse<TResponse>>,
3255) -> String {
3256 match event {
3257 WireBridgeEvent::SessionMismatch { expected, actual } => {
3258 format!("{expected}: session mismatch: {actual}")
3259 }
3260 WireBridgeEvent::OutOfOrder { seq, expected } => {
3261 format!("wireBridge: out-of-order seq {seq}, expected {expected}")
3262 }
3263 WireBridgeEvent::LateReceipt {
3264 receipt,
3265 ack_for_seq,
3266 } => format!("wireBridge: late {receipt:?} for seq {ack_for_seq}"),
3267 WireBridgeEvent::Invalid { error } => error.clone(),
3268 _ => "wireBridge: bridge error".to_owned(),
3269 }
3270}
3271
3272fn remote_responder_events_node<TRequest, TResponse>(
3273 graph: &Graph,
3274 bridge: &WireBridgeBundle<RemoteCallResponse<TResponse>, RemoteCallRequest<TRequest>>,
3275 name: &str,
3276 handlers: Rc<HashMap<String, RemoteResponderHandler<TRequest, TResponse>>>,
3277 reject_unknown: bool,
3278) -> Node<RemoteResponderEvent<TRequest, TResponse>>
3279where
3280 TRequest: Clone + 'static,
3281 TResponse: Clone + 'static,
3282{
3283 let session_id = bridge.inbound.session_id().to_owned();
3284 graph.node_opts::<RemoteResponderEvent<TRequest, TResponse>, _>(
3285 vec![bridge.inbound.erased()],
3286 move |ctx| {
3287 let mut cursor = ctx
3288 .state_get::<RemoteResponderCursor>()
3289 .map_or_else(RemoteResponderCursor::default, |cursor| (*cursor).clone());
3290 for value in raw_data(ctx, 0) {
3291 let ingress = match value.downcast::<WireBridgeIngress<RemoteCallRequest<TRequest>>>()
3292 {
3293 Ok(ingress) => (*ingress).clone(),
3294 Err(_) => {
3295 ctx.emit(RemoteResponderEvent::<TRequest, TResponse>::Invalid {
3296 error: "remoteResponder: inbound DATA must carry a wire bridge ingress fact"
3297 .to_owned(),
3298 });
3299 continue;
3300 }
3301 };
3302 let (request, seq) =
3303 match reduce_remote_responder_ingress(&mut cursor, ingress, &session_id) {
3304 RemoteResponderInbound::Request { request, seq } => (request, seq),
3305 RemoteResponderInbound::Consumed => continue,
3306 RemoteResponderInbound::Invalid { error } => {
3307 ctx.emit(RemoteResponderEvent::<TRequest, TResponse>::Invalid {
3308 error,
3309 });
3310 continue;
3311 }
3312 };
3313 ctx.emit(RemoteResponderEvent::<TRequest, TResponse>::Request {
3314 request: request.clone(),
3315 seq,
3316 });
3317 let Some(handler) = handlers.get(&request.operation) else {
3318 if reject_unknown {
3319 let error =
3320 format!("remoteResponder: unknown operation '{}'", request.operation);
3321 ctx.emit(remote_responder_rejected_event::<TRequest, TResponse>(
3322 request, error,
3323 ));
3324 }
3325 continue;
3326 };
3327 let response = match catch_unwind(AssertUnwindSafe(|| handler(&request))) {
3328 Ok(Ok(payload)) => RemoteCallResponse::Result {
3329 operation: request.operation.clone(),
3330 request_id: request.request_id.clone(),
3331 payload,
3332 },
3333 Ok(Err(error)) => RemoteCallResponse::Error {
3334 operation: request.operation.clone(),
3335 request_id: request.request_id.clone(),
3336 error,
3337 },
3338 Err(panic) => {
3339 if let Some(message) = panic_message(&panic) {
3340 if is_graph_invariant_panic(&message) {
3341 resume_unwind(panic);
3342 }
3343 RemoteCallResponse::Error {
3344 operation: request.operation.clone(),
3345 request_id: request.request_id.clone(),
3346 error: format!("remoteResponder: handler threw: {message}"),
3347 }
3348 } else {
3349 RemoteCallResponse::Error {
3350 operation: request.operation.clone(),
3351 request_id: request.request_id.clone(),
3352 error: "remoteResponder: handler threw".to_owned(),
3353 }
3354 }
3355 }
3356 };
3357 let command = WireBridgeCommand::Send {
3358 payload: response,
3359 idempotency_key: Some(format!(
3360 "{}:{}:response",
3361 session_id, request.request_id
3362 )),
3363 request_id: Some(request.request_id.clone()),
3364 };
3365 ctx.emit(RemoteResponderEvent::<TRequest, TResponse>::Response {
3366 request_id: request.request_id,
3367 operation: request.operation,
3368 command,
3369 });
3370 }
3371 ctx.state_set(cursor);
3372 },
3373 no_terminal_graph_opts(format!("{name}/events")),
3374 )
3375}
3376
3377fn remote_responder_rejected_event<TRequest, TResponse>(
3378 request: RemoteCallRequest<TRequest>,
3379 error: String,
3380) -> RemoteResponderEvent<TRequest, TResponse> {
3381 let command = WireBridgeCommand::Send {
3382 payload: RemoteCallResponse::Error {
3383 operation: request.operation.clone(),
3384 request_id: request.request_id.clone(),
3385 error: error.clone(),
3386 },
3387 idempotency_key: None,
3388 request_id: Some(request.request_id.clone()),
3389 };
3390 RemoteResponderEvent::Rejected {
3391 request_id: Some(request.request_id),
3392 operation: Some(request.operation),
3393 error,
3394 command: Some(command),
3395 }
3396}
3397
3398fn remote_responder_response_commands_node<TRequest, TResponse>(
3399 graph: &Graph,
3400 events: &Node<RemoteResponderEvent<TRequest, TResponse>>,
3401 name: &str,
3402) -> Node<WireBridgeCommand<RemoteCallResponse<TResponse>>>
3403where
3404 TRequest: Clone + 'static,
3405 TResponse: Clone + 'static,
3406{
3407 graph.node_opts::<WireBridgeCommand<RemoteCallResponse<TResponse>>, _>(
3408 vec![events.erased()],
3409 |ctx| {
3410 for event in ctx.batch::<RemoteResponderEvent<TRequest, TResponse>>(0) {
3411 match event.as_ref() {
3412 RemoteResponderEvent::Response { command, .. } => ctx.emit(command.clone()),
3413 RemoteResponderEvent::Rejected {
3414 command: Some(command),
3415 ..
3416 } => ctx.emit(command.clone()),
3417 _ => {}
3418 }
3419 }
3420 },
3421 no_terminal_graph_opts(format!("{name}/responseCommands")),
3422 )
3423}
3424
3425fn remote_responder_requests_node<TRequest, TResponse>(
3426 graph: &Graph,
3427 events: &Node<RemoteResponderEvent<TRequest, TResponse>>,
3428 name: &str,
3429) -> Node<RemoteCallRequest<TRequest>>
3430where
3431 TRequest: Clone + 'static,
3432 TResponse: Clone + 'static,
3433{
3434 graph.node_opts::<RemoteCallRequest<TRequest>, _>(
3435 vec![events.erased()],
3436 |ctx| {
3437 for event in ctx.batch::<RemoteResponderEvent<TRequest, TResponse>>(0) {
3438 if let RemoteResponderEvent::Request { request, .. } = event.as_ref() {
3439 ctx.emit(request.clone());
3440 }
3441 }
3442 },
3443 no_terminal_graph_opts(format!("{name}/requests")),
3444 )
3445}
3446
3447fn remote_responder_status_node<TRequest, TResponse>(
3448 graph: &Graph,
3449 events: &Node<RemoteResponderEvent<TRequest, TResponse>>,
3450 name: &str,
3451) -> Node<RemoteResponderStatus>
3452where
3453 TRequest: Clone + 'static,
3454 TResponse: Clone + 'static,
3455{
3456 graph.node_opts::<RemoteResponderStatus, _>(
3457 vec![events.erased()],
3458 |ctx| {
3459 let mut status = ctx
3460 .state_get::<RemoteResponderStatus>()
3461 .map_or_else(initial_remote_responder_status, |status| (*status).clone());
3462 for event in ctx.batch::<RemoteResponderEvent<TRequest, TResponse>>(0) {
3463 match event.as_ref() {
3464 RemoteResponderEvent::Response {
3465 request_id,
3466 operation,
3467 command,
3468 } => {
3469 status.request_id = Some(request_id.clone());
3470 status.operation = Some(operation.clone());
3471 if response_command_error(command).is_some() {
3472 status.state = RemoteResponderStatusState::Rejected;
3473 status.rejected = status.rejected.saturating_add(1);
3474 } else {
3475 status.state = RemoteResponderStatusState::Responded;
3476 status.handled = status.handled.saturating_add(1);
3477 }
3478 }
3479 RemoteResponderEvent::Rejected {
3480 request_id,
3481 operation,
3482 ..
3483 } => {
3484 status.state = RemoteResponderStatusState::Rejected;
3485 status.request_id = request_id.clone();
3486 status.operation = operation.clone();
3487 status.rejected = status.rejected.saturating_add(1);
3488 }
3489 RemoteResponderEvent::Invalid { .. } => {
3490 status.state = RemoteResponderStatusState::Errored;
3491 status.errors = status.errors.saturating_add(1);
3492 }
3493 RemoteResponderEvent::Request { request, .. } => {
3494 status.request_id = Some(request.request_id.clone());
3495 status.operation = Some(request.operation.clone());
3496 }
3497 }
3498 }
3499 ctx.state_set(status.clone());
3500 ctx.emit(status);
3501 },
3502 no_terminal_graph_opts(format!("{name}/status")),
3503 )
3504}
3505
3506fn initial_remote_responder_status() -> RemoteResponderStatus {
3507 RemoteResponderStatus {
3508 state: RemoteResponderStatusState::Idle,
3509 operation: None,
3510 request_id: None,
3511 handled: 0,
3512 rejected: 0,
3513 errors: 0,
3514 }
3515}
3516
3517fn remote_responder_errors_node<TRequest, TResponse>(
3518 graph: &Graph,
3519 events: &Node<RemoteResponderEvent<TRequest, TResponse>>,
3520 name: &str,
3521) -> Node<RemoteCallError>
3522where
3523 TRequest: Clone + 'static,
3524 TResponse: Clone + 'static,
3525{
3526 graph.node_opts::<RemoteCallError, _>(
3527 vec![events.erased()],
3528 |ctx| {
3529 for event in ctx.batch::<RemoteResponderEvent<TRequest, TResponse>>(0) {
3530 match event.as_ref() {
3531 RemoteResponderEvent::Response {
3532 request_id,
3533 operation,
3534 command,
3535 } => {
3536 if let Some(error) = response_command_error(command) {
3537 ctx.emit(RemoteCallError {
3538 operation: Some(operation.clone()),
3539 request_id: Some(request_id.clone()),
3540 error,
3541 });
3542 }
3543 }
3544 RemoteResponderEvent::Rejected {
3545 request_id,
3546 operation,
3547 error,
3548 ..
3549 } => ctx.emit(RemoteCallError {
3550 operation: operation.clone(),
3551 request_id: request_id.clone(),
3552 error: error.clone(),
3553 }),
3554 RemoteResponderEvent::Invalid { error } => ctx.emit(RemoteCallError {
3555 operation: None,
3556 request_id: None,
3557 error: error.clone(),
3558 }),
3559 _ => {}
3560 }
3561 }
3562 },
3563 no_terminal_graph_opts(format!("{name}/errors")),
3564 )
3565}
3566
3567fn response_command_error<T>(command: &WireBridgeCommand<RemoteCallResponse<T>>) -> Option<String> {
3568 match command {
3569 WireBridgeCommand::Send {
3570 payload: RemoteCallResponse::Error { error, .. },
3571 ..
3572 } => Some(error.clone()),
3573 _ => None,
3574 }
3575}
3576
3577fn no_terminal_graph_opts(name: impl Into<String>) -> GraphNodeOpts {
3578 let mut opts = GraphNodeOpts::named(name);
3579 opts.node.partial = true;
3580 opts.node.complete_when_deps_complete = false;
3581 opts.node.error_when_deps_error = false;
3582 opts
3583}
3584
3585fn panic_message(panic: &Box<dyn std::any::Any + Send>) -> Option<String> {
3586 panic.downcast_ref::<String>().cloned().or_else(|| {
3587 panic
3588 .downcast_ref::<&'static str>()
3589 .map(|message| (*message).to_owned())
3590 })
3591}
3592
3593fn is_graph_invariant_panic(message: &str) -> bool {
3594 message.contains("R-reentrancy")
3595 || message.contains("R-rewire")
3596 || message.contains("D22")
3597 || message.contains("same graph")
3598 || message.contains("different graph")
3599}
3600
3601fn attach_wire_bridge_command_source<TOutbound, TInbound>(
3602 bridge: &WireBridgeBundle<TOutbound, TInbound>,
3603 source: Core,
3604) where
3605 TOutbound: Clone + 'static,
3606 TInbound: Clone + 'static,
3607{
3608 attach_wire_bridge_command_source_parts(&bridge.command, &bridge.command_sources, source);
3609}
3610
3611fn attach_wire_bridge_inbound_source<TOutbound, TInbound>(
3612 bridge: &WireBridgeBundle<TOutbound, TInbound>,
3613 source: Core,
3614) where
3615 TOutbound: Clone + 'static,
3616 TInbound: Clone + 'static,
3617{
3618 attach_wire_bridge_inbound_source_parts(&bridge.inbound, &bridge.inbound_sources, source);
3619}
3620
3621fn attach_wire_bridge_command_source_parts<TOutbound>(
3622 command: &Node<WireBridgeCommand<TOutbound>>,
3623 sources: &Rc<RefCell<Vec<Core>>>,
3624 source: Core,
3625) where
3626 TOutbound: Clone + 'static,
3627{
3628 let previous = sources.borrow().clone();
3629 {
3630 let mut current = sources.borrow_mut();
3631 if !current.iter().any(|candidate| candidate.ptr_eq(&source)) {
3632 current.push(source.clone());
3633 }
3634 }
3635 let command_sources = sources.borrow().clone();
3636 let source_count = command_sources.len();
3637 let rewire = catch_unwind(AssertUnwindSafe(|| {
3638 command.replace_deps(
3639 command_sources,
3640 wire_bridge_command_body::<TOutbound>(source_count),
3641 );
3642 }));
3643 if let Err(panic) = rewire {
3644 *sources.borrow_mut() = previous.clone();
3645 command.replace_deps(
3646 previous,
3647 wire_bridge_command_body::<TOutbound>(sources.borrow().len()),
3648 );
3649 resume_unwind(panic);
3650 }
3651}
3652
3653fn detach_wire_bridge_command_source<TOutbound>(
3654 command: &Node<WireBridgeCommand<TOutbound>>,
3655 sources: &Rc<RefCell<Vec<Core>>>,
3656 source: Core,
3657) where
3658 TOutbound: Clone + 'static,
3659{
3660 let previous = sources.borrow().clone();
3661 if !previous.iter().any(|candidate| candidate.ptr_eq(&source)) {
3662 return;
3663 }
3664 let next = previous
3665 .iter()
3666 .filter(|candidate| !candidate.ptr_eq(&source))
3667 .cloned()
3668 .collect::<Vec<_>>();
3669 *sources.borrow_mut() = next.clone();
3670 let rewire = catch_unwind(AssertUnwindSafe(|| {
3671 command.replace_deps(
3672 next,
3673 wire_bridge_command_body::<TOutbound>(sources.borrow().len()),
3674 );
3675 }));
3676 if let Err(panic) = rewire {
3677 *sources.borrow_mut() = previous.clone();
3678 command.replace_deps(
3679 previous,
3680 wire_bridge_command_body::<TOutbound>(sources.borrow().len()),
3681 );
3682 resume_unwind(panic);
3683 }
3684}
3685
3686fn attach_wire_bridge_inbound_source_parts<TInbound>(
3687 inbound: &WireBridgeInbound<TInbound>,
3688 sources: &Rc<RefCell<Vec<Core>>>,
3689 source: Core,
3690) where
3691 TInbound: Clone + 'static,
3692{
3693 let previous = sources.borrow().clone();
3694 {
3695 let mut current = sources.borrow_mut();
3696 if !current.iter().any(|candidate| candidate.ptr_eq(&source)) {
3697 current.push(source.clone());
3698 }
3699 }
3700 let inbound_sources = sources.borrow().clone();
3701 let source_count = inbound_sources.len();
3702 let rewire = catch_unwind(AssertUnwindSafe(|| {
3703 inbound.node.replace_deps(
3704 inbound_sources,
3705 wire_bridge_inbound_body::<TInbound>(source_count),
3706 );
3707 }));
3708 if let Err(panic) = rewire {
3709 *sources.borrow_mut() = previous.clone();
3710 inbound.node.replace_deps(
3711 previous,
3712 wire_bridge_inbound_body::<TInbound>(sources.borrow().len()),
3713 );
3714 resume_unwind(panic);
3715 }
3716}
3717
3718fn detach_wire_bridge_inbound_source<TInbound>(
3719 inbound: &WireBridgeInbound<TInbound>,
3720 sources: &Rc<RefCell<Vec<Core>>>,
3721 source: Core,
3722) where
3723 TInbound: Clone + 'static,
3724{
3725 let previous = sources.borrow().clone();
3726 if !previous.iter().any(|candidate| candidate.ptr_eq(&source)) {
3727 return;
3728 }
3729 let next = previous
3730 .iter()
3731 .filter(|candidate| !candidate.ptr_eq(&source))
3732 .cloned()
3733 .collect::<Vec<_>>();
3734 *sources.borrow_mut() = next.clone();
3735 let rewire = catch_unwind(AssertUnwindSafe(|| {
3736 inbound.node.replace_deps(
3737 next,
3738 wire_bridge_inbound_body::<TInbound>(sources.borrow().len()),
3739 );
3740 }));
3741 if let Err(panic) = rewire {
3742 *sources.borrow_mut() = previous.clone();
3743 inbound.node.replace_deps(
3744 previous,
3745 wire_bridge_inbound_body::<TInbound>(sources.borrow().len()),
3746 );
3747 resume_unwind(panic);
3748 }
3749}
3750
3751fn wire_bridge_inbound_body<TInbound: Clone + 'static>(
3752 source_count: usize,
3753) -> impl Fn(&Ctx) + 'static {
3754 move |ctx: &Ctx| {
3755 for index in 0..source_count {
3756 for ingress in ctx.batch::<WireBridgeIngress<TInbound>>(index) {
3757 ctx.emit((*ingress).clone());
3758 }
3759 }
3760 }
3761}
3762
3763fn wire_bridge_command_body<T: Clone + 'static>(source_count: usize) -> impl Fn(&Ctx) + 'static {
3764 move |ctx: &Ctx| {
3765 for index in 0..source_count {
3766 for command in ctx.batch::<WireBridgeCommand<T>>(index) {
3767 ctx.emit((*command).clone());
3768 }
3769 }
3770 }
3771}
3772
3773fn wire_bridge_events_node<TOutbound, TInbound>(
3774 graph: &Graph,
3775 command: &Node<WireBridgeCommand<TOutbound>>,
3776 inbound: &Node<WireBridgeIngress<TInbound>>,
3777 name: String,
3778 opts: WireBridgeOptions,
3779) -> Node<WireBridgeEvent<TOutbound, TInbound>>
3780where
3781 TOutbound: Clone + 'static,
3782 TInbound: Clone + 'static,
3783{
3784 let session_id = opts.session_id.clone();
3785 let policy = opts.retry.clone();
3786 let now = opts
3787 .now_ms
3788 .clone()
3789 .unwrap_or_else(|| Rc::new(|| 0_u64) as Rc<dyn Fn() -> u64>);
3790 let event_opts = GraphNodeOpts {
3791 name: Some(format!("{name}/events")),
3792 node: NodeOpts {
3793 partial: true,
3794 complete_when_deps_complete: false,
3795 error_when_deps_error: false,
3796 terminal_as_real_input: true,
3797 ..NodeOpts::default()
3798 },
3799 ..GraphNodeOpts::default()
3800 };
3801 graph.node_opts::<WireBridgeEvent<TOutbound, TInbound>, _>(
3802 vec![command.erased(), inbound.erased()],
3803 move |ctx| {
3804 let state = bridge_state::<TOutbound>(ctx);
3805 state.borrow_mut().active = true;
3806 install_cleanup(ctx, state.clone());
3807 for value in raw_data(ctx, 1) {
3808 match value.downcast::<WireBridgeIngress<TInbound>>() {
3809 Ok(ingress) => process_inbound(
3810 ctx,
3811 &state,
3812 (*ingress).clone(),
3813 &session_id,
3814 ),
3815 Err(_) => ctx.emit(WireBridgeEvent::<TOutbound, TInbound>::Invalid {
3816 error: "wireBridge: inbound DATA must carry a wire bridge envelope"
3817 .to_owned(),
3818 }),
3819 }
3820 }
3821 for value in raw_data(ctx, 0) {
3822 match value.downcast::<WireBridgeCommand<TOutbound>>() {
3823 Ok(command) => {
3824 process_command::<TOutbound, TInbound>(
3825 ctx,
3826 &state,
3827 (*command).clone(),
3828 &opts,
3829 &policy,
3830 &now,
3831 );
3832 }
3833 Err(_) => ctx.emit(WireBridgeEvent::<TOutbound, TInbound>::Invalid {
3834 error: "wireBridge: command DATA must carry a wire bridge command"
3835 .to_owned(),
3836 }),
3837 }
3838 }
3839 if let Some(terminal) = ctx.terminal(1) {
3840 let error = match terminal {
3841 DepTerminal::Complete => format!(
3842 "{session_id}: inbound protocol COMPLETE is local misuse; remote completion must arrive as a DATA envelope fact"
3843 ),
3844 DepTerminal::Error(error) => format!(
3845 "{session_id}: inbound protocol ERROR {error} is local misuse; remote errors must arrive as DATA envelope facts"
3846 ),
3847 };
3848 ctx.emit(WireBridgeEvent::<TOutbound, TInbound>::Invalid { error });
3849 }
3850 },
3851 event_opts,
3852 )
3853}
3854
3855fn bridge_state<T: Clone + 'static>(ctx: &Ctx) -> Rc<RefCell<BridgeState<T>>> {
3856 if let Some(state) = ctx.state_get::<RefCell<BridgeState<T>>>() {
3857 return state;
3858 }
3859 let state = RefCell::new(BridgeState::<T> {
3860 active: true,
3861 cleanup_installed: false,
3862 next_seq: 1,
3863 cursor: 0,
3864 remote_cursor: 0,
3865 pending: BTreeMap::new(),
3866 });
3867 ctx.state_set(state);
3868 ctx.state_get::<RefCell<BridgeState<T>>>()
3869 .expect("bridge state was just installed")
3870}
3871
3872fn install_cleanup<T: Clone + 'static>(ctx: &Ctx, state: Rc<RefCell<BridgeState<T>>>) {
3873 if state.borrow().cleanup_installed {
3874 return;
3875 }
3876 state.borrow_mut().cleanup_installed = true;
3877 ctx.on_deactivation(move || {
3878 let mut state = state.borrow_mut();
3879 state.active = false;
3880 state.cleanup_installed = false;
3881 state.pending.clear();
3882 });
3883}
3884
3885fn raw_data(ctx: &Ctx, dep: usize) -> Vec<AnyValue> {
3886 ctx.wave_data()
3887 .get(dep)
3888 .map(|waves| {
3889 waves
3890 .iter()
3891 .flat_map(|wave| {
3892 wave.iter().filter_map(|item| match item {
3893 WaveData::Data(value) => Some(value.clone()),
3894 WaveData::Sentinel => None,
3895 })
3896 })
3897 .collect()
3898 })
3899 .unwrap_or_default()
3900}
3901
3902fn process_command<TOutbound, TInbound>(
3903 ctx: &Ctx,
3904 state: &Rc<RefCell<BridgeState<TOutbound>>>,
3905 command: WireBridgeCommand<TOutbound>,
3906 opts: &WireBridgeOptions,
3907 policy: &RetryPolicy,
3908 now: &Rc<dyn Fn() -> u64>,
3909) where
3910 TOutbound: Clone + 'static,
3911 TInbound: Clone + 'static,
3912{
3913 match command {
3914 WireBridgeCommand::Start {
3915 idempotency_key,
3916 request_id,
3917 } => emit_outbound::<TOutbound, TInbound>(
3918 ctx,
3919 state,
3920 OutboundSpec {
3921 envelope_type: WireBridgeEnvelopeType::Start,
3922 payload: None,
3923 idempotency_key,
3924 request_id,
3925 ack_for_seq: None,
3926 track_ack: true,
3927 clear_pending_first: false,
3928 },
3929 opts,
3930 policy,
3931 now,
3932 ),
3933 WireBridgeCommand::Send {
3934 payload,
3935 idempotency_key,
3936 request_id,
3937 } => emit_outbound::<TOutbound, TInbound>(
3938 ctx,
3939 state,
3940 OutboundSpec {
3941 envelope_type: WireBridgeEnvelopeType::Data,
3942 payload: Some(WireBridgePayload::Data(payload)),
3943 idempotency_key,
3944 request_id,
3945 ack_for_seq: None,
3946 track_ack: true,
3947 clear_pending_first: false,
3948 },
3949 opts,
3950 policy,
3951 now,
3952 ),
3953 WireBridgeCommand::Ack {
3954 ack_for_seq,
3955 idempotency_key,
3956 request_id,
3957 } => {
3958 if ack_for_seq == 0 {
3959 ctx.emit(WireBridgeEvent::<TOutbound, TInbound>::Invalid {
3960 error: "wireBridge: ack command ack_for_seq must be positive".to_owned(),
3961 });
3962 return;
3963 }
3964 emit_outbound::<TOutbound, TInbound>(
3965 ctx,
3966 state,
3967 OutboundSpec {
3968 envelope_type: WireBridgeEnvelopeType::Ack,
3969 payload: None,
3970 idempotency_key,
3971 request_id,
3972 ack_for_seq: Some(ack_for_seq),
3973 track_ack: false,
3974 clear_pending_first: false,
3975 },
3976 opts,
3977 policy,
3978 now,
3979 );
3980 }
3981 WireBridgeCommand::Nack {
3982 ack_for_seq,
3983 error,
3984 idempotency_key,
3985 request_id,
3986 } => {
3987 if ack_for_seq == 0 {
3988 ctx.emit(WireBridgeEvent::<TOutbound, TInbound>::Invalid {
3989 error: "wireBridge: nack command ack_for_seq must be positive".to_owned(),
3990 });
3991 return;
3992 }
3993 emit_outbound::<TOutbound, TInbound>(
3994 ctx,
3995 state,
3996 OutboundSpec {
3997 envelope_type: WireBridgeEnvelopeType::Nack,
3998 payload: Some(WireBridgePayload::Error(error)),
3999 idempotency_key,
4000 request_id,
4001 ack_for_seq: Some(ack_for_seq),
4002 track_ack: false,
4003 clear_pending_first: false,
4004 },
4005 opts,
4006 policy,
4007 now,
4008 );
4009 }
4010 WireBridgeCommand::Close {
4011 reason,
4012 idempotency_key,
4013 } => {
4014 emit_outbound::<TOutbound, TInbound>(
4015 ctx,
4016 state,
4017 OutboundSpec {
4018 envelope_type: WireBridgeEnvelopeType::Close,
4019 payload: Some(WireBridgePayload::Close { reason }),
4020 idempotency_key,
4021 request_id: None,
4022 ack_for_seq: None,
4023 track_ack: true,
4024 clear_pending_first: true,
4025 },
4026 opts,
4027 policy,
4028 now,
4029 );
4030 }
4031 WireBridgeCommand::AckTimeout {
4032 seq,
4033 attempt,
4034 observed_at_ms,
4035 } => process_ack_timeout_command::<TOutbound, TInbound>(
4036 ctx,
4037 state,
4038 AckTimeoutCommandInput {
4039 seq,
4040 attempt,
4041 observed_at_ms,
4042 },
4043 opts,
4044 policy,
4045 now,
4046 ),
4047 }
4048}
4049
4050struct AckTimeoutCommandInput {
4051 seq: u64,
4052 attempt: u32,
4053 observed_at_ms: Option<u64>,
4054}
4055
4056struct OutboundSpec<T> {
4057 envelope_type: WireBridgeEnvelopeType,
4058 payload: Option<WireBridgePayload<T>>,
4059 idempotency_key: Option<String>,
4060 request_id: Option<String>,
4061 ack_for_seq: Option<u64>,
4062 track_ack: bool,
4063 clear_pending_first: bool,
4064}
4065
4066fn emit_outbound<TOutbound, TInbound>(
4067 ctx: &Ctx,
4068 state: &Rc<RefCell<BridgeState<TOutbound>>>,
4069 spec: OutboundSpec<TOutbound>,
4070 opts: &WireBridgeOptions,
4071 policy: &RetryPolicy,
4072 now: &Rc<dyn Fn() -> u64>,
4073) where
4074 TOutbound: Clone + 'static,
4075 TInbound: Clone + 'static,
4076{
4077 let Some((seq, cursor, next_seq)) = ({
4078 let state = state.borrow();
4079 if state.next_seq == u64::MAX {
4080 None
4081 } else {
4082 Some((state.next_seq, state.cursor, state.next_seq + 1))
4083 }
4084 }) else {
4085 ctx.emit(WireBridgeEvent::<TOutbound, TInbound>::Invalid {
4086 error: format!("{}: next outbound seq exceeded u64::MAX", opts.session_id),
4087 });
4088 return;
4089 };
4090 let envelope = match wire_bridge_envelope(WireBridgeEnvelopeInput {
4091 session_id: opts.session_id.clone(),
4092 envelope_type: spec.envelope_type,
4093 seq,
4094 cursor,
4095 payload: spec.payload,
4096 idempotency_key: spec.idempotency_key,
4097 attempt: 1,
4098 max_attempts: policy.max_attempts,
4099 timestamp_ms: Some(now()),
4100 ack_for_seq: spec.ack_for_seq,
4101 request_id: spec.request_id,
4102 }) {
4103 Ok(envelope) => envelope,
4104 Err(error) => {
4105 ctx.emit(WireBridgeEvent::<TOutbound, TInbound>::Invalid {
4106 error: error.to_string(),
4107 });
4108 return;
4109 }
4110 };
4111 if spec.clear_pending_first {
4112 clear_pending(state);
4113 }
4114 state.borrow_mut().next_seq = next_seq;
4115 ctx.emit(WireBridgeEvent::<TOutbound, TInbound>::Outbound {
4116 envelope: envelope.clone(),
4117 });
4118 if spec.track_ack {
4119 state.borrow_mut().pending.insert(
4120 envelope.metadata.seq,
4121 PendingEnvelope {
4122 envelope,
4123 timeout_reported_attempt: None,
4124 retry_due_at_ms: None,
4125 },
4126 );
4127 }
4128}
4129
4130fn clear_pending<T>(state: &Rc<RefCell<BridgeState<T>>>) {
4131 state.borrow_mut().pending.clear();
4132}
4133
4134fn process_ack_timeout_command<TOutbound, TInbound>(
4135 ctx: &Ctx,
4136 state: &Rc<RefCell<BridgeState<TOutbound>>>,
4137 input: AckTimeoutCommandInput,
4138 opts: &WireBridgeOptions,
4139 policy: &RetryPolicy,
4140 now: &Rc<dyn Fn() -> u64>,
4141) where
4142 TOutbound: Clone + 'static,
4143 TInbound: Clone + 'static,
4144{
4145 let AckTimeoutCommandInput {
4146 seq,
4147 attempt,
4148 observed_at_ms,
4149 } = input;
4150 if seq == 0 {
4151 ctx.emit(WireBridgeEvent::<TOutbound, TInbound>::Invalid {
4152 error: "wireBridge: ack-timeout command seq must be positive".to_owned(),
4153 });
4154 return;
4155 }
4156 if attempt == 0 {
4157 ctx.emit(WireBridgeEvent::<TOutbound, TInbound>::Invalid {
4158 error: "wireBridge: ack-timeout command attempt must be positive".to_owned(),
4159 });
4160 return;
4161 }
4162 let current = {
4163 let state_ref = state.borrow();
4164 let Some(pending) = state_ref.pending.get(&seq) else {
4165 return;
4166 };
4167 if pending.envelope.metadata.attempt != attempt {
4168 return;
4169 }
4170 pending.envelope.clone()
4171 };
4172 let retry_due = {
4173 let pending = state.borrow();
4174 if let Some(pending) = pending.pending.get(&seq) {
4175 if pending.timeout_reported_attempt == Some(attempt) {
4176 if let (Some(retry_due_at_ms), Some(observed_at_ms)) =
4177 (pending.retry_due_at_ms, observed_at_ms)
4178 {
4179 if observed_at_ms < retry_due_at_ms {
4180 return;
4181 }
4182 }
4183 true
4184 } else {
4185 false
4186 }
4187 } else {
4188 false
4189 }
4190 };
4191 if retry_due {
4192 emit_retry_outbound::<TOutbound, TInbound>(ctx, state, policy, now, current);
4193 return;
4194 }
4195 ctx.emit(WireBridgeEvent::<TOutbound, TInbound>::Timeout { seq, attempt });
4196 if !policy.should_retry(attempt) {
4197 state.borrow_mut().pending.remove(&seq);
4198 ctx.emit(WireBridgeEvent::<TOutbound, TInbound>::Exhausted {
4199 seq,
4200 attempt,
4201 error: format!("{}: ack timeout for seq {seq}", opts.session_id),
4202 });
4203 return;
4204 }
4205 let next_attempt = attempt.saturating_add(1);
4206 let delay_ms = policy.next_delay_ms(next_attempt).unwrap_or_default();
4207 if let Some(pending) = state.borrow_mut().pending.get_mut(&seq) {
4208 pending.timeout_reported_attempt = Some(attempt);
4209 if delay_ms > 0 {
4210 pending.retry_due_at_ms = observed_at_ms.map(|ms| ms.saturating_add(delay_ms));
4211 }
4212 }
4213 ctx.emit(WireBridgeEvent::<TOutbound, TInbound>::Retry {
4214 seq,
4215 attempt: next_attempt,
4216 delay_ms,
4217 error: format!("{}: ack timeout for seq {seq}", opts.session_id),
4218 });
4219 let should_wait = state
4220 .borrow()
4221 .pending
4222 .get(&seq)
4223 .and_then(|pending| pending.retry_due_at_ms)
4224 .is_some();
4225 if !should_wait {
4226 emit_retry_outbound::<TOutbound, TInbound>(ctx, state, policy, now, current);
4227 }
4228}
4229
4230fn emit_retry_outbound<TOutbound, TInbound>(
4231 ctx: &Ctx,
4232 state: &Rc<RefCell<BridgeState<TOutbound>>>,
4233 policy: &RetryPolicy,
4234 now: &Rc<dyn Fn() -> u64>,
4235 current: WireBridgeEnvelope<TOutbound>,
4236) where
4237 TOutbound: Clone + 'static,
4238 TInbound: Clone + 'static,
4239{
4240 let seq = current.metadata.seq;
4241 let attempt = current.metadata.attempt.saturating_add(1);
4242 let cursor = state.borrow().cursor;
4243 let timestamp_ms = now();
4244 let retry = wire_bridge_envelope(WireBridgeEnvelopeInput {
4245 session_id: current.session_id.clone(),
4246 envelope_type: current.envelope_type,
4247 seq,
4248 cursor,
4249 payload: current.payload.clone(),
4250 idempotency_key: Some(current.metadata.idempotency_key.clone()),
4251 attempt,
4252 max_attempts: policy.max_attempts,
4253 timestamp_ms: Some(timestamp_ms),
4254 ack_for_seq: current.metadata.ack_for_seq,
4255 request_id: current.metadata.request_id.clone(),
4256 })
4257 .expect("retry envelope keeps validated metadata");
4258 if let Some(pending) = state.borrow_mut().pending.get_mut(&seq) {
4259 pending.envelope = retry.clone();
4260 pending.timeout_reported_attempt = None;
4261 pending.retry_due_at_ms = None;
4262 }
4263 ctx.emit(WireBridgeEvent::<TOutbound, TInbound>::Outbound { envelope: retry });
4264}
4265
4266fn process_inbound<TOutbound, TInbound>(
4267 ctx: &Ctx,
4268 state: &Rc<RefCell<BridgeState<TOutbound>>>,
4269 ingress: WireBridgeIngress<TInbound>,
4270 session_id: &str,
4271) where
4272 TOutbound: Clone + 'static,
4273 TInbound: Clone + 'static,
4274{
4275 let envelope = match ingress {
4276 WireBridgeIngress::Envelope(envelope) => envelope,
4277 WireBridgeIngress::Invalid(error) => {
4278 ctx.emit(WireBridgeEvent::<TOutbound, TInbound>::Invalid { error });
4279 return;
4280 }
4281 };
4282 if let Err(error) = validate_inbound_envelope(&envelope) {
4283 ctx.emit(WireBridgeEvent::<TOutbound, TInbound>::Invalid {
4284 error: error.to_string(),
4285 });
4286 return;
4287 }
4288 if envelope.session_id != session_id {
4289 ctx.emit(WireBridgeEvent::<TOutbound, TInbound>::SessionMismatch {
4290 expected: session_id.to_owned(),
4291 actual: envelope.session_id,
4292 });
4293 return;
4294 }
4295 let seq = envelope.metadata.seq;
4296 let early_event = {
4297 let mut state_mut = state.borrow_mut();
4298 let expected = state_mut.cursor.saturating_add(1);
4299 if seq <= state_mut.cursor {
4300 Some(WireBridgeEvent::<TOutbound, TInbound>::Duplicate {
4301 seq,
4302 cursor: state_mut.cursor,
4303 })
4304 } else if seq > expected {
4305 Some(WireBridgeEvent::<TOutbound, TInbound>::OutOfOrder { seq, expected })
4306 } else if envelope.metadata.cursor < state_mut.remote_cursor {
4307 Some(WireBridgeEvent::<TOutbound, TInbound>::Invalid {
4308 error: format!(
4309 "{session_id}: inbound cursor {} regressed below {}",
4310 envelope.metadata.cursor, state_mut.remote_cursor
4311 ),
4312 })
4313 } else {
4314 state_mut.remote_cursor = envelope.metadata.cursor;
4315 state_mut.cursor = seq;
4316 None
4317 }
4318 };
4319 if let Some(event) = early_event {
4320 ctx.emit(event);
4321 return;
4322 }
4323 ctx.emit(WireBridgeEvent::<TOutbound, TInbound>::Cursor { cursor: seq });
4324 ctx.emit(WireBridgeEvent::<TOutbound, TInbound>::Inbound {
4325 envelope: envelope.clone(),
4326 });
4327 match envelope.envelope_type {
4328 WireBridgeEnvelopeType::Ack => process_ack(ctx, state, envelope),
4329 WireBridgeEnvelopeType::Nack => process_nack(ctx, state, envelope),
4330 WireBridgeEnvelopeType::Start
4331 | WireBridgeEnvelopeType::Data
4332 | WireBridgeEnvelopeType::Status
4333 | WireBridgeEnvelopeType::Error
4334 | WireBridgeEnvelopeType::Close => {}
4335 }
4336}
4337
4338fn validate_inbound_envelope<T>(
4339 envelope: &WireBridgeEnvelope<T>,
4340) -> Result<(), WireBridgeEnvelopeError> {
4341 if envelope.session_id.is_empty() {
4342 return Err(WireBridgeEnvelopeError::EmptySessionId);
4343 }
4344 if envelope.metadata.seq == 0 {
4345 return Err(WireBridgeEnvelopeError::ZeroSeq);
4346 }
4347 if envelope.metadata.idempotency_key.is_empty() {
4348 return Err(WireBridgeEnvelopeError::EmptyIdempotencyKey);
4349 }
4350 if envelope.metadata.attempt == 0 {
4351 return Err(WireBridgeEnvelopeError::ZeroAttempt);
4352 }
4353 if envelope.metadata.max_attempts < envelope.metadata.attempt {
4354 return Err(WireBridgeEnvelopeError::MaxAttemptsBeforeAttempt);
4355 }
4356 if envelope.metadata.ack_for_seq == Some(0) {
4357 return Err(WireBridgeEnvelopeError::ZeroAckForSeq);
4358 }
4359 if matches!(
4360 envelope.envelope_type,
4361 WireBridgeEnvelopeType::Ack | WireBridgeEnvelopeType::Nack
4362 ) && envelope.metadata.ack_for_seq.is_none()
4363 {
4364 return Err(WireBridgeEnvelopeError::MissingAckForSeq);
4365 }
4366 validate_payload_for_type(envelope.envelope_type, &envelope.payload)?;
4367 Ok(())
4368}
4369
4370fn validate_payload_for_type<T>(
4371 envelope_type: WireBridgeEnvelopeType,
4372 payload: &Option<WireBridgePayload<T>>,
4373) -> Result<(), WireBridgeEnvelopeError> {
4374 match envelope_type {
4375 WireBridgeEnvelopeType::Data => match payload {
4376 Some(WireBridgePayload::Data(_)) => Ok(()),
4377 Some(_) => Err(WireBridgeEnvelopeError::PayloadTypeMismatch),
4378 None => Err(WireBridgeEnvelopeError::MissingPayload),
4379 },
4380 WireBridgeEnvelopeType::Nack | WireBridgeEnvelopeType::Error => match payload {
4381 Some(WireBridgePayload::Error(_)) => Ok(()),
4382 Some(_) => Err(WireBridgeEnvelopeError::PayloadTypeMismatch),
4383 None => Err(WireBridgeEnvelopeError::MissingPayload),
4384 },
4385 WireBridgeEnvelopeType::Status => match payload {
4386 Some(WireBridgePayload::Status(_)) => Ok(()),
4387 Some(_) => Err(WireBridgeEnvelopeError::PayloadTypeMismatch),
4388 None => Err(WireBridgeEnvelopeError::MissingPayload),
4389 },
4390 WireBridgeEnvelopeType::Close => match payload {
4391 Some(WireBridgePayload::Close { .. }) => Ok(()),
4392 Some(_) => Err(WireBridgeEnvelopeError::PayloadTypeMismatch),
4393 None => Err(WireBridgeEnvelopeError::MissingPayload),
4394 },
4395 WireBridgeEnvelopeType::Start | WireBridgeEnvelopeType::Ack => match payload {
4396 Some(_) => Err(WireBridgeEnvelopeError::UnexpectedPayload),
4397 None => Ok(()),
4398 },
4399 }
4400}
4401
4402fn process_ack<TOutbound, TInbound>(
4403 ctx: &Ctx,
4404 state: &Rc<RefCell<BridgeState<TOutbound>>>,
4405 envelope: WireBridgeEnvelope<TInbound>,
4406) where
4407 TOutbound: Clone + 'static,
4408 TInbound: Clone + 'static,
4409{
4410 let ack_for_seq = envelope
4411 .metadata
4412 .ack_for_seq
4413 .expect("validated ack has ack_for_seq");
4414 let pending = state.borrow_mut().pending.remove(&ack_for_seq);
4415 match pending {
4416 Some(pending) => {
4417 ctx.emit(WireBridgeEvent::<TOutbound, TInbound>::Ack {
4418 ack_for_seq,
4419 envelope,
4420 outbound: pending.envelope,
4421 });
4422 }
4423 None => ctx.emit(WireBridgeEvent::<TOutbound, TInbound>::LateReceipt {
4424 receipt: WireBridgeReceipt::Ack,
4425 ack_for_seq,
4426 }),
4427 }
4428}
4429
4430fn process_nack<TOutbound, TInbound>(
4431 ctx: &Ctx,
4432 state: &Rc<RefCell<BridgeState<TOutbound>>>,
4433 envelope: WireBridgeEnvelope<TInbound>,
4434) where
4435 TOutbound: Clone + 'static,
4436 TInbound: Clone + 'static,
4437{
4438 let ack_for_seq = envelope
4439 .metadata
4440 .ack_for_seq
4441 .expect("validated nack has ack_for_seq");
4442 let pending = state.borrow_mut().pending.remove(&ack_for_seq);
4443 let error = payload_error_string(&envelope.payload, "remote nack");
4444 match pending {
4445 Some(pending) => {
4446 ctx.emit(WireBridgeEvent::<TOutbound, TInbound>::Nack {
4447 ack_for_seq,
4448 envelope,
4449 outbound: pending.envelope,
4450 error,
4451 });
4452 }
4453 None => ctx.emit(WireBridgeEvent::<TOutbound, TInbound>::LateReceipt {
4454 receipt: WireBridgeReceipt::Nack,
4455 ack_for_seq,
4456 }),
4457 }
4458}
4459
4460fn project_outbound<TOutbound, TInbound>(
4461 graph: &Graph,
4462 events: &Node<WireBridgeEvent<TOutbound, TInbound>>,
4463 name: &str,
4464) -> Node<WireBridgeEnvelope<TOutbound>>
4465where
4466 TOutbound: Clone + 'static,
4467 TInbound: Clone + 'static,
4468{
4469 graph.node_opts::<WireBridgeEnvelope<TOutbound>, _>(
4470 vec![events.erased()],
4471 |ctx| {
4472 for event in ctx.batch::<WireBridgeEvent<TOutbound, TInbound>>(0) {
4473 if let WireBridgeEvent::Outbound { envelope } = event.as_ref() {
4474 ctx.emit(envelope.clone());
4475 }
4476 }
4477 },
4478 GraphNodeOpts::named(format!("{name}/outbound")),
4479 )
4480}
4481
4482fn project_acks<TOutbound, TInbound>(
4483 graph: &Graph,
4484 events: &Node<WireBridgeEvent<TOutbound, TInbound>>,
4485 name: &str,
4486) -> Node<WireBridgeAck<TInbound>>
4487where
4488 TOutbound: Clone + 'static,
4489 TInbound: Clone + 'static,
4490{
4491 graph.node_opts::<WireBridgeAck<TInbound>, _>(
4492 vec![events.erased()],
4493 |ctx| {
4494 for event in ctx.batch::<WireBridgeEvent<TOutbound, TInbound>>(0) {
4495 if let WireBridgeEvent::Ack {
4496 ack_for_seq,
4497 envelope,
4498 ..
4499 } = event.as_ref()
4500 {
4501 ctx.emit(WireBridgeAck {
4502 ack_for_seq: *ack_for_seq,
4503 envelope: envelope.clone(),
4504 });
4505 }
4506 }
4507 },
4508 GraphNodeOpts::named(format!("{name}/acks")),
4509 )
4510}
4511
4512fn project_nacks<TOutbound, TInbound>(
4513 graph: &Graph,
4514 events: &Node<WireBridgeEvent<TOutbound, TInbound>>,
4515 name: &str,
4516) -> Node<WireBridgeNack<TInbound>>
4517where
4518 TOutbound: Clone + 'static,
4519 TInbound: Clone + 'static,
4520{
4521 graph.node_opts::<WireBridgeNack<TInbound>, _>(
4522 vec![events.erased()],
4523 |ctx| {
4524 for event in ctx.batch::<WireBridgeEvent<TOutbound, TInbound>>(0) {
4525 if let WireBridgeEvent::Nack {
4526 ack_for_seq,
4527 envelope,
4528 error,
4529 ..
4530 } = event.as_ref()
4531 {
4532 ctx.emit(WireBridgeNack {
4533 ack_for_seq: *ack_for_seq,
4534 envelope: envelope.clone(),
4535 error: error.clone(),
4536 });
4537 }
4538 }
4539 },
4540 GraphNodeOpts::named(format!("{name}/nacks")),
4541 )
4542}
4543
4544fn project_status<TOutbound, TInbound>(
4545 graph: &Graph,
4546 events: &Node<WireBridgeEvent<TOutbound, TInbound>>,
4547 name: &str,
4548 session_id: String,
4549) -> Node<WireBridgeStatus>
4550where
4551 TOutbound: Clone + 'static,
4552 TInbound: Clone + 'static,
4553{
4554 graph.node_opts::<WireBridgeStatus, _>(
4555 vec![events.erased()],
4556 move |ctx| {
4557 let mut next = ctx
4558 .state_get::<WireBridgeStatus>()
4559 .map_or_else(|| initial_status(&session_id), |status| (*status).clone());
4560 for event in ctx.batch::<WireBridgeEvent<TOutbound, TInbound>>(0) {
4561 next = reduce_status(next, event.as_ref());
4562 }
4563 ctx.state_set(next.clone());
4564 ctx.emit(next);
4565 },
4566 GraphNodeOpts::named(format!("{name}/status")),
4567 )
4568}
4569
4570fn initial_status(session_id: &str) -> WireBridgeStatus {
4571 WireBridgeStatus {
4572 session_id: session_id.to_owned(),
4573 state: WireBridgeStatusState::Idle,
4574 cursor: 0,
4575 next_seq: 1,
4576 pending: 0,
4577 attempts: 0,
4578 acked: 0,
4579 nacked: 0,
4580 errors: 0,
4581 last_seq: None,
4582 last_delay_ms: None,
4583 }
4584}
4585
4586fn reduce_status<TOutbound, TInbound>(
4587 mut current: WireBridgeStatus,
4588 event: &WireBridgeEvent<TOutbound, TInbound>,
4589) -> WireBridgeStatus {
4590 match event {
4591 WireBridgeEvent::Outbound { envelope } => {
4592 current.state = match envelope.envelope_type {
4593 WireBridgeEnvelopeType::Start => WireBridgeStatusState::Started,
4594 WireBridgeEnvelopeType::Close => WireBridgeStatusState::Closed,
4595 WireBridgeEnvelopeType::Data
4596 | WireBridgeEnvelopeType::Ack
4597 | WireBridgeEnvelopeType::Nack
4598 | WireBridgeEnvelopeType::Status
4599 | WireBridgeEnvelopeType::Error => WireBridgeStatusState::Open,
4600 };
4601 current.next_seq = current
4602 .next_seq
4603 .max(envelope.metadata.seq.saturating_add(1));
4604 if envelope.envelope_type == WireBridgeEnvelopeType::Close {
4605 current.pending = if envelope.metadata.attempt == 1 { 1 } else { 0 };
4606 } else if should_track_ack(envelope.envelope_type) && envelope.metadata.attempt == 1 {
4607 current.pending = current.pending.saturating_add(1);
4608 }
4609 if should_track_ack(envelope.envelope_type) {
4610 current.attempts = current.attempts.saturating_add(1);
4611 }
4612 current.last_seq = Some(envelope.metadata.seq);
4613 }
4614 WireBridgeEvent::Ack {
4615 envelope, outbound, ..
4616 } => {
4617 current.state = if outbound.envelope_type == WireBridgeEnvelopeType::Close {
4618 WireBridgeStatusState::Closed
4619 } else {
4620 WireBridgeStatusState::Open
4621 };
4622 current.pending = current.pending.saturating_sub(1);
4623 current.acked = current.acked.saturating_add(1);
4624 current.last_seq = Some(envelope.metadata.seq);
4625 }
4626 WireBridgeEvent::Nack { envelope, .. } => {
4627 current.state = WireBridgeStatusState::Errored;
4628 current.pending = current.pending.saturating_sub(1);
4629 current.nacked = current.nacked.saturating_add(1);
4630 current.errors = current.errors.saturating_add(1);
4631 current.last_seq = Some(envelope.metadata.seq);
4632 }
4633 WireBridgeEvent::Retry { seq, delay_ms, .. } => {
4634 current.state = WireBridgeStatusState::Waiting;
4635 current.last_seq = Some(*seq);
4636 current.last_delay_ms = Some(*delay_ms);
4637 }
4638 WireBridgeEvent::Exhausted { seq, .. } => {
4639 current.state = WireBridgeStatusState::Exhausted;
4640 current.pending = current.pending.saturating_sub(1);
4641 current.errors = current.errors.saturating_add(1);
4642 current.last_seq = Some(*seq);
4643 }
4644 WireBridgeEvent::Cursor { cursor } => current.cursor = *cursor,
4645 WireBridgeEvent::OutOfOrder { seq, .. } => {
4646 current.state = WireBridgeStatusState::Errored;
4647 current.errors = current.errors.saturating_add(1);
4648 current.last_seq = Some(*seq);
4649 }
4650 WireBridgeEvent::SessionMismatch { .. }
4651 | WireBridgeEvent::LateReceipt { .. }
4652 | WireBridgeEvent::Invalid { .. } => {
4653 current.state = WireBridgeStatusState::Errored;
4654 current.errors = current.errors.saturating_add(1);
4655 }
4656 WireBridgeEvent::Inbound { envelope } => match envelope.envelope_type {
4657 WireBridgeEnvelopeType::Error => {
4658 current.state = WireBridgeStatusState::Errored;
4659 current.errors = current.errors.saturating_add(1);
4660 current.last_seq = Some(envelope.metadata.seq);
4661 }
4662 WireBridgeEnvelopeType::Close => {
4663 current.state = WireBridgeStatusState::Closed;
4664 current.last_seq = Some(envelope.metadata.seq);
4665 }
4666 WireBridgeEnvelopeType::Start
4667 | WireBridgeEnvelopeType::Data
4668 | WireBridgeEnvelopeType::Ack
4669 | WireBridgeEnvelopeType::Nack
4670 | WireBridgeEnvelopeType::Status => {}
4671 },
4672 WireBridgeEvent::Timeout { .. } | WireBridgeEvent::Duplicate { .. } => {}
4673 }
4674 current
4675}
4676
4677fn project_errors<TOutbound, TInbound>(
4678 graph: &Graph,
4679 events: &Node<WireBridgeEvent<TOutbound, TInbound>>,
4680 name: &str,
4681) -> Node<String>
4682where
4683 TOutbound: Clone + 'static,
4684 TInbound: Clone + 'static,
4685{
4686 let name = name.to_owned();
4687 let node_name = name.clone();
4688 graph.node_opts::<String, _>(
4689 vec![events.erased()],
4690 move |ctx| {
4691 for event in ctx.batch::<WireBridgeEvent<TOutbound, TInbound>>(0) {
4692 match event.as_ref() {
4693 WireBridgeEvent::Nack { error, .. }
4694 | WireBridgeEvent::Exhausted { error, .. }
4695 | WireBridgeEvent::Invalid { error } => ctx.emit(error.clone()),
4696 WireBridgeEvent::OutOfOrder { seq, expected } => ctx.emit(format!(
4697 "{name}: inbound seq {seq} arrived before expected seq {expected}"
4698 )),
4699 WireBridgeEvent::SessionMismatch { expected, actual } => ctx.emit(format!(
4700 "{name}: inbound session {actual} did not match expected {expected}"
4701 )),
4702 WireBridgeEvent::LateReceipt {
4703 receipt,
4704 ack_for_seq,
4705 } => ctx.emit(format!(
4706 "{name}: late {receipt:?} for unknown or completed ack_for_seq {ack_for_seq}"
4707 )),
4708 WireBridgeEvent::Inbound { envelope }
4709 if envelope.envelope_type == WireBridgeEnvelopeType::Error =>
4710 {
4711 ctx.emit(payload_error_string(
4712 &envelope.payload,
4713 "remote error envelope",
4714 ));
4715 }
4716 WireBridgeEvent::Outbound { .. }
4717 | WireBridgeEvent::Inbound { .. }
4718 | WireBridgeEvent::Ack { .. }
4719 | WireBridgeEvent::Timeout { .. }
4720 | WireBridgeEvent::Retry { .. }
4721 | WireBridgeEvent::Cursor { .. }
4722 | WireBridgeEvent::Duplicate { .. } => {}
4723 }
4724 }
4725 },
4726 GraphNodeOpts::named(format!("{node_name}/errors")),
4727 )
4728}
4729
4730fn project_cursor<TOutbound, TInbound>(
4731 graph: &Graph,
4732 events: &Node<WireBridgeEvent<TOutbound, TInbound>>,
4733 name: &str,
4734) -> Node<u64>
4735where
4736 TOutbound: Clone + 'static,
4737 TInbound: Clone + 'static,
4738{
4739 graph.node_opts::<u64, _>(
4740 vec![events.erased()],
4741 |ctx| {
4742 for event in ctx.batch::<WireBridgeEvent<TOutbound, TInbound>>(0) {
4743 if let WireBridgeEvent::Cursor { cursor } = event.as_ref() {
4744 ctx.emit(*cursor);
4745 }
4746 }
4747 },
4748 GraphNodeOpts::named(format!("{name}/cursor")),
4749 )
4750}
4751
4752fn project_attempts<TOutbound, TInbound>(
4753 graph: &Graph,
4754 events: &Node<WireBridgeEvent<TOutbound, TInbound>>,
4755 name: &str,
4756) -> Node<WireBridgeAttempt>
4757where
4758 TOutbound: Clone + 'static,
4759 TInbound: Clone + 'static,
4760{
4761 graph.node_opts::<WireBridgeAttempt, _>(
4762 vec![events.erased()],
4763 |ctx| {
4764 for event in ctx.batch::<WireBridgeEvent<TOutbound, TInbound>>(0) {
4765 if let WireBridgeEvent::Outbound { envelope } = event.as_ref() {
4766 if should_track_ack(envelope.envelope_type) {
4767 ctx.emit(WireBridgeAttempt {
4768 seq: envelope.metadata.seq,
4769 attempt: envelope.metadata.attempt,
4770 max_attempts: envelope.metadata.max_attempts,
4771 });
4772 }
4773 }
4774 }
4775 },
4776 GraphNodeOpts::named(format!("{name}/attempts")),
4777 )
4778}
4779
4780fn should_track_ack(envelope_type: WireBridgeEnvelopeType) -> bool {
4781 matches!(
4782 envelope_type,
4783 WireBridgeEnvelopeType::Start
4784 | WireBridgeEnvelopeType::Data
4785 | WireBridgeEnvelopeType::Close
4786 )
4787}
4788
4789fn payload_error_string<T>(payload: &Option<WireBridgePayload<T>>, fallback: &str) -> String {
4790 match payload {
4791 Some(WireBridgePayload::Error(error)) => error.clone(),
4792 Some(WireBridgePayload::Data(_))
4793 | Some(WireBridgePayload::Status(_))
4794 | Some(WireBridgePayload::Close { .. })
4795 | None => fallback.to_owned(),
4796 }
4797}
4798
4799#[cfg(test)]
4800mod tests {
4801 use super::*;
4802 use crate::graph::graph;
4803
4804 fn envelope<T>(
4805 session_id: &str,
4806 envelope_type: WireBridgeEnvelopeType,
4807 seq: u64,
4808 cursor: u64,
4809 payload: Option<WireBridgePayload<T>>,
4810 ack_for_seq: Option<u64>,
4811 ) -> WireBridgeEnvelope<T> {
4812 wire_bridge_envelope(WireBridgeEnvelopeInput {
4813 session_id: session_id.to_owned(),
4814 envelope_type,
4815 seq,
4816 cursor,
4817 payload,
4818 idempotency_key: None,
4819 attempt: 1,
4820 max_attempts: 1,
4821 timestamp_ms: None,
4822 ack_for_seq,
4823 request_id: None,
4824 })
4825 .expect("test envelope is valid")
4826 }
4827
4828 #[test]
4829 fn wire_bridge_envelope_validates_metadata_and_idempotency() {
4830 assert_eq!(
4831 wire_bridge_idempotency_key("session-a", 7),
4832 canonical_tuple_key(&["session-a", "7"])
4833 );
4834 let env = wire_bridge_envelope(WireBridgeEnvelopeInput {
4835 session_id: "session-a".to_owned(),
4836 envelope_type: WireBridgeEnvelopeType::Data,
4837 seq: 7,
4838 cursor: 3,
4839 payload: Some(WireBridgePayload::Data("ok".to_owned())),
4840 idempotency_key: None,
4841 attempt: 2,
4842 max_attempts: 4,
4843 timestamp_ms: Some(10),
4844 ack_for_seq: None,
4845 request_id: Some("req-1".to_owned()),
4846 })
4847 .expect("valid envelope");
4848 assert_eq!(
4849 env.metadata.idempotency_key,
4850 canonical_tuple_key(&["session-a", "7"])
4851 );
4852 assert_eq!(env.metadata.request_id.as_deref(), Some("req-1"));
4853 assert!(matches!(
4854 wire_bridge_envelope::<()>(WireBridgeEnvelopeInput {
4855 session_id: "session-a".to_owned(),
4856 envelope_type: WireBridgeEnvelopeType::Data,
4857 seq: 1,
4858 cursor: 0,
4859 payload: None,
4860 idempotency_key: None,
4861 attempt: 1,
4862 max_attempts: 1,
4863 timestamp_ms: None,
4864 ack_for_seq: None,
4865 request_id: None,
4866 }),
4867 Err(WireBridgeEnvelopeError::MissingPayload)
4868 ));
4869 assert!(matches!(
4870 wire_bridge_envelope::<()>(WireBridgeEnvelopeInput {
4871 session_id: "session-a".to_owned(),
4872 envelope_type: WireBridgeEnvelopeType::Error,
4873 seq: 1,
4874 cursor: 0,
4875 payload: Some(WireBridgePayload::Status("wrong".to_owned())),
4876 idempotency_key: None,
4877 attempt: 1,
4878 max_attempts: 1,
4879 timestamp_ms: None,
4880 ack_for_seq: None,
4881 request_id: None,
4882 }),
4883 Err(WireBridgeEnvelopeError::PayloadTypeMismatch)
4884 ));
4885 assert!(matches!(
4886 wire_bridge_envelope::<()>(WireBridgeEnvelopeInput {
4887 session_id: "session-a".to_owned(),
4888 envelope_type: WireBridgeEnvelopeType::Data,
4889 seq: 0,
4890 cursor: 0,
4891 payload: None,
4892 idempotency_key: None,
4893 attempt: 1,
4894 max_attempts: 1,
4895 timestamp_ms: None,
4896 ack_for_seq: None,
4897 request_id: None,
4898 }),
4899 Err(WireBridgeEnvelopeError::ZeroSeq)
4900 ));
4901 assert!(matches!(
4902 wire_bridge_envelope::<()>(WireBridgeEnvelopeInput {
4903 session_id: "session-a".to_owned(),
4904 envelope_type: WireBridgeEnvelopeType::Ack,
4905 seq: 1,
4906 cursor: 0,
4907 payload: None,
4908 idempotency_key: None,
4909 attempt: 1,
4910 max_attempts: 1,
4911 timestamp_ms: None,
4912 ack_for_seq: None,
4913 request_id: None,
4914 }),
4915 Err(WireBridgeEnvelopeError::MissingAckForSeq)
4916 ));
4917 }
4918
4919 #[test]
4920 fn wire_bridge_commands_emit_ordered_outbound_facts_and_describe_topology() {
4921 let g = graph();
4922 let bridge = wire_bridge::<String, String>(
4923 &g,
4924 WireBridgeOptions {
4925 name: Some("bridge".to_owned()),
4926 session_id: "session-a".to_owned(),
4927 now_ms: Some(Rc::new(|| 42)),
4928 ..WireBridgeOptions::new("session-a")
4929 },
4930 );
4931 let _outbound = bridge.outbound.subscribe(|_| {});
4932 let _status = bridge.status.subscribe(|_| {});
4933 let _attempts = bridge.attempts.subscribe(|_| {});
4934
4935 bridge.start();
4936 bridge.send("payload".to_owned(), None, Some("req-1".to_owned()));
4937 bridge.ack(3, None, None);
4938 bridge.nack(4, "bad", None, None);
4939 bridge.close(Some("done".to_owned()), None);
4940
4941 assert_eq!(
4942 bridge.outbound.cache(),
4943 Some(WireBridgeEnvelope {
4944 session_id: "session-a".to_owned(),
4945 envelope_type: WireBridgeEnvelopeType::Close,
4946 payload: Some(WireBridgePayload::Close {
4947 reason: Some("done".to_owned()),
4948 }),
4949 metadata: WireBridgeMetadata {
4950 seq: 5,
4951 cursor: 0,
4952 idempotency_key: canonical_tuple_key(&["session-a", "5"]),
4953 attempt: 1,
4954 max_attempts: 1,
4955 timestamp_ms: Some(42),
4956 ack_for_seq: None,
4957 request_id: None,
4958 },
4959 })
4960 );
4961 assert_eq!(
4962 bridge.status.cache().unwrap().state,
4963 WireBridgeStatusState::Closed
4964 );
4965 let snap = g.describe();
4966 let mut ids = snap
4967 .nodes
4968 .iter()
4969 .map(|node| node.id.clone())
4970 .filter(|id| id.starts_with("bridge/"))
4971 .collect::<Vec<_>>();
4972 ids.sort();
4973 assert_eq!(
4974 ids,
4975 vec![
4976 "bridge/acks",
4977 "bridge/attempts",
4978 "bridge/command",
4979 "bridge/cursor",
4980 "bridge/errors",
4981 "bridge/events",
4982 "bridge/inbound",
4983 "bridge/nacks",
4984 "bridge/outbound",
4985 "bridge/status",
4986 ]
4987 );
4988 assert!(snap
4989 .edges
4990 .iter()
4991 .any(|edge| edge.from == "bridge/command" && edge.to == "bridge/events"));
4992 assert!(snap
4993 .edges
4994 .iter()
4995 .any(|edge| edge.from == "bridge/inbound" && edge.to == "bridge/events"));
4996 }
4997
4998 #[test]
4999 fn inbound_ack_advances_cursor_and_clears_pending() {
5000 let g = graph();
5001 let bridge =
5002 wire_bridge::<String, String>(&g, WireBridgeOptions::named("session-a", "bridge"));
5003 let _acks = bridge.acks.subscribe(|_| {});
5004 let _cursor = bridge.cursor.subscribe(|_| {});
5005 let _status = bridge.status.subscribe(|_| {});
5006
5007 bridge.send("work".to_owned(), None, None);
5008 bridge.inbound.set(envelope(
5009 "session-a",
5010 WireBridgeEnvelopeType::Ack,
5011 1,
5012 1,
5013 None,
5014 Some(1),
5015 ));
5016
5017 assert_eq!(bridge.cursor.cache(), Some(1));
5018 assert_eq!(bridge.acks.cache().unwrap().ack_for_seq, 1);
5019 let status = bridge.status.cache().unwrap();
5020 assert_eq!(status.pending, 0);
5021 assert_eq!(status.acked, 1);
5022 }
5023
5024 #[test]
5025 fn inbound_nack_error_and_status_are_graph_visible() {
5026 let g = graph();
5027 let bridge =
5028 wire_bridge::<String, String>(&g, WireBridgeOptions::named("session-a", "bridge"));
5029 let _nacks = bridge.nacks.subscribe(|_| {});
5030 let _errors = bridge.errors.subscribe(|_| {});
5031 let _status = bridge.status.subscribe(|_| {});
5032
5033 bridge.send("work".to_owned(), None, None);
5034 bridge.inbound.set(envelope(
5035 "session-a",
5036 WireBridgeEnvelopeType::Nack,
5037 1,
5038 1,
5039 Some(WireBridgePayload::Error("remote failed".to_owned())),
5040 Some(1),
5041 ));
5042
5043 assert_eq!(bridge.nacks.cache().unwrap().ack_for_seq, 1);
5044 assert_eq!(bridge.errors.cache(), Some("remote failed".to_owned()));
5045 let status = bridge.status.cache().unwrap();
5046 assert_eq!(status.state, WireBridgeStatusState::Errored);
5047 assert_eq!(status.nacked, 1);
5048 }
5049
5050 #[test]
5051 fn inbound_duplicate_out_of_order_late_and_session_mismatch_are_visible() {
5052 let g = graph();
5053 let bridge =
5054 wire_bridge::<String, String>(&g, WireBridgeOptions::named("session-a", "bridge"));
5055 let seen = Rc::new(RefCell::new(Vec::new()));
5056 let seen_sink = seen.clone();
5057 let _events = bridge.events.subscribe(move |msg| {
5058 if let Message::Data(value) = msg {
5059 if let Some(event) = value.downcast_ref::<WireBridgeEvent<String, String>>() {
5060 seen_sink.borrow_mut().push(event.clone());
5061 }
5062 }
5063 });
5064
5065 bridge.inbound.set(envelope(
5066 "session-b",
5067 WireBridgeEnvelopeType::Data,
5068 1,
5069 0,
5070 Some(WireBridgePayload::Data("bad-session".to_owned())),
5071 None,
5072 ));
5073 bridge.inbound.set(envelope(
5074 "session-a",
5075 WireBridgeEnvelopeType::Data,
5076 2,
5077 0,
5078 Some(WireBridgePayload::Data("early".to_owned())),
5079 None,
5080 ));
5081 bridge.inbound.set(envelope(
5082 "session-a",
5083 WireBridgeEnvelopeType::Data,
5084 1,
5085 0,
5086 Some(WireBridgePayload::Data("ok".to_owned())),
5087 None,
5088 ));
5089 bridge.inbound.set(envelope(
5090 "session-a",
5091 WireBridgeEnvelopeType::Data,
5092 1,
5093 0,
5094 Some(WireBridgePayload::Data("dup".to_owned())),
5095 None,
5096 ));
5097 bridge.inbound.set(envelope(
5098 "session-a",
5099 WireBridgeEnvelopeType::Ack,
5100 2,
5101 1,
5102 None,
5103 Some(99),
5104 ));
5105
5106 let events = seen.borrow();
5107 assert!(events.iter().any(|event| matches!(
5108 event,
5109 WireBridgeEvent::SessionMismatch { expected, actual }
5110 if expected == "session-a" && actual == "session-b"
5111 )));
5112 assert!(events.iter().any(|event| matches!(
5113 event,
5114 WireBridgeEvent::OutOfOrder {
5115 seq: 2,
5116 expected: 1
5117 }
5118 )));
5119 assert!(events
5120 .iter()
5121 .any(|event| matches!(event, WireBridgeEvent::Duplicate { seq: 1, cursor: 1 })));
5122 assert!(events.iter().any(|event| matches!(
5123 event,
5124 WireBridgeEvent::LateReceipt {
5125 receipt: WireBridgeReceipt::Ack,
5126 ack_for_seq: 99
5127 }
5128 )));
5129 }
5130
5131 #[test]
5132 fn malformed_inbound_and_command_reject_as_bridge_error_facts() {
5133 let g = graph();
5134 let bridge =
5135 wire_bridge::<String, String>(&g, WireBridgeOptions::named("session-a", "bridge"));
5136 let _errors = bridge.errors.subscribe(|_| {});
5137 let _outbound = bridge.outbound.subscribe(|_| {});
5138
5139 bridge
5140 .command
5141 .down(vec![Message::Data(Rc::new("not-a-command".to_owned()))]);
5142 bridge
5143 .inbound
5144 .down(vec![Message::Data(Rc::new("not-an-envelope".to_owned()))]);
5145
5146 assert!(bridge
5147 .errors
5148 .cache()
5149 .unwrap()
5150 .contains("inbound DATA must carry"));
5151 assert!(bridge.outbound.cache().is_none());
5152 }
5153
5154 #[test]
5155 fn inbound_vec_is_split_into_single_receipt_local_waves() {
5156 let g = graph();
5157 let bridge =
5158 wire_bridge::<String, String>(&g, WireBridgeOptions::named("session-a", "bridge"));
5159 let batch_sizes = Rc::new(RefCell::new(Vec::new()));
5160 let batch_sizes_sink = batch_sizes.clone();
5161 let observed = g.node_opts::<usize, _>(
5162 vec![bridge.inbound.erased()],
5163 move |ctx| {
5164 let len = ctx.batch::<WireBridgeIngress<String>>(0).len();
5165 batch_sizes_sink.borrow_mut().push(len);
5166 ctx.emit(len);
5167 },
5168 GraphNodeOpts::named("inbound-batch-sizes"),
5169 );
5170 let _observed = observed.subscribe(|_| {});
5171
5172 bridge.inbound.down(vec![
5173 Message::Data(Rc::new(envelope(
5174 "session-a",
5175 WireBridgeEnvelopeType::Data,
5176 1,
5177 0,
5178 Some(WireBridgePayload::Data("one".to_owned())),
5179 None,
5180 ))),
5181 Message::Data(Rc::new(envelope(
5182 "session-a",
5183 WireBridgeEnvelopeType::Data,
5184 2,
5185 0,
5186 Some(WireBridgePayload::Data("two".to_owned())),
5187 None,
5188 ))),
5189 ]);
5190
5191 assert_eq!(*batch_sizes.borrow(), vec![1, 1]);
5192 }
5193
5194 #[test]
5195 fn invalid_close_command_does_not_clear_pending_ack() {
5196 let g = graph();
5197 let bridge =
5198 wire_bridge::<String, String>(&g, WireBridgeOptions::named("session-a", "bridge"));
5199 let _acks = bridge.acks.subscribe(|_| {});
5200 let _errors = bridge.errors.subscribe(|_| {});
5201 let _status = bridge.status.subscribe(|_| {});
5202
5203 bridge.send("work".to_owned(), None, None);
5204 bridge.close(Some("done".to_owned()), Some(String::new()));
5205 assert!(bridge
5206 .errors
5207 .cache()
5208 .unwrap()
5209 .contains("idempotency_key must be non-empty"));
5210
5211 bridge.inbound.set(envelope(
5212 "session-a",
5213 WireBridgeEnvelopeType::Ack,
5214 1,
5215 1,
5216 None,
5217 Some(1),
5218 ));
5219
5220 assert_eq!(bridge.acks.cache().unwrap().ack_for_seq, 1);
5221 let status = bridge.status.cache().unwrap();
5222 assert_eq!(status.pending, 0);
5223 assert_eq!(status.acked, 1);
5224 }
5225
5226 #[test]
5227 fn remote_protocol_error_does_not_terminalize_local_bridge_node() {
5228 let g = graph();
5229 let bridge =
5230 wire_bridge::<String, String>(&g, WireBridgeOptions::named("session-a", "bridge"));
5231 let _errors = bridge.errors.subscribe(|_| {});
5232 let _cursor = bridge.cursor.subscribe(|_| {});
5233
5234 bridge
5235 .inbound
5236 .down(vec![Message::Error("remote protocol error".into())]);
5237 bridge.inbound.set(envelope(
5238 "session-a",
5239 WireBridgeEnvelopeType::Data,
5240 1,
5241 0,
5242 Some(WireBridgePayload::Data("still-live".to_owned())),
5243 None,
5244 ));
5245
5246 assert_eq!(bridge.cursor.cache(), Some(1));
5247 assert_ne!(bridge.events.status(), crate::node::Status::Errored);
5248 assert_ne!(bridge.status.status(), crate::node::Status::Errored);
5249 }
5250
5251 #[test]
5252 fn remote_error_envelope_and_close_are_facts_not_local_terminals() {
5253 let g = graph();
5254 let bridge =
5255 wire_bridge::<String, String>(&g, WireBridgeOptions::named("session-a", "bridge"));
5256 let _errors = bridge.errors.subscribe(|_| {});
5257 let _status = bridge.status.subscribe(|_| {});
5258
5259 bridge.inbound.set(envelope(
5260 "session-a",
5261 WireBridgeEnvelopeType::Error,
5262 1,
5263 0,
5264 Some(WireBridgePayload::Error("remote failed".to_owned())),
5265 None,
5266 ));
5267 assert_eq!(bridge.errors.cache(), Some("remote failed".to_owned()));
5268 assert_eq!(
5269 bridge.status.cache().unwrap().state,
5270 WireBridgeStatusState::Errored
5271 );
5272 assert_ne!(bridge.status.status(), crate::node::Status::Errored);
5273
5274 bridge.close(None, None);
5275 let status = bridge.status.cache().unwrap();
5276 assert_eq!(status.state, WireBridgeStatusState::Closed);
5277 assert_eq!(status.pending, 1);
5278 assert_ne!(bridge.status.status(), crate::node::Status::Completed);
5279
5280 bridge.inbound.set(envelope(
5281 "session-a",
5282 WireBridgeEnvelopeType::Ack,
5283 2,
5284 1,
5285 None,
5286 Some(1),
5287 ));
5288 let status = bridge.status.cache().unwrap();
5289 assert_eq!(status.state, WireBridgeStatusState::Closed);
5290 assert_eq!(status.pending, 0);
5291 assert_eq!(status.acked, 1);
5292 assert_ne!(bridge.status.status(), crate::node::Status::Completed);
5293 }
5294
5295 #[test]
5296 fn explicit_ack_timeout_command_retries_and_exhausts_without_hidden_driver() {
5297 let g = graph();
5298 let bridge = wire_bridge::<String, String>(
5299 &g,
5300 WireBridgeOptions {
5301 name: Some("bridge".to_owned()),
5302 session_id: "session-a".to_owned(),
5303 retry: RetryPolicy::new(
5304 2,
5305 crate::resilience::BackoffPolicy::Constant { delay_ms: 10 },
5306 ),
5307 now_ms: Some(Rc::new(|| 1000)),
5308 },
5309 );
5310 let _outbound = bridge.outbound.subscribe(|_| {});
5311 let _attempts = bridge.attempts.subscribe(|_| {});
5312 let _errors = bridge.errors.subscribe(|_| {});
5313 let _status = bridge.status.subscribe(|_| {});
5314
5315 bridge.send("work".to_owned(), None, None);
5316 assert_eq!(bridge.attempts.cache().unwrap().attempt, 1);
5317 bridge
5318 .command
5319 .down(vec![data_msg(WireBridgeCommand::<String>::AckTimeout {
5320 seq: 1,
5321 attempt: 1,
5322 observed_at_ms: Some(1000),
5323 })]);
5324 assert_eq!(
5325 bridge.status.cache().unwrap().state,
5326 WireBridgeStatusState::Waiting
5327 );
5328 assert_eq!(bridge.attempts.cache().unwrap().attempt, 1);
5329 bridge
5330 .command
5331 .down(vec![data_msg(WireBridgeCommand::<String>::AckTimeout {
5332 seq: 1,
5333 attempt: 1,
5334 observed_at_ms: Some(1005),
5335 })]);
5336 assert_eq!(bridge.attempts.cache().unwrap().attempt, 1);
5337 bridge
5338 .command
5339 .down(vec![data_msg(WireBridgeCommand::<String>::AckTimeout {
5340 seq: 1,
5341 attempt: 1,
5342 observed_at_ms: Some(1010),
5343 })]);
5344 assert_eq!(bridge.attempts.cache().unwrap().attempt, 2);
5345 bridge
5346 .command
5347 .down(vec![data_msg(WireBridgeCommand::<String>::AckTimeout {
5348 seq: 1,
5349 attempt: 2,
5350 observed_at_ms: Some(1020),
5351 })]);
5352 assert_eq!(
5353 bridge.status.cache().unwrap().state,
5354 WireBridgeStatusState::Exhausted
5355 );
5356 assert_eq!(
5357 bridge.errors.cache(),
5358 Some("session-a: ack timeout for seq 1".to_owned())
5359 );
5360 }
5361
5362 #[test]
5363 fn stale_ack_timeout_command_is_fail_closed_noop_after_ack() {
5364 let g = graph();
5365 let bridge = wire_bridge::<String, String>(
5366 &g,
5367 WireBridgeOptions {
5368 name: Some("bridge".to_owned()),
5369 session_id: "session-a".to_owned(),
5370 ..WireBridgeOptions::new("session-a")
5371 },
5372 );
5373 let _acks = bridge.acks.subscribe(|_| {});
5374 let _status = bridge.status.subscribe(|_| {});
5375 let _attempts = bridge.attempts.subscribe(|_| {});
5376
5377 bridge.send("work".to_owned(), None, None);
5378 bridge.inbound.set(envelope(
5379 "session-a",
5380 WireBridgeEnvelopeType::Ack,
5381 1,
5382 1,
5383 None,
5384 Some(1),
5385 ));
5386
5387 assert_eq!(bridge.acks.cache().unwrap().ack_for_seq, 1);
5388 bridge
5389 .command
5390 .down(vec![data_msg(WireBridgeCommand::<String>::AckTimeout {
5391 seq: 1,
5392 attempt: 1,
5393 observed_at_ms: Some(1000),
5394 })]);
5395 let status = bridge.status.cache().unwrap();
5396 assert_eq!(status.pending, 0);
5397 assert_eq!(status.acked, 1);
5398 assert_eq!(status.last_seq, Some(1));
5399 assert_eq!(bridge.attempts.cache().unwrap().attempt, 1);
5400 }
5401
5402 #[test]
5403 fn malformed_ack_timeout_command_is_invalid_fact_not_terminal() {
5404 let g = graph();
5405 let bridge = wire_bridge::<String, String>(
5406 &g,
5407 WireBridgeOptions {
5408 name: Some("bridge".to_owned()),
5409 session_id: "session-a".to_owned(),
5410 ..WireBridgeOptions::new("session-a")
5411 },
5412 );
5413 let _errors = bridge.errors.subscribe(|_| {});
5414
5415 bridge.send("work".to_owned(), None, None);
5416 bridge
5417 .command
5418 .down(vec![data_msg(WireBridgeCommand::<String>::AckTimeout {
5419 seq: 1,
5420 attempt: 0,
5421 observed_at_ms: Some(1000),
5422 })]);
5423
5424 assert_eq!(
5425 bridge.errors.cache(),
5426 Some("wireBridge: ack-timeout command attempt must be positive".to_owned())
5427 );
5428 assert_ne!(bridge.events.status(), crate::node::Status::Errored);
5429
5430 bridge
5431 .command
5432 .down(vec![data_msg(WireBridgeCommand::<String>::AckTimeout {
5433 seq: 0,
5434 attempt: 1,
5435 observed_at_ms: Some(1000),
5436 })]);
5437 assert_eq!(
5438 bridge.errors.cache(),
5439 Some("wireBridge: ack-timeout command seq must be positive".to_owned())
5440 );
5441 assert_ne!(bridge.events.status(), crate::node::Status::Errored);
5442 }
5443
5444 #[test]
5445 fn unknown_ack_timeout_command_is_fail_closed_noop_without_pending_state() {
5446 let g = graph();
5447 let bridge = wire_bridge::<String, String>(
5448 &g,
5449 WireBridgeOptions {
5450 name: Some("bridge".to_owned()),
5451 session_id: "session-a".to_owned(),
5452 ..WireBridgeOptions::new("session-a")
5453 },
5454 );
5455 let _outbound = bridge.outbound.subscribe(|_| {});
5456 let _attempts = bridge.attempts.subscribe(|_| {});
5457 let _errors = bridge.errors.subscribe(|_| {});
5458 let _status = bridge.status.subscribe(|_| {});
5459
5460 bridge
5461 .command
5462 .down(vec![data_msg(WireBridgeCommand::<String>::AckTimeout {
5463 seq: 99,
5464 attempt: 1,
5465 observed_at_ms: Some(1000),
5466 })]);
5467
5468 assert!(bridge.outbound.cache().is_none());
5469 assert!(bridge.attempts.cache().is_none());
5470 assert!(bridge.errors.cache().is_none());
5471 assert!(bridge
5472 .status
5473 .cache()
5474 .is_none_or(|status| status.state == WireBridgeStatusState::Idle));
5475 assert_ne!(bridge.events.status(), crate::node::Status::Errored);
5476 }
5477
5478 #[test]
5479 fn remote_call_orphan_response_is_visible_and_not_buffered_for_future_request() {
5480 let g = graph();
5481 let bridge = wire_bridge::<RemoteCallRequest<String>, RemoteCallResponse<String>>(
5482 &g,
5483 WireBridgeOptions::named("session-a", "bridge"),
5484 );
5485 let remote = remote_call::<String, String>(&g, &bridge);
5486 let _results = remote.results.subscribe(|_| {});
5487 let _errors = remote.errors.subscribe(|_| {});
5488 let _status = remote.status.subscribe(|_| {});
5489
5490 bridge.inbound.set(envelope(
5491 "session-a",
5492 WireBridgeEnvelopeType::Data,
5493 1,
5494 0,
5495 Some(WireBridgePayload::Data(RemoteCallResponse::Result {
5496 operation: "upper".to_owned(),
5497 request_id: "req-1".to_owned(),
5498 payload: "STALE".to_owned(),
5499 })),
5500 None,
5501 ));
5502 remote.call("upper", "req-1", "hello".to_owned());
5503
5504 assert!(remote.results.cache().is_none());
5505 assert_eq!(
5506 remote.errors.cache(),
5507 Some(RemoteCallError {
5508 operation: Some("upper".to_owned()),
5509 request_id: Some("req-1".to_owned()),
5510 error: "remote_call: orphan response for unknown or completed request".to_owned(),
5511 })
5512 );
5513 assert_eq!(
5514 remote.status.cache().unwrap().state,
5515 RemoteCallStatusState::Requested
5516 );
5517
5518 bridge.inbound.set(envelope(
5519 "session-a",
5520 WireBridgeEnvelopeType::Data,
5521 2,
5522 1,
5523 Some(WireBridgePayload::Data(RemoteCallResponse::Result {
5524 operation: "upper".to_owned(),
5525 request_id: "req-1".to_owned(),
5526 payload: "HELLO".to_owned(),
5527 })),
5528 None,
5529 ));
5530
5531 assert_eq!(
5532 remote.results.cache(),
5533 Some(RemoteCallResult {
5534 operation: "upper".to_owned(),
5535 request_id: "req-1".to_owned(),
5536 payload: "HELLO".to_owned(),
5537 })
5538 );
5539 }
5540
5541 #[test]
5542 fn remote_call_requires_operation_match_and_status_response_is_non_terminal() {
5543 let g = graph();
5544 let bridge = wire_bridge::<RemoteCallRequest<String>, RemoteCallResponse<String>>(
5545 &g,
5546 WireBridgeOptions::named("session-a", "bridge"),
5547 );
5548 let remote = remote_call::<String, String>(&g, &bridge);
5549 let _responses = remote.responses.subscribe(|_| {});
5550 let _results = remote.results.subscribe(|_| {});
5551 let _errors = remote.errors.subscribe(|_| {});
5552 let _status = remote.status.subscribe(|_| {});
5553
5554 remote.call("upper", "req-1", "hello".to_owned());
5555 bridge.inbound.set(envelope(
5556 "session-a",
5557 WireBridgeEnvelopeType::Data,
5558 1,
5559 1,
5560 Some(WireBridgePayload::Data(RemoteCallResponse::Result {
5561 operation: "lower".to_owned(),
5562 request_id: "req-1".to_owned(),
5563 payload: "wrong".to_owned(),
5564 })),
5565 None,
5566 ));
5567
5568 assert!(remote.results.cache().is_none());
5569 assert_eq!(
5570 remote.errors.cache(),
5571 Some(RemoteCallError {
5572 operation: Some("upper".to_owned()),
5573 request_id: Some("req-1".to_owned()),
5574 error:
5575 "remote_call: response operation 'lower' did not match pending operation 'upper'"
5576 .to_owned(),
5577 })
5578 );
5579
5580 bridge.inbound.set(envelope(
5581 "session-a",
5582 WireBridgeEnvelopeType::Data,
5583 2,
5584 1,
5585 Some(WireBridgePayload::Data(RemoteCallResponse::Status {
5586 operation: "upper".to_owned(),
5587 request_id: "req-1".to_owned(),
5588 status: "working".to_owned(),
5589 })),
5590 None,
5591 ));
5592 assert_eq!(
5593 remote.responses.cache(),
5594 Some(RemoteCallResponse::Status {
5595 operation: "upper".to_owned(),
5596 request_id: "req-1".to_owned(),
5597 status: "working".to_owned(),
5598 })
5599 );
5600 assert_eq!(remote.status.cache().unwrap().pending, 1);
5601
5602 bridge.inbound.set(envelope(
5603 "session-a",
5604 WireBridgeEnvelopeType::Data,
5605 3,
5606 1,
5607 Some(WireBridgePayload::Data(RemoteCallResponse::Result {
5608 operation: "upper".to_owned(),
5609 request_id: "req-1".to_owned(),
5610 payload: "HELLO".to_owned(),
5611 })),
5612 None,
5613 ));
5614 assert_eq!(
5615 remote.results.cache(),
5616 Some(RemoteCallResult {
5617 operation: "upper".to_owned(),
5618 request_id: "req-1".to_owned(),
5619 payload: "HELLO".to_owned(),
5620 })
5621 );
5622 assert_eq!(remote.status.cache().unwrap().pending, 0);
5623 }
5624
5625 #[test]
5626 fn remote_call_duplicate_request_id_is_visible_and_does_not_corrupt_pending() {
5627 let g = graph();
5628 let bridge = wire_bridge::<RemoteCallRequest<String>, RemoteCallResponse<String>>(
5629 &g,
5630 WireBridgeOptions::named("session-a", "bridge"),
5631 );
5632 let remote = remote_call::<String, String>(&g, &bridge);
5633 let _results = remote.results.subscribe(|_| {});
5634 let _errors = remote.errors.subscribe(|_| {});
5635 let _status = remote.status.subscribe(|_| {});
5636
5637 remote.call("upper", "req-1", "first".to_owned());
5638 remote.call("upper", "req-1", "second".to_owned());
5639
5640 assert_eq!(
5641 remote.errors.cache(),
5642 Some(RemoteCallError {
5643 operation: Some("upper".to_owned()),
5644 request_id: Some("req-1".to_owned()),
5645 error: "remote_call: duplicate in-flight request_id 'req-1'".to_owned(),
5646 })
5647 );
5648 assert_eq!(remote.status.cache().unwrap().pending, 1);
5649
5650 bridge.inbound.set(envelope(
5651 "session-a",
5652 WireBridgeEnvelopeType::Data,
5653 1,
5654 1,
5655 Some(WireBridgePayload::Data(RemoteCallResponse::Result {
5656 operation: "upper".to_owned(),
5657 request_id: "req-1".to_owned(),
5658 payload: "FIRST".to_owned(),
5659 })),
5660 None,
5661 ));
5662
5663 assert_eq!(
5664 remote.results.cache(),
5665 Some(RemoteCallResult {
5666 operation: "upper".to_owned(),
5667 request_id: "req-1".to_owned(),
5668 payload: "FIRST".to_owned(),
5669 })
5670 );
5671 assert_eq!(remote.status.cache().unwrap().pending, 0);
5672 }
5673}