diff --git a/Orleans.slnx b/Orleans.slnx index e4047b5f194..877afac409d 100644 --- a/Orleans.slnx +++ b/Orleans.slnx @@ -42,6 +42,7 @@ + @@ -152,6 +153,7 @@ + diff --git a/docs/site/src/content/docs/resources/nuget-packages.md b/docs/site/src/content/docs/resources/nuget-packages.md index 6858d4ef0c9..157908666a6 100644 --- a/docs/site/src/content/docs/resources/nuget-packages.md +++ b/docs/site/src/content/docs/resources/nuget-packages.md @@ -7,7 +7,7 @@ ms.topic: reference # Orleans NuGet packages -All official packages use the `Microsoft.Orleans` prefix and are published on [NuGet.org](https://www.nuget.org/profiles/Orleans). Keep Orleans package versions aligned within an application. +Official packages use the `Microsoft.Orleans` prefix. Find released packages on [NuGet.org](https://www.nuget.org/profiles/Orleans). Keep Orleans package versions aligned within an application. ## Start with a metapackage @@ -76,6 +76,7 @@ Memory persistence distributes records across cluster storage grains but isn't d | [Microsoft.Orleans.Reminders.Redis](https://www.nuget.org/packages/Microsoft.Orleans.Reminders.Redis) | Redis reminders. | | [Microsoft.Orleans.DurableJobs](https://www.nuget.org/packages/Microsoft.Orleans.DurableJobs) | Distributed scheduling for durable one-time jobs. | | [Microsoft.Orleans.DurableJobs.AzureStorage](https://www.nuget.org/packages/Microsoft.Orleans.DurableJobs.AzureStorage) | Azure Blob Storage for durable jobs. | +| `Microsoft.Orleans.Messaging` | Experimental grain-scoped inbox and outbox messaging contracts, awaiting their first NuGet publication. | Use reminders for recurring durable callbacks and durable jobs for scheduled one-time work. Grain timers are activation-scoped and use the core runtime rather than a provider package. diff --git a/docs/site/src/data/unpublished-api-packages.json b/docs/site/src/data/unpublished-api-packages.json index 64d185c77e2..49cf020943e 100644 --- a/docs/site/src/data/unpublished-api-packages.json +++ b/docs/site/src/data/unpublished-api-packages.json @@ -1,4 +1,6 @@ { "description": "Generated API assemblies which are not currently published as standalone NuGet packages.", - "packages": {} + "packages": { + "Microsoft.Orleans.Messaging": "The new package is awaiting its first NuGet publication." + } } diff --git a/src/Orleans.Core/Orleans.Core.csproj b/src/Orleans.Core/Orleans.Core.csproj index 303f4a85626..9d993320826 100644 --- a/src/Orleans.Core/Orleans.Core.csproj +++ b/src/Orleans.Core/Orleans.Core.csproj @@ -76,6 +76,7 @@ + diff --git a/src/Orleans.Journaling/DurableDictionary.cs b/src/Orleans.Journaling/DurableDictionary.cs index 8aeed395024..83f3ddd8ddb 100644 --- a/src/Orleans.Journaling/DurableDictionary.cs +++ b/src/Orleans.Journaling/DurableDictionary.cs @@ -16,7 +16,7 @@ public interface IDurableDictionary : IDictionary wh [DebuggerTypeProxy(typeof(IDurableDictionaryDebugView<,>))] [DebuggerDisplay("Count = {Count}")] -internal class DurableDictionary : IDurableDictionary, IStateMachine, IDurableDictionaryCommandHandler where TKey : notnull +internal class DurableDictionary : IDurableDictionary, IStateMachine, IReportsStagedChanges, IDurableDictionaryCommandHandler where TKey : notnull { private readonly IDurableDictionaryCommandCodec _codec; private readonly Dictionary _items = []; diff --git a/src/Orleans.Journaling/DurableList.cs b/src/Orleans.Journaling/DurableList.cs index 57ede9747db..6b46bca9897 100644 --- a/src/Orleans.Journaling/DurableList.cs +++ b/src/Orleans.Journaling/DurableList.cs @@ -27,7 +27,7 @@ public interface IDurableList : IList [DebuggerTypeProxy(typeof(IDurableCollectionDebugView<>))] [DebuggerDisplay("Count = {Count}")] -internal sealed class DurableList : IDurableList, IStateMachine, IDurableListCommandHandler +internal sealed class DurableList : IDurableList, IStateMachine, IReportsStagedChanges, IDurableListCommandHandler { private readonly IDurableListCommandCodec _codec; private readonly List _items = []; diff --git a/src/Orleans.Journaling/DurableQueue.cs b/src/Orleans.Journaling/DurableQueue.cs index f24a2543e3e..49ddf0cb0ca 100644 --- a/src/Orleans.Journaling/DurableQueue.cs +++ b/src/Orleans.Journaling/DurableQueue.cs @@ -65,7 +65,7 @@ public interface IDurableQueue : IEnumerable, IReadOnlyCollection [DebuggerTypeProxy(typeof(DurableQueueDebugView<>))] [DebuggerDisplay("Count = {Count}")] -internal sealed class DurableQueue : IDurableQueue, IStateMachine, IDurableQueueCommandHandler +internal sealed class DurableQueue : IDurableQueue, IStateMachine, IReportsStagedChanges, IDurableQueueCommandHandler { private readonly IDurableQueueCommandCodec _codec; private readonly Queue _items = new(); diff --git a/src/Orleans.Journaling/DurableSet.cs b/src/Orleans.Journaling/DurableSet.cs index 8efd87b4502..a883e4c397f 100644 --- a/src/Orleans.Journaling/DurableSet.cs +++ b/src/Orleans.Journaling/DurableSet.cs @@ -40,7 +40,7 @@ public interface IDurableSet : ISet, IReadOnlyCollection, IReadOnlySet< [DebuggerTypeProxy(typeof(IDurableCollectionDebugView<>))] [DebuggerDisplay("Count = {Count}")] -internal sealed class DurableSet : IDurableSet, IStateMachine, IDurableSetCommandHandler +internal sealed class DurableSet : IDurableSet, IStateMachine, IReportsStagedChanges, IDurableSetCommandHandler { private readonly IDurableSetCommandCodec _codec; private readonly HashSet _items = []; diff --git a/src/Orleans.Journaling/DurableTaskCompletionSource.cs b/src/Orleans.Journaling/DurableTaskCompletionSource.cs index 760d65ca000..af37e01bdcf 100644 --- a/src/Orleans.Journaling/DurableTaskCompletionSource.cs +++ b/src/Orleans.Journaling/DurableTaskCompletionSource.cs @@ -42,7 +42,7 @@ public interface IDurableTaskCompletionSource } [DebuggerDisplay("Status = {Status}")] -internal sealed class DurableTaskCompletionSource : IDurableTaskCompletionSource, IStateMachine, IDurableTaskCompletionSourceCommandHandler +internal sealed class DurableTaskCompletionSource : IDurableTaskCompletionSource, IStateMachine, IReportsStagedChanges, IDurableTaskCompletionSourceCommandHandler { private readonly IDurableTaskCompletionSourceCommandCodec _codec; private readonly DeepCopier _copier; @@ -52,6 +52,7 @@ internal sealed class DurableTaskCompletionSource : IDurableTaskCompletionSou private DurableTaskCompletionSourceStatus _status; private T? _value; private Exception? _exception; + private JournalStreamWriter _writer; public DurableTaskCompletionSource( [ServiceKey] string key, @@ -90,6 +91,7 @@ public bool TrySetResult(T value) } _status = DurableTaskCompletionSourceStatus.Completed; + _writer.MarkModified(); _value = _copier.Copy(value); return true; } @@ -102,6 +104,7 @@ public bool TrySetException(Exception exception) } _status = DurableTaskCompletionSourceStatus.Faulted; + _writer.MarkModified(); _exception = _exceptionCopier.Copy(exception); return true; } @@ -114,6 +117,7 @@ public bool TrySetCanceled() } _status = DurableTaskCompletionSourceStatus.Canceled; + _writer.MarkModified(); return true; } @@ -154,6 +158,7 @@ private void OnValuePersisted() void IStateMachine.Reset(JournalStreamWriter writer) { + _writer = writer; _status = DurableTaskCompletionSourceStatus.Pending; _value = default; _exception = null; @@ -198,12 +203,14 @@ private void WriteState(JournalStreamWriter writer) void IDurableTaskCompletionSourceCommandHandler.ApplyCompleted(T value) { _status = DurableTaskCompletionSourceStatus.Completed; + _writer.MarkModified(); _value = value; } void IDurableTaskCompletionSourceCommandHandler.ApplyFaulted(Exception exception) { _status = DurableTaskCompletionSourceStatus.Faulted; + _writer.MarkModified(); _exception = exception; } diff --git a/src/Orleans.Journaling/DurableValue.cs b/src/Orleans.Journaling/DurableValue.cs index cc6a39614ad..caaeeaba4ab 100644 --- a/src/Orleans.Journaling/DurableValue.cs +++ b/src/Orleans.Journaling/DurableValue.cs @@ -16,11 +16,12 @@ public interface IDurableValue } [DebuggerDisplay("{Value}")] -internal sealed class DurableValue : IDurableValue, IStateMachine, IDurableValueCommandHandler +internal sealed class DurableValue : IDurableValue, IStateMachine, IReportsStagedChanges, IDurableValueCommandHandler { private readonly IDurableValueCommandCodec _codec; private T? _value; private bool _isDirty; + private JournalStreamWriter _writer; public DurableValue( [ServiceKey] string key, @@ -54,7 +55,11 @@ public T? Value private void OnValuePersisted() => OnPersisted?.Invoke(); - public void OnModified() => _isDirty = true; + public void OnModified() + { + _isDirty = true; + _writer.MarkModified(); + } void IStateMachine.ReplayEntry(JournalEntry entry, JournalReplayContext context) => context.GetRequiredCommandCodec(entry.FormatKey, _codec).Apply(entry.Reader, this); @@ -64,6 +69,7 @@ void IStateMachine.ReplayEntry(JournalEntry entry, JournalReplayContext context) void IStateMachine.Reset(JournalStreamWriter writer) { + _writer = writer; _value = default; _isDirty = false; } diff --git a/src/Orleans.Journaling/IJournaledStateManager.cs b/src/Orleans.Journaling/IJournaledStateManager.cs index 13d369ecacf..c56ae37188d 100644 --- a/src/Orleans.Journaling/IJournaledStateManager.cs +++ b/src/Orleans.Journaling/IJournaledStateManager.cs @@ -52,6 +52,9 @@ public interface IJournaledStateManager : IAsyncDisposable /// Stage mutations only after establishing that they are safe to commit. Pending changes are shared /// by all callers using this manager. Storage acknowledgement establishes durability. /// A failed journal operation fences the manager; recovery requires a new manager and state machine instances. + /// Before capture, the manager prepares each registered state through . + /// A preparation failure reports and retains pending changes + /// for an explicit retry. /// Cancellation stops the caller's wait; an already queued write continues to its storage outcome. /// /// The token used to cancel the caller's wait. @@ -81,4 +84,5 @@ public interface IJournaledStateManager : IAsyncDisposable /// concurrent writers and should not be used for correctness decisions. /// long PendingWriteByteCount => -1; + } diff --git a/src/Orleans.Journaling/IReportsStagedChanges.cs b/src/Orleans.Journaling/IReportsStagedChanges.cs new file mode 100644 index 00000000000..e183c3a3b26 --- /dev/null +++ b/src/Orleans.Journaling/IReportsStagedChanges.cs @@ -0,0 +1,12 @@ +namespace Orleans.Journaling; + +/// +/// Marks a state whose every change advances : the state commits a +/// journal entry when the change is made, or calls . +/// +/// +/// A state which journals changes it can't observe, such as an object mutated in place and serialized whole when a write +/// captures it, doesn't implement this interface. While one is registered, the manager can't tell whether a state staged +/// a change, so is negative. +/// +internal interface IReportsStagedChanges; diff --git a/src/Orleans.Journaling/IStateMachine.cs b/src/Orleans.Journaling/IStateMachine.cs index bf3fd61204e..5f1cde9828e 100644 --- a/src/Orleans.Journaling/IStateMachine.cs +++ b/src/Orleans.Journaling/IStateMachine.cs @@ -15,6 +15,7 @@ namespace Orleans.Journaling; /// /// /// When the application requests a write, the journaled state manager calls +/// to establish the state's persistence prerequisites, then /// (and occasionally ) to materialize /// the pending changes, then flushes the journal to durable storage. /// @@ -68,6 +69,25 @@ public interface IStateMachine /// void OnRecoveryCompleted() { } + /// + /// Establishes the persistence prerequisites for this state's pending changes. + /// + /// The token which cancels preparation when the journal owner stops. + /// A completion representing readiness to capture this state. + /// + /// The manager prepares every registered state before capturing each append or snapshot, whether or not the state + /// has pending changes, so a state can establish prerequisites for changes which other states hold, or fail a write + /// whose pending changes must not commit. Preparation is idempotent. After asynchronous + /// preparation completes, the manager repeats preparation for all states to include changes + /// staged on interleaved turns. Prepared prerequisites complete synchronously on subsequent calls + /// and remain valid through capture. A complete synchronous pass flows directly into capture. + /// Preparation completes without awaiting another operation on its own journal manager, + /// since that operation is serialized behind the current write. + /// A failure reports and retains pending changes + /// for an explicit write retry. + /// + ValueTask PrepareWriteAsync(CancellationToken cancellationToken = default) => ValueTask.CompletedTask; + /// /// Writes pending state changes to the journal. /// diff --git a/src/Orleans.Journaling/JournalBufferWriter.cs b/src/Orleans.Journaling/JournalBufferWriter.cs index 60a745e0d98..316442e34e4 100644 --- a/src/Orleans.Journaling/JournalBufferWriter.cs +++ b/src/Orleans.Journaling/JournalBufferWriter.cs @@ -24,6 +24,7 @@ public abstract class JournalBufferWriter : IDisposable, IBufferWriter #endif private readonly ArcBufferWriter _buffer = new(); private int _committedLength; + private long _stagedChangeCount; private bool _hasActiveEntry; private bool _disposed; @@ -269,6 +270,28 @@ internal void AppendPreservedEntry(JournalStreamId streamId, IPreservedJournalEn } } + /// + /// Gets or sets a count which advances whenever a state stages a change and never decreases between writes. + /// + /// The manager restores it after a write captures pending changes, which stages nothing new. + internal long StagedChangeCount + { + get + { + lock (_lock) return _stagedChangeCount; + } + set + { + lock (_lock) _stagedChangeCount = value; + } + } + + /// Records a change which a state stages in memory and journals when a write captures it. + internal void MarkModified() + { + lock (_lock) _stagedChangeCount++; + } + internal void CommitActiveEntry(JournalStreamId streamId) { lock (_lock) @@ -279,6 +302,7 @@ internal void CommitActiveEntry(JournalStreamId streamId) { FinishEntry(streamId); _committedLength = _buffer.Length; + _stagedChangeCount++; } catch { diff --git a/src/Orleans.Journaling/JournalStreamWriter.cs b/src/Orleans.Journaling/JournalStreamWriter.cs index f5895bd7b7d..2ffdf8fca02 100644 --- a/src/Orleans.Journaling/JournalStreamWriter.cs +++ b/src/Orleans.Journaling/JournalStreamWriter.cs @@ -26,6 +26,9 @@ internal JournalStreamWriter(JournalStreamId id, JournalBufferWriter writer) internal bool IsInitialized => _writer is not null; + /// Records a change which the state journals when a write captures it. + internal void MarkModified() => _writer?.MarkModified(); + /// /// Appends a format-owned entry for retired or unknown state preservation. /// diff --git a/src/Orleans.Journaling/JournaledStateManager.cs b/src/Orleans.Journaling/JournaledStateManager.cs index 3f69bc77186..7cd4b99dd14 100644 --- a/src/Orleans.Journaling/JournaledStateManager.cs +++ b/src/Orleans.Journaling/JournaledStateManager.cs @@ -18,6 +18,9 @@ internal partial class JournaledStateManager : IJournaledStateManager, IJournalS #endif private readonly Dictionary _states = new(StringComparer.Ordinal); private readonly Dictionary _statesMap = []; + + // Whether a registered state journals changes which StagedChangeCount can't observe. + private bool _hasUnreportedChanges; private readonly JournaledStateManagerShared _shared; private readonly IJournalStorage _storage; private readonly IGrainContext? _grainContext; @@ -147,6 +150,7 @@ public void RegisterStateMachine(string name, IStateMachine stateMachine) } _states.Add(name, stateMachine); + if (stateMachine is not IReportsStagedChanges) _hasUnreportedChanges = true; _workQueue.Enqueue(new RegisterStateWorkItem(name)); } @@ -282,6 +286,31 @@ private async Task WorkLoop() "The journaled state operation was queued before deletion reset its state."); } + if (workItem is AppendJournalWorkItem or WriteSnapshotWorkItem) + { + try + { + for (var prepared = false; !prepared;) + { + await PrepareStates(ref prepared, _shutdownCancellation.Token).ConfigureAwait(true); + } + + _shutdownCancellation.Token.ThrowIfCancellationRequested(); + } + catch (OperationCanceledException) when (_shutdownCancellation.IsCancellationRequested) + { + workItem.TrySetCanceled(_shutdownCancellation.Token); + return; + } + catch (Exception exception) + { + LogStatePreparationFailed(_shared.Logger, exception); + storageActivity?.SetStatus(ActivityStatusCode.Error, "Journal state preparation failed."); + workItem.SetException(new JournaledStatePreCommitException(exception)); + continue; + } + } + // Note that the implementation of each command is inlined to avoid allocating unnecessary async states. // We are ok sacrificing some code organization for performance in the inner loop. switch (workItem) @@ -317,6 +346,8 @@ private async Task WorkLoop() lock (_lock) { + // Capture journals changes which states already staged, so it doesn't count as staging. + var stagedChangeCount = _journalWriter.StagedChangeCount; if (isSnapshot) { using var snapshotWriter = _shared.JournalFormat.CreateWriter(); @@ -402,6 +433,8 @@ private async Task WorkLoop() hasBufferToConsume = true; } } + + _journalWriter.StagedChangeCount = stagedChangeCount; } _shared.Instruments.OnGather( @@ -599,6 +632,28 @@ private async Task WorkLoop() } } + // Completes when every state is synchronously ready; otherwise returns the first incomplete preparation. + private ValueTask PrepareStates(ref bool prepared, CancellationToken cancellationToken) + { + prepared = true; + lock (_lock) + { + foreach (var state in _states.Values) + { + var preparation = state.PrepareWriteAsync(cancellationToken); + if (!preparation.IsCompletedSuccessfully) + { + prepared = false; + return preparation; + } + + preparation.GetAwaiter().GetResult(); + } + } + + return ValueTask.CompletedTask; + } + private void Fence(Exception exception) { lock (_lock) @@ -1057,6 +1112,18 @@ private void BindState(string name, uint id) public long PendingWriteByteCount => _journalWriter.CommittedLength; + /// + /// Gets a count which advances whenever a registered state stages a change and never decreases, or a negative value + /// when a registered state doesn't report its changes. + /// + /// + /// Every state which implements advances it when it is modified. A write captures + /// changes without advancing it. Comparing two readings taken on the activation's scheduler shows whether any state + /// staged a change between them. Orleans.Messaging uses it to tell whether a failed handler attempt staged changes, + /// and treats a negative value as staged. + /// + internal long StagedChangeCount => _hasUnreportedChanges ? -1 : _journalWriter.StagedChangeCount; + void ILifecycleParticipant.Participate(IGrainLifecycle observer) => observer.Subscribe(GrainLifecycleStage.SetupState, this); @@ -1327,7 +1394,7 @@ private sealed class RetiredStateTracker( /// This keeps buffering entries and dumps them back into the journal upon compaction. /// [DebuggerDisplay("RetiredStateMachine Id = {StreamId.Value}")] - private sealed class RetiredStateMachine(JournalStreamId streamId) : IStateMachine + private sealed class RetiredStateMachine(JournalStreamId streamId) : IStateMachine, IReportsStagedChanges { private readonly List _preservedEntries = []; @@ -1355,6 +1422,11 @@ void IStateMachine.WritePendingEntries(JournalStreamWriter writer) { } Message = "Error processing work items.")] private static partial void LogErrorProcessingWorkItems(ILogger logger, Exception exception); + [LoggerMessage( + Level = LogLevel.Error, + Message = "Journal state preparation failed; pending changes remain staged.")] + private static partial void LogStatePreparationFailed(ILogger logger, Exception exception); + [LoggerMessage( Level = LogLevel.Information, Message = "State \"{Name}\" was not found. I have substituted a placeholder for graceful time-based retirement.")] diff --git a/src/Orleans.Journaling/JournaledStatePreCommitException.cs b/src/Orleans.Journaling/JournaledStatePreCommitException.cs new file mode 100644 index 00000000000..55a0cbbbae5 --- /dev/null +++ b/src/Orleans.Journaling/JournaledStatePreCommitException.cs @@ -0,0 +1,21 @@ +namespace Orleans.Journaling; + +/// +/// Reports a state preparation failure which prevented a journal write. +/// +/// +/// Pending changes remain staged and safe to commit. The caller can restore the prerequisite +/// and explicitly retry persistence, or retire its owner and recover from durable state. +/// +[GenerateSerializer] +public sealed class JournaledStatePreCommitException : Exception +{ + /// + /// Initializes a state preparation failure. + /// + /// The original preparation failure. + public JournaledStatePreCommitException(Exception innerException) + : base("The journal write was prevented by a state preparation failure.", innerException) + { + } +} diff --git a/src/Orleans.Journaling/Orleans.Journaling.csproj b/src/Orleans.Journaling/Orleans.Journaling.csproj index 7af1350de12..30a0adad54b 100644 --- a/src/Orleans.Journaling/Orleans.Journaling.csproj +++ b/src/Orleans.Journaling/Orleans.Journaling.csproj @@ -27,6 +27,8 @@ + + diff --git a/src/Orleans.Journaling/README.md b/src/Orleans.Journaling/README.md index 09c18463568..395a57a95e8 100644 --- a/src/Orleans.Journaling/README.md +++ b/src/Orleans.Journaling/README.md @@ -207,15 +207,29 @@ provider and requested state name. - `Reset`: reset in-memory state and bind a `JournalStreamWriter`. - `ReplayEntry`: reconstruct state from a recorded operation. +- `PrepareWriteAsync`: establish asynchronous persistence prerequisites for pending state. - `WritePendingEntries`: emit pending operations into the supplied writer. - `WriteSnapshot`: emit a reconstructible snapshot into the supplied writer. - `OnRecoveryCompleted`: finish reconstruction before application use. - `OnWriteCompleted`: publish effects that depend on storage acknowledgement. +Command-backed states can use `IDurableValueCommandCodec` to encode a +complete command before applying it locally. A snapshot is a command in the same +application-defined schema which reconstructs the whole state. Annotate command data +for Orleans serialization and supply System.Text.Json metadata when using the JSON codec. + Recovery resets state machines and replays durable entries. A failed initialization can be retried on the same manager and registered states. `JournalReplayContext.ResolveStateMachine` routes entries to the state machine for their stream. +Write preparation runs before append or snapshot capture. +Each state prepares its own prerequisites idempotently. An asynchronous completion +leaves prepared prerequisites synchronously ready; the manager then rechecks all states +to include mutations staged on interleaved turns. A complete synchronous pass flows +directly into capture. Preparation failures retain pending changes for an explicit +write retry and report `JournaledStatePreCommitException`. The owner's shutdown token +cancels preparation, and disposal drains its actual completion. + `IJournaledStateManager` is independent of the grain-facing `IDurableStateManager` and extends `IAsyncDisposable`. Its owner API provides `RegisterStateMachine`, `TryGetStateMachine`, `InitializeAsync`, `WriteStateAsync`, whole-journal `DeleteStateAsync`, and `PendingWriteByteCount`. diff --git a/src/Orleans.Messaging/Configuration/DurableMessagingOptions.cs b/src/Orleans.Messaging/Configuration/DurableMessagingOptions.cs new file mode 100644 index 00000000000..fb1e04c7bef --- /dev/null +++ b/src/Orleans.Messaging/Configuration/DurableMessagingOptions.cs @@ -0,0 +1,258 @@ +using System; + +namespace Orleans.Messaging.Configuration; + +/// +/// Configuration options for durable messaging inboxes, outboxes, and the messaging pump. +/// +/// +/// +/// These options control inbox capacity, deduplication tracking, inbox and outbox retry and +/// dead-letter behavior, and pump batch sizes. +/// +/// +/// Transport is at-least-once. Retained processed records and a durable admission floor deduplicate logical sends. +/// Configuration values affect delivery availability, memory usage, throughput, and recovery characteristics. +/// +/// +public class DurableMessagingOptions +{ + internal const int MaximumBackoffExponent = 6; + internal const int MaximumBackoffMultiplier = 1 << MaximumBackoffExponent; + + // The runtime multiplies this base delay by at most MaximumBackoffMultiplier. + // Keep the expanded delay within the timer implementation's uint-millisecond limit. + private static readonly TimeSpan MaxSupportedRetryDelay = + TimeSpan.FromTicks( + TimeSpan.FromMilliseconds(uint.MaxValue - 1).Ticks + / MaximumBackoffMultiplier); + + /// + /// Gets or sets the maximum number of pending inbox messages. + /// When this limit is reached, new deliveries return DeliveryResult.Backpressured(). + /// + /// + /// + /// A lower value (e.g., 100) provides stronger backpressure but may reduce throughput. + /// A higher value (e.g., 10,000) allows more buffering but increases memory usage and recovery time. + /// + /// + /// The inbox capacity is checked before accepting new messages. Messages are persisted to durable + /// storage, so capacity limits affect both in-memory state and storage I/O during recovery. + /// + /// + /// A message with an time counts toward this limit from its acceptance until it + /// completes, so size the limit for the messages a grain holds scheduled as well as those it is processing. + /// + /// + /// Messages a grain sends to itself are accepted regardless of this limit, because sending is synchronous. They count + /// toward it until they complete, so they can cause deliveries from other grains to be backpressured. + /// + /// + /// + /// The maximum inbox capacity. Must be greater than zero. Defaults to 1000. + /// + public int MaxCapacity { get; set; } = 1000; + + /// + /// Gets or sets how long after a message's send time the receiving inbox retains its processed record and admits + /// deliveries of it. A tracked processed message is answered with DeliveryResult.Duplicate(), or, when an + /// inbox sink rejected it on delivery, with the same DeliveryResult.DeadLettered reason and code. + /// + /// + /// + /// A longer window (e.g., 30 days) retains more processed records and admits older sends, increasing + /// memory usage and storage I/O. A shorter window (e.g., 1 hour) expires older sends sooner. + /// + /// + /// The inbox advances a journaled admission floor to its current clock minus this window; the floor only advances + /// and survives recovery and migration. Pending messages stay until they are processed. A processed record becomes + /// eligible for removal once its is at or below the durable floor. A message the + /// inbox doesn't track whose send time is at or below the floor receives , + /// including after a backward clock jump. The inbox never accepts a delivered send time later than its clock plus + /// this window: it rejects such a delivery without staging anything when the message is untracked and above the + /// admission floor, or the time would extend its record. A grain's messages to itself use its clock, or the first time above the floor after a backward + /// clock jump. Every processed record therefore becomes eligible for removal within twice this window of the + /// receiver's clock, or just after the floor when a self-send follows a backward jump. + /// + /// + /// Clock jumps and skew change when messages expire and how long records are retained; deduplication follows the + /// journaled floor and each send's fixed time. Size this window to exceed by the + /// delivery delays and clock variation the application tolerates, and configure it uniformly across the cluster. + /// + /// + /// + /// The deduplication window. Must be greater than . Defaults to 7 days. + /// + public TimeSpan DeduplicationWindow { get; set; } = TimeSpan.FromDays(7); + + /// + /// Gets or sets the delay before the first retry of a failed outbox delivery or inbox processing attempt. + /// + /// + /// + /// The outbox doubles this interval after each failed delivery attempt, including when the target inbox returns + /// DeliveryResult.Backpressured(), up to 64 times this interval. The inbox applies the same backoff after a + /// handler fails. + /// + /// + /// A shorter delay (e.g., 100ms) enables faster recovery when the target processes messages quickly, + /// but may increase CPU usage during sustained backpressure. A longer delay (e.g., 5 seconds) + /// reduces retry overhead but increases latency for message delivery. + /// + /// + /// For high-throughput scenarios where quick recovery from backpressure is important, + /// consider values between 100-500ms. For less time-sensitive workloads, 1-5 seconds is appropriate. + /// + /// + /// + /// The initial retry interval. Must be greater than zero. Defaults to 1 second. + /// + public TimeSpan RetryInitialInterval { get; set; } = TimeSpan.FromSeconds(1); + + /// + /// Gets or sets how long the inbox retries a message whose handler attempts throw before the message is + /// dead-lettered. + /// + /// + /// The timeout starts with the first failed attempt, and the last attempt runs when it elapses. A handler exception + /// leaves the message's outcome unknown, so retries span transient faults, such as outages and deployments which + /// fix a bug; a handler calls IInboxHandlerContext.Fail to dead-letter a message at once. A retrying message + /// keeps its place in . The timeout and retry backoff follow the Durable Jobs clock, which + /// schedules the wake-up for each retry. + /// + /// + /// Must be greater than or equal to , which dead-letters a message at its first failure. + /// Defaults to 6 hours. + /// + public TimeSpan InboxRetryTimeout { get; set; } = TimeSpan.FromHours(6); + + /// + /// Gets or sets how long after a message's send time the outbox attempts delivery before it dead-letters the + /// message, including while the receiver is backpressured or reports no inbox handler. + /// + /// + /// + /// The outbox journals the message's when it stages the send; retries, + /// backpressure, and recovery keep it. The timeout follows the Durable Jobs clock; a jump can end retries early or + /// prolong them. A timeout or leaves the receiver's earlier delivery outcome + /// unknown. A source's stages application responses atomically with + /// retirement; see . + /// + /// + /// The outbox logs each DeliveryResult.HandlerNotFound() answer as an error. + /// + /// + /// + /// Must be greater than zero and less than , so the retry budget ends within the + /// receiver's retention window. Defaults to 6 hours. + /// + public TimeSpan OutboxRetryTimeout { get; set; } = TimeSpan.FromHours(6); + + /// + /// Gets or sets how long inbox and outbox dead letters are retained. + /// + public TimeSpan DeadLetterRetentionPeriod { get; set; } = TimeSpan.FromDays(30); + + /// + /// Gets or sets the maximum number of dead letters retained per inbox and per outbox. + /// + public int MaxRetainedDeadLetters { get; set; } = 1000; + + /// + /// Gets or sets the maximum number of inbox messages processed by one messaging pump pass. + /// + public int InboxBatchSize { get; set; } = 32; + + /// + /// Gets or sets the maximum number of outbox delivery attempts in flight for one grain. + /// + /// + /// The pump starts another attempt as soon as one completes, so a slow receiver holds one slot without delaying + /// messages to other receivers. + /// + public int OutboxBatchSize { get; set; } = 32; + + /// + /// Validates the configuration values and throws if any are invalid. + /// + /// + /// Thrown if is less than or equal to zero, + /// or if is less than or equal to , + /// or if a retry, retention, or batch option is outside its supported range. + /// + + /// + /// This method is typically called by the dependency injection container during service registration + /// to ensure configuration values are valid before the system starts. + /// + public void Validate() + { + + if (MaxCapacity <= 0) + { + throw new ArgumentOutOfRangeException(nameof(MaxCapacity), MaxCapacity, "MaxCapacity must be greater than zero."); + } + + if (DeduplicationWindow <= TimeSpan.Zero) + { + throw new ArgumentOutOfRangeException(nameof(DeduplicationWindow), DeduplicationWindow, "DeduplicationWindow must be greater than TimeSpan.Zero."); + } + + if (RetryInitialInterval <= TimeSpan.Zero) + { + throw new ArgumentOutOfRangeException(nameof(RetryInitialInterval), RetryInitialInterval, "RetryInitialInterval must be greater than TimeSpan.Zero."); + } + if (RetryInitialInterval > MaxSupportedRetryDelay) + { + throw new ArgumentOutOfRangeException( + nameof(RetryInitialInterval), + RetryInitialInterval, + $"RetryInitialInterval must be less than or equal to {MaxSupportedRetryDelay}."); + } + + if (InboxRetryTimeout < TimeSpan.Zero) + { + throw new ArgumentOutOfRangeException(nameof(InboxRetryTimeout), InboxRetryTimeout, "InboxRetryTimeout must be greater than or equal to TimeSpan.Zero."); + } + + if (OutboxRetryTimeout <= TimeSpan.Zero) + { + throw new ArgumentOutOfRangeException(nameof(OutboxRetryTimeout), OutboxRetryTimeout, "OutboxRetryTimeout must be greater than TimeSpan.Zero."); + } + + if (OutboxRetryTimeout >= DeduplicationWindow) + { + throw new ArgumentOutOfRangeException( + nameof(OutboxRetryTimeout), + OutboxRetryTimeout, + "OutboxRetryTimeout must be less than DeduplicationWindow."); + } + + if (DeadLetterRetentionPeriod <= TimeSpan.Zero) + { + throw new ArgumentOutOfRangeException( + nameof(DeadLetterRetentionPeriod), + DeadLetterRetentionPeriod, + "DeadLetterRetentionPeriod must be greater than TimeSpan.Zero."); + } + + if (MaxRetainedDeadLetters <= 0) + { + throw new ArgumentOutOfRangeException( + nameof(MaxRetainedDeadLetters), + MaxRetainedDeadLetters, + "MaxRetainedDeadLetters must be greater than zero."); + } + + if (InboxBatchSize <= 0) + { + throw new ArgumentOutOfRangeException(nameof(InboxBatchSize), InboxBatchSize, "InboxBatchSize must be greater than zero."); + } + + if (OutboxBatchSize <= 0) + { + throw new ArgumentOutOfRangeException(nameof(OutboxBatchSize), OutboxBatchSize, "OutboxBatchSize must be greater than zero."); + } + } +} diff --git a/src/Orleans.Messaging/DeadLetterRetention.cs b/src/Orleans.Messaging/DeadLetterRetention.cs new file mode 100644 index 00000000000..e2388a608e9 --- /dev/null +++ b/src/Orleans.Messaging/DeadLetterRetention.cs @@ -0,0 +1,41 @@ +using System; +using System.Collections.Generic; +using System.Linq; + +namespace Orleans.Messaging; + +internal static class DeadLetterRetention +{ + public static bool Compact( + IDictionary entries, + DateTimeOffset now, + TimeSpan retentionPeriod, + int maxRetainedEntries, + Func getTimestamp, + int reservedCapacity = 0) + where TKey : notnull + { + var removed = false; + foreach (var entry in entries + .Where(entry => MessagingTime.IsExpired(now, getTimestamp(entry.Value), retentionPeriod)) + .ToList()) + { + entries.Remove(entry.Key); + removed = true; + } + + var removeCount = entries.Count + reservedCapacity - maxRetainedEntries; + if (removeCount <= 0) + { + return removed; + } + + foreach (var entry in entries.OrderBy(entry => getTimestamp(entry.Value)).Take(removeCount).ToList()) + { + entries.Remove(entry.Key); + removed = true; + } + + return removed; + } +} diff --git a/src/Orleans.Messaging/DeliveryResult.cs b/src/Orleans.Messaging/DeliveryResult.cs new file mode 100644 index 00000000000..3b250362c4b --- /dev/null +++ b/src/Orleans.Messaging/DeliveryResult.cs @@ -0,0 +1,77 @@ +using Orleans.Serialization; + +namespace Orleans.Messaging; + +/// +/// Result of attempting to deliver a message to an inbox. +/// +[GenerateSerializer, Alias("Orleans.Messaging.DeliveryResult")] +public readonly struct DeliveryResult +{ + /// + /// The status of the delivery attempt. + /// + [Id(0)] + public DeliveryStatus Status { get; init; } + + /// + /// Optional diagnostic message (e.g., reason for rejection). + /// + [Id(1)] + public string? Message { get; init; } + + /// + /// Gets the code which identifies why the receiver dead-lettered the message: an application-defined code from an + /// inbox message sink, such as "orders/duplicate-order"; + /// when the receiver reported none. + /// + /// + /// Messaging carries the code to the inbox dead letter and to the sender's + /// without interpreting it. + /// + [Id(2)] + public string? Code { get; init; } + + /// + /// Creates a result indicating the message was accepted and persisted to inbox. + /// + public static DeliveryResult Accepted() => new() { Status = DeliveryStatus.Accepted }; + + /// + /// Creates a result indicating the message was a duplicate. + /// + public static DeliveryResult Duplicate() => new() { Status = DeliveryStatus.Duplicate }; + + /// + /// Creates a result indicating the inbox is at capacity. + /// + public static DeliveryResult Backpressured() => new() { Status = DeliveryStatus.Backpressured }; + + /// + /// Creates a result indicating the receiving inbox has no registered handler. + /// + public static DeliveryResult HandlerNotFound() => new() + { + Status = DeliveryStatus.HandlerNotFound, + Message = "No inbox handler is registered." + }; + + /// + /// Creates a result indicating the message was dead-lettered. + /// + /// The diagnostic reason. + /// The application-defined code which identifies the rejection, if any. + public static DeliveryResult DeadLettered(string reason, string? code = null) => new() + { + Status = DeliveryStatus.DeadLettered, + Message = reason, + Code = code + }; + + /// Creates a terminal result for an expired send whose earlier delivery outcome is unknown. + public static DeliveryResult Expired() => new() + { + Status = DeliveryStatus.Expired, + Message = "The message was sent before the receiver's deduplication window; its earlier delivery outcome is unknown." + }; +} diff --git a/src/Orleans.Messaging/DeliveryStatus.cs b/src/Orleans.Messaging/DeliveryStatus.cs new file mode 100644 index 00000000000..7beb2fb6885 --- /dev/null +++ b/src/Orleans.Messaging/DeliveryStatus.cs @@ -0,0 +1,50 @@ +namespace Orleans.Messaging; + +/// +/// Status codes for delivery attempts. +/// +/// Zero is not a valid status, so a default reports no outcome. +public enum DeliveryStatus +{ + /// + /// Message was accepted and persisted to inbox. + /// + Accepted = 1, + + /// + /// The inbox already holds the message or retains its processed record. + /// + /// + /// A duplicate of a message which an inbox message sink rejected receives that rejection instead: the original + /// result, with its reason and . + /// + Duplicate = 2, + + /// + /// Inbox is at capacity; sender should retry later. + /// + Backpressured = 3, + + /// + /// The receiving inbox has no registered handler. + /// + HandlerNotFound = 4, + + /// + /// The receiver rejected the message permanently and moved it to its dead-letter store. + /// + /// + /// A redelivery while its processed record is retained receives the same result. + /// + DeadLettered = 5, + + /// + /// The message was sent at or before the receiver's durable admission floor, and the receiver retains no outcome + /// for it. + /// + /// + /// This attempt stages no message effects. An earlier attempt can have been accepted, so the sender treats the + /// delivery outcome as unknown and retires the message as . + /// + Expired = 6 +} diff --git a/src/Orleans.Messaging/Diagnostics/MessagingEvents.cs b/src/Orleans.Messaging/Diagnostics/MessagingEvents.cs new file mode 100644 index 00000000000..12a88ee0df8 --- /dev/null +++ b/src/Orleans.Messaging/Diagnostics/MessagingEvents.cs @@ -0,0 +1,196 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; +using System.Runtime.CompilerServices; +using Orleans.Runtime; + +namespace Orleans.Messaging.Diagnostics; + +/// +/// Provides the diagnostic listener and event payload types for durable messaging events. +/// +/// +/// +/// These types are public but may change between minor versions. They are intended for advanced scenarios such as +/// simulation testing and diagnostics. +/// +/// +/// Each event is written synchronously on the activation which raises it, at the transition which it describes, so an +/// observer sees the grain's state at that transition. An exception thrown by an observer propagates to the code which +/// raised the event; documents how a delivery treats one. +/// +/// +public static class MessagingEvents +{ + /// The name of the diagnostic listener for durable messaging events. + public const string ListenerName = "Orleans.Messaging"; + + private static readonly DiagnosticListener Listener = new(ListenerName); + + /// Gets an observable sequence of all durable messaging events. + public static IObservable AllEvents { get; } = new Observable(); + + /// The base class used for durable messaging diagnostic events. + /// The activation whose inbox or outbox raised the event. + public abstract class MessagingEvent(IGrainContext grainContext) + { + /// The activation whose inbox or outbox raised the event. + public readonly IGrainContext GrainContext = grainContext; + } + + /// + /// Event payload for a new delivery whose send time is above the inbox's admission floor, before the inbox + /// stages message effects. + /// + /// The receiving activation. + /// The delivered message. + /// + /// An exception thrown by an observer fails the delivery attempt like an inbox message sink exception: the inbox + /// stages nothing, and the sender's outbox retries the message, which tests use to inject delivery faults. + /// + public sealed class InboxMessageReceiving(IGrainContext grainContext, InboxMessage message) : MessagingEvent(grainContext) + { + /// The delivered message. + public readonly InboxMessage Message = message; + } + + /// + /// Event payload for a failed delivery attempt whose retry the outbox staged. + /// + /// The sending activation. + /// The message's ID. + /// The receiving grain. + /// The number of failed delivery attempts so far. + /// The time, on the Durable Jobs clock, from which the outbox attempts the delivery again. + /// The attempt's failure. + public sealed class OutboxRetryScheduled( + IGrainContext grainContext, + MessageId messageId, + GrainId receiverId, + int attemptCount, + DateTimeOffset nextAttemptAt, + string error) : MessagingEvent(grainContext) + { + /// The message's ID. + public readonly MessageId MessageId = messageId; + + /// The receiving grain. + public readonly GrainId ReceiverId = receiverId; + + /// The number of failed delivery attempts so far. + public readonly int AttemptCount = attemptCount; + + /// The time, on the Durable Jobs clock, from which the outbox attempts the delivery again. + public readonly DateTimeOffset NextAttemptAt = nextAttemptAt; + + /// The attempt's failure. + public readonly string Error = error; + } + + /// + /// Event payload for an outbox message whose retirement the outbox staged: the message was delivered, its receiver + /// rejected it, or the outbox dead-lettered it. + /// + /// The sending activation. + /// The message's ID. + /// How the message retired. + /// + /// The retirement commits with the sending grain's next journal write. A reference message's source is notified + /// first, so its changes for the retirement are staged when observers run. + /// + public sealed class OutboxMessageRetired( + IGrainContext grainContext, + MessageId messageId, + OutboxMessageRetirement retirement) : MessagingEvent(grainContext) + { + /// The message's ID. + public readonly MessageId MessageId = messageId; + + /// How the message retired. + public readonly OutboxMessageRetirement Retirement = retirement; + } + + internal static void EmitInboxMessageReceiving(IGrainContext grainContext, InboxMessage message) + { + if (!Listener.IsEnabled(nameof(InboxMessageReceiving))) + { + return; + } + + Emit(grainContext, message); + + [MethodImpl(MethodImplOptions.NoInlining)] + static void Emit(IGrainContext grainContext, InboxMessage message) => + Write(nameof(InboxMessageReceiving), new InboxMessageReceiving(grainContext, message)); + } + + internal static void EmitOutboxRetryScheduled( + IGrainContext grainContext, + MessageId messageId, + GrainId receiverId, + int attemptCount, + DateTimeOffset nextAttemptAt, + string error) + { + if (!Listener.IsEnabled(nameof(OutboxRetryScheduled))) + { + return; + } + + Emit(grainContext, messageId, receiverId, attemptCount, nextAttemptAt, error); + + [MethodImpl(MethodImplOptions.NoInlining)] + static void Emit( + IGrainContext grainContext, + MessageId messageId, + GrainId receiverId, + int attemptCount, + DateTimeOffset nextAttemptAt, + string error) => + Write( + nameof(OutboxRetryScheduled), + new OutboxRetryScheduled(grainContext, messageId, receiverId, attemptCount, nextAttemptAt, error)); + } + + internal static void EmitOutboxMessageRetired(IGrainContext grainContext, MessageId messageId, OutboxMessageRetirement retirement) + { + if (!Listener.IsEnabled(nameof(OutboxMessageRetired))) + { + return; + } + + Emit(grainContext, messageId, retirement); + + [MethodImpl(MethodImplOptions.NoInlining)] + static void Emit(IGrainContext grainContext, MessageId messageId, OutboxMessageRetirement retirement) => + Write(nameof(OutboxMessageRetired), new OutboxMessageRetired(grainContext, messageId, retirement)); + } + + [UnconditionalSuppressMessage( + "Trimming", + "IL2026", + Justification = "Event payloads are sealed types whose public fields and properties the generic annotation preserves, and subscribers consume the typed payload rather than reflecting over the members of the values it carries.")] + private static void Write<[DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicFields | DynamicallyAccessedMemberTypes.PublicProperties)] TEvent>(string name, TEvent payload) + where TEvent : MessagingEvent => Listener.Write(name, payload); + + private sealed class Observable : IObservable + { + public IDisposable Subscribe(IObserver observer) => Listener.Subscribe(new Observer(observer)); + + private sealed class Observer(IObserver observer) : IObserver> + { + public void OnCompleted() => observer.OnCompleted(); + + public void OnError(Exception error) => observer.OnError(error); + + public void OnNext(KeyValuePair value) + { + if (value.Value is MessagingEvent evt) + { + observer.OnNext(evt); + } + } + } + } +} diff --git a/src/Orleans.Messaging/Envelope.cs b/src/Orleans.Messaging/Envelope.cs new file mode 100644 index 00000000000..2da2671d3d1 --- /dev/null +++ b/src/Orleans.Messaging/Envelope.cs @@ -0,0 +1,51 @@ +using System; +using System.Collections.Generic; +using System.Text.Json.Serialization; + +namespace Orleans.Messaging; + +/// Contains a command identity, payload, subject, optional application headers, and an optional delivery time. +/// +/// Message identity and payload are required; an empty payload is valid. Raw protocols can omit +/// subject and headers. Prepare the payload and headers before publishing a message and keep them +/// unchanged afterward. Struct assignment shares these objects; ordinary Orleans deep copying +/// and deserialization produce independent payload arrays, header dictionaries, and header value arrays. +/// +[GenerateSerializer, Alias("Orleans.Messaging.Envelope")] +public readonly struct Envelope +{ + /// Gets the exact application-defined command identity within the receiving inbox. + [Id(0)] + public required MessageId MessageId { get; init; } + + /// Gets the required application payload bytes, including an empty payload. + [Id(1)] + public required byte[] Payload { get; init; } + + /// Gets the optional, nonempty ordinal application protocol subject. + [Id(2)] + public string? Subject { get; init; } + + /// Gets optional application-defined named byte-array headers. + /// Use an ordinal dictionary for case-sensitive header names. Keys are nonempty, well-formed UTF-16 strings, and arrays are non-null. + [Id(3)] + public Dictionary? Headers { get; init; } + + /// Gets the optional earliest time at which the receiving inbox processes the message. + /// + /// + /// The outbox delivers the message immediately. The receiving inbox accepts, deduplicates, and persists it + /// as usual, then holds it until this time is reached on the Durable Jobs clock, and one durable wake-up + /// resumes the grain when the message becomes due. A message a grain sends to itself is accepted by its own + /// inbox in the sending journal write. processes the message as soon as + /// its acceptance is durable. + /// + /// + /// The time is part of the command: reusing a pending message ID with a different value is a conflicting command. + /// The message counts toward the inbox MaxCapacity from its acceptance until it completes. + /// + /// + [Id(4)] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public DateTimeOffset? DeliverAt { get; init; } +} diff --git a/src/Orleans.Messaging/EnvelopeEquivalence.cs b/src/Orleans.Messaging/EnvelopeEquivalence.cs new file mode 100644 index 00000000000..bdbf42079f8 --- /dev/null +++ b/src/Orleans.Messaging/EnvelopeEquivalence.cs @@ -0,0 +1,18 @@ +using System; + +namespace Orleans.Messaging; + +internal static class EnvelopeEquivalence +{ + public static bool AreEquivalent(OutboxMessage left, OutboxMessage right) => + left.ReceiverId == right.ReceiverId && AreSameCommand(left.Envelope, right.Envelope); + + public static bool AreSameCommand(InboxMessage left, InboxMessage right) => AreSameCommand(left.Envelope, right.Envelope); + + public static bool AreSameCommand(Envelope left, Envelope right) => + left.MessageId == right.MessageId + && left.Payload.Length == right.Payload.Length + && string.Equals(left.Subject, right.Subject, StringComparison.Ordinal) + && left.DeliverAt == right.DeliverAt + && left.Payload.AsSpan().SequenceEqual(right.Payload); +} diff --git a/src/Orleans.Messaging/EnvelopeValidation.cs b/src/Orleans.Messaging/EnvelopeValidation.cs new file mode 100644 index 00000000000..f69a2375636 --- /dev/null +++ b/src/Orleans.Messaging/EnvelopeValidation.cs @@ -0,0 +1,53 @@ +using System; +using System.Text; + +namespace Orleans.Messaging; + +internal static class EnvelopeValidation +{ + internal const int MaxMessageIdBytes = 1024; + internal const int MaxSubjectBytes = 256; + private static readonly UTF8Encoding Utf8 = new(false, true); + + // External admission and recovery validate generated values before publishing shared mutations. + public static void Validate(Envelope envelope) + { + ValidateMessageId(envelope.MessageId); + if (envelope.Payload is null) throw new ArgumentException("The payload must not be null.", nameof(envelope)); + if (envelope.Subject is not null) ValidateSubject(envelope.Subject); + if (envelope.Headers is { } headers) + { + foreach (var header in headers) + { + if (string.IsNullOrEmpty(header.Key)) throw new ArgumentException("Header keys must not be empty.", nameof(envelope)); + if (!HierarchicalKey.IsWellFormedUtf16(header.Key)) + throw new ArgumentException("Header keys must not contain unpaired surrogate characters.", nameof(envelope)); + if (header.Value is null) throw new ArgumentException("Header values must not be null.", nameof(envelope)); + } + } + } + + public static void Validate(InboxMessage message) => Validate(message.Envelope); + + public static void Validate(OutboxMessage message) + { + if (message.ReceiverId.IsDefault) throw new ArgumentException("The outgoing receiver must not be default.", nameof(message)); + Validate(message.Envelope); + } + + internal static void ValidateMessageId(MessageId messageId) + { + if (messageId.IsDefault) throw new ArgumentException("The message ID must not be unset.", nameof(messageId)); + if (Utf8.GetByteCount(messageId.ToString()) > MaxMessageIdBytes) + throw new ArgumentException($"The message ID exceeds {MaxMessageIdBytes} UTF-8 bytes.", nameof(messageId)); + } + + public static void ValidateSubject(string subject) + { + ArgumentException.ThrowIfNullOrEmpty(subject); + if (!HierarchicalKey.IsWellFormedUtf16(subject)) + throw new ArgumentException("The subject must not contain unpaired surrogate characters.", nameof(subject)); + if (Utf8.GetByteCount(subject) > MaxSubjectBytes) + throw new ArgumentException($"The subject exceeds {MaxSubjectBytes} UTF-8 bytes.", nameof(subject)); + } +} diff --git a/src/Orleans.Messaging/HierarchicalKey.cs b/src/Orleans.Messaging/HierarchicalKey.cs new file mode 100644 index 00000000000..fe8f9801341 --- /dev/null +++ b/src/Orleans.Messaging/HierarchicalKey.cs @@ -0,0 +1,350 @@ +using System; +using System.Diagnostics.CodeAnalysis; +using System.Runtime.CompilerServices; +using System.Text; +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace Orleans.Messaging; + +/// +/// An immutable, ordinal application identity formed from nonempty hierarchical segments. +/// +/// +/// Create constructs literal segments and escapes slash and backslash characters exactly once. +/// Parse reads the canonical escaped path. The key wraps its canonical string; equality and hashing use that string. Applications preserve the identity across retries. +/// Every key is well-formed UTF-16: surrogate characters appear only as high-low pairs, so distinct keys keep distinct serialized forms. +/// +[Immutable, Alias("Orleans.Messaging.HierarchicalKey")] +[JsonConverter(typeof(HierarchicalKeyJsonConverter))] +public readonly struct HierarchicalKey : ISpanFormattable, IEquatable, IParsable, ISpanParsable +{ + /// The escape character used within canonical segments. + public const char EscapeCharacter = '\\'; + + /// The separator between canonical segments. + public const char SegmentSeparator = '/'; + + private const char HighSurrogateStart = '\uD800'; + private const char LowSurrogateEnd = '\uDFFF'; + + private readonly string? _value; + + private HierarchicalKey(string canonical) => _value = canonical; + + /// Serializes hierarchical identities as canonical escaped JSON strings. + public sealed class HierarchicalKeyJsonConverter : JsonConverter + { + /// Creates a hierarchical identity JSON converter. + public HierarchicalKeyJsonConverter() { } + + /// + public override HierarchicalKey Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + if (reader.TokenType == JsonTokenType.Null) return default; + if (reader.TokenType is JsonTokenType.String or JsonTokenType.PropertyName + && TryParse(reader.GetString(), null, out var result)) return result; + throw new JsonException("A hierarchical identity must be a canonical escaped path."); + } + + /// + public override void Write(Utf8JsonWriter writer, HierarchicalKey value, JsonSerializerOptions options) + { + if (value.IsDefault) writer.WriteNullValue(); + else writer.WriteStringValue(value.ToString()); + } + + /// + public override HierarchicalKey ReadAsPropertyName(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) => + Read(ref reader, typeToConvert, options); + + /// + public override void WriteAsPropertyName(Utf8JsonWriter writer, HierarchicalKey value, JsonSerializerOptions options) + { + if (value.IsDefault) throw new JsonException("A hierarchical dictionary key must be set."); + writer.WritePropertyName(value.ToString()); + } + } + + /// Gets whether this value is unset. + public bool IsDefault => _value is null; + + /// Gets the canonical path length in UTF-16 characters. + public int Length => _value?.Length ?? 0; + + /// Creates one literal segment, escaping slash and backslash characters. + /// The nonempty literal segment. + /// The segment identity. + /// is null. + /// is empty or contains an unpaired surrogate. + public static HierarchicalKey Create(string value) + { + ValidateSegment(value); + return new(Escape(value)); + } + + /// Creates a hierarchy from literal segments in root-first order. + /// The nonempty literal segments. + /// A flat canonical identity which owns its immutable backing string. + /// A segment is null. + /// The input or a segment is empty, or a segment contains an unpaired surrogate. + public static HierarchicalKey Create(params ReadOnlySpan values) + { + if (values.IsEmpty) + { + throw new ArgumentException("Values must not be empty.", nameof(values)); + } + var builder = new StringBuilder(); + foreach (var value in values) + { + ValidateSegment(value); + if (builder.Length > 0) builder.Append(SegmentSeparator); + AppendEscaped(builder, value); + } + return new(builder.ToString()); + } + + /// Appends one literal child segment. + /// The nonempty literal child segment. + /// The child identity. + /// This key is unset. + /// is null. + /// is empty or contains an unpaired surrogate. + public HierarchicalKey CreateChildKey(string value) + { + EnsureSet(); + ValidateSegment(value); + return new(string.Concat(_value, "/", Escape(value))); + } + + /// Composes this hierarchy with an already constructed suffix hierarchy. + /// The constructed suffix. + /// A flat concatenated identity preserving both paths' segment boundaries. + /// This key is unset. + /// is unset. + public HierarchicalKey Append(HierarchicalKey suffix) + { + EnsureSet(); + if (suffix.IsDefault) throw new ArgumentException("The suffix must not be unset.", nameof(suffix)); + return new(string.Concat(_value, "/", suffix._value)); + } + + /// Gets the immediate parent, or null for a root or unset key. + /// The parent identity when this key has more than one segment. + public HierarchicalKey? GetParent() + { + if (_value is null) return null; + var lastSeparator = LastSeparator(_value); + return lastSeparator < 0 ? null : new HierarchicalKey(_value[..lastSeparator]); + } + + /// Tests whether this key is the other's immediate child. + /// The potential parent. + /// Whether there is exactly one additional segment. + public bool IsChildOf(HierarchicalKey other) => other.IsParentOf(this); + + /// Tests whether this key is the other's immediate parent. + /// The potential child. + /// Whether the other key extends this key by exactly one segment. + public bool IsParentOf(HierarchicalKey other) => IsAncestorOf(other) && LastSeparator(other._value!) == Length; + + /// Tests whether this key is equal to or an ancestor of the other key. + /// The identity to inspect. + /// Whether the full prefix consists of equal ordinal segments. Unset keys return false. + public bool IsAncestorOf(HierarchicalKey other) => !IsDefault && !other.IsDefault + && (Equals(other) || other.Length > Length && other._value![Length] == SegmentSeparator + && other._value.StartsWith(_value!, StringComparison.Ordinal)); + + /// + public static HierarchicalKey Parse(string s, IFormatProvider? provider = null) + { + ArgumentNullException.ThrowIfNull(s); + return TryParse(s, provider, out var result) ? result : throw new FormatException("The value is not a valid canonical hierarchical key."); + } + + /// + public static HierarchicalKey Parse(ReadOnlySpan s, IFormatProvider? provider = null) => + TryParse(s, provider, out var result) ? result : throw new FormatException("The value is not a valid canonical hierarchical key."); + + /// + public static bool TryParse([NotNullWhen(true)] string? s, IFormatProvider? provider, out HierarchicalKey result) + { + if (s is not null && IsCanonical(s)) + { + result = new(s); + return true; + } + result = default; + return false; + } + + /// + public static bool TryParse(ReadOnlySpan s, IFormatProvider? provider, out HierarchicalKey result) + { + if (IsCanonical(s)) + { + result = new(new string(s)); + return true; + } + result = default; + return false; + } + + /// + public bool Equals(HierarchicalKey other) => string.Equals(_value, other._value, StringComparison.Ordinal); + + + /// + public override bool Equals(object? obj) => obj is HierarchicalKey other && Equals(other); + + /// + public override int GetHashCode() => _value is null ? 0 : StringComparer.Ordinal.GetHashCode(_value); + + /// Compares complete ordinal key values. + /// The first identity. + /// The second identity. + /// Whether the identities are equal. + public static bool operator ==(HierarchicalKey left, HierarchicalKey right) => left.Equals(right); + + /// Compares complete ordinal key values for inequality. + /// The first identity. + /// The second identity. + /// Whether the identities differ. + public static bool operator !=(HierarchicalKey left, HierarchicalKey right) => !left.Equals(right); + + /// + public override string ToString() => _value ?? string.Empty; + + /// + public string ToString(string? format, IFormatProvider? formatProvider) => ToString(); + + /// + public bool TryFormat(Span destination, out int charsWritten, ReadOnlySpan format, IFormatProvider? provider) + { + if (ToString().AsSpan().TryCopyTo(destination)) + { + charsWritten = Length; + return true; + } + charsWritten = 0; + return false; + } + + /// Enumerates escaped canonical segment spans in root-first order. + /// An allocation-free segment enumerator. + public SegmentEnumerator GetEnumerator() => new(ToString().AsSpan()); + + private void EnsureSet() + { + if (IsDefault) throw new InvalidOperationException("The key must not be unset."); + } + + private static void ValidateSegment(string value, [CallerArgumentExpression(nameof(value))] string? paramName = null) + { + ArgumentException.ThrowIfNullOrEmpty(value, paramName); + if (!IsWellFormedUtf16(value)) throw new ArgumentException("The value must not contain unpaired surrogate characters.", paramName); + } + + // Orleans string serialization replaces unpaired surrogates, so accepting them would let distinct keys collide on the wire. + internal static bool IsWellFormedUtf16(ReadOnlySpan value) + { + var i = value.IndexOfAnyInRange(HighSurrogateStart, LowSurrogateEnd); + if (i < 0) return true; + for (; i < value.Length; i++) + { + if (char.IsHighSurrogate(value[i])) + { + if (++i == value.Length || !char.IsLowSurrogate(value[i])) return false; + } + else if (char.IsLowSurrogate(value[i])) + { + return false; + } + } + return true; + } + + private static string Escape(string value) + { + if (value.AsSpan().IndexOfAny(EscapeCharacter, SegmentSeparator) < 0) return value; + var builder = new StringBuilder(value.Length); + AppendEscaped(builder, value); + return builder.ToString(); + } + + private static void AppendEscaped(StringBuilder builder, string value) + { + foreach (var character in value) + { + if (character is EscapeCharacter or SegmentSeparator) builder.Append(EscapeCharacter); + builder.Append(character); + } + } + + private static int LastSeparator(ReadOnlySpan path) + { + var lastSeparator = -1; + for (var i = 0; i < path.Length; i++) + { + if (path[i] == EscapeCharacter) i++; + else if (path[i] == SegmentSeparator) lastSeparator = i; + } + return lastSeparator; + } + + private static bool IsCanonical(ReadOnlySpan path) + { + var segmentLength = 0; + for (var i = 0; i < path.Length; i++) + { + if (path[i] == EscapeCharacter) + { + if (++i == path.Length || path[i] is not (EscapeCharacter or SegmentSeparator)) return false; + } + else if (path[i] == SegmentSeparator) + { + if (segmentLength == 0) return false; + segmentLength = 0; + continue; + } + else if (char.IsSurrogate(path[i])) + { + if (!char.IsHighSurrogate(path[i]) || ++i == path.Length || !char.IsLowSurrogate(path[i])) return false; + } + segmentLength++; + } + return segmentLength != 0; + } + + /// Enumerates borrowed spans of the immutable canonical key. + public ref struct SegmentEnumerator + { + private readonly ReadOnlySpan _path; + private int _next; + + internal SegmentEnumerator(ReadOnlySpan path) => _path = path; + + /// Gets the current escaped canonical segment. + public ReadOnlySpan Current { get; private set; } + + /// Advances to the next segment. + /// Whether a segment is available. + public bool MoveNext() + { + if (_next >= _path.Length) + { + Current = default; + return false; + } + var start = _next; + for (; _next < _path.Length; _next++) + { + if (_path[_next] == EscapeCharacter) _next++; + else if (_path[_next] == SegmentSeparator) break; + } + Current = _path[start.._next]; + _next++; + return true; + } + } +} diff --git a/src/Orleans.Messaging/HierarchicalKeyCodec.cs b/src/Orleans.Messaging/HierarchicalKeyCodec.cs new file mode 100644 index 00000000000..98a43ecea81 --- /dev/null +++ b/src/Orleans.Messaging/HierarchicalKeyCodec.cs @@ -0,0 +1,48 @@ +using System; +using System.Buffers; +using Orleans.Serialization; +using Orleans.Serialization.Buffers; +using Orleans.Serialization.Cloning; +using Orleans.Serialization.Codecs; +using Orleans.Serialization.WireProtocol; + +namespace Orleans.Messaging; + +[RegisterSerializer] +internal sealed class HierarchicalKeyCodec(IFieldCodec strings) : IFieldCodec +{ + public void WriteField(ref Writer writer, uint fieldIdDelta, + Type? expectedType, HierarchicalKey value) where TBufferWriter : IBufferWriter + { + ReferenceCodec.MarkValueField(writer.Session); + writer.WriteFieldHeader(fieldIdDelta, expectedType, typeof(HierarchicalKey), WireType.TagDelimited); + strings.WriteField(ref writer, 0, typeof(string), value.IsDefault ? null! : value.ToString()); + writer.WriteEndObject(); + } + + public HierarchicalKey ReadValue(ref Reader reader, Field field) + { + field.EnsureWireTypeTagDelimited(); + ReferenceCodec.MarkValueField(reader.Session); + HierarchicalKey result = default; + uint fieldId = 0; + while (true) + { + var header = reader.ReadFieldHeader(); + if (header.IsEndBaseOrEndObject) return result; + fieldId += header.FieldIdDelta; + if (fieldId == 0) + { + var canonical = strings.ReadValue(ref reader, header); + result = canonical is null ? default : HierarchicalKey.Parse(canonical); + } + else reader.ConsumeUnknownField(header); + } + } +} + +[RegisterCopier] +internal sealed class HierarchicalKeyCopier : IDeepCopier +{ + public HierarchicalKey DeepCopy(HierarchicalKey input, CopyContext context) => input; +} diff --git a/src/Orleans.Messaging/Hosting/MessageTypeExtensions.cs b/src/Orleans.Messaging/Hosting/MessageTypeExtensions.cs new file mode 100644 index 00000000000..64344d5740e --- /dev/null +++ b/src/Orleans.Messaging/Hosting/MessageTypeExtensions.cs @@ -0,0 +1,38 @@ +using System; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Orleans.Messaging; +using Orleans.Serialization; + +namespace Orleans.Hosting; + +/// Extensions for registering typed durable message subjects. +public static class MessageTypeExtensions +{ + /// Registers a singleton subject binding, keyed by its ordinal subject. + /// The subject's payload contract. + /// The service collection. + /// The exact application protocol subject. + /// The service collection. + /// A subject binding is already registered. + public static IServiceCollection AddMessageType(this IServiceCollection services, string subject) + { + ArgumentNullException.ThrowIfNull(services); + EnvelopeValidation.ValidateSubject(subject); + foreach (var descriptor in services) + { + if (descriptor.IsKeyedService + && descriptor.ServiceType.IsGenericType + && descriptor.ServiceType.GetGenericTypeDefinition() == typeof(MessageType<>) + && Equals(descriptor.ServiceKey, subject)) + { + throw new InvalidOperationException($"A payload contract is already registered for durable message subject '{subject}'."); + } + } + + services.TryAddSingleton>(static provider => provider.GetRequiredService().GetSerializer()); + services.AddKeyedSingleton>(subject, + (provider, _) => new MessageType(subject, provider.GetRequiredService>())); + return services; + } +} diff --git a/src/Orleans.Messaging/Hosting/MessagingExtensions.cs b/src/Orleans.Messaging/Hosting/MessagingExtensions.cs new file mode 100644 index 00000000000..f5002f7fb46 --- /dev/null +++ b/src/Orleans.Messaging/Hosting/MessagingExtensions.cs @@ -0,0 +1,101 @@ +using System; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Options; +using Orleans.Configuration; +using Orleans.Messaging; +using Orleans.Messaging.Configuration; +using Orleans.DurableJobs; +using Orleans.Journaling; +using Orleans.Journaling.Json; + +namespace Orleans.Hosting; + +/// +/// Extensions for configuring messaging. +/// +public static class MessagingExtensions +{ + /// + /// Adds inbox and outbox messaging support to the silo. + /// + /// + /// Grains implementing or deriving from + /// bind their scoped messaging services during activation setup, before journal recovery. + /// + /// The silo builder. + /// The optional action used to configure inbox and outbox behavior. + /// The silo builder. + public static ISiloBuilder AddMessaging(this ISiloBuilder builder, Action? configureOptions = null) + { + builder.AddDurableJobs(); + return builder.ConfigureServices(services => services.AddMessaging(configureOptions)); + } + + /// + /// Adds inbox and outbox messaging services. + /// + /// + /// Grains implementing or deriving from + /// bind their scoped messaging services during activation setup, before journal recovery. + /// + /// The service collection. + /// The optional action used to configure inbox and outbox behavior. + /// The service collection. + public static IServiceCollection AddMessaging(this IServiceCollection services, Action? configureOptions = null) + { + services.AddDurableJobs(); + services.TryAddSingleton(TimeProvider.System); + + // The pump job is each grain's only durable wake-up for committed messaging work, so it retries indefinitely + // while other jobs keep the configured policy. + services.AddOptions().PostConfigure(static (options, services) => + { + var shouldRetry = options.ShouldRetry; + var clock = services.GetRequiredKeyedService(DurableJobTimeProviderNames.DurableJobs); + var messaging = services.GetRequiredService>(); + options.ShouldRetry = (context, exception) => string.Equals(context.Job.Name, MessagingPump.JobName, StringComparison.Ordinal) + ? MessagingPump.GetRetryTime(context, clock.GetUtcNow(), messaging.Value.RetryInitialInterval) + : shouldRetry(context, exception); + }); + services.Configure(options => + { + options.AddTypeInfoResolver(InboxJsonContext.Default); + options.AddTypeInfoResolver(OutboxJsonContext.Default); + }); + + var optionsBuilder = services.AddOptions(); + if (configureOptions is not null) + { + optionsBuilder.Configure(configureOptions); + } + + optionsBuilder.Validate( + options => + { + try + { + options.Validate(); + return true; + } + catch (ArgumentException) + { + return false; + } + }, + "DurableMessagingOptions validation failed."); + + services.ConfigureNamedOptionForLogging(Options.DefaultName); + services.TryAddSingleton(); + services.TryAddSingleton(); + services.TryAddScoped(); + services.TryAddScoped(); + services.Replace(ServiceDescriptor.Scoped(static sp => sp.GetRequiredService())); + services.TryAddScoped(); + services.Replace(ServiceDescriptor.Scoped(static sp => sp.GetRequiredService())); + services.TryAddScoped(); + services.TryAddEnumerable( + ServiceDescriptor.Singleton()); + return services; + } +} diff --git a/src/Orleans.Messaging/IInbox.cs b/src/Orleans.Messaging/IInbox.cs new file mode 100644 index 00000000000..db6cb3a8655 --- /dev/null +++ b/src/Orleans.Messaging/IInbox.cs @@ -0,0 +1,41 @@ +namespace Orleans.Messaging; + +/// Registers handlers for the messages delivered to a grain's inbox. +/// +/// +/// Each message is dispatched to the first registered handler which can handle it. Handlers can be registered at any +/// time. registers typed subject handlers. +/// +/// +/// A handler decodes the body, performs asynchronous I/O, validates local results, and checks cancellation before +/// the first shared business or journaled mutation. From that mutation until it returns, it executes synchronously: +/// it applies safe-to-commit business changes, stages outgoing messages through , and calls +/// or . The runtime persists the +/// outcome with those changes. Failures before a terminal operation follow the inbox retry and dead-letter policy. +/// +/// +public interface IInbox +{ + /// Registers a handler. + /// The handler. + /// This inbox. + IInbox Register(IInboxHandler handler); + + /// Registers a sink which accepts messages directly into application-owned journaled state. + /// The sink. + /// This inbox. + /// + /// Messages are offered to sinks before inbox acceptance and handler dispatch; see . + /// Register sinks during construction or activation setup, before the grain receives messages. + /// + IInbox Register(IInboxMessageSink sink); + + /// Registers a source of durable work which the inbox's wake-up keeps reactivating the grain for. + /// The work source. + /// This inbox. + /// + /// See . Register work sources during construction or activation setup, before the + /// grain activates. + /// + IInbox Register(IInboxWorkSource source); +} diff --git a/src/Orleans.Messaging/IInboxHandler.cs b/src/Orleans.Messaging/IInboxHandler.cs new file mode 100644 index 00000000000..9d5d707a9ac --- /dev/null +++ b/src/Orleans.Messaging/IInboxHandler.cs @@ -0,0 +1,24 @@ +using System.Threading; +using System.Threading.Tasks; + +namespace Orleans.Messaging; + +/// Handles inbox messages which it accepts. +/// +/// The inbox dispatches each message to the first registered handler whose returns +/// . Selection receives only the message, so a handler stages a terminal outcome only +/// from , after the inbox selects it. +/// +public interface IInboxHandler +{ + /// Determines whether this handler handles the message. + /// The message to route. + /// when this handler handles the message. + bool CanHandle(InboxMessage message); + + /// Handles the message. + /// The message and its terminal operations. + /// The attempt token. + /// A task which completes when handling returns. + ValueTask HandleAsync(IInboxHandlerContext context, CancellationToken cancellationToken); +} diff --git a/src/Orleans.Messaging/IInboxHandlerContext.cs b/src/Orleans.Messaging/IInboxHandlerContext.cs new file mode 100644 index 00000000000..d7217376bd6 --- /dev/null +++ b/src/Orleans.Messaging/IInboxHandlerContext.cs @@ -0,0 +1,54 @@ +namespace Orleans.Messaging; + +/// +/// Exposes the received message and its attempt-scoped terminal outcome. +/// +public interface IInboxHandlerContext +{ + /// + /// Gets the received message and its immutable application headers and payload. + /// + /// + /// The context keeps the message available through actual handler completion, including + /// after Complete or Fail removes the pending message. Keep its payload and headers unchanged while handling the command. + /// + InboxMessage Message { get; } + + /// + /// Synchronously stages inbox completion and transport deduplication for the active attempt. + /// + /// + /// Perform fallible and asynchronous work first, then apply business mutations and stage outgoing messages before + /// calling Complete in the same synchronous final block. Return from the handler without further awaits. An attempt + /// which stages changes and then ends without an outcome fails the activation, which discards those changes, and + /// the message runs again after recovery from the journal. + /// The runtime owns the subsequent journal write and acknowledgement. + /// Repeated completion within the same active attempt coalesces. Completion retains its + /// logical outcome when cancellation arrives after the final block starts. + /// + /// + /// The context is retired or belongs to another activation or handler attempt. + /// + void Complete(); + + /// + /// Synchronously stages a permanent processing failure, retaining the message as a dead letter + /// and recording its identity for transport deduplication. + /// + /// The nonblank diagnostic reason for the permanent failure. + /// + /// Stage any safe-to-commit business changes and outgoing messages before this operation, + /// then return synchronously. The runtime persists the dead letter and completion together + /// and owns their journal acknowledgement. The dead letter records the current attempt count. + /// Repeated calls with the same reason coalesce; Complete preserves an already staged dead letter. + /// Cancellation after staging preserves the terminal outcome. + /// Use an application reply and Complete for normal business rejection outcomes. + /// + /// The reason is blank. + /// The reason is null. + /// + /// The context is retired or belongs to another activation or attempt, or a different + /// terminal outcome has already been staged. + /// + void Fail(string reason); +} diff --git a/src/Orleans.Messaging/IInboxMessageSink.cs b/src/Orleans.Messaging/IInboxMessageSink.cs new file mode 100644 index 00000000000..a902651b153 --- /dev/null +++ b/src/Orleans.Messaging/IInboxMessageSink.cs @@ -0,0 +1,73 @@ +namespace Orleans.Messaging; + +/// +/// Accepts delivered messages directly into application-owned journaled state, so their payload is journaled once. +/// +/// +/// +/// After validation and deduplication, the inbox offers each message to the registered sinks in registration order, +/// including messages the grain sends to itself and messages with . The first sink +/// which takes the message decides its . A sink which reports +/// has synchronously staged the message's effects in its own journaled state. The +/// inbox then journals only the message's processed record, without its payload, in the same journal write, and reports +/// the result once that write is durable. Run effects which depend on the write, such as starting work, from +/// . +/// +/// +/// A sink-owned message never occupies inbox capacity and has no inbox retry lifecycle: the sink owns its processing, +/// including honoring the message's . Messages which every sink declines are accepted by +/// the inbox and dispatched to instances once their has +/// passed; decline a scheduled message to have the inbox hold it until then. +/// +/// +/// Sinks must be idempotent. The inbox deduplicates the message ID while it retains the processed record, for +/// DeduplicationWindow after the message's . Delayed retries after that receive +/// before reaching a sink. A message whose journal write fails is redelivered to +/// the next activation, which recovers the sink's state from the journal. +/// +/// +public interface IInboxMessageSink +{ + /// Takes a message into the sink's journaled state, or declines it. + /// The validated, deduplicated message. + /// + /// When this method returns , the delivery result which the inbox reports to the sender. + /// + /// + /// when the sink takes the message and reports ; + /// when it declines the message without staging anything. + /// + /// + /// + /// Runs synchronously on the activation scheduler. Validate the message before staging any change. The sink reports + /// one of these statuses: + /// + /// + /// + /// : the sink staged the message's effects, and the inbox records the message + /// as processed in the same write. + /// + /// + /// : the sink already holds the message and stages nothing. The inbox reports + /// it once any write in progress, which can hold the original, is durable. + /// + /// + /// : the sink is at capacity and stages nothing, so the sender retries + /// later. A message the grain sends to itself can't wait for a retry, so this status fails that send. + /// + /// + /// : the sink rejects the message and stages nothing. The inbox + /// dead-letters it with its payload and the result's reason and , records it as + /// processed, and returns the reason and code to the sender. The processed record keeps the reason and code, so a + /// duplicate of the message, such as a retry whose first answer was lost, receives the same rejection. + /// + /// + /// + /// A sink exception, or any other status, means the outcome is unknown, so the inbox stages nothing and fails the + /// delivery: the sender's outbox retries it with backoff until OutboxRetryTimeout, and a send to the grain + /// itself throws to its caller. A sink which throws must leave no staged change. Return + /// to reject a message permanently. + /// + /// + bool TryAccept(InboxMessage message, out DeliveryResult result); +} diff --git a/src/Orleans.Messaging/IInboxWorkSource.cs b/src/Orleans.Messaging/IInboxWorkSource.cs new file mode 100644 index 00000000000..7a3a367a9b8 --- /dev/null +++ b/src/Orleans.Messaging/IInboxWorkSource.cs @@ -0,0 +1,34 @@ +using System; + +namespace Orleans.Messaging; + +/// +/// Reports durable work which a component holds outside the inbox, such as messages which an +/// accepted, so the inbox's durable wake-up reactivates the grain while that work remains. +/// +/// +/// +/// The inbox keeps one durable wake-up job per grain while it, the outbox, or any registered work source holds work. +/// Before each journal write, the inbox consults the work sources and ensures the wake-up, so a write which makes work +/// pending also guarantees that the grain is reactivated after a deactivation or crash. Hold the work in journaled +/// state of the grain's journal, and resume it when the grain activates: a wake-up reactivates the grain, and the work +/// source's owner continues its work from the recovered state. +/// +/// +/// Each wake-up job run consults the work sources again. It reschedules after the shortest delay any source reports, +/// and the wake-up ends once no source, the inbox, nor the outbox holds work. Wake-ups may be early, late, or duplicated, +/// including while the grain is active and running the work. +/// +/// +public interface IInboxWorkSource +{ + /// Gets the delay before the grain must be woken for this source's pending work. + /// + /// when the source holds no pending work; otherwise, the delay after which a wake-up should + /// reactivate the grain if it isn't active. A delay of or less requests a prompt + /// wake-up. For work which runs as soon as the grain is active, return the longest delay acceptable before a + /// deactivated grain resumes it. + /// + /// Runs synchronously on the activation scheduler, before each journal write and on each wake-up. + TimeSpan? GetWakeupDelay(); +} diff --git a/src/Orleans.Messaging/IMessagingDiagnostics.cs b/src/Orleans.Messaging/IMessagingDiagnostics.cs new file mode 100644 index 00000000000..a7f462161ed --- /dev/null +++ b/src/Orleans.Messaging/IMessagingDiagnostics.cs @@ -0,0 +1,116 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using Orleans.Runtime; + +namespace Orleans.Messaging; + +/// +/// Provides operational access to a grain's messaging state. +/// +public interface IMessagingDiagnostics +{ + /// + /// Gets messages which failed during inbox processing. + /// + IReadOnlyList> InboxDeadLetters { get; } + + /// + /// Gets messages which could not be delivered from the outbox. + /// + /// + /// A message whose failed to provide it has no materialized message: its record + /// carries only the , with an empty payload and a default + /// , and its reason records the source's failure. + /// + IReadOnlyList> OutboxDeadLetters { get; } + + /// + /// Stages removal of an inbox dead letter. + /// + /// The receiver-local command identity. + /// when the dead letter existed and was removed. + /// + /// The removal becomes durable with the grain's next journal write. + /// + bool RemoveInboxDeadLetter(MessageId messageId); + + /// + /// Stages removal of an outbox dead letter. + /// + /// The message identifier. + /// when the dead letter existed and was removed. + /// + /// The removal becomes durable with the grain's next journal write. + /// + bool RemoveOutboxDeadLetter(MessageId messageId); +} + +/// +/// Describes a dead-lettered durable message. +/// +/// The incoming or outgoing message type. +[GenerateSerializer, Alias("Orleans.Messaging.DeadLetter`1")] +public sealed class DeadLetter +{ + /// + /// Gets the message. + /// + /// + /// Treat the message's encoded headers as immutable. Application diagnostic DTOs use + /// ordinary RPC copying when returning message buffers. + /// + [Id(0)] + public required TMessage Message { get; init; } + + /// + /// Gets when the message was dead-lettered. + /// + [Id(1)] + public DateTimeOffset DeadLetteredAt { get; init; } + + /// + /// Gets the terminal failure reason. + /// + [Id(2)] + public required string Reason { get; init; } + + /// + /// Gets the number of attempts made. + /// + [Id(3)] + public int AttemptCount { get; init; } + + /// + /// Gets the application-defined code which an inbox message sink reported when it dead-lettered the message, or + /// . + /// + [Id(4)] + public string? Code { get; init; } +} + +internal sealed class MessagingDiagnostics(Inbox inbox, Outbox outbox) : IMessagingDiagnostics +{ + public IReadOnlyList> InboxDeadLetters => + inbox.DeadLetterState.Values.Select(static entry => new DeadLetter + { + Message = entry.Message, + DeadLetteredAt = entry.DeadLetteredAt, + Reason = entry.Reason, + AttemptCount = entry.AttemptCount, + Code = entry.Code + }).ToList(); + + public IReadOnlyList> OutboxDeadLetters => + outbox.DeadLetterState.Values.Select(static entry => new DeadLetter + { + Message = entry.Message, + DeadLetteredAt = entry.DeadLetteredAt, + Reason = entry.Reason, + AttemptCount = entry.AttemptCount + }).ToList(); + + public bool RemoveInboxDeadLetter(MessageId messageId) => inbox.RemoveDeadLetter(messageId); + + public bool RemoveOutboxDeadLetter(MessageId messageId) => outbox.RemoveDeadLetter(messageId); +} diff --git a/src/Orleans.Messaging/IMessagingExtension.cs b/src/Orleans.Messaging/IMessagingExtension.cs new file mode 100644 index 00000000000..fff97ffb613 --- /dev/null +++ b/src/Orleans.Messaging/IMessagingExtension.cs @@ -0,0 +1,48 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using Orleans; +using Orleans.Concurrency; +using Orleans.Runtime; +using Orleans.Serialization; + +namespace Orleans.Messaging; + +/// +/// The grain extension through which other grains exchange durable messages with this grain. +/// +[Alias("Orleans.Messaging.IMessagingExtension")] +public interface IMessagingExtension : IGrainExtension +{ + /// + /// Delivers a message to this grain's inbox. + /// + /// The received command with a required identity and payload; its receiver is this grain. + /// Cancels the caller's wait for delivery. + /// + /// Direct admission shares the payload array, header dictionary, and header value arrays under the immutable-publication contract. + /// RPC copying and deserialization isolate payload arrays, header dictionaries, and header value arrays using ordinary Orleans serialization. + /// A successful result means the inbox durably accepted the message, or already processed it. + /// Callers retrying a delivery reuse the message's . + /// + /// Deliveries interleave with the grain's other requests. A delivery runs no handler: it validates, deduplicates, + /// and stages the message's acceptance synchronously, then waits for that journal write, and the messaging pump runs + /// handlers later. An inbox message sink runs synchronously within the delivery, so it can run while a request is + /// suspended at an await, as handlers can. Interleaving lets concurrent deliveries share journal writes, and lets a + /// message reach a grain whose running request waits for it. + /// + /// + /// Result indicating delivery/processing status. + /// + /// has an unset message ID, exceeds identity admission limits, or has invalid payload or headers. + /// Optional standard metadata is validated where its contract is consumed. + /// + /// + /// The message's is later than the receiver's clock plus DeduplicationWindow, + /// and the message is untracked and above the admission floor, or that send time would extend its record. The inbox + /// stages nothing for it. + /// + [Alias("DeliverAsync")] + [AlwaysInterleave] + ValueTask DeliverAsync(InboxMessage message, CancellationToken cancellationToken = default); +} diff --git a/src/Orleans.Messaging/IMessagingGrain.cs b/src/Orleans.Messaging/IMessagingGrain.cs new file mode 100644 index 00000000000..718d6952c0f --- /dev/null +++ b/src/Orleans.Messaging/IMessagingGrain.cs @@ -0,0 +1,14 @@ +namespace Orleans.Messaging; + +/// +/// Identifies grain implementations which initialize inbox and outbox services during activation setup. +/// +/// +/// Implement this local capability on a grain class, an application base class, or an application grain interface. +/// With messaging services registered, selected activations validate their execution model and bind +/// messaging state before journal recovery. Grains deriving from +/// receive the same setup automatically. +/// +public interface IMessagingGrain +{ +} diff --git a/src/Orleans.Messaging/IOutbox.cs b/src/Orleans.Messaging/IOutbox.cs new file mode 100644 index 00000000000..1478b2c837b --- /dev/null +++ b/src/Orleans.Messaging/IOutbox.cs @@ -0,0 +1,102 @@ +using System; + +namespace Orleans.Messaging; + +/// +/// Stages opaque durable messages alongside a grain's journaled business state. +/// +/// +/// The outbox state machine's asynchronous write preparation establishes a durable self-wakeup before capture. +/// Its write-completion callback makes captured messages eligible for dispatch. +/// Each wakeup checks the grain's messages and their retry deadlines. Messages remain pending until the destination +/// acknowledges durable acceptance, recognizes a duplicate, or reports a terminal delivery outcome. +/// The outbox journals each send's ; every attempt, backpressure, and recovery reuse +/// it, and retries end OutboxRetryTimeout after it. Applications use and +/// to stage crash-recoverable responses to terminal outcomes in the +/// same journal write as retirement. +/// Applications namespace command identities and define ordering in their protocols. +/// +public interface IOutbox +{ + /// + /// Synchronously stages an outgoing command for the grain's next journal write. + /// + /// The fully built outgoing message and required receiver. + /// + /// Direct sends share the payload array, header dictionary, and header value arrays. Keep these objects unchanged after publication. + /// Equivalent repeated identities retain the original message and headers; payload, optional subject, + /// optional , and destination must match. Custom headers do not change pending intent equality. Conflicts fail explicitly. + /// Inbox handlers stage outgoing + /// messages in their synchronous final block before calling + /// or . + /// Ordinary callers persist staged messages using their journaled state manager. + /// An explicit write retry retains pending business changes and messages after a scheduling failure. + /// A message addressed to the sending grain bypasses the outbox: the grain's own inbox stages its acceptance + /// for the same journal write, applying the inbox identity rules, so a completed ID is a duplicate and a different + /// pending command with the ID conflicts. The inbox admits such a message regardless of its capacity and of handler + /// registration, and the send fails when an reports + /// for it. + /// + /// The envelope has an unset identity, invalid subject or destination, or exceeds identity/subject limits. + /// + /// The identity conflicts, an inbox sink backpressured a message addressed to the sending grain, or the outbox is + /// unavailable. + /// + void Send(OutboxMessage message); + + /// + /// Synchronously stages, for the grain's next journal write, a message which a registered + /// provides. + /// + /// The message ID, which a registered source provides. + /// + /// The outbox journals the message ID and delivery state, while the source holds the payload and receiver, and + /// delivers the message like one staged through . Stage the message in the source's + /// state before this call: the outbox resolves the source now, reads and validates its message, and fails when none + /// provides it; an exception from the source propagates and nothing is staged. After the send, a source which fails + /// to provide a valid message dead-letters the entry instead; see . Sending a + /// pending ID again coalesces with the pending message. A message which a source addresses to + /// the sending grain is rejected; send it through , which stages it in the grain's + /// own inbox. + /// A retry timeout or reports + /// to the source; the receiver's earlier outcome is unknown, + /// so the application reconciles it before starting a new send. + /// + /// + /// The ID is unset or too long, or the source's message is invalid or addressed to the sending grain. + /// + /// + /// The ID is pending as an ordinary message, no registered source provides it, the source returned a different ID, + /// or the outbox is unavailable. + /// + void Send(MessageId messageId); + + /// + /// Synchronously stages, for the grain's next journal write, the removal of a pending message which a registered + /// provides. + /// + /// The ID of a message sent through . + /// + /// when the outbox staged the removal; when no such message is pending. + /// + /// + /// Stage the source's release of the payload in the same journal write. The outbox makes no further delivery attempt + /// and doesn't call . An attempt already in flight can still deliver the + /// message and still reads the payload array, header dictionary, and header value arrays which + /// returned, so release them by dropping references rather than + /// reusing them. + /// + /// + /// The ID is pending as an ordinary message, or the outbox is unavailable. + /// + bool Withdraw(MessageId messageId); + + /// Registers a source which provides messages sent through . + /// The source. + /// This outbox. + /// + /// Register sources during construction or activation setup, before journal recovery; see + /// . + /// + IOutbox Register(IOutboxMessageSource source); +} diff --git a/src/Orleans.Messaging/IOutboxMessageSource.cs b/src/Orleans.Messaging/IOutboxMessageSource.cs new file mode 100644 index 00000000000..98fbd10ec66 --- /dev/null +++ b/src/Orleans.Messaging/IOutboxMessageSource.cs @@ -0,0 +1,150 @@ +namespace Orleans.Messaging; + +/// +/// Provides outgoing messages whose payload and receiver the application stores in its own journaled state. +/// +/// +/// +/// stages a reference entry which carries the message ID and delivery +/// state, but no payload or receiver, so the payload is journaled once, in the application's state. The outbox keeps +/// every delivery semantic of an ordinary message: delivery after the sending write is durable, retries with backoff, +/// dead-lettering after OutboxRetryTimeout, wake-ups, and metrics. +/// +/// +/// The outbox calls when the message is sent and before every delivery attempt. The first +/// registered source which returns the message owns it for the activation's lifetime; for an entry recovered from the +/// journal, the first source which returns the message or throws owns it. When the entry retires, the outbox calls the +/// owner's once for that entry in the current activation, synchronously, while it stages the +/// entry's removal. The source's journaled changes and the removal commit in the same journal write. If the activation +/// fails before that write commits, recovery restores the pending entry and the source's prior journaled state, and +/// retiring the recovered entry invokes again. +/// +/// +/// A source withdraws a pending message through , for example after the application times +/// it out or an administrator removes it, and releases its payload in the same journal write. The outbox removes the +/// entry without calling and makes no further delivery attempt. An attempt already in flight can +/// still deliver the message and reads the buffers which returned, so keep them unchanged: +/// release them by dropping references rather than reusing them. +/// +/// +/// A source which can't provide a pending message after its send, for example because its stored payload no longer +/// decodes, throws from ; an invalid message is handled the same way. The outbox then +/// dead-letters that entry alone, without delivering it, and calls with +/// ; the grain's messaging and its other messages are +/// unaffected. A recovered entry which no registered source provides is dead-lettered the same way, without an owner to +/// notify. Because no valid message exists, the dead letter's message carries only its , +/// with an empty payload and a default , and its reason records the failure. +/// +/// +/// All calls run on the activation scheduler. Register sources during activation setup, before journal recovery: a +/// recovered reference entry fails the activation when no source is registered. While a source provides a message, it +/// returns the same message, envelope and receiver alike, for every call. +/// +/// +public interface IOutboxMessageSource +{ + /// Materializes a message for its send or a delivery attempt. + /// The ID of a reference entry. + /// + /// When this method returns , a valid message whose envelope's + /// is and whose receiver is another grain. + /// + /// + /// when this source provides the message; when the message isn't + /// this source's. + /// + /// + /// The outbox validates the message. During , an exception or an invalid + /// message fails the send, and nothing is staged. Afterwards, the entry is durable: an exception, an invalid message, + /// including one with a different or addressed to the sending grain, or + /// from the source which owns the entry dead-letters the entry with its failure as the + /// reason, counting the failed materialization as a delivery attempt, and the outbox then calls + /// with . + /// + bool TryGetMessage(MessageId messageId, out OutboxMessage message); + + /// Notifies the source that the outbox is staging a message's retirement, so the source can release its payload. + /// The ID of the retired reference entry. + /// How the message retired, and the dead letter's reason for a dead-lettered message. + /// + /// + /// Stage the release and other durable effects synchronously in journaled state; they commit atomically with the + /// entry's removal. This callback runs once for the entry in the current activation. Recovery can invoke it again + /// when the retirement write has not committed, starting from the source's recovered journaled state. + /// Make any effects outside that journal write idempotent across repeated calls. + /// Stage a durable application outcome or follow-up work here for + /// and + /// , so recovery preserves the application's response even + /// after diagnostic dead letters are compacted. A dead-lettered send can have been accepted by an earlier attempt; + /// reconcile that uncertain outcome using application-level idempotency. + /// + /// + /// Before this call, the outbox copies a dead-lettered message's payload and header values into its dead letter, + /// unless the source failed to provide the message, so the source can release or reuse its buffers here. Don't depend + /// on decoding the payload here: this call also retires a message whose payload the source can no longer provide. An + /// exception fails the activation's messaging, which deactivates the grain. + /// + /// + void OnRetired(MessageId messageId, OutboxMessageRetirement retirement); +} + +/// Describes how an outbox message retired. +/// The outbox creates retirements; sources read them in . +public readonly struct OutboxMessageRetirement +{ + /// Gets how the message retired. + public OutboxMessageRetirementStatus Status { get; init; } + + /// + /// Gets the dead letter's reason for a message which the receiver rejected or the outbox dead-lettered, such as the + /// rejection which the receiver reported; for a delivered message. + /// + public string? Reason { get; init; } + + /// + /// Gets the application-defined code which the receiver's inbox message sink reported with a rejection, as + /// ; when the receiver reported none or the message didn't + /// retire as . + /// + public string? Code { get; init; } + + /// Creates a retirement of a message which the receiver durably accepted or reported as a duplicate. + public static OutboxMessageRetirement Delivered() => new() { Status = OutboxMessageRetirementStatus.Delivered }; + + /// Creates a retirement of a message which the outbox dead-lettered for . + /// The dead letter's reason. + public static OutboxMessageRetirement DeadLettered(string reason) => new() + { + Status = OutboxMessageRetirementStatus.DeadLettered, + Reason = reason + }; + + /// Creates a retirement of a message which the receiver dead-lettered for . + /// The reason which the receiver reported. + /// The application-defined code which the receiver reported, if any. + public static OutboxMessageRetirement Rejected(string reason, string? code = null) => new() + { + Status = OutboxMessageRetirementStatus.Rejected, + Reason = reason, + Code = code + }; +} + +/// Describes how an outbox message retired. +public enum OutboxMessageRetirementStatus +{ + /// The receiver durably accepted the message, or reported it as a duplicate. + Delivered = 1, + + /// + /// The outbox dead-lettered the message: its delivery attempts didn't succeed before its retry limits elapsed, or its + /// source failed to provide it, or the receiver answered . The receiver's + /// earlier delivery outcome is unknown: an earlier attempt can have been durably accepted. + /// + DeadLettered = 2, + + /// + /// The receiver durably rejected the message, with the reason and code which it reported. + /// + Rejected = 3 +} diff --git a/src/Orleans.Messaging/Inbox.State.cs b/src/Orleans.Messaging/Inbox.State.cs new file mode 100644 index 00000000000..363f9e4d67d --- /dev/null +++ b/src/Orleans.Messaging/Inbox.State.cs @@ -0,0 +1,344 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text.Json.Serialization; +using System.Threading; +using System.Threading.Tasks; +using Orleans.Journaling; + +namespace Orleans.Messaging; + +internal sealed partial class Inbox +{ + private readonly Dictionary _messages = []; + private readonly Dictionary _processed = []; + private readonly Dictionary _deadLetters = []; + private JournalStreamWriter _writer; + + // Forgetting an identity requires a durable floor which excludes its latest observed send time. + private DateTimeOffset _admissionFloor = DateTimeOffset.MinValue; + private DateTimeOffset _capturedAdmissionFloor = DateTimeOffset.MinValue; + private DateTimeOffset _durableAdmissionFloor = DateTimeOffset.MinValue; + + // Acceptances and completions are durable once a completed write captured their sequence number. + private long _lastSequence; + private long _capturedSequence; + private long _durableSequence; + private long _lastCompletionSequence; + private bool _capturedSnapshot; + + // Whether the last inbox command in the journal records that an activation stopped without an interrupted attempt. + private bool _stoppedCleanly; + + // Whether this activation finished its recovery bookkeeping, so its stop can be recorded as clean. + private bool _recovered; + + // The earliest due time, on the Durable Jobs clock, of accepted or retrying messages staged since the last capture. + private DateTimeOffset? _stagedWakeup; + + private bool IsDurable(InboxEntry entry) => entry.Sequence <= _durableSequence; + + // Whether every processed record which deduplication relies on is durable. + private bool AreCompletionsDurable => _lastCompletionSequence <= _durableSequence; + + // The handler attempt in progress, whose staged changes the inbox discards if it fails. + private HandlerExecution? _execution; + + // Every journal write prepares every state, so a write which makes a message or a work source's work pending ensures + // its wake-up before it can become durable. After messaging fails, in-memory state may hold changes which must not + // commit, such as a failed handler's, so every later write fails and recovery starts from the journal. + ValueTask IStateMachine.PrepareWriteAsync(CancellationToken cancellationToken) + { + if (_pump.IsFailed) + { + return ValueTask.FromException(new InvalidOperationException( + "Messaging failed for this activation, so its uncommitted changes are discarded by recovery from the journal.")); + } + + var dueTime = _stagedWakeup; + if (GetWorkSourceDelay() is { } delay) + { + var workDueTime = MessagingTime.AddClamped(Shared.JobTimeProvider.GetUtcNow(), delay); + if (dueTime is null || workDueTime < dueTime) dueTime = workDueTime; + } + + return dueTime is { } due ? _pump.EnsureWakeupAsync(due, cancellationToken) : ValueTask.CompletedTask; + } + + private void OnWriteCaptured(bool snapshot = false) + { + _capturedSequence = _lastSequence; + _capturedAdmissionFloor = _admissionFloor; + _capturedSnapshot = snapshot; + _stagedWakeup = null; + } + + void IStateMachine.OnWriteCompleted() + { + _durableSequence = _capturedSequence; + _durableAdmissionFloor = _capturedAdmissionFloor; + // Floor-only appends stay constant-time; snapshots reclaim history after their admission floor is durable. + if (_capturedSnapshot) CompactProcessed(); + _pump.Signal(); + } + + private DateTimeOffset GetAdmissionFloor(DateTimeOffset now) + { + var nowTicks = now.UtcTicks; + var candidate = new DateTimeOffset(Math.Max(0, nowTicks - Options.DeduplicationWindow.Ticks), TimeSpan.Zero); + return candidate > _admissionFloor ? candidate : _admissionFloor; + } + + private static bool IsAtOrBelowFloor(InboxProcessedEntry entry, DateTimeOffset floor) => + entry.SentAt <= floor; + + private void CompactProcessed() + { + foreach (var (key, entry) in _processed) + { + if (entry.Sequence <= _durableSequence && IsAtOrBelowFloor(entry, _durableAdmissionFloor)) _processed.Remove(key); + } + } + + private void Emit(InboxCommand command) + { + Shared.InboxCodec.WriteSet(command, _writer); + Apply(command); + var dueTime = command switch + { + { Kind: InboxCommandKind.Accept, Message: { } message } => message.Envelope.DeliverAt ?? DateTimeOffset.MinValue, + { Kind: InboxCommandKind.Retry, State: { } state } => state.NextAttemptAt, + { Kind: InboxCommandKind.ExtendSentAt, MessageId: var key } when _messages.TryGetValue(key, out var pending) => GetDueTime(pending), + _ => (DateTimeOffset?)null + }; + if (dueTime is { } due && (_stagedWakeup is null || due < _stagedWakeup)) _stagedWakeup = due; + } + + void IStateMachine.Reset(JournalStreamWriter writer) + { + ReportDepth(-_messages.Count); + _writer = writer; + _messages.Clear(); + _processed.Clear(); + _deadLetters.Clear(); + _admissionFloor = _capturedAdmissionFloor = _durableAdmissionFloor = DateTimeOffset.MinValue; + _stagedWakeup = null; + _capturedSnapshot = false; + _stoppedCleanly = false; + _lastSequence = _capturedSequence = _durableSequence = _lastCompletionSequence = 0; + } + + void IStateMachine.ReplayEntry(JournalEntry entry, JournalReplayContext context) => + context.GetRequiredCommandCodec(entry.FormatKey, Shared.InboxCodec).Apply(entry.Reader, this); + + void IStateMachine.OnRecoveryCompleted() + { + foreach (var (key, entry) in _messages) + { + EnvelopeValidation.Validate(entry.Message); + ValidateRecoveredSentAt(entry.Message.SentAt); + if (key != entry.Message.Envelope.MessageId) + { + throw new InvalidOperationException("The recovered inbox key does not match its command identity."); + } + } + + foreach (var (key, letter) in _deadLetters) + { + EnvelopeValidation.Validate(letter.Message); + ValidateRecoveredSentAt(letter.Message.SentAt); + if (key != letter.Message.Envelope.MessageId) + { + throw new InvalidOperationException("The recovered inbox dead-letter key does not match its command identity."); + } + } + + foreach (var entry in _processed.Values) ValidateRecoveredSentAt(entry.SentAt); + + _capturedSequence = _durableSequence = _lastSequence; + _capturedAdmissionFloor = _durableAdmissionFloor = _admissionFloor; + CompactProcessed(); + } + + void IStateMachine.WritePendingEntries(JournalStreamWriter writer) => OnWriteCaptured(); + + void IStateMachine.WriteSnapshot(JournalStreamWriter writer) + { + // The floor and compacted history commit atomically. Keep the in-memory records until acknowledgement, so + // an interleaving delivery still deduplicates while the snapshot's admission floor is not yet durable. + var floor = GetAdmissionFloor(Shared.JobTimeProvider.GetUtcNow()); + Shared.InboxCodec.WriteSet(new() + { + Kind = InboxCommandKind.Snapshot, + Snapshot = new() + { + Messages = _messages.Values.ToArray(), + Processed = _processed.Values.Where(entry => !IsAtOrBelowFloor(entry, floor)).ToArray(), + DeadLetters = _deadLetters.Values.ToArray(), + AdmissionFloor = floor, + StoppedCleanly = _stoppedCleanly + } + }, writer); + _admissionFloor = floor; + OnWriteCaptured(snapshot: true); + } + + void IDurableValueCommandHandler.ApplySet(InboxCommand command) => Apply(command); + + private void Apply(InboxCommand command) + { + if (command.Kind != InboxCommandKind.Snapshot) _stoppedCleanly = command.Kind == InboxCommandKind.CleanStop; + switch (command.Kind) + { + case InboxCommandKind.Accept: + var message = command.Message ?? throw new InvalidOperationException("An accept command requires its message."); + _messages.Add(message.Envelope.MessageId, new InboxEntry { Message = message, State = new(), Sequence = ++_lastSequence }); + _processed.Remove(message.Envelope.MessageId); + ReportDepth(1); + break; + case InboxCommandKind.Retry: + _messages[command.MessageId].State = command.State ?? throw new InvalidOperationException("A retry command requires its state."); + break; + case InboxCommandKind.Complete: + if (_messages.Remove(command.MessageId)) ReportDepth(-1); + + _lastCompletionSequence = ++_lastSequence; + _processed[command.MessageId] = new() + { + MessageId = command.MessageId, + SentAt = command.SentAt, + Rejection = command.Rejection, + Sequence = _lastSequence + }; + if (command.DeadLetter is { } letter) _deadLetters[command.MessageId] = letter; + break; + case InboxCommandKind.AdvanceAdmissionFloor: + AdvanceAdmissionFloor(command.AdmissionFloor); + break; + case InboxCommandKind.ExtendSentAt: + // Pending entries are present in every snapshot; processed extensions use completion upserts. + if (!_messages.TryGetValue(command.MessageId, out var pending)) + { + throw new InvalidOperationException("An ExtendSentAt command requires its pending entry."); + } + if (command.SentAt > pending.Message.SentAt) + { + pending.Message = pending.Message with { SentAt = command.SentAt }; + pending.Sequence = ++_lastSequence; + } + break; + case InboxCommandKind.RemoveDeadLetters: + foreach (var key in command.RemovedKeys!) _deadLetters.Remove(key); + break; + case InboxCommandKind.CleanStop: + case InboxCommandKind.Resume: + break; + case InboxCommandKind.Snapshot: + var snapshot = command.Snapshot ?? throw new InvalidOperationException("A snapshot command requires its state."); + AdvanceAdmissionFloor(snapshot.AdmissionFloor); + _stoppedCleanly = snapshot.StoppedCleanly; + var previousCount = _messages.Count; + _messages.Clear(); + _processed.Clear(); + _deadLetters.Clear(); + foreach (var restored in snapshot.Messages) _messages.Add(restored.Message.Envelope.MessageId, restored); + + ReportDepth(_messages.Count - previousCount); + + foreach (var processed in snapshot.Processed) + { + _processed.Add(processed.MessageId, processed); + } + + foreach (var restored in snapshot.DeadLetters) _deadLetters.Add(restored.Message.Envelope.MessageId, restored); + break; + default: + throw new NotSupportedException($"Inbox command '{command.Kind}' is not supported."); + } + } + + private void AdvanceAdmissionFloor(DateTimeOffset floor) + { + if (floor < _admissionFloor) throw new InvalidOperationException("The inbox admission floor cannot move backward."); + _admissionFloor = floor; + } + + private static void ValidateRecoveredSentAt(DateTimeOffset sentAt) + { + // Admission requires a time above a floor which is at least MinValue, so default cannot be admitted state. + if (sentAt == DateTimeOffset.MinValue) throw new InvalidOperationException("A recovered inbox entry requires its admitted send time."); + } +} + +internal enum InboxCommandKind +{ + Accept = 1, + Retry = 2, + Complete = 3, + RemoveDeadLetters = 4, + Snapshot = 6, + AdvanceAdmissionFloor = 7, + ExtendSentAt = 8, + + // An activation stopped with no handler attempt interrupted, so recovery charges no attempt. + CleanStop = 9, + + // A recovered activation consumed a clean stop, so a later unclean stop is charged. + Resume = 10 +} + +[GenerateSerializer, Alias("Orleans.Messaging.InboxCommand")] +internal sealed class InboxCommand +{ + [Id(0)] public InboxCommandKind Kind { get; init; } + [Id(1)] public InboxMessage? Message { get; init; } + [Id(2)] public MessageId MessageId { get; init; } + [Id(3)] public InboxMessageState? State { get; init; } + [Id(5)] public InboxDeadLetter? DeadLetter { get; init; } + [Id(6)] public MessageId[]? RemovedKeys { get; init; } + [Id(7)] public InboxSnapshot? Snapshot { get; init; } + [Id(8)] public InboxRejection? Rejection { get; init; } + [Id(9)] public DateTimeOffset SentAt { get; init; } + [Id(10)] public DateTimeOffset AdmissionFloor { get; init; } +} + +[GenerateSerializer, Alias("Orleans.Messaging.InboxSnapshot")] +internal sealed class InboxSnapshot +{ + [Id(0)] public InboxEntry[] Messages { get; init; } = []; + [Id(1)] public InboxProcessedEntry[] Processed { get; init; } = []; + [Id(2)] public InboxDeadLetter[] DeadLetters { get; init; } = []; + [Id(3)] public DateTimeOffset AdmissionFloor { get; init; } = DateTimeOffset.MinValue; + [Id(4)] public bool StoppedCleanly { get; init; } +} + +[GenerateSerializer, Alias("Orleans.Messaging.InboxProcessedEntry")] +internal sealed class InboxProcessedEntry +{ + [Id(0)] public MessageId MessageId { get; init; } + [Id(2)] public InboxRejection? Rejection { get; init; } + [Id(3)] public required DateTimeOffset SentAt { get; init; } + [field: NonSerialized] + [JsonIgnore] public long Sequence { get; init; } +} + +/// The answer which an inbox sink gave when it rejected a message on delivery. +[GenerateSerializer, Alias("Orleans.Messaging.InboxRejection")] +internal sealed class InboxRejection +{ + [Id(0)] public required string Reason { get; init; } + [Id(1)] public string? Code { get; init; } +} + +[GenerateSerializer, Alias("Orleans.Messaging.InboxEntry")] +internal sealed class InboxEntry +{ + [Id(0)] public required InboxMessage Message { get; set; } + [Id(1)] public required InboxMessageState State { get; set; } + [field: NonSerialized] + [JsonIgnore] public long Sequence { get; set; } +} + +[JsonSerializable(typeof(InboxCommand))] +[JsonSerializable(typeof(InboxSnapshot))] +internal sealed partial class InboxJsonContext : JsonSerializerContext; diff --git a/src/Orleans.Messaging/Inbox.cs b/src/Orleans.Messaging/Inbox.cs new file mode 100644 index 00000000000..31ec2206584 --- /dev/null +++ b/src/Orleans.Messaging/Inbox.cs @@ -0,0 +1,849 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; +using System.Linq; +using System.Runtime.ExceptionServices; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Orleans.Journaling; +using Orleans.Messaging.Configuration; +using Orleans.Messaging.Diagnostics; +using Orleans.Runtime; + +namespace Orleans.Messaging; + +/// +/// Accepts, deduplicates, and processes durable incoming messages for one grain. +/// +/// +/// All state transitions run on the activation scheduler, so each check-then-emit sequence is atomic. +/// Accepted messages are processed only after their acceptance is durable and their , +/// if any, has passed. +/// +internal sealed partial class Inbox : IInbox, IMessagingExtension, IStateMachine, IReportsStagedChanges, IDurableValueCommandHandler +{ + private readonly MessagingPump _pump; + private readonly InboxDispatcher _handlers = new([]); + private readonly List _sinks = []; + private readonly List _workSources = []; + + public Inbox(MessagingPump pump) + { + _pump = pump; + pump.StateManager.RegisterStateMachine(MessagingStateNames.Inbox, this); + pump.Register(this); + } + + private MessagingShared Shared => _pump.Shared; + private DurableMessagingOptions Options => _pump.Shared.Options; + private ILogger Logger => _pump.Shared.InboxLogger; + private TimeProvider TimeProvider => _pump.Shared.TimeProvider; + private IGrainContext GrainContext => _pump.GrainContext; + + internal int Count => _messages.Count; + + internal IReadOnlyDictionary MessageState => _messages; + + internal IReadOnlyDictionary DeadLetterState => _deadLetters; + + internal InboxDispatcher Handlers => _handlers; + + public IInbox Register(IInboxHandler handler) + { + _handlers.Add(handler); + + // Pending messages wait for a handler without holding a wake-up, so a late registration resumes them. + if (_messages.Count > 0) _pump.RequestWakeup(); + return this; + } + + public IInbox Register(IInboxMessageSink sink) + { + ArgumentNullException.ThrowIfNull(sink); + _sinks.Add(sink); + return this; + } + + public IInbox Register(IInboxWorkSource source) + { + ArgumentNullException.ThrowIfNull(source); + _workSources.Add(source); + return this; + } + + public async ValueTask DeliverAsync(InboxMessage message, CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + _pump.ShutdownToken.ThrowIfCancellationRequested(); + EnvelopeValidation.Validate(message); + + var key = message.Envelope.MessageId; + var receiverTime = Shared.JobTimeProvider.GetUtcNow(); + if (message.SentAt > MessagingTime.AddClamped(receiverTime, Options.DeduplicationWindow)) + { + // Recorded outcomes and expiration decisions survive clock rollback. Bound only a time we would store. + var wouldStoreTime = _processed.TryGetValue(key, out var processed) + ? message.SentAt > processed.SentAt + : _messages.TryGetValue(key, out var pending) + ? message.SentAt > pending.Message.SentAt + : message.SentAt > GetAdmissionFloor(receiverTime); + if (wouldStoreTime) + { + LogSentAtTooFarAhead(Logger, key, GrainContext.GrainId, message.SentAt, receiverTime); + throw new ArgumentOutOfRangeException(nameof(message), message.SentAt, + "SentAt must not be later than the receiver's clock plus DeduplicationWindow."); + } + } + + // Delivery is a grain call, so the activation is servicing requests. + _pump.EnsureStarted(); + try + { + var stagedBeforeDuplicate = _pump.StagedChangeCount; + if (TryStageDuplicate(message, out var pending)) + { + var rejection = pending is null ? _processed[key].Rejection : null; + ExcludeFromHandlerAttempt(stagedBeforeDuplicate); + // The sender may retire its copy once this returns, so the acceptance or the processed record must be + // durable. A completion staged by a handler or a sink can still be in flight. + if (pending is null ? !AreCompletionsDurable : !IsDurable(pending)) await PersistForCallerAsync(cancellationToken).ConfigureAwait(true); + if (pending is not null) _pump.Signal(); + + Shared.Instruments.OnInboxMessageReceived(Shared.GrainType, "duplicate"); + + // A sender whose answer was lost delivers again, so it receives the rejection which a sink first answered. + return rejection is not null + ? DeliveryResult.DeadLettered(rejection.Reason, rejection.Code) + : DeliveryResult.Duplicate(); + } + + // Retained identities precede expiration: a lost acknowledgement may belong to pending or completed work. + // Only a durable, non-decreasing floor permits forgetting that evidence, regardless of later clock motion. + var floor = GetAdmissionFloor(receiverTime); + if (message.SentAt <= floor) + { + var stagedBeforeExpiration = _pump.StagedChangeCount; + if (message.SentAt > _admissionFloor) + { + Emit(new() { Kind = InboxCommandKind.AdvanceAdmissionFloor, AdmissionFloor = floor }); + ExcludeFromHandlerAttempt(stagedBeforeExpiration); + } + if (_admissionFloor > _durableAdmissionFloor) await PersistForCallerAsync(cancellationToken).ConfigureAwait(true); + Shared.Instruments.OnInboxMessageReceived(Shared.GrainType, "expired"); + LogMessageExpired(Logger, key, GrainContext.GrainId, message.SentAt, _admissionFloor); + return DeliveryResult.Expired(); + } + + MessagingEvents.EmitInboxMessageReceiving(GrainContext, message); + + // Sink-owned messages never occupy inbox capacity and need no handler. A sink's duplicate can still be staged, + // so every outcome except backpressure is reported once the write is durable. + var stagedBefore = _pump.StagedChangeCount; + if (StageSinkAcceptance(message) is { } sinkResult) + { + ExcludeFromHandlerAttempt(stagedBefore); + if (sinkResult.Status != DeliveryStatus.Backpressured) await PersistForCallerAsync(cancellationToken).ConfigureAwait(true); + return sinkResult; + } + + // Every pending message, scheduled or not, holds capacity until it completes. + if (_messages.Count >= Options.MaxCapacity) + { + Shared.Instruments.OnInboxMessageReceived(Shared.GrainType, "backpressured"); + return DeliveryResult.Backpressured(); + } + + if (_handlers.IsEmpty) + { + Shared.Instruments.OnInboxMessageReceived(Shared.GrainType, "handler_not_found"); + LogHandlerNotFound(Logger, key, GrainContext.GrainId); + return DeliveryResult.HandlerNotFound(); + } + + Emit(new() { Kind = InboxCommandKind.Accept, Message = message }); + ExcludeFromHandlerAttempt(stagedBefore); + await PersistForCallerAsync(cancellationToken).ConfigureAwait(true); + Shared.Instruments.OnInboxMessageReceived(Shared.GrainType, "accepted"); + LogMessageAccepted(Logger, key, GrainContext.GrainId); + return DeliveryResult.Accepted(); + } + catch (Exception exception) when (exception is not OperationCanceledException || !cancellationToken.IsCancellationRequested) + { + LogDeliveryOperationFailed(Logger, exception, key, GrainContext.GrainId); + throw; + } + } + + /// + /// Attributes changes which a delivery staged since to the delivery, which interleaves + /// with a handler attempt waiting at an await, so the attempt's failure doesn't treat them as its own. + /// + private void ExcludeFromHandlerAttempt(long stagedBefore) + { + if (_execution is { } execution) execution.StagedChanges += _pump.StagedChangeCount - stagedBefore; + } + + // Whether a state staged a change during the attempt which isn't a delivery's; a manager which doesn't track staged + // changes may have. + private bool StagedDuring(HandlerExecution execution) => + _pump.StagedChangeCount is var count && (count < 0 || count != execution.StagedChanges); + + /// + /// Stages acceptance of a message which this grain sends to itself, for the caller's next journal write. + /// + /// + /// + /// Applies the identity rules of synchronously and without writing: a completed or + /// equivalent pending message is a duplicate, and a different pending command with the same ID conflicts. + /// The acceptance becomes durable atomically with the caller's other staged state, after which the inbox processes + /// the message as usual. When the pump is not running yet, the durable wake-up which the write ensures for new + /// acceptances starts it once the activation services calls, so handlers never run during activation. + /// + /// + /// A synchronous send cannot report backpressure or a missing handler, so the message is admitted regardless of + /// and of handler registration, and a sink which backpressures + /// it fails the send. The message still occupies capacity for deliveries from other grains until it completes, and + /// processing waits for a handler to be registered. + /// + /// + /// + /// The inbox holds a different pending command with the same ID, or an inbox sink backpressured the message or + /// reported an unsupported status. + /// + internal void StageLocal(InboxMessage message) + { + // Reuse the outbox's clock observation: a fresh read could observe another rollback. + var now = message.SentAt; + // A local send commits with the inbox, so it can use the first admitted time after a clock rollback. + if (message.SentAt <= _admissionFloor) + { + if (_admissionFloor == DateTimeOffset.MaxValue) throw new InvalidOperationException("The inbox admission floor leaves no representable send time for a new local message."); + message = message with { SentAt = _admissionFloor.AddTicks(1) }; + } + Debug.Assert(message.SentAt <= MessagingTime.AddClamped(now, Options.DeduplicationWindow) + || message.SentAt <= _admissionFloor.AddTicks(1)); + if (TryStageDuplicate(message, out _)) + { + Shared.Instruments.OnInboxMessageReceived(Shared.GrainType, "duplicate"); + return; + } + + if (StageSinkAcceptance(message) is { } sinkResult) + { + if (sinkResult.Status == DeliveryStatus.Backpressured) + { + throw new InvalidOperationException( + $"An inbox message sink backpressured message '{message.Envelope.MessageId}', which the grain sent to itself."); + } + + return; + } + + try + { + Emit(new() { Kind = InboxCommandKind.Accept, Message = message }); + } + catch (Exception exception) + { + LogStagingFailed(Logger, exception); + _pump.Fail(exception, "Inbox staging failed."); + throw; + } + + Shared.Instruments.OnInboxMessageReceived(Shared.GrainType, "accepted"); + LogMessageAccepted(Logger, message.Envelope.MessageId, GrainContext.GrainId); + } + + /// + /// Offers a message to the registered sinks and stages the outcome which the first sink to take the message reports. + /// + /// The delivery result, or when every sink declines the message. + /// + /// An accepted message journals only its processed record, in the write which commits the sink's staged changes. + /// A duplicate or backpressured message stages nothing. A sink which dead-letters the message stages nothing itself, + /// so the message is dead-lettered with its payload and recorded as processed. A sink exception, or an unsupported + /// status, leaves the outcome unknown: it propagates to the caller with nothing staged, so the sender retries. + /// + /// A sink reported an unsupported delivery status. + private DeliveryResult? StageSinkAcceptance(InboxMessage message) + { + var key = message.Envelope.MessageId; + foreach (var sink in _sinks) + { + if (!sink.TryAccept(message, out var result)) continue; + if (result.Status is DeliveryStatus.Duplicate or DeliveryStatus.Backpressured) + { + Shared.Instruments.OnInboxMessageReceived(Shared.GrainType, result.Status == DeliveryStatus.Duplicate ? "duplicate" : "backpressured"); + return result; + } + + if (result.Status is not (DeliveryStatus.Accepted or DeliveryStatus.DeadLettered)) + { + throw new InvalidOperationException($"The inbox message sink reported the unsupported delivery status '{result.Status}'."); + } + + var accepted = result.Status == DeliveryStatus.Accepted; + var reason = result.Message ?? "The inbox message sink dead-lettered the message."; + try + { + if (accepted) + { + StageCompletion(message, deadLetter: null); + } + else + { + LogSinkDeadLettered(Logger, key, GrainContext.GrainId, reason, result.Code); + StageDeadLetter(message, reason, attemptCount: 1, new() { Reason = reason, Code = result.Code }); + } + } + catch (Exception exception) + { + // The sink's staged changes cannot commit without the processed record. + LogStagingFailed(Logger, exception); + _pump.Fail(exception, "Inbox staging failed."); + throw; + } + + Shared.Instruments.OnInboxMessageReceived(Shared.GrainType, accepted ? "accepted" : "dead_lettered"); + Shared.Instruments.OnInboxMessageProcessed(Shared.GrainType, accepted ? "success" : "dead_lettered"); + if (!accepted) return DeliveryResult.DeadLettered(reason, result.Code); + LogMessageAccepted(Logger, key, GrainContext.GrainId); + return DeliveryResult.Accepted(); + } + + return null; + } + + /// + /// Recognizes a retained or pending identity and stages any extension of its observed send time. + /// + /// The received message. + /// The pending entry, or when the message was completed or is new. + private bool TryStageDuplicate(InboxMessage message, out InboxEntry? pending) + { + var key = message.Envelope.MessageId; + if (_processed.TryGetValue(key, out var processed)) + { + ExtendSentAt(message, processed.SentAt); + pending = null; + return true; + } + + if (_messages.TryGetValue(key, out pending)) + { + if (!EnvelopeEquivalence.AreSameCommand(pending.Message, message)) + { + throw new InvalidOperationException( + $"The inbox already contains a different command with message ID '{key}'."); + } + + ExtendSentAt(message, pending.Message.SentAt); + return true; + } + + return false; + } + + private void ExtendSentAt(InboxMessage message, DateTimeOffset recordedSentAt) + { + if (message.SentAt <= recordedSentAt) return; + var key = message.Envelope.MessageId; + try + { + if (_processed.TryGetValue(key, out var processed)) + { + // An in-flight snapshot can omit the old record, so replay needs the complete decision, not a delta. + StageCompletion(message, deadLetter: null, rejection: processed.Rejection); + } + else + { + Debug.Assert(_messages.ContainsKey(key)); + Emit(new() { Kind = InboxCommandKind.ExtendSentAt, MessageId = key, SentAt = message.SentAt }); + } + } + catch (Exception exception) + { + LogStagingFailed(Logger, exception); + _pump.Fail(exception, "Inbox retention update failed."); + throw; + } + } + + /// + /// Processes one batch of durable, due messages and returns whether any message was processed. + /// + internal async ValueTask ProcessReadyAsync(CancellationToken cancellationToken) + { + if (_handlers.IsEmpty || _messages.Count == 0) + { + return false; + } + + var jobNow = Shared.JobTimeProvider.GetUtcNow(); + var batch = new List(Math.Min(Options.InboxBatchSize, _messages.Count)); + foreach (var entry in _messages.Values) + { + if (IsDurable(entry) && GetDelay(entry, jobNow) == TimeSpan.Zero) + { + batch.Add(entry); + if (batch.Count == Options.InboxBatchSize) break; + } + } + + if (batch.Count == 0) + { + return false; + } + + foreach (var entry in batch) + { + cancellationToken.ThrowIfCancellationRequested(); + await ProcessAsync(entry, cancellationToken).ConfigureAwait(true); + } + + await PersistAsync().ConfigureAwait(true); + return true; + } + + private async ValueTask ProcessAsync(InboxEntry entry, CancellationToken cancellationToken) + { + if (!IsCurrent(entry)) + { + return; + } + + var start = Stopwatch.GetTimestamp(); + var execution = new HandlerExecution(this, entry) { Active = true, StagedChanges = _pump.StagedChangeCount }; + _execution = execution; + Exception? error = null; + try + { + await _handlers.HandleAsync(new InboxHandlerContext(entry.Message, execution), cancellationToken).ConfigureAwait(true); + execution.ThrowIfRejected(); + if (!execution.Completed) + { + throw new InvalidOperationException("Inbox handlers must call Complete or Fail before returning successfully."); + } + } + catch (Exception) when (execution.Rejection is { } rejection) + { + _pump.Fail(rejection.SourceException, "Inbox operation failed."); + rejection.Throw(); + } + catch (Exception exception) when (execution.Completed) + { + LogHandlerException(Logger, exception, entry.Message.Envelope.MessageId, GrainContext.GrainId); + } + catch (Exception exception) when (StagedDuring(execution)) + { + // The attempt ended without an outcome after it staged changes. Pending changes share one journal buffer, so a + // retry record would commit them; failing the activation discards them, and the message runs again after + // recovery from the journal. + LogHandlerException(Logger, exception, entry.Message.Envelope.MessageId, GrainContext.GrainId); + var failure = new InvalidOperationException( + $"The inbox handler for message '{entry.Message.Envelope.MessageId}' failed after staging changes, so the activation discards them.", + exception); + _pump.Fail(failure, "An inbox handler failed after staging changes."); + if (exception is OperationCanceledException && cancellationToken.IsCancellationRequested) throw; + throw failure; + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception exception) + { + error = exception; + LogHandlerException(Logger, exception, entry.Message.Envelope.MessageId, GrainContext.GrainId); + } + finally + { + execution.Active = false; + _execution = null; + } + + string status; + if (error is not null) + { + if (!IsCurrent(entry)) + { + return; + } + + status = StageFailedAttempt(entry, error.Message, error.ToString(), Shared.JobTimeProvider.GetUtcNow()); + } + else + { + status = execution.DeadLetterReason is null ? "success" : "dead_lettered"; + } + + Shared.Instruments.OnInboxMessageProcessed(Shared.GrainType, status); + Shared.Instruments.OnInboxProcessingDuration(Stopwatch.GetElapsedTime(start), Shared.GrainType); + } + + /// + /// Stages a failed attempt's outcome: a retry with backoff, or a dead letter once InboxRetryTimeout has elapsed + /// since the message's first failure. Returns the processing status for metrics. + /// + /// + /// An unknown outcome retries until the timeout; the last attempt runs when it elapses, and Fail(reason) + /// dead-letters a message at once. Retry deadlines follow the Durable Jobs clock, which schedules the next wake-up. + /// An interrupted attempt starts the retry window but dead-letters only a message whose window an earlier failure + /// started, so one unclean stop never dead-letters a message by itself. + /// + private string StageFailedAttempt(InboxEntry entry, string reason, string detail, DateTimeOffset now, bool interrupted = false) + { + var attempts = checked(entry.State.AttemptCount + 1); + var expirationTime = entry.State.ExpirationTime ?? MessagingTime.AddClamped(now, Options.InboxRetryTimeout); + if (expirationTime <= now && (!interrupted || entry.State.ExpirationTime is not null)) + { + StageDeadLetter(entry.Message, reason, attempts); + return "dead_lettered"; + } + + var retryAt = MessagingTime.AddClamped(now, MessagingTime.GetBackoff(Options.RetryInitialInterval, attempts)); + Emit(new() + { + Kind = InboxCommandKind.Retry, + MessageId = entry.Message.Envelope.MessageId, + State = new() + { + AttemptCount = attempts, + LastError = detail, + NextAttemptAt = retryAt < expirationTime ? retryAt : expirationTime, + ExpirationTime = expirationTime + } + }); + return "retry"; + } + + private bool IsCurrent(InboxEntry entry) => + _messages.TryGetValue(entry.Message.Envelope.MessageId, out var current) && ReferenceEquals(current, entry); + + private void StageCompletion(InboxMessage message, InboxDeadLetter? deadLetter, InboxRejection? rejection = null) => + Emit(new() + { + Kind = InboxCommandKind.Complete, + MessageId = message.Envelope.MessageId, + SentAt = message.SentAt, + DeadLetter = deadLetter, + Rejection = rejection + }); + + private void StageDeadLetter(InboxMessage message, string reason, int attemptCount, InboxRejection? rejection = null) + { + var now = TimeProvider.GetUtcNow(); + var key = message.Envelope.MessageId; + CompactDeadLetters(now, reservedCapacity: _deadLetters.ContainsKey(key) ? 0 : 1); + StageCompletion(message, new InboxDeadLetter + { + Message = message, + DeadLetteredAt = now, + Reason = reason, + AttemptCount = attemptCount, + Code = rejection?.Code + }, rejection); + } + + private bool CompactDeadLetters(DateTimeOffset now, int reservedCapacity = 0) + { + if (_deadLetters.Count == 0) return false; + var retained = new Dictionary(_deadLetters); + if (!DeadLetterRetention.Compact(retained, now, Options.DeadLetterRetentionPeriod, Options.MaxRetainedDeadLetters, + static entry => entry.DeadLetteredAt, reservedCapacity)) + { + return false; + } + + Emit(new() + { + Kind = InboxCommandKind.RemoveDeadLetters, + RemovedKeys = _deadLetters.Keys.Where(key => !retained.ContainsKey(key)).ToArray() + }); + return true; + } + + internal bool RemoveDeadLetter(MessageId messageId) + { + _pump.ShutdownToken.ThrowIfCancellationRequested(); + if (!_deadLetters.ContainsKey(messageId)) return false; + Emit(new() { Kind = InboxCommandKind.RemoveDeadLetters, RemovedKeys = [messageId] }); + return true; + } + + /// + /// Returns the delay until the next message or work source is due, or when neither the inbox + /// nor any work source holds work which this activation can perform. Messages wait without a wake-up until a handler + /// is registered. + /// + /// The current Durable Jobs time, which the caller uses to schedule the wake-up. + internal TimeSpan? GetNextWakeupDelay(DateTimeOffset jobNow) + { + var next = GetWorkSourceDelay(); + if (_messages.Count == 0 || _handlers.IsEmpty || next <= TimeSpan.Zero) return next; + foreach (var entry in _messages.Values) + { + // Capture follows synchronous preparation; its wake-up covers the write until acknowledgement. + if (entry.Sequence > _capturedSequence) continue; + var delay = GetDelay(entry, jobNow); + if (delay == TimeSpan.Zero) return delay; + if (next is null || delay < next) next = delay; + } + + return next; + } + + // Work sources may report any non-positive delay to request a prompt wake-up; it is normalized to zero so that due + // times derived from it stay representable. + private TimeSpan? GetWorkSourceDelay() + { + TimeSpan? next = null; + foreach (var source in _workSources) + { + if (source.GetWakeupDelay() is not { } delay) continue; + if (delay <= TimeSpan.Zero) return TimeSpan.Zero; + if (next is null || delay < next) next = delay; + } + + return next; + } + + // DeliverAt and retry deadlines are in the Durable Jobs time domain, which schedules wake-ups. + private static TimeSpan GetDelay(InboxEntry entry, DateTimeOffset jobNow) + { + var dueTime = GetDueTime(entry); + return dueTime > jobNow ? dueTime - jobNow : TimeSpan.Zero; + } + + private static DateTimeOffset GetDueTime(InboxEntry entry) + { + var dueTime = entry.Message.Envelope.DeliverAt ?? DateTimeOffset.MinValue; + return entry.State.NextAttemptAt is { } retryAt && retryAt > dueTime ? retryAt : dueTime; + } + + internal async Task StartAsync(CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + if (GrainContext.GrainInstance is not DurableGrain and not IMessagingGrain) + { + throw new InvalidOperationException("Inbox activation requires IMessagingGrain or DurableGrain."); + } + + var compacted = CompactDeadLetters(TimeProvider.GetUtcNow()); + if (ChargeInterruptedAttempts() | compacted) + { + await PersistAsync().ConfigureAwait(true); + } + + _recovered = true; + } + + /// + /// Charges one failed attempt to each due message unless the previous activation journaled a clean stop, and returns + /// whether it staged anything. + /// + /// + /// An activation which fails after a handler staged changes, or whose silo crashes, stops without recording the + /// attempt's outcome. Charging it at recovery keeps backoff and InboxRetryTimeout bounding every failure path + /// without a journal write on the happy path. A crash also charges due messages which hadn't been dispatched, so + /// attempts can overcount by one per unclean stop; a charge alone never dead-letters a message whose retry window + /// hadn't started. The charge commits before the pump runs any handler. + /// + private bool ChargeInterruptedAttempts() + { + if (_messages.Count == 0) return false; + if (_stoppedCleanly) + { + // Consume the marker, so a later unclean stop of this activation is charged. + Emit(new() { Kind = InboxCommandKind.Resume }); + return true; + } + + var now = Shared.JobTimeProvider.GetUtcNow(); + var charged = 0; + foreach (var entry in _messages.Values.ToArray()) + { + // A message which isn't due couldn't have been in an attempt. + if (GetDelay(entry, now) != TimeSpan.Zero) continue; + var status = StageFailedAttempt(entry, InterruptedAttemptReason, InterruptedAttemptReason, now, interrupted: true); + Shared.Instruments.OnInboxMessageProcessed(Shared.GrainType, status); + charged++; + } + + if (charged > 0) LogInterruptedAttemptsCharged(Logger, GrainContext.GrainId, charged); + return charged > 0; + } + + private const string InterruptedAttemptReason = "The activation stopped without recording the outcome of an attempt."; + + /// + /// Journals that this activation stopped with no handler attempt interrupted, so recovery charges none. + /// + /// + /// The pump calls this after its loop ends. A messaging failure, such as a handler which failed after staging + /// changes, records nothing, so recovery charges the attempt. A failed write leaves the stop unclean, which charges + /// at most one extra attempt. + /// + internal async Task RecordCleanStopAsync() + { + if (!_recovered || _pump.IsFailed || _stoppedCleanly || _messages.Count == 0) return; + try + { + Emit(new() { Kind = InboxCommandKind.CleanStop }); + await _pump.StateManager.WriteStateAsync(CancellationToken.None).ConfigureAwait(true); + } + catch (Exception exception) + { + LogCleanStopFailed(Logger, exception, GrainContext.GrainId); + } + } + + /// Releases the reported depth once the activation's messaging state is discarded. + internal void ReleaseDepth() => ReportDepth(-_messages.Count); + + private void ReportDepth(int delta) => Shared.Instruments.OnInboxDepthChanged(Shared.GrainType, delta); + + /// Persists staged changes while ends only the caller's wait. + private ValueTask PersistForCallerAsync(CancellationToken cancellationToken) + { + var write = PersistAsync(); + if (write.IsCompleted || !cancellationToken.CanBeCanceled) + { + return write; + } + + // The write continues to its outcome after the caller stops waiting; PersistAsync reports failures through the pump. + var task = write.AsTask(); + task.Ignore(); + return new(task.WaitAsync(cancellationToken)); + } + + private async ValueTask PersistAsync() + { + try + { + await _pump.StateManager.WriteStateAsync(CancellationToken.None).ConfigureAwait(true); + } + catch (Exception exception) + { + _pump.Fail(exception, "Inbox persistence failed."); + throw; + } + } + + /// + /// Tracks one handler attempt and stages its terminal outcome. + /// + internal sealed class HandlerExecution(Inbox owner, InboxEntry entry) + { + public bool Active { get; set; } + + /// + /// Gets or sets the journal's staged change count when the attempt started, advanced past changes which + /// interleaved deliveries staged. + /// + public long StagedChanges { get; set; } + + public bool Completed { get; private set; } + + public string? DeadLetterReason { get; private set; } + + public ExceptionDispatchInfo? Rejection { get; private set; } + + public void Complete() + { + ValidateAttempt(); + if (Completed) + { + return; + } + + try + { + owner.StageCompletion(entry.Message, deadLetter: null); + Completed = true; + } + catch (Exception exception) + { + Reject(exception); + } + } + + public void Fail(string reason) + { + ValidateAttempt(); + ArgumentException.ThrowIfNullOrWhiteSpace(reason); + if (Completed) + { + if (string.Equals(DeadLetterReason, reason, StringComparison.Ordinal)) + { + return; + } + + Reject(new InvalidOperationException("The inbox handler has already staged a different terminal outcome.")); + } + + try + { + owner.StageDeadLetter(entry.Message, reason, checked(entry.State.AttemptCount + 1)); + DeadLetterReason = reason; + Completed = true; + } + catch (Exception exception) + { + Reject(exception); + } + } + + public void ThrowIfRejected() => Rejection?.Throw(); + + [DoesNotReturn] + public void Reject(Exception exception) + { + Rejection ??= ExceptionDispatchInfo.Capture(exception); + Rejection.Throw(); + } + + private void ValidateAttempt() + { + ThrowIfRejected(); + if (!Active) + { + Reject(new InvalidOperationException("The inbox handler context belongs to an inactive or different attempt.")); + } + } + } + + [LoggerMessage(Level = LogLevel.Error, EventName = "DeliveryOperationFailed", + Message = "Inbox delivery of message {MessageId} to {GrainId} failed")] + private static partial void LogDeliveryOperationFailed(ILogger logger, Exception exception, MessageId messageId, GrainId grainId); + + [LoggerMessage(Level = LogLevel.Error, Message = "Inbox command staging failed.")] + private static partial void LogStagingFailed(ILogger logger, Exception exception); + + [LoggerMessage(Level = LogLevel.Warning, Message = "Inbox for {ReceiverId} charged an interrupted attempt to {Count} due messages: its previous activation stopped without recording their outcome.")] + private static partial void LogInterruptedAttemptsCharged(ILogger logger, GrainId receiverId, int count); + + [LoggerMessage(Level = LogLevel.Warning, Message = "Inbox for {ReceiverId} couldn't record a clean stop, so its next activation charges one attempt to each due message.")] + private static partial void LogCleanStopFailed(ILogger logger, Exception exception, GrainId receiverId); + + [LoggerMessage(Level = LogLevel.Warning, Message = "Inbox for {ReceiverId} expired message {MessageId}: its send time {SentAt} is at or below the durable admission floor {AdmissionFloor}. Its earlier delivery outcome is unknown.")] + private static partial void LogMessageExpired(ILogger logger, MessageId messageId, GrainId receiverId, DateTimeOffset sentAt, DateTimeOffset admissionFloor); + + [LoggerMessage(Level = LogLevel.Warning, Message = "Inbox for {ReceiverId} rejected message {MessageId}: its send time {SentAt} is later than receiver time {ReceiverTime} plus DeduplicationWindow.")] + private static partial void LogSentAtTooFarAhead(ILogger logger, MessageId messageId, GrainId receiverId, DateTimeOffset sentAt, DateTimeOffset receiverTime); + + [LoggerMessage(Level = LogLevel.Warning, Message = "No inbox handler registered for message {MessageId} to {ReceiverId}")] + private static partial void LogHandlerNotFound(ILogger logger, MessageId messageId, GrainId receiverId); + + [LoggerMessage(Level = LogLevel.Information, Message = "Accepted message {MessageId} to {ReceiverId}")] + private static partial void LogMessageAccepted(ILogger logger, MessageId messageId, GrainId receiverId); + + [LoggerMessage(Level = LogLevel.Error, Message = "Handler threw exception for message {MessageId} to {ReceiverId}")] + private static partial void LogHandlerException(ILogger logger, Exception exception, MessageId messageId, GrainId receiverId); + + [LoggerMessage(Level = LogLevel.Warning, Message = "Inbox message sink dead-lettered message {MessageId} to {ReceiverId} with code {Code}: {Reason}")] + private static partial void LogSinkDeadLettered(ILogger logger, MessageId messageId, GrainId receiverId, string reason, string? code); +} diff --git a/src/Orleans.Messaging/InboxDispatcher.cs b/src/Orleans.Messaging/InboxDispatcher.cs new file mode 100644 index 00000000000..7464fb10baa --- /dev/null +++ b/src/Orleans.Messaging/InboxDispatcher.cs @@ -0,0 +1,37 @@ +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; + +namespace Orleans.Messaging; + +/// Dispatches each inbox message to the first registered handler which can handle it. +internal readonly struct InboxDispatcher +{ + private readonly List _handlers; + + internal InboxDispatcher(List handlers) => _handlers = handlers; + + internal bool IsEmpty => _handlers.Count == 0; + + internal void Add(IInboxHandler handler) + { + ArgumentNullException.ThrowIfNull(handler); + _handlers.Add(handler); + } + + internal ValueTask HandleAsync(IInboxHandlerContext context, CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + foreach (var handler in _handlers) + { + if (handler.CanHandle(context.Message)) + { + return handler.HandleAsync(context, cancellationToken); + } + } + + throw new InvalidOperationException( + $"No handler is registered for message subject '{context.Message.Envelope.Subject ?? ""}'."); + } +} diff --git a/src/Orleans.Messaging/InboxExtensions.cs b/src/Orleans.Messaging/InboxExtensions.cs new file mode 100644 index 00000000000..a06ed484b88 --- /dev/null +++ b/src/Orleans.Messaging/InboxExtensions.cs @@ -0,0 +1,128 @@ +using System; +using System.Threading; +using System.Threading.Tasks; + +namespace Orleans.Messaging; + +/// Registers typed subject handlers with an . +/// Stateful overloads store their argument directly and support static delegates. +public static class InboxExtensions +{ + /// Registers a synchronous subject handler. + /// The body contract. + /// The inbox. + /// The subject and serializer binding. + /// The synchronous body and context delegate. + /// The inbox. + public static IInbox Register(this IInbox inbox, MessageType messageType, Action handler) + { + ArgumentNullException.ThrowIfNull(inbox); + ArgumentNullException.ThrowIfNull(messageType); + ArgumentNullException.ThrowIfNull(handler); + return inbox.Register(new SynchronousHandler(messageType, handler)); + } + + /// Registers a synchronous subject handler with an explicit argument. + /// The body contract. + /// The handler argument type. + /// The inbox. + /// The subject and serializer binding. + /// The argument stored with the handler. + /// The synchronous body, argument, and context delegate. + /// The inbox. + /// Use a static delegate to pass state without a captured closure. + public static IInbox Register( + this IInbox inbox, MessageType messageType, TArg argument, Action handler) + { + ArgumentNullException.ThrowIfNull(inbox); + ArgumentNullException.ThrowIfNull(messageType); + ArgumentNullException.ThrowIfNull(handler); + return inbox.Register(new SynchronousHandler(messageType, argument, handler)); + } + + /// Registers a subject handler supporting asynchronous preparation. + /// The body contract. + /// The inbox. + /// The subject and serializer binding. + /// The body, context, and attempt-token delegate. + /// The inbox. + public static IInbox Register( + this IInbox inbox, MessageType messageType, Func handler) + { + ArgumentNullException.ThrowIfNull(inbox); + ArgumentNullException.ThrowIfNull(messageType); + ArgumentNullException.ThrowIfNull(handler); + return inbox.Register(new AsynchronousHandler(messageType, handler)); + } + + /// Registers an asynchronous subject handler with an explicit argument. + /// The body contract. + /// The handler argument type. + /// The inbox. + /// The subject and serializer binding. + /// The argument stored with the handler. + /// The body, argument, context, and attempt-token delegate. + /// The inbox. + /// Use a static delegate to pass state without a captured closure. + public static IInbox Register( + this IInbox inbox, MessageType messageType, TArg argument, + Func handler) + { + ArgumentNullException.ThrowIfNull(inbox); + ArgumentNullException.ThrowIfNull(messageType); + ArgumentNullException.ThrowIfNull(handler); + return inbox.Register(new AsynchronousHandler(messageType, argument, handler)); + } + + private abstract class Handler(MessageType messageType) : IInboxHandler + { + public bool CanHandle(InboxMessage message) => + string.Equals(message.Envelope.Subject, messageType.Subject, StringComparison.Ordinal); + + public ValueTask HandleAsync(IInboxHandlerContext context, CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + // The delegate owns the decoded body, including when cancellation arrives during decoding. + var body = messageType.Decode(context.Message.Envelope); + return Invoke(body, context, cancellationToken); + } + + protected abstract ValueTask Invoke(T body, IInboxHandlerContext context, CancellationToken cancellationToken); + } + + private sealed class SynchronousHandler( + MessageType messageType, Action handler) : Handler(messageType) + { + protected override ValueTask Invoke(T body, IInboxHandlerContext context, CancellationToken cancellationToken) + { + handler(body, context); + return ValueTask.CompletedTask; + } + } + + private sealed class SynchronousHandler( + MessageType messageType, TArg argument, Action handler) : Handler(messageType) + { + protected override ValueTask Invoke(T body, IInboxHandlerContext context, CancellationToken cancellationToken) + { + handler(body, argument, context); + return ValueTask.CompletedTask; + } + } + + private sealed class AsynchronousHandler( + MessageType messageType, + Func handler) : Handler(messageType) + { + protected override ValueTask Invoke(T body, IInboxHandlerContext context, CancellationToken cancellationToken) => + handler(body, context, cancellationToken); + } + + private sealed class AsynchronousHandler( + MessageType messageType, TArg argument, + Func handler) : Handler(messageType) + { + protected override ValueTask Invoke(T body, IInboxHandlerContext context, CancellationToken cancellationToken) => + handler(body, argument, context, cancellationToken); + } +} diff --git a/src/Orleans.Messaging/InboxHandlerContext.cs b/src/Orleans.Messaging/InboxHandlerContext.cs new file mode 100644 index 00000000000..ed8b74b2857 --- /dev/null +++ b/src/Orleans.Messaging/InboxHandlerContext.cs @@ -0,0 +1,10 @@ +namespace Orleans.Messaging; + +internal sealed class InboxHandlerContext(InboxMessage message, Inbox.HandlerExecution execution) : IInboxHandlerContext +{ + public InboxMessage Message { get; } = message; + + public void Complete() => execution.Complete(); + + public void Fail(string reason) => execution.Fail(reason); +} diff --git a/src/Orleans.Messaging/InboxMessage.cs b/src/Orleans.Messaging/InboxMessage.cs new file mode 100644 index 00000000000..099c6f4e916 --- /dev/null +++ b/src/Orleans.Messaging/InboxMessage.cs @@ -0,0 +1,50 @@ +using System; + +namespace Orleans.Messaging; + +/// A received command whose destination is the grain owning the inbox. +[GenerateSerializer, Alias("Orleans.Messaging.InboxMessage")] +public readonly struct InboxMessage +{ + /// Gets the command identity, payload, and application metadata. + [Id(0)] + public required Envelope Envelope { get; init; } + + /// + /// Gets the time, on the sending grain's Durable Jobs clock, at which this logical send was staged. + /// + /// + /// + /// The outbox journals this value when it stages the send, and every delivery attempt and recovery reuses it. + /// Direct callers of likewise reuse it when retrying a command. + /// + /// + /// The receiving inbox keeps each pending message until it is processed, including a message whose + /// is later. It retains each processed record for DeduplicationWindow after + /// the latest send time it has observed for that message ID. A delivery of a tracked message receives its recorded + /// outcome; when its send time is later than the recorded one, the inbox durably extends the record to that time + /// before answering, and an earlier or equal send time leaves the record unchanged. Before removing records, the inbox + /// journals an admission floor, which only advances. It answers a message it doesn't track with + /// , staging nothing for it, when the message's send time is at or below that + /// floor. + /// + /// + /// The floor survives recovery and migration, so a delayed retry of a removed message stays expired even after a + /// clock moves backward. Clock jumps and skew change when messages expire and how long records are retained. + /// Once a record is removed, sending its command again with a later send time is admitted as a new delivery, so + /// applications reconcile an uncertain outcome before doing so. + /// + /// + /// The inbox never accepts a delivered send time later than its own clock plus DeduplicationWindow. It rejects + /// such a delivery with an , staging nothing, when the message is untracked + /// and above the admission floor or the send time would extend its record, so the sender retries it within its retry + /// budget. A tracked message whose send time is equal to or earlier than its record's still receives its recorded + /// outcome, and an untracked message at or below the floor still receives . + /// A message a grain sends to itself uses its clock, or the first time above the admission floor after a backward + /// clock jump. Every processed record therefore becomes eligible for removal within twice the window of the + /// receiver's clock, or just after the floor when a self-send follows a backward jump. + /// + /// + [Id(1)] + public required DateTimeOffset SentAt { get; init; } +} diff --git a/src/Orleans.Messaging/MessageId.cs b/src/Orleans.Messaging/MessageId.cs new file mode 100644 index 00000000000..f6082990280 --- /dev/null +++ b/src/Orleans.Messaging/MessageId.cs @@ -0,0 +1,122 @@ +using System; +using System.Diagnostics.CodeAnalysis; +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace Orleans.Messaging; + +/// +/// Identifies one message by an application-supplied . +/// +/// +/// The receiver deduplicates messages by this identity, so applications reuse it when retrying a send. +/// Equality, hashing, parsing, and formatting use the key's canonical escaped path. +/// +[Immutable, Alias("Orleans.Messaging.MessageId")] +[JsonConverter(typeof(MessageIdJsonConverter))] +public readonly struct MessageId : ISpanFormattable, IEquatable, IParsable, ISpanParsable +{ + /// Creates a message identity from a hierarchical key. + /// The application identity of the message. + public MessageId(HierarchicalKey key) => Key = key; + + /// Gets the hierarchical key which identifies the message. + public HierarchicalKey Key { get; } + + /// Gets whether this value is unset. + public bool IsDefault => Key.IsDefault; + + /// Creates a message identity from literal segments in root-first order. + /// The nonempty literal segments. + /// The message identity. + /// A segment is null. + /// The input or a segment is empty, or a segment contains an unpaired surrogate. + public static MessageId Create(params ReadOnlySpan segments) => new(HierarchicalKey.Create(segments)); + + /// + public static MessageId Parse(string s, IFormatProvider? provider = null) => new(HierarchicalKey.Parse(s, provider)); + + /// + public static MessageId Parse(ReadOnlySpan s, IFormatProvider? provider = null) => new(HierarchicalKey.Parse(s, provider)); + + /// + public static bool TryParse([NotNullWhen(true)] string? s, IFormatProvider? provider, out MessageId result) + { + var parsed = HierarchicalKey.TryParse(s, provider, out var key); + result = new(key); + return parsed; + } + + /// + public static bool TryParse(ReadOnlySpan s, IFormatProvider? provider, out MessageId result) + { + var parsed = HierarchicalKey.TryParse(s, provider, out var key); + result = new(key); + return parsed; + } + + /// + public bool Equals(MessageId other) => Key.Equals(other.Key); + + /// + public override bool Equals(object? obj) => obj is MessageId other && Equals(other); + + /// + public override int GetHashCode() => Key.GetHashCode(); + + /// Compares message identities. + /// The first identity. + /// The second identity. + /// Whether the identities are equal. + public static bool operator ==(MessageId left, MessageId right) => left.Equals(right); + + /// Compares message identities for inequality. + /// The first identity. + /// The second identity. + /// Whether the identities differ. + public static bool operator !=(MessageId left, MessageId right) => !left.Equals(right); + + /// + public override string ToString() => Key.ToString(); + + /// + public string ToString(string? format, IFormatProvider? formatProvider) => Key.ToString(); + + /// + public bool TryFormat(Span destination, out int charsWritten, ReadOnlySpan format, IFormatProvider? provider) => + Key.TryFormat(destination, out charsWritten, format, provider); + + /// Serializes message identities as canonical escaped JSON strings. + public sealed class MessageIdJsonConverter : JsonConverter + { + /// Creates a message identity JSON converter. + public MessageIdJsonConverter() { } + + /// + public override MessageId Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + if (reader.TokenType == JsonTokenType.Null) return default; + if (reader.TokenType is JsonTokenType.String or JsonTokenType.PropertyName + && TryParse(reader.GetString(), null, out var result)) return result; + throw new JsonException("A message identity must be a canonical escaped path."); + } + + /// + public override void Write(Utf8JsonWriter writer, MessageId value, JsonSerializerOptions options) + { + if (value.IsDefault) writer.WriteNullValue(); + else writer.WriteStringValue(value.ToString()); + } + + /// + public override MessageId ReadAsPropertyName(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) => + Read(ref reader, typeToConvert, options); + + /// + public override void WriteAsPropertyName(Utf8JsonWriter writer, MessageId value, JsonSerializerOptions options) + { + if (value.IsDefault) throw new JsonException("A message identity dictionary key must be set."); + writer.WritePropertyName(value.ToString()); + } + } +} diff --git a/src/Orleans.Messaging/MessageIdCodec.cs b/src/Orleans.Messaging/MessageIdCodec.cs new file mode 100644 index 00000000000..1a04a7b6704 --- /dev/null +++ b/src/Orleans.Messaging/MessageIdCodec.cs @@ -0,0 +1,48 @@ +using System; +using System.Buffers; +using Orleans.Serialization; +using Orleans.Serialization.Buffers; +using Orleans.Serialization.Cloning; +using Orleans.Serialization.Codecs; +using Orleans.Serialization.WireProtocol; + +namespace Orleans.Messaging; + +[RegisterSerializer] +internal sealed class MessageIdCodec(IFieldCodec strings) : IFieldCodec +{ + public void WriteField(ref Writer writer, uint fieldIdDelta, + Type? expectedType, MessageId value) where TBufferWriter : IBufferWriter + { + ReferenceCodec.MarkValueField(writer.Session); + writer.WriteFieldHeader(fieldIdDelta, expectedType, typeof(MessageId), WireType.TagDelimited); + strings.WriteField(ref writer, 0, typeof(string), value.IsDefault ? null! : value.ToString()); + writer.WriteEndObject(); + } + + public MessageId ReadValue(ref Reader reader, Field field) + { + field.EnsureWireTypeTagDelimited(); + ReferenceCodec.MarkValueField(reader.Session); + MessageId result = default; + uint fieldId = 0; + while (true) + { + var header = reader.ReadFieldHeader(); + if (header.IsEndBaseOrEndObject) return result; + fieldId += header.FieldIdDelta; + if (fieldId == 0) + { + var canonical = strings.ReadValue(ref reader, header); + result = canonical is null ? default : MessageId.Parse(canonical); + } + else reader.ConsumeUnknownField(header); + } + } +} + +[RegisterCopier] +internal sealed class MessageIdCopier : IDeepCopier +{ + public MessageId DeepCopy(MessageId input, CopyContext context) => input; +} diff --git a/src/Orleans.Messaging/MessageState.cs b/src/Orleans.Messaging/MessageState.cs new file mode 100644 index 00000000000..b7c30b89d60 --- /dev/null +++ b/src/Orleans.Messaging/MessageState.cs @@ -0,0 +1,79 @@ +using System; + +namespace Orleans.Messaging; + +[GenerateSerializer, Alias("Orleans.Messaging.InboxMessageState")] +internal sealed class InboxMessageState +{ + [Id(0)] + public int AttemptCount { get; set; } + + /// Gets or sets the time, on the Durable Jobs clock, from which the inbox attempts the message again. + [Id(1)] + public DateTimeOffset? NextAttemptAt { get; set; } + + [Id(2)] + public string? LastError { get; set; } + + /// + /// Gets or sets when a failing message's retries end on the Durable Jobs clock, InboxRetryTimeout after its + /// first failure. + /// + [Id(3)] + public DateTimeOffset? ExpirationTime { get; set; } +} + +[GenerateSerializer, Alias("Orleans.Messaging.OutboxMessageState")] +internal sealed class OutboxMessageState +{ + [Id(0)] + public int AttemptCount { get; set; } + + [Id(1)] + public DateTimeOffset? NextAttemptAt { get; set; } + + [Id(2)] + public string? LastError { get; set; } + + /// + /// Gets or sets the logical send's staging time on the Durable Jobs clock. Every attempt preserves this value; + /// retries end OutboxRetryTimeout after it. A recovered entry requires this journaled value. + /// + [Id(3)] + public DateTimeOffset? SentAt { get; set; } +} + +[GenerateSerializer, Alias("Orleans.Messaging.InboxDeadLetter")] +internal sealed class InboxDeadLetter +{ + [Id(0)] + public required InboxMessage Message { get; init; } + + [Id(1)] + public required DateTimeOffset DeadLetteredAt { get; init; } + + [Id(2)] + public required string Reason { get; init; } + + [Id(3)] + public int AttemptCount { get; init; } + + [Id(4)] + public string? Code { get; init; } +} + +[GenerateSerializer, Alias("Orleans.Messaging.OutboxDeadLetter")] +internal sealed class OutboxDeadLetter +{ + [Id(0)] + public required OutboxMessage Message { get; init; } + + [Id(1)] + public required DateTimeOffset DeadLetteredAt { get; init; } + + [Id(2)] + public required string Reason { get; init; } + + [Id(3)] + public int AttemptCount { get; init; } +} diff --git a/src/Orleans.Messaging/MessageType.cs b/src/Orleans.Messaging/MessageType.cs new file mode 100644 index 00000000000..c8fb9914671 --- /dev/null +++ b/src/Orleans.Messaging/MessageType.cs @@ -0,0 +1,100 @@ +using System; +using Orleans.Runtime; +using Orleans.Serialization; + +namespace Orleans.Messaging; + +/// +/// Binds an exact message subject to an Orleans payload serializer. +/// +/// The subject's payload contract. +/// +/// Bindings are immutable and can be shared across activations. Decode during handler preparation; +/// decoded resource-bearing values retain their ordinary application-managed ownership. +/// +public sealed class MessageType +{ + private readonly Serializer _serializer; + + /// Creates a subject and payload-type binding. + /// The ordinal protocol subject. + /// The payload serializer. + public MessageType(string subject, Serializer serializer) + { + EnvelopeValidation.ValidateSubject(subject); + ArgumentNullException.ThrowIfNull(serializer); + Subject = subject; + _serializer = serializer; + } + + /// Creates a subject and payload-type binding using the payload serializer from . + /// The ordinal protocol subject. + /// The serializer which provides the payload serializer. + /// + /// Use this overload to define a protocol's bindings together in one application class, constructed from the + /// service. + /// + public MessageType(string subject, Serializer serializer) + : this(subject, (serializer ?? throw new ArgumentNullException(nameof(serializer))).GetSerializer()) + { + } + + /// Gets the exact protocol subject. + public string Subject { get; } + + /// Decodes an envelope using this subject's payload contract. + /// The envelope containing the serialized payload. + /// The nonnull decoded payload. + /// The subject is absent or different, or the payload is a serialized null value. + public T Decode(Envelope envelope) + { + if (envelope.Subject is null) + { + throw new ArgumentException( + $"Message subject '{Subject}' is required.", nameof(envelope)); + } + if (!string.Equals(Subject, envelope.Subject, StringComparison.Ordinal)) + { + throw new ArgumentException( + $"Expected message subject '{Subject}', but received '{envelope.Subject}'.", + nameof(envelope)); + } + + return _serializer.Deserialize(envelope.Payload) + ?? throw new ArgumentException($"Message subject '{Subject}' requires a nonnull payload.", nameof(envelope)); + } + + /// Encodes a typed body into an outgoing message. + /// The stable application command identity. + /// The destination inbox. + /// The nonnull body to serialize. + /// + /// The optional earliest time, on the Durable Jobs clock, at which the receiving inbox processes the message. + /// See . + /// + /// An outgoing message with the encoded payload. + /// + /// Serialization completes before publication. Keep the published payload bytes unchanged. + /// + public OutboxMessage Create(MessageId messageId, GrainId receiverId, T body, DateTimeOffset? deliverAt = null) + { + if (body is null) throw new ArgumentNullException(nameof(body)); + if (receiverId.IsDefault) + { + throw new ArgumentException("The receiving grain ID must be set.", nameof(receiverId)); + } + EnvelopeValidation.ValidateMessageId(messageId); + var payload = _serializer.SerializeToArray(body); + return new OutboxMessage + { + Envelope = new Envelope + { + MessageId = messageId, + Subject = Subject, + Payload = payload, + DeliverAt = deliverAt + }, + ReceiverId = receiverId + }; + } +} diff --git a/src/Orleans.Messaging/MessagingActivationValidator.cs b/src/Orleans.Messaging/MessagingActivationValidator.cs new file mode 100644 index 00000000000..5a8279b4d9a --- /dev/null +++ b/src/Orleans.Messaging/MessagingActivationValidator.cs @@ -0,0 +1,20 @@ +using System; +using Orleans.Concurrency; +using Orleans.Runtime; + +namespace Orleans.Messaging; + +internal static class MessagingActivationValidator +{ + // The public attribute exposes the runtime's internal, sealed strategy type. + private static readonly Type _statelessWorkerPlacementType = new StatelessWorkerAttribute().PlacementStrategy.GetType(); + + public static void Validate(Type grainClass, PlacementStrategy placementStrategy) + { + if (placementStrategy.GetType() == _statelessWorkerPlacementType) + { + throw new InvalidOperationException( + $"Messaging requires one activation per grain identity, but grain type '{grainClass}' is a stateless worker."); + } + } +} diff --git a/src/Orleans.Messaging/MessagingGrainTypeConfigurator.cs b/src/Orleans.Messaging/MessagingGrainTypeConfigurator.cs new file mode 100644 index 00000000000..b67b53ff0f9 --- /dev/null +++ b/src/Orleans.Messaging/MessagingGrainTypeConfigurator.cs @@ -0,0 +1,28 @@ +using System; +using Microsoft.Extensions.DependencyInjection; +using Orleans.Journaling; +using Orleans.Metadata; +using Orleans.Runtime; + +namespace Orleans.Messaging; + +internal sealed class MessagingGrainTypeConfigurator(GrainClassMap grainClasses, IServiceProvider services) : IConfigureGrainTypeComponents +{ + public void Configure(GrainType grainType, GrainProperties properties, GrainTypeSharedContext shared) + { + if (grainClasses.TryGetGrainClass(grainType, out var grainClass) + && (typeof(DurableGrain).IsAssignableFrom(grainClass) || typeof(IMessagingGrain).IsAssignableFrom(grainClass))) + { + var placementStrategy = shared.PlacementStrategy; + shared.SetComponent(new MessagingShared(services, grainType)); + shared.AddActivationSetup(context => + { + MessagingActivationValidator.Validate(grainClass, placementStrategy); + var activationServices = context.ActivationServices; + var inbox = activationServices.GetRequiredService(); + _ = activationServices.GetRequiredService(); + context.GetComponent()!.GetOrSetExtension(() => inbox); + }); + } + } +} diff --git a/src/Orleans.Messaging/MessagingInstruments.cs b/src/Orleans.Messaging/MessagingInstruments.cs new file mode 100644 index 00000000000..e6aba94026a --- /dev/null +++ b/src/Orleans.Messaging/MessagingInstruments.cs @@ -0,0 +1,84 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics.Metrics; +using Orleans.Runtime; + +namespace Orleans.Messaging; + +internal sealed class MessagingInstruments(OrleansInstruments instruments) +{ + private const string MillisecondsUnit = "ms"; + private const string GrainTypeTagName = "grain_type"; + private const string StatusTagName = "status"; + + private readonly Counter _inboxMessagesReceived = instruments.Meter.CreateCounter("orleans-messaging-inbox-messages-received"); + private readonly Counter _inboxMessagesProcessed = instruments.Meter.CreateCounter("orleans-messaging-inbox-messages-processed"); + private readonly Counter _outboxMessagesSent = instruments.Meter.CreateCounter("orleans-messaging-outbox-messages-sent"); + private readonly Counter _outboxMessagesDelivered = instruments.Meter.CreateCounter("orleans-messaging-outbox-messages-delivered"); + private readonly Histogram _inboxProcessingDuration = instruments.Meter.CreateHistogram("orleans-messaging-inbox-processing-duration", MillisecondsUnit); + private readonly Histogram _outboxDeliveryDuration = instruments.Meter.CreateHistogram("orleans-messaging-outbox-delivery-duration", MillisecondsUnit); + private readonly UpDownCounter _inboxDepth = instruments.Meter.CreateUpDownCounter("orleans-messaging-inbox-depth"); + private readonly UpDownCounter _outboxDepth = instruments.Meter.CreateUpDownCounter("orleans-messaging-outbox-depth"); + + internal static MessagingInstruments CreateForDirectConstruction() => new(new OrleansInstruments(new DirectMeterFactory())); + + internal void OnInboxDepthChanged(string grainType, int delta) => AddDepth(_inboxDepth, grainType, delta); + + internal void OnOutboxDepthChanged(string grainType, int delta) => AddDepth(_outboxDepth, grainType, delta); + + private static void AddDepth(UpDownCounter counter, string grainType, int delta) + { + if (delta != 0 && counter.Enabled) + { + counter.Add(delta, new KeyValuePair(GrainTypeTagName, grainType)); + } + } + + internal void OnInboxMessageReceived(string grainType, string status) => + Add(_inboxMessagesReceived, grainType, status); + + internal void OnInboxMessageProcessed(string grainType, string status) => + Add(_inboxMessagesProcessed, grainType, status); + + internal void OnInboxProcessingDuration(TimeSpan duration, string grainType) => + Record(_inboxProcessingDuration, duration, grainType); + + internal void OnOutboxMessageSent(string grainType) + { + if (_outboxMessagesSent.Enabled) + { + _outboxMessagesSent.Add(1, new KeyValuePair(GrainTypeTagName, grainType)); + } + } + + internal void OnOutboxMessageDelivered(string grainType, string status) => + Add(_outboxMessagesDelivered, grainType, status); + + internal void OnOutboxDeliveryDuration(TimeSpan duration, string grainType) => + Record(_outboxDeliveryDuration, duration, grainType); + + private static void Add(Counter counter, string grainType, string status) + { + if (counter.Enabled) + { + counter.Add(1, new(GrainTypeTagName, grainType), new(StatusTagName, status)); + } + } + + private static void Record(Histogram histogram, TimeSpan duration, string grainType) + { + if (histogram.Enabled) + { + histogram.Record(Math.Max(0, duration.TotalMilliseconds), new KeyValuePair(GrainTypeTagName, grainType)); + } + } + + private sealed class DirectMeterFactory : IMeterFactory + { + public Meter Create(MeterOptions options) => new(options); + + public void Dispose() + { + } + } +} diff --git a/src/Orleans.Messaging/MessagingPump.cs b/src/Orleans.Messaging/MessagingPump.cs new file mode 100644 index 00000000000..402be588775 --- /dev/null +++ b/src/Orleans.Messaging/MessagingPump.cs @@ -0,0 +1,388 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Orleans.DurableJobs; +using Orleans.Journaling; +using Orleans.Runtime; + +namespace Orleans.Messaging; + +/// +/// Drives inbox processing and outbox delivery for one activation. +/// +/// +/// A single loop runs on the activation scheduler and services both queues whenever it is signaled. +/// One durable job keeps waking the grain while either queue or a registered holds work +/// which the activation can perform; inbox messages wait for a handler without a wake-up. +/// Work staged with an earlier due time than the current wake-up schedules an earlier one, and the job which a newer, +/// earlier wake-up supersedes completes when it runs. +/// Wake-ups may be early, late, or duplicated: every pass derives the work to perform from current state. +/// The wake-up is crash insurance for work which the live loop performs as soon as it is signaled, so work which is +/// already due needs a wake-up within , and a write schedules a job only when no tracked or +/// scheduling job is due by then. A running job re-arms itself once per run, so a busy grain performs no job operation +/// per write. The job retries indefinitely when an activation fails (see ), and every +/// activation re-arms it from recovered state, so committed work always keeps a wake-up. +/// Wake-up bookkeeping runs on the activation scheduler: journal writes prepare on it and serialize with job runs. +/// The loop keeps the activation alive while a pass runs over queued work and while outbox deliveries are in flight. +/// Work which only becomes due later relies on the durable job, so the activation can be collected while it waits. +/// +internal sealed partial class MessagingPump : IDurableJobFeatureHandler, ILifecycleObserver +{ + internal const string JobName = "orleans.messaging.pump"; + internal static readonly TimeSpan RecheckDelay = TimeSpan.FromSeconds(1); + private readonly SingleWaiterAutoResetEvent _signal = new() { RunContinuationsAsynchronously = true }; + private readonly CancellationTokenSource _shutdown = new(); + private Inbox? _inbox; + private Outbox? _outbox; + private Task? _loop; + private TaskCompletionSource? _nextPass; + private Task? _scheduling; + private DateTimeOffset _schedulingDueTime; + private bool _keepingAlive; + + // The job which wakes the grain next, if any, and its due time. A job run which completes clears it. + private string? _wakeupJobId; + private DateTimeOffset _wakeupDueTime; + + // Advances whenever a job run completes. A schedule requested in an earlier epoch may belong to a job which has + // already run and completed, so it no longer guarantees a wake-up. + private int _wakeupEpoch; + private int _schedulingEpoch; + + public MessagingPump(IGrainContext grainContext, IJournaledStateManager stateManager, IDurableJobHandlerRegistry jobHandlers) + { + GrainContext = grainContext; + StateManager = stateManager; + Shared = grainContext.GetComponent() + ?? throw new InvalidOperationException($"Messaging is not configured for grain type '{grainContext.GrainId.Type}'."); + jobHandlers.Register(this); + grainContext.ObservableLifecycle.Subscribe(nameof(MessagingPump), GrainLifecycleStage.Activate, this); + + // Wake-ups for due work fire up to RecheckDelay later, so the loop starts once activation completes and performs + // recovered and newly staged work without waiting for one. + grainContext.ObservableLifecycle.Subscribe(nameof(MessagingPump), GrainLifecycleStage.Last, _ => + { + EnsureStarted(); + return Task.CompletedTask; + }, static _ => Task.CompletedTask); + + // Recovery reports depth during SetupState, so releasing it there covers activations which fail before Activate. + grainContext.ObservableLifecycle.Subscribe(nameof(MessagingPump), GrainLifecycleStage.SetupState, static _ => Task.CompletedTask, _ => + { + _inbox?.ReleaseDepth(); + _outbox?.ReleaseDepth(); + return Task.CompletedTask; + }); + } + + public IGrainContext GrainContext { get; } + + public IJournaledStateManager StateManager { get; } + + /// + /// Gets the state manager's count of staged changes, or a negative value when a replacement manager doesn't track them. + /// + public long StagedChangeCount => StateManager is JournaledStateManager manager ? manager.StagedChangeCount : -1; + + public MessagingShared Shared { get; } + + /// Gets the token which is canceled when messaging stops for this activation. + public CancellationToken ShutdownToken => _shutdown.Token; + + internal void Register(Inbox inbox) => _inbox = inbox; + + internal void Register(Outbox outbox) => _outbox = outbox; + + internal Inbox Inbox => _inbox ?? throw new InvalidOperationException("The messaging inbox has not been registered."); + + private Outbox Outbox => _outbox ?? throw new InvalidOperationException("The messaging outbox has not been registered."); + + public bool CanHandle(string jobName) => string.Equals(jobName, JobName, StringComparison.Ordinal); + + /// Requests a processing pass. + public void Signal() + { + if (!_shutdown.IsCancellationRequested) _signal.Signal(); + } + + /// Requests a processing pass and ensures a durable wake-up for work which has become processable. + public void RequestWakeup() + { + if (_shutdown.IsCancellationRequested) return; + _signal.Signal(); + RepairWakeupAsync().Ignore(); + } + + /// Starts the loop once activation completes, when the activation services grain calls. + public void EnsureStarted() + { + if (_loop is null && !_shutdown.IsCancellationRequested) + { + _loop = RunAsync(); + } + } + + /// + /// Ensures a durable job will wake this grain no later than , or within + /// for work which is due sooner, on the Durable Jobs clock. + /// + public async ValueTask EnsureWakeupAsync(DateTimeOffset dueTime, CancellationToken cancellationToken) + { + // The live loop performs work as soon as it is due, so any tracked or scheduling wake-up within RecheckDelay + // covers it should the activation fail. + var earliest = MessagingTime.AddClamped(Shared.JobTimeProvider.GetUtcNow(), RecheckDelay); + if (dueTime < earliest) dueTime = earliest; + if (_wakeupJobId is not null && _wakeupDueTime <= dueTime) return; + if (_scheduling is null || _schedulingEpoch != _wakeupEpoch || _schedulingDueTime > dueTime) + { + _schedulingEpoch = _wakeupEpoch; + _schedulingDueTime = dueTime; + _scheduling = Shared.JobManager.ScheduleJobAsync(new ScheduleJobRequest + { + Target = GrainContext.GrainId, + JobName = JobName, + DueTime = _schedulingDueTime + }, _shutdown.Token); + } + + var scheduling = _scheduling; + var epoch = _schedulingEpoch; + try + { + var job = await scheduling.WaitAsync(cancellationToken).ConfigureAwait(true); + if (epoch == _wakeupEpoch && (_wakeupJobId is null || job.DueTime < _wakeupDueTime)) + { + _wakeupJobId = job.Id; + _wakeupDueTime = job.DueTime; + } + } + finally + { + if (scheduling.IsCompleted && ReferenceEquals(_scheduling, scheduling)) _scheduling = null; + } + } + + public async ValueTask ExecuteJobAsync(IJobRunContext context, CancellationToken cancellationToken) + { + // A running job keeps rescheduling itself until both queues and every work source are empty. + var jobId = context.Job.Id; + if (_wakeupJobId is null) + { + _wakeupJobId = jobId; + _wakeupDueTime = context.Job.DueTime; + } + EnsureStarted(); + var pass = _nextPass ??= new(TaskCreationOptions.RunContinuationsAsynchronously); + _signal.Signal(); + try + { + await pass.Task.WaitAsync(cancellationToken).ConfigureAwait(true); + // A failed pass can leave only staged removals; its job must survive the activation's failure. + _shutdown.Token.ThrowIfCancellationRequested(); + } + catch + { + // Durable Jobs retries the failed run (see GetRetryTime), which wakes a recovered activation. + CompleteWakeup(jobId); + throw; + } + + var now = Shared.JobTimeProvider.GetUtcNow(); + if (GetNextWakeupDelay(now) is not { } delay) + { + CompleteWakeup(jobId); + return DurableJobRunResult.Completed; + } + + var next = MessagingTime.AddClamped(now, delay > TimeSpan.Zero ? delay : RecheckDelay); + if (_wakeupJobId is not null && _wakeupJobId != jobId && _wakeupDueTime <= next) + { + // Another job wakes the grain first, and its run reschedules for any remaining work. + CompleteWakeup(jobId); + return DurableJobRunResult.Completed; + } + + _wakeupJobId = jobId; + _wakeupDueTime = next; + return DurableJobRunResult.RescheduleAt(next); + } + + private void CompleteWakeup(string jobId) + { + if (_wakeupJobId == jobId) _wakeupJobId = null; + _wakeupEpoch++; + } + + // The delay until the inbox, the outbox, or a work source next holds work, or null when none does. + private TimeSpan? GetNextWakeupDelay(DateTimeOffset now) => + (Inbox.GetNextWakeupDelay(now), Outbox.GetNextWakeupDelay(now)) switch + { + ({ } inbox, { } outbox) => inbox < outbox ? inbox : outbox, + (var inbox, var outbox) => inbox ?? outbox + }; + + private async Task RepairWakeupAsync() + { + try + { + await EnsureWakeupAsync(Shared.JobTimeProvider.GetUtcNow(), CancellationToken.None).ConfigureAwait(true); + } + catch (Exception exception) when (!_shutdown.IsCancellationRequested) + { + // Recovery re-establishes the wake-up for remaining work. + Fail(exception, "Scheduling the messaging wake-up failed."); + } + } + + private async Task RunAsync() + { + // Callers start the loop from grain calls and job executions; queue the first pass to the activation + // scheduler so it never runs on, and nests within, the caller's stack. + await Task.Yield(); + var cancellationToken = _shutdown.Token; + while (!cancellationToken.IsCancellationRequested) + { + SetKeepAlive(Inbox.Count > 0 || Outbox.Count > 0); + var pass = _nextPass; + _nextPass = null; + var progress = false; + try + { + progress = await Inbox.ProcessReadyAsync(cancellationToken).ConfigureAwait(true); + progress |= await Outbox.AdvanceAsync(cancellationToken).ConfigureAwait(true); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + } + catch (Exception exception) + { + // Expected failures are handled within the pass, so an escaping exception leaves state inconsistent with the journal. + progress = false; + LogPassFailed(Shared.PumpLogger, exception, GrainContext.GrainId); + Fail(exception, "Messaging processing pass failed."); + } + + pass?.TrySetResult(); + if (!progress) + { + SetKeepAlive(Outbox.IsDelivering); + await _signal.WaitAsync().ConfigureAwait(true); + } + } + + SetKeepAlive(false); + _nextPass?.TrySetCanceled(cancellationToken); + } + + private void SetKeepAlive(bool keepAlive) + { + if (keepAlive == _keepingAlive || GrainContext is not IActivationKeepAlive activation) return; + _keepingAlive = keepAlive; + if (keepAlive) + { + activation.AcquireKeepAlive(); + } + else + { + activation.ReleaseKeepAlive(); + } + } + + public async Task OnStart(CancellationToken cancellationToken) + { + await Outbox.StartAsync(cancellationToken).ConfigureAwait(true); + await Inbox.StartAsync(cancellationToken).ConfigureAwait(true); + + // Recovered work keeps a wake-up at its real due time, even if the job which woke this activation completes. + var now = Shared.JobTimeProvider.GetUtcNow(); + if (GetNextWakeupDelay(now) is { } delay) + { + await EnsureWakeupAsync(MessagingTime.AddClamped(now, delay), cancellationToken).ConfigureAwait(true); + } + } + + /// + /// Returns when Durable Jobs retries a pump job whose run failed, which it does indefinitely with backoff on the + /// Durable Jobs clock. + /// + /// + /// The job is the grain's only durable wake-up for committed work, so giving up after a bounded number of attempts + /// would strand that work after a long outage until an unrelated call reactivated the grain. + /// + internal static DateTimeOffset GetRetryTime(IJobRunContext context, DateTimeOffset now, TimeSpan initialInterval) => + MessagingTime.AddClamped(now, MessagingTime.GetBackoff(initialInterval, context.DequeueCount)); + + public async Task OnStop(CancellationToken cancellationToken) + { + var scheduling = _scheduling; + Stop(); + + // Grain calls finish before deactivation stops the lifecycle, so the loop and outbox deliveries own every remaining write. + if (_loop is { } loop) + { + await loop.ConfigureAwait(ConfigureAwaitOptions.ContinueOnCapturedContext | ConfigureAwaitOptions.SuppressThrowing); + } + + if (scheduling is not null) + { + await ((Task)scheduling).ConfigureAwait(ConfigureAwaitOptions.ContinueOnCapturedContext | ConfigureAwaitOptions.SuppressThrowing); + } + + await Outbox.StopAsync().ConfigureAwait(true); + + // The loop and the outbox's deliveries have ended, so no attempt is in progress unless messaging failed. + await Inbox.RecordCleanStopAsync().ConfigureAwait(true); + } + + /// Stops processing for this activation. + internal void Stop() + { + if (_shutdown.IsCancellationRequested) return; + try + { + _shutdown.Cancel(); + } + catch (AggregateException exception) + { + LogCancellationFailed(Shared.PumpLogger, exception); + } + finally + { + _signal.Signal(); + } + } + + /// Gets whether messaging failed, so the activation's uncommitted changes must not commit. + internal bool IsFailed { get; private set; } + + /// Stops messaging and deactivates the grain after an unrecoverable failure. + /// + /// In-memory state may no longer match what the activation may commit, so every later journal write fails and the + /// next activation recovers from the journal. + /// + internal void Fail(Exception exception, string description) + { + IsFailed = true; + if (_shutdown.IsCancellationRequested) return; + Stop(); + try + { + GrainContext.Deactivate(new DeactivationReason(DeactivationReasonCode.ApplicationError, exception, description)); + } + catch (Exception deactivationException) + { + LogDeactivationFailed(Shared.PumpLogger, deactivationException); + } + } + + + [LoggerMessage(Level = LogLevel.Error, Message = "Messaging processing pass failed for grain {GrainId}")] + private static partial void LogPassFailed(ILogger logger, Exception exception, GrainId grainId); + + [LoggerMessage(Level = LogLevel.Error, Message = "Requesting deactivation after a messaging failure failed.")] + private static partial void LogDeactivationFailed(ILogger logger, Exception exception); + + [LoggerMessage(Level = LogLevel.Error, Message = "A messaging cancellation callback failed during shutdown.")] + private static partial void LogCancellationFailed(ILogger logger, Exception exception); +} diff --git a/src/Orleans.Messaging/MessagingShared.cs b/src/Orleans.Messaging/MessagingShared.cs new file mode 100644 index 00000000000..23fb8626ff4 --- /dev/null +++ b/src/Orleans.Messaging/MessagingShared.cs @@ -0,0 +1,64 @@ +using System; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Orleans.DurableJobs; +using Orleans.Journaling; +using Orleans.Messaging.Configuration; +using Orleans.Runtime; + +namespace Orleans.Messaging; + +/// +/// Services and settings shared by every messaging activation of one grain type. +/// +/// +/// stores one instance as a grain-type component, so activations +/// resolve it as an component instead of copying it per activation. +/// +internal sealed class MessagingShared +{ + public MessagingShared(IServiceProvider services, GrainType grainType) + { + GrainType = grainType.ToString(); + Options = services.GetRequiredService>().Value; + TimeProvider = services.GetRequiredService(); + JobTimeProvider = services.GetRequiredKeyedService(DurableJobTimeProviderNames.DurableJobs); + JobManager = services.GetRequiredService(); + Transport = services.GetRequiredService(); + Instruments = services.GetRequiredService(); + var loggers = services.GetRequiredService(); + InboxLogger = loggers.CreateLogger(); + OutboxLogger = loggers.CreateLogger(); + PumpLogger = loggers.CreateLogger(); + var format = services.GetRequiredService>().Value.JournalFormatKey; + InboxCodec = services.GetRequiredKeyedService>(format); + OutboxCodec = services.GetRequiredKeyedService>(format); + } + + /// Gets the grain type metric tag. + public string GrainType { get; } + + public DurableMessagingOptions Options { get; } + + public TimeProvider TimeProvider { get; } + + /// Gets the durable jobs clock, which defines the time domain of wake-up due times. + public TimeProvider JobTimeProvider { get; } + + public ILocalDurableJobManager JobManager { get; } + + public IOutboxTransport Transport { get; } + + public MessagingInstruments Instruments { get; } + + public ILogger InboxLogger { get; } + + public ILogger OutboxLogger { get; } + + public ILogger PumpLogger { get; } + + public IDurableValueCommandCodec InboxCodec { get; } + + public IDurableValueCommandCodec OutboxCodec { get; } +} diff --git a/src/Orleans.Messaging/MessagingStateNames.cs b/src/Orleans.Messaging/MessagingStateNames.cs new file mode 100644 index 00000000000..d9cd0bb37c9 --- /dev/null +++ b/src/Orleans.Messaging/MessagingStateNames.cs @@ -0,0 +1,9 @@ +namespace Orleans.Messaging; + +internal static class MessagingStateNames +{ + private const string Prefix = "orleans.messaging."; + + public const string Inbox = Prefix + "inbox"; + public const string Outbox = Prefix + "outbox"; +} diff --git a/src/Orleans.Messaging/MessagingTime.cs b/src/Orleans.Messaging/MessagingTime.cs new file mode 100644 index 00000000000..2ae576d1936 --- /dev/null +++ b/src/Orleans.Messaging/MessagingTime.cs @@ -0,0 +1,28 @@ +using System; + +namespace Orleans.Messaging; + +internal static class MessagingTime +{ + public static bool IsExpired(DateTimeOffset now, DateTimeOffset timestamp, TimeSpan retention) + { + var nowTicks = now.UtcTicks; + var timestampTicks = timestamp.UtcTicks; + return nowTicks >= timestampTicks + && nowTicks - timestampTicks >= retention.Ticks; + } + + /// Returns the delay before retry , doubling from the initial interval to a cap. + /// The delay before the first retry. + /// The one-based number of the failed attempt. + public static TimeSpan GetBackoff(TimeSpan initialInterval, int attempt) => + TimeSpan.FromTicks(initialInterval.Ticks + * (1L << Math.Clamp(attempt - 1, 0, Configuration.DurableMessagingOptions.MaximumBackoffExponent))); + + public static DateTimeOffset AddClamped(DateTimeOffset timestamp, TimeSpan duration) + { + var utcTicks = timestamp.UtcDateTime.Ticks; + var remainingTicks = DateTimeOffset.MaxValue.Ticks - utcTicks; + return new DateTimeOffset(utcTicks + Math.Min(duration.Ticks, remainingTicks), TimeSpan.Zero); + } +} diff --git a/src/Orleans.Messaging/Orleans.Messaging.csproj b/src/Orleans.Messaging/Orleans.Messaging.csproj new file mode 100644 index 00000000000..54cd69d5f14 --- /dev/null +++ b/src/Orleans.Messaging/Orleans.Messaging.csproj @@ -0,0 +1,40 @@ + + + Microsoft.Orleans.Messaging + Microsoft Orleans Messaging + Inbox and outbox messaging for Microsoft Orleans. + $(PackageTags) Messaging Inbox Outbox + README.md + true + $(DefaultTargetFrameworks) + disable + enable + $(NoWarn);ORLEANSEXP005 + $(VersionSuffix).alpha.1 + alpha.1 + + + true + + + + true + true + true + + + + + + + + + + + + + + + + + diff --git a/src/Orleans.Messaging/OrleansContracts.txt b/src/Orleans.Messaging/OrleansContracts.txt new file mode 100644 index 00000000000..71a144be71d --- /dev/null +++ b/src/Orleans.Messaging/OrleansContracts.txt @@ -0,0 +1,14 @@ +# This file is generated by the Orleans contract analyzer. +# To regenerate this project from the repository root: +# dotnet format PATH_TO_PROJECT.csproj analyzers --severity info --diagnostics ORLEANS0016 ORLEANS0017 ORLEANS0018 ORLEANS0019 ORLEANS0020 ORLEANS0022 ORLEANS0023 ORLEANS0024 +# Run the command once per contract project; do not pass a .sln or .slnx path. +# Verify with: dotnet build PATH_TO_PROJECT.csproj +# The regeneration command edits this manifest only; it does not change source attributes. +# OrleansContracts format: 2 +# Method lines use: wire-identity: CLR-signature. +# The identity is the identifier Orleans uses at runtime, whether generated or declared in source. +# Review every diff: identity or signature changes can break wire compatibility during rolling upgrades. +# Details: https://aka.ms/orleans/OrleansContracts.txt + +interface [GrainInterfaceType("Orleans.Messaging.IMessagingExtension")] Orleans.Messaging.IMessagingExtension [Version(0)] + DeliverAsync: DeliverAsync(Orleans.Messaging.InboxMessage, System.Threading.CancellationToken) -> ValueTask diff --git a/src/Orleans.Messaging/Outbox.State.cs b/src/Orleans.Messaging/Outbox.State.cs new file mode 100644 index 00000000000..46ea3a2f3f7 --- /dev/null +++ b/src/Orleans.Messaging/Outbox.State.cs @@ -0,0 +1,180 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text.Json.Serialization; +using System.Threading; +using System.Threading.Tasks; +using Orleans.Journaling; + +namespace Orleans.Messaging; + +internal sealed partial class Outbox +{ + // Entries are durable once a completed write captured their sequence number. + private long _lastSequence; + private long _capturedSequence; + private long _durableSequence; + + ValueTask IStateMachine.PrepareWriteAsync(CancellationToken cancellationToken) => + _lastSequence != _capturedSequence + ? _pump.EnsureWakeupAsync(Shared.JobTimeProvider.GetUtcNow(), cancellationToken) + : ValueTask.CompletedTask; + + private void OnWriteCaptured() => _capturedSequence = _lastSequence; + + void IStateMachine.OnWriteCompleted() + { + _durableSequence = _capturedSequence; + _pump.Signal(); + } + + private OutboxCommand CreateSnapshot() => new() + { + Kind = OutboxCommandKind.Snapshot, + Snapshot = new() + { + Messages = _messages.Values.ToArray(), + DeadLetters = _deadLetters.Values.ToArray() + } + }; + + private void Emit(OutboxCommand command) + { + Shared.OutboxCodec.WriteSet(command, _writer); + Apply(command); + } + + void IStateMachine.Reset(JournalStreamWriter writer) + { + _writer = writer; + ResetState(); + } + + void IStateMachine.ReplayEntry(JournalEntry entry, JournalReplayContext context) => + context.GetRequiredCommandCodec(entry.FormatKey, Shared.OutboxCodec).Apply(entry.Reader, this); + + void IStateMachine.OnRecoveryCompleted() => OnRecoveryCompleted(); + void IStateMachine.WritePendingEntries(JournalStreamWriter writer) => OnWriteCaptured(); + + void IStateMachine.WriteSnapshot(JournalStreamWriter writer) + { + Shared.OutboxCodec.WriteSet(CreateSnapshot(), writer); + OnWriteCaptured(); + } + + void IDurableValueCommandHandler.ApplySet(OutboxCommand command) => Apply(command); + + private void Apply(OutboxCommand command) + { + switch (command.Kind) + { + case OutboxCommandKind.Enqueue: + var state = command.State ?? throw new InvalidOperationException("An enqueue command requires its delivery state."); + if (state.SentAt is null) throw new InvalidOperationException("An enqueue command requires its send time."); + var entry = new OutboxEntry { Message = command.Message, Reference = command.Reference, State = state, Sequence = ++_lastSequence }; + _messages.Add(entry.MessageId, entry); + ReportDepth(1); + break; + case OutboxCommandKind.Delivery: + if (command.State is { } retry) _messages[command.MessageId].State = retry; + else if (_messages.Remove(command.MessageId)) ReportDepth(-1); + if (command.DeadLetter is { } letter) _deadLetters[command.MessageId] = letter; + break; + case OutboxCommandKind.RemoveDeadLetters: + foreach (var key in command.RemovedDeadLetters!) _deadLetters.Remove(key); + break; + case OutboxCommandKind.Snapshot: + var snapshot = command.Snapshot ?? throw new InvalidOperationException("A snapshot command requires its state."); + var previousCount = _messages.Count; + _messages.Clear(); + _deadLetters.Clear(); + foreach (var restored in snapshot.Messages) _messages.Add(restored.MessageId, restored); + foreach (var restored in snapshot.DeadLetters) _deadLetters.Add(restored.Message.Envelope.MessageId, restored); + ReportDepth(_messages.Count - previousCount); + break; + default: + throw new NotSupportedException($"Outbox command '{command.Kind}' is not supported."); + } + } + + internal bool RemoveDeadLetter(MessageId id) + { + _pump.ShutdownToken.ThrowIfCancellationRequested(); + if (!_deadLetters.ContainsKey(id)) return false; + Emit(new() { Kind = OutboxCommandKind.RemoveDeadLetters, RemovedDeadLetters = [id] }); + return true; + } +} + +internal enum OutboxCommandKind +{ + Enqueue = 1, + Delivery = 2, + RemoveDeadLetters = 3, + Snapshot = 4 +} + +[GenerateSerializer, Alias("Orleans.Messaging.OutboxCommand")] +internal sealed class OutboxCommand +{ + [Id(0)] public OutboxCommandKind Kind { get; init; } + [Id(1)] public OutboxMessage? Message { get; init; } + [Id(2)] public MessageId MessageId { get; init; } + [Id(3)] public OutboxMessageState? State { get; init; } + [Id(4)] public OutboxDeadLetter? DeadLetter { get; init; } + [Id(5)] public MessageId[]? RemovedDeadLetters { get; init; } + [Id(6)] public OutboxSnapshot? Snapshot { get; init; } + + // An enqueue command sets this instead of Message for a message which an IOutboxMessageSource provides. + [Id(7)] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public OutboxMessageReference? Reference { get; init; } +} + +/// Identifies a message which an provides. +[GenerateSerializer, Alias("Orleans.Messaging.OutboxMessageReference")] +internal readonly struct OutboxMessageReference +{ + [Id(0)] public required MessageId MessageId { get; init; } +} + +[GenerateSerializer, Alias("Orleans.Messaging.OutboxSnapshot")] +internal sealed class OutboxSnapshot +{ + [Id(0)] public OutboxEntry[] Messages { get; init; } = []; + [Id(1)] public OutboxDeadLetter[] DeadLetters { get; init; } = []; +} + +[GenerateSerializer, Alias("Orleans.Messaging.OutboxEntry")] +internal sealed class OutboxEntry +{ + // Exactly one of Message and Reference is set. A reference entry journals no payload or receiver: its source provides the message. + [Id(0)] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public OutboxMessage? Message { get; init; } + + [Id(1)] public required OutboxMessageState State { get; set; } + + [Id(2)] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public OutboxMessageReference? Reference { get; init; } + + [field: NonSerialized] + [JsonIgnore] public long Sequence { get; init; } + + /// + /// Gets the source which owns a reference entry's message, resolved when it is sent or first materialized after + /// recovery; for an ordinary entry or a recovered reference which no source provides. + /// + [field: NonSerialized] + [JsonIgnore] public IOutboxMessageSource? Source { get; set; } + + [JsonIgnore] + public MessageId MessageId => Message is { } message ? message.Envelope.MessageId + : Reference is { } reference ? reference.MessageId + : throw new InvalidOperationException("An outbox entry requires its message or message reference."); +} + +[JsonSerializable(typeof(OutboxCommand))] +[JsonSerializable(typeof(OutboxSnapshot))] +internal sealed partial class OutboxJsonContext : JsonSerializerContext; diff --git a/src/Orleans.Messaging/Outbox.cs b/src/Orleans.Messaging/Outbox.cs new file mode 100644 index 00000000000..4def0773afa --- /dev/null +++ b/src/Orleans.Messaging/Outbox.cs @@ -0,0 +1,782 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; +using System.Linq; +using System.Runtime.ExceptionServices; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Orleans.Messaging.Configuration; +using Orleans.Messaging.Diagnostics; +using Orleans.Journaling; +using Orleans.Runtime; + +namespace Orleans.Messaging; + +/// +/// Delivers journal-acknowledged messages to their receivers' inboxes. +/// +internal sealed partial class Outbox : IStateMachine, IReportsStagedChanges, IDurableValueCommandHandler, IOutbox +{ + private readonly MessagingPump _pump; + private JournalStreamWriter _writer; + private readonly Dictionary _messages = []; + private readonly Dictionary _deadLetters = []; + private readonly List _sources = []; + + // Entries whose delivery attempt's outcome is not applied yet, the outcomes of completed attempts, and the number of + // attempts still running. An entry stays in flight until its outcome is applied, so it never has two attempts. + private readonly HashSet _inFlight = new(ReferenceEqualityComparer.Instance); + private readonly List _outcomes = []; + private int _running; + private TaskCompletionSource? _drained; + + public Outbox(MessagingPump pump) + { + _pump = pump; + pump.StateManager.RegisterStateMachine(MessagingStateNames.Outbox, this); + pump.Register(this); + } + + private MessagingShared Shared => _pump.Shared; + private DurableMessagingOptions Options => _pump.Shared.Options; + private ILogger Logger => _pump.Shared.OutboxLogger; + private TimeProvider TimeProvider => Shared.TimeProvider; + private TimeProvider JobTimeProvider => Shared.JobTimeProvider; + private GrainId GrainId => _pump.GrainContext.GrainId; + + internal IReadOnlyDictionary MessageState => _messages; + internal IReadOnlyDictionary DeadLetterState => _deadLetters; + internal int Count => _messages.Count; + + /// Gets whether delivery attempts are in flight or awaiting application by the pump. + internal bool IsDelivering => _inFlight.Count > 0; + + public IOutbox Register(IOutboxMessageSource source) + { + ArgumentNullException.ThrowIfNull(source); + _sources.Add(source); + return this; + } + + public void Send(OutboxMessage message) + { + _pump.ShutdownToken.ThrowIfCancellationRequested(); + EnvelopeValidation.Validate(message); + if (_messages.TryGetValue(message.Envelope.MessageId, out var existing)) + { + if (existing.Message is not { } pending || !EnvelopeEquivalence.AreEquivalent(pending, message)) + { + throw new InvalidOperationException( + $"The outbox already contains a different envelope with message ID '{message.Envelope.MessageId}'."); + } + return; + } + + if (message.ReceiverId == GrainId) + { + // The grain's own inbox accepts the message in the caller's journal write, so the outbox never holds it. + _pump.Inbox.StageLocal(message.ToInboxMessage(JobTimeProvider.GetUtcNow())); + Shared.Instruments.OnOutboxMessageSent(Shared.GrainType); + return; + } + + try + { + Emit(new() { Kind = OutboxCommandKind.Enqueue, Message = message, State = CreateState() }); + Shared.Instruments.OnOutboxMessageSent(Shared.GrainType); + } + catch (Exception exception) + { + FailStaging(exception); + } + } + + public void Send(MessageId messageId) + { + _pump.ShutdownToken.ThrowIfCancellationRequested(); + EnvelopeValidation.ValidateMessageId(messageId); + if (_messages.TryGetValue(messageId, out var existing)) + { + // The source is authoritative for a pending reference entry's message, so a repeated send coalesces. + if (existing.Reference is null) + { + throw new InvalidOperationException( + $"The outbox already contains a different message with message ID '{messageId}'."); + } + return; + } + + if (FindSource(messageId, out _) is not { } source) + { + throw new InvalidOperationException($"No registered outbox message source provides message '{messageId}'."); + } + + try + { + Emit(new() + { + Kind = OutboxCommandKind.Enqueue, + Reference = new() { MessageId = messageId }, + State = CreateState() + }); + _messages[messageId].Source = source; + Shared.Instruments.OnOutboxMessageSent(Shared.GrainType); + } + catch (Exception exception) + { + FailStaging(exception); + } + } + + public bool Withdraw(MessageId messageId) + { + _pump.ShutdownToken.ThrowIfCancellationRequested(); + if (!_messages.TryGetValue(messageId, out var entry)) return false; + if (entry.Reference is null) + { + throw new InvalidOperationException($"Message '{messageId}' is pending as an ordinary outbox message, which can't be withdrawn."); + } + + try + { + Emit(new() { Kind = OutboxCommandKind.Delivery, MessageId = messageId }); + } + catch (Exception exception) + { + FailStaging(exception); + } + + Shared.Instruments.OnOutboxMessageDelivered(Shared.GrainType, "withdrawn"); + LogMessageWithdrawn(Logger, messageId); + return true; + } + + // The sending write fixes the logical send time for every attempt and recovery. + private OutboxMessageState CreateState() => + new() { SentAt = JobTimeProvider.GetUtcNow() }; + + private DateTimeOffset GetExpirationTime(OutboxMessageState state) => + MessagingTime.AddClamped(state.SentAt!.Value, Options.OutboxRetryTimeout); + + /// Returns the first registered source which provides the message, and its validated message. + private IOutboxMessageSource? FindSource(MessageId messageId, out OutboxMessage message) + { + foreach (var source in _sources) + { + if (TryGetSourceMessage(source, messageId, out message)) return source; + } + + message = default; + return null; + } + + /// + /// Returns the entry's message, materializing a reference entry's message from its owning source, or returns + /// for a recovered entry which no registered source provides. Throws when the owning source + /// throws, returns an invalid message, or no longer provides the message. + /// + private bool TryGetMessage(OutboxEntry entry, out OutboxMessage message) + { + if (entry.Message is { } ordinary) + { + message = ordinary; + return true; + } + + if (entry.Source is { } owner) + { + if (TryGetSourceMessage(owner, entry.MessageId, out message)) return true; + + // The owner provided the message when it was sent, and withdrawal goes through IOutbox.Withdraw. + throw new InvalidOperationException( + $"The outbox message source which owns message '{entry.MessageId}' no longer provides it; withdraw a message with IOutbox.Withdraw."); + } + + // A recovered entry's owner is the first source which doesn't decline it, including one which fails to provide + // it, so that source is notified when the entry is dead-lettered. + foreach (var source in _sources) + { + entry.Source = source; + if (TryGetSourceMessage(source, entry.MessageId, out message)) return true; + } + + entry.Source = null; + message = default; + return false; + } + + private bool TryGetSourceMessage(IOutboxMessageSource source, MessageId messageId, out OutboxMessage message) + { + if (!source.TryGetMessage(messageId, out message)) return false; + EnvelopeValidation.Validate(message); + if (message.Envelope.MessageId != messageId) + { + throw new InvalidOperationException( + $"The outbox message source returned message ID '{message.Envelope.MessageId}' for message '{messageId}'."); + } + if (message.ReceiverId == GrainId) + { + // The grain's own inbox would hold the payload, so a source cannot keep it as the single copy. + throw new ArgumentException( + $"The outbox message source addressed message '{messageId}' to the sending grain; send its message with Send(OutboxMessage) instead.", + nameof(messageId)); + } + + return true; + } + + /// Notifies the source of a retired reference entry, so its release commits with the entry's removal. + private void Retire(OutboxEntry entry, OutboxMessageRetirement retirement) + { + // A recovered reference which no source provides has no owner to notify. + entry.Source?.OnRetired(entry.MessageId, retirement); + MessagingEvents.EmitOutboxMessageRetired(_pump.GrainContext, entry.MessageId, retirement); + } + + /// + /// Creates the message of a dead letter whose source failed to provide it: the message ID with an empty payload and + /// a default receiver, which no valid outgoing message has. + /// + private static OutboxMessage CreateUnmaterializedMessage(MessageId messageId) => + new() { Envelope = new() { MessageId = messageId, Payload = [] }, ReceiverId = default }; + + /// Copies the payload and header values of a message which a source owns. + private static OutboxMessage CopySourceMessage(OutboxMessage message) + { + var envelope = message.Envelope; + Dictionary? headers = null; + if (envelope.Headers is { } sourceHeaders) + { + headers = new(sourceHeaders.Count, sourceHeaders.Comparer); + foreach (var (name, value) in sourceHeaders) headers.Add(name, value.AsSpan().ToArray()); + } + + return message with { Envelope = envelope with { Payload = envelope.Payload.AsSpan().ToArray(), Headers = headers } }; + } + + private static bool IsUnmaterialized(OutboxMessage message) => + message.ReceiverId.IsDefault + && message.Envelope is { Payload.Length: 0, Subject: null, Headers: null, DeliverAt: null }; + + private void OnRecoveryCompleted() + { + if (_sources.Count == 0 && _messages.Values.Any(static entry => entry.Reference is not null)) + { + throw new InvalidOperationException( + "The outbox recovered messages which a source provides, but no outbox message source is registered. Register outbox message sources during activation setup, before journal recovery."); + } + + foreach (var (key, entry) in _messages) + { + if (entry.State.SentAt is null) + { + throw new InvalidOperationException("A recovered outbox entry requires its send time."); + } + + if (entry.Message is { } message) + { + if (entry.Reference is not null) + { + throw new InvalidOperationException("A recovered outbox entry has both a message and a message reference."); + } + + EnvelopeValidation.Validate(message); + } + else + { + EnvelopeValidation.ValidateMessageId(key); + } + + if (key != entry.MessageId) + { + throw new InvalidOperationException("The recovered outbox key does not match its command identity."); + } + if (entry.Message?.ReceiverId == GrainId) + { + throw new InvalidOperationException("A recovered outbox message is addressed to its own grain, whose inbox accepts such messages directly."); + } + } + foreach (var (key, letter) in _deadLetters) + { + if (IsUnmaterialized(letter.Message)) + { + EnvelopeValidation.ValidateMessageId(letter.Message.Envelope.MessageId); + } + else + { + EnvelopeValidation.Validate(letter.Message); + } + + if (key != letter.Message.Envelope.MessageId) + { + throw new InvalidOperationException("The recovered outbox dead-letter key does not match its message identity."); + } + if (letter.Message.ReceiverId == GrainId) + { + throw new InvalidOperationException("A recovered outbox dead letter is addressed to its own grain, whose inbox accepts such messages directly."); + } + } + _capturedSequence = _durableSequence = _lastSequence; + } + + private void ResetState() + { + ReportDepth(-_messages.Count); + _messages.Clear(); + _deadLetters.Clear(); + _lastSequence = _capturedSequence = _durableSequence = 0; + } + + [DoesNotReturn] + private void FailStaging(Exception exception) + { + LogStagingFailed(Logger, exception); + _pump.Fail(exception, "Outbox staging failed."); + ExceptionDispatchInfo.Throw(exception); + } + + private async ValueTask PersistAsync() + { + try + { + await _pump.StateManager.WriteStateAsync(CancellationToken.None).ConfigureAwait(true); + } + catch (Exception exception) + { + _pump.Fail(exception, "Outbox persistence failed."); + throw; + } + } + + /// + /// Selects up to durable entries which are due and have no attempt in flight: the + /// messages to attempt, and the entries to dead-letter without an attempt. + /// + private (List<(OutboxEntry Entry, OutboxMessage Message)>? Candidates, List? Unavailable) SelectMessages(int capacity) + { + var now = JobTimeProvider.GetUtcNow(); + List<(OutboxEntry Entry, OutboxMessage Message)>? candidates = null; + List? unavailable = null; + var selected = 0; + foreach (var entry in _messages.Values) + { + if (entry.Sequence > _durableSequence || _inFlight.Contains(entry) || entry.State.NextAttemptAt > now) continue; + + // Entries which are only removed count toward the capacity too, so a pass removes at most a batch of entries. + if (++selected > capacity) break; + OutboxMessage? message = null; + Exception? error = null; + try + { + if (TryGetMessage(entry, out var provided)) message = provided; + } + catch (Exception exception) + { + // The entry is durable, so a source which can't provide a valid message dead-letters only that entry. + error = exception; + } + + // Backpressure and failed attempts share the absolute retry budget established by the sending write. + if (GetExpirationTime(entry.State) <= now) (unavailable ??= []).Add(new(entry, UnavailableKind.Expired, message, error)); + else if (error is not null) (unavailable ??= []).Add(new(entry, UnavailableKind.Failed, null, error)); + else if (message is { } candidate) (candidates ??= []).Add((entry, candidate)); + else (unavailable ??= []).Add(new(entry, UnavailableKind.Unclaimed, null, null)); + } + + if (candidates is null) LogNoMessages(Logger, Count); + else LogDeliveringMessages(Logger, candidates.Count); + return (candidates, unavailable); + } + + private async Task DeliverAsync(OutboxEntry entry, OutboxMessage message, CancellationToken cancellationToken) + { + var start = Stopwatch.GetTimestamp(); + try + { + // Retention and admission use the original send time, including for delayed attempts and recovered entries. + var delivery = new InboxMessage + { + Envelope = message.Envelope, + SentAt = entry.State.SentAt!.Value + }; + var result = await Shared.Transport.DeliverAsync(message.ReceiverId, delivery, cancellationToken).ConfigureAwait(true); + return new(entry, message, result, null, Stopwatch.GetElapsedTime(start)); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + return new(entry, message, null, "Delivery was canceled.", Stopwatch.GetElapsedTime(start)); + } + catch (Exception exception) + { + if (Logger.IsEnabled(LogLevel.Error)) + { + LogDeliveryError(Logger, exception, message.Envelope.MessageId, message.ReceiverId); + } + return new(entry, message, null, exception.ToString(), Stopwatch.GetElapsedTime(start)); + } + } + + /// + /// Applies completed delivery attempts and starts attempts for due messages, keeping up to + /// attempts in flight, and returns whether it changed state. + /// + /// + /// Each attempt completes on its own, so a slow receiver holds one slot without delaying messages to other receivers. + /// + internal async ValueTask AdvanceAsync(CancellationToken cancellationToken) + { + var progress = false; + if (_outcomes.Count > 0) + { + cancellationToken.ThrowIfCancellationRequested(); + var outcomes = _outcomes.ToArray(); + _outcomes.Clear(); + foreach (var outcome in outcomes) _inFlight.Remove(outcome.Entry); + await ApplyDeliveriesAsync(outcomes).ConfigureAwait(true); + progress = true; + } + + var capacity = Options.OutboxBatchSize - _inFlight.Count; + if (capacity <= 0) return progress; + var (candidates, unavailable) = SelectMessages(capacity); + + // Attempts start before any await, so no withdrawal or source change interleaves between materializing a message + // and calling its receiver. + if (candidates is not null) + { + foreach (var (entry, message) in candidates) RunAttemptAsync(entry, message).Ignore(); + } + + if (unavailable is not null) + { + await DeadLetterUnavailableAsync(unavailable).ConfigureAwait(true); + progress = true; + } + + return progress; + } + + private async Task RunAttemptAsync(OutboxEntry entry, OutboxMessage message) + { + _inFlight.Add(entry); + _running++; + var outcome = await DeliverAsync(entry, message, _pump.ShutdownToken).ConfigureAwait(true); + _outcomes.Add(outcome); + if (--_running == 0 && _drained is { } drained) + { + _drained = null; + drained.TrySetResult(); + } + + _pump.Signal(); + } + + private async ValueTask ApplyDeliveriesAsync(DeliveryOutcome[] outcomes) + { + int applied = 0, delivered = 0, backpressured = 0; + try + { + foreach (var outcome in outcomes) + { + // An entry which was withdrawn, or removed and sent again, since its attempt started has a stale outcome. + var id = outcome.Entry.MessageId; + if (!_messages.TryGetValue(id, out var current) || !ReferenceEquals(current, outcome.Entry)) continue; + + var status = outcome.Result?.Status; + if (status == DeliveryStatus.Expired) + { + // Expired means this send is at or below the admission floor; earlier acceptance remains unknown. + var reason = outcome.Result!.Value.Message ?? "The message was sent before the receiver's deduplication window; its earlier delivery outcome is unknown."; + StageDeadLetter( + current, + current.Reference is null ? outcome.Message : CopySourceMessage(outcome.Message), + reason, + checked(current.State.AttemptCount + 1)); + } + else if (status is DeliveryStatus.Accepted or DeliveryStatus.Duplicate or DeliveryStatus.DeadLettered) + { + Emit(new() { Kind = OutboxCommandKind.Delivery, MessageId = id }); + Retire(current, status == DeliveryStatus.DeadLettered + ? OutboxMessageRetirement.Rejected(outcome.Result!.Value.Message ?? "The receiver dead-lettered the message.", outcome.Result.Value.Code) + : OutboxMessageRetirement.Delivered()); + delivered++; + } + else + { + if (status == DeliveryStatus.Backpressured) backpressured++; + var attemptCount = checked(current.State.AttemptCount + 1); + var error = outcome.Error ?? status switch + { + DeliveryStatus.Backpressured => "The receiver is backpressured.", + DeliveryStatus.HandlerNotFound => outcome.Result?.Message ?? "The receiver has no registered inbox handler.", + _ => $"Unexpected delivery status '{status}'." + }; + if (PrepareRetry(outcome, attemptCount, error) is { } retry) + { + Emit(new() { Kind = OutboxCommandKind.Delivery, MessageId = id, State = retry }); + MessagingEvents.EmitOutboxRetryScheduled( + _pump.GrainContext, id, outcome.Message.ReceiverId, attemptCount, retry.NextAttemptAt!.Value, error); + } + else + { + // The source may release a reference entry's payload and headers when it retires, so the dead letter keeps a copy. + StageDeadLetter(current, current.Reference is null ? outcome.Message : CopySourceMessage(outcome.Message), error, attemptCount); + } + } + + RecordDeliveryMetrics(outcome); + applied++; + } + } + catch (Exception exception) + { + FailStaging(exception); + } + + if (applied == 0) return; + await PersistAsync().ConfigureAwait(true); + LogDeliveryComplete(Logger, delivered, backpressured, applied - delivered - backpressured, Count); + } + + /// + /// Returns the retry state after a failed attempt, or when the retry timeout has elapsed. + /// + private OutboxMessageState? PrepareRetry(DeliveryOutcome outcome, int attemptCount, string error) + { + var now = JobTimeProvider.GetUtcNow(); + var expirationTime = GetExpirationTime(outcome.Entry.State); + if (expirationTime <= now) return null; + + // The next attempt runs no later than the timeout, when the outbox dead-letters the message instead. + var retryAt = MessagingTime.AddClamped(now, MessagingTime.GetBackoff(Options.RetryInitialInterval, attemptCount)); + return new OutboxMessageState + { + AttemptCount = attemptCount, + LastError = error, + SentAt = outcome.Entry.State.SentAt, + NextAttemptAt = retryAt < expirationTime ? retryAt : expirationTime + }; + } + + /// + /// Dead-letters entries without an attempt: those whose retry timeout elapsed, those whose owning sources failed to + /// provide a valid message, and recovered references which no registered source provides. + /// + private async ValueTask DeadLetterUnavailableAsync(List unavailable) + { + try + { + foreach (var (entry, kind, message, error) in unavailable) + { + var id = entry.MessageId; + switch (kind) + { + case UnavailableKind.Expired: + LogMessageExpired(Logger, id); + StageDeadLetter( + entry, + message is not { } expired ? CreateUnmaterializedMessage(id) : entry.Reference is null ? expired : CopySourceMessage(expired), + entry.State.LastError is { } last + ? $"The outbox retry timeout elapsed after the last attempt failed: {last}" + : "The outbox retry timeout elapsed before the message was delivered.", + entry.State.AttemptCount); + Shared.Instruments.OnOutboxMessageDelivered(Shared.GrainType, "expired"); + break; + case UnavailableKind.Failed: + LogSourceFailed(Logger, error!, id); + StageDeadLetter(entry, CreateUnmaterializedMessage(id), + $"The outbox message source failed to provide the message: {error}", checked(entry.State.AttemptCount + 1)); + Shared.Instruments.OnOutboxMessageDelivered(Shared.GrainType, "source_error"); + break; + default: + LogMessageUnprovided(Logger, id); + StageDeadLetter(entry, CreateUnmaterializedMessage(id), + "No registered outbox message source provides the message.", entry.State.AttemptCount); + Shared.Instruments.OnOutboxMessageDelivered(Shared.GrainType, "unprovided"); + break; + } + } + } + catch (Exception exception) + { + FailStaging(exception); + } + + await PersistAsync().ConfigureAwait(true); + } + + /// + /// Stages the removal of with its dead letter, and retires it with its owning source, if any, + /// in the same journal write. + /// + private void StageDeadLetter( + OutboxEntry entry, OutboxMessage message, string reason, int attemptCount, OutboxMessageRetirement? retirement = null) + { + var now = TimeProvider.GetUtcNow(); + var id = entry.MessageId; + CompactDeadLetters(now, reservedCapacity: _deadLetters.ContainsKey(id) ? 0 : 1); + Emit(new() + { + Kind = OutboxCommandKind.Delivery, + MessageId = id, + DeadLetter = new() { Message = message, DeadLetteredAt = now, Reason = reason, AttemptCount = attemptCount } + }); + Retire(entry, retirement ?? OutboxMessageRetirement.DeadLettered(reason)); + } + + private bool CompactDeadLetters(DateTimeOffset now, int reservedCapacity = 0) + { + if (_deadLetters.Count == 0) return false; + var retained = new Dictionary(_deadLetters); + if (!DeadLetterRetention.Compact(retained, now, Options.DeadLetterRetentionPeriod, Options.MaxRetainedDeadLetters, + static entry => entry.DeadLetteredAt, reservedCapacity)) + { + return false; + } + + Emit(new() + { + Kind = OutboxCommandKind.RemoveDeadLetters, + RemovedDeadLetters = _deadLetters.Keys.Where(key => !retained.ContainsKey(key)).ToArray() + }); + return true; + } + private void RecordDeliveryMetrics(DeliveryOutcome outcome) + { + var message = outcome.Message; + var status = outcome.Result?.Status; + var metricStatus = status switch + { + DeliveryStatus.Accepted => "accepted", + DeliveryStatus.Duplicate => "duplicate", + DeliveryStatus.DeadLettered => "deadlettered", + DeliveryStatus.Backpressured => "backpressured", + DeliveryStatus.HandlerNotFound => "handler_not_found", + DeliveryStatus.Expired => "expired", + null => "error", + _ => status.Value.ToString().ToLowerInvariant() + }; + Shared.Instruments.OnOutboxMessageDelivered(Shared.GrainType, metricStatus); + Shared.Instruments.OnOutboxDeliveryDuration(outcome.Duration, Shared.GrainType); + switch (status) + { + case DeliveryStatus.Accepted: + case DeliveryStatus.Duplicate: + case DeliveryStatus.DeadLettered: + case DeliveryStatus.Expired: + if (Logger.IsEnabled(LogLevel.Debug)) + { + LogMessageDelivered(Logger, message.Envelope.MessageId, message.ReceiverId, status.Value); + } + break; + case DeliveryStatus.Backpressured: + LogDeliveryBackpressured(Logger, message.Envelope.MessageId, message.ReceiverId); + break; + case DeliveryStatus.HandlerNotFound: + if (Logger.IsEnabled(LogLevel.Error)) + { + LogDeliveryHandlerNotFound(Logger, message.Envelope.MessageId, message.ReceiverId, outcome.Result?.Message); + } + break; + case { } unexpected: + if (Logger.IsEnabled(LogLevel.Warning)) + { + LogUnexpectedDeliveryStatus(Logger, unexpected, message.Envelope.MessageId, message.ReceiverId); + } + break; + } + } + + /// + /// Returns the delay until the next captured delivery attempt is due, or when none remain. + /// + /// The current Durable Jobs time. + internal TimeSpan? GetNextWakeupDelay(DateTimeOffset now) + { + TimeSpan? next = null; + foreach (var entry in _messages.Values) + { + // Capture follows synchronous preparation; its wake-up covers the write until acknowledgement. + if (entry.Sequence > _capturedSequence) continue; + var at = entry.State.NextAttemptAt ?? now; + var delay = at > now ? at - now : TimeSpan.Zero; + if (delay == TimeSpan.Zero) return delay; + if (next is null || delay < next) next = delay; + } + return next; + } + + internal async Task StartAsync(CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + try + { + if (!CompactDeadLetters(TimeProvider.GetUtcNow())) return; + } + catch (Exception exception) + { + FailStaging(exception); + } + + await PersistAsync().ConfigureAwait(true); + } + + /// Waits for in-flight deliveries, which observe the pump's shutdown token. + internal Task StopAsync() => _running == 0 ? Task.CompletedTask : (_drained ??= new(TaskCreationOptions.RunContinuationsAsynchronously)).Task; + /// Releases the reported depth once the activation's messaging state is discarded. + internal void ReleaseDepth() => ReportDepth(-_messages.Count); + + private void ReportDepth(int delta) => Shared.Instruments.OnOutboxDepthChanged(Shared.GrainType, delta); + + [LoggerMessage(Level = LogLevel.Error, Message = "Outbox command staging failed.")] + private static partial void LogStagingFailed(ILogger logger, Exception exception); + + [LoggerMessage(Level = LogLevel.Debug, Message = "No durable messages to deliver (all {Count} messages are still pending)")] + private static partial void LogNoMessages(ILogger logger, int count); + + [LoggerMessage(Level = LogLevel.Debug, Message = "Delivering {Count} durable messages from outbox")] + private static partial void LogDeliveringMessages(ILogger logger, int count); + + [LoggerMessage(Level = LogLevel.Debug, Message = "Delivered message {MessageId} to {ReceiverId} (Status: {Status})")] + private static partial void LogMessageDelivered(ILogger logger, MessageId messageId, GrainId receiverId, DeliveryStatus status); + + [LoggerMessage(Level = LogLevel.Error, Message = "No inbox handler for message {MessageId} to {ReceiverId}: {Message}")] + private static partial void LogDeliveryHandlerNotFound(ILogger logger, MessageId messageId, GrainId receiverId, string? message); + + [LoggerMessage(Level = LogLevel.Debug, Message = "Backpressured delivering message {MessageId} to {ReceiverId}, will retry later")] + private static partial void LogDeliveryBackpressured(ILogger logger, MessageId messageId, GrainId receiverId); + + [LoggerMessage(Level = LogLevel.Warning, Message = "Unexpected delivery status {Status} for message {MessageId} to {ReceiverId}")] + private static partial void LogUnexpectedDeliveryStatus(ILogger logger, DeliveryStatus status, MessageId messageId, GrainId receiverId); + + [LoggerMessage(Level = LogLevel.Error, Message = "Error delivering message {MessageId} to {ReceiverId}")] + private static partial void LogDeliveryError(ILogger logger, Exception exception, MessageId messageId, GrainId receiverId); + + [LoggerMessage(Level = LogLevel.Information, Message = "Removed outbox message {MessageId}, which its source withdrew")] + private static partial void LogMessageWithdrawn(ILogger logger, MessageId messageId); + + [LoggerMessage(Level = LogLevel.Warning, Message = "Dead-lettered recovered outbox message {MessageId}, which no registered outbox message source provides")] + private static partial void LogMessageUnprovided(ILogger logger, MessageId messageId); + + [LoggerMessage(Level = LogLevel.Warning, Message = "Dead-lettered outbox message {MessageId}, whose retry timeout elapsed")] + private static partial void LogMessageExpired(ILogger logger, MessageId messageId); + + [LoggerMessage(Level = LogLevel.Error, Message = "The outbox message source failed to provide message {MessageId}; the message is dead-lettered")] + private static partial void LogSourceFailed(ILogger logger, Exception exception, MessageId messageId); + + [LoggerMessage(Level = LogLevel.Debug, Message = "Outbox delivery complete: {DeliveredCount} delivered, {BackpressuredCount} backpressured, {FailedCount} failed, {RemainingCount} remaining")] + private static partial void LogDeliveryComplete(ILogger logger, int deliveredCount, int backpressuredCount, int failedCount, int remainingCount); + + private readonly record struct DeliveryOutcome( + OutboxEntry Entry, + OutboxMessage Message, + DeliveryResult? Result, + string? Error, + TimeSpan Duration); + + private enum UnavailableKind { Expired, Failed, Unclaimed } + + private readonly record struct Unavailable(OutboxEntry Entry, UnavailableKind Kind, OutboxMessage? Message, Exception? Error); +} diff --git a/src/Orleans.Messaging/OutboxExtensions.cs b/src/Orleans.Messaging/OutboxExtensions.cs new file mode 100644 index 00000000000..a757847e7e1 --- /dev/null +++ b/src/Orleans.Messaging/OutboxExtensions.cs @@ -0,0 +1,114 @@ +using System; +using System.Collections.Generic; +using Orleans.Runtime; + +namespace Orleans.Messaging; + +/// Extensions for encoding and staging typed messages and replies. +public static class OutboxExtensions +{ + /// Encodes and synchronously stages one typed command. + /// The body contract. + /// The owning grain's outbox. + /// The subject and serializer binding. + /// The stable application command identity. + /// The destination inbox. + /// The body to encode. + /// + /// The optional earliest time, on the Durable Jobs clock, at which the receiving inbox processes the message. + /// The outbox still delivers the message immediately. See . + /// + /// + /// Serialization finishes before staging begins. Compute fallible business results first, + /// then send, apply the business state, and complete in one synchronous final block. + /// Payload and header arrays remain available through the staged message's ordinary managed references. + /// + public static void Send( + this IOutbox outbox, MessageType messageType, + MessageId messageId, GrainId destination, T body, DateTimeOffset? deliverAt = null) + { + ArgumentNullException.ThrowIfNull(outbox); + ArgumentNullException.ThrowIfNull(messageType); + var message = messageType.Create(messageId, destination, body, deliverAt); + outbox.Send(message); + } + + /// Encodes and synchronously stages one typed command to a grain reference. + /// The body contract. + /// The owning grain's outbox. + /// The subject and serializer binding. + /// The stable application command identity. + /// The destination grain. + /// The body to encode. + /// + /// The optional earliest time, on the Durable Jobs clock, at which the receiving inbox processes the message. + /// The outbox still delivers the message immediately. See . + /// + public static void Send( + this IOutbox outbox, MessageType messageType, + MessageId messageId, IAddressable destination, T body, DateTimeOffset? deliverAt = null) + { + ArgumentNullException.ThrowIfNull(destination); + outbox.Send(messageType, messageId, destination.GetGrainId(), body, deliverAt); + } + + /// Encodes and synchronously stages typed commands in enumeration order. + /// The body contract. + /// The owning grain's outbox. + /// The subject and serializer binding. + /// The application-defined identities, destinations, and bodies. + /// + /// Enumerates once and encodes every command before staging any, so an enumeration or encoding failure stages + /// nothing. If staging fails, earlier messages remain staged under their original identities with their encoded + /// payload and headers. Persistence follows the grain's ordinary journal write. + /// + public static void Send( + this IOutbox outbox, MessageType messageType, + IEnumerable<(MessageId MessageId, GrainId Destination, T Body)> messages) + { + ArgumentNullException.ThrowIfNull(outbox); + ArgumentNullException.ThrowIfNull(messageType); + ArgumentNullException.ThrowIfNull(messages); + var encoded = new List(); + foreach (var (messageId, destination, body) in messages) + { + encoded.Add(messageType.Create(messageId, destination, body, deliverAt: null)); + } + + foreach (var message in encoded) + { + outbox.Send(message); + } + } + + /// Encodes and stages a reply under the received command's deterministic result child. + /// The reply body contract. + /// The replying grain's outbox. + /// The reply subject and serializer binding. + /// The current received message and completion context. + /// The application-selected reply destination. + /// The reply body. + /// The reply identity is context.Message.Envelope.MessageId.Key.CreateChildKey("result"). + public static void SendReply( + this IOutbox outbox, MessageType messageType, + IInboxHandlerContext context, GrainId destination, T body) + { + ArgumentNullException.ThrowIfNull(context); + outbox.Send(messageType, new MessageId(context.Message.Envelope.MessageId.Key.CreateChildKey("result")), destination, body); + } + + /// Encodes and stages a deterministic result reply to a grain reference. + /// The reply body contract. + /// The replying grain's outbox. + /// The reply subject and serializer binding. + /// The current received message and completion context. + /// The application-selected replying grain reference. + /// The reply body. + public static void SendReply( + this IOutbox outbox, MessageType messageType, + IInboxHandlerContext context, IAddressable destination, T body) + { + ArgumentNullException.ThrowIfNull(destination); + outbox.SendReply(messageType, context, destination.GetGrainId(), body); + } +} diff --git a/src/Orleans.Messaging/OutboxMessage.cs b/src/Orleans.Messaging/OutboxMessage.cs new file mode 100644 index 00000000000..0f97fdddcb7 --- /dev/null +++ b/src/Orleans.Messaging/OutboxMessage.cs @@ -0,0 +1,22 @@ +using System; +using Orleans.Runtime; + +namespace Orleans.Messaging; + +/// An outgoing command with destination routing separate from its common envelope. +[GenerateSerializer, Alias("Orleans.Messaging.OutboxMessage")] +public readonly struct OutboxMessage +{ + /// Gets the command identity, payload, and application metadata. + [Id(0)] + public required Envelope Envelope { get; init; } + + /// Gets the required destination grain identity. + [Id(1)] + public required GrainId ReceiverId { get; init; } + + /// Creates the incoming view by sharing the envelope, payload, and headers. + /// The logical send's time, which every delivery of this command reuses. + /// A received message whose destination is implied by the receiving grain. + public InboxMessage ToInboxMessage(DateTimeOffset sentAt) => new() { Envelope = Envelope, SentAt = sentAt }; +} diff --git a/src/Orleans.Messaging/OutboxTransport.cs b/src/Orleans.Messaging/OutboxTransport.cs new file mode 100644 index 00000000000..48071cd5d7c --- /dev/null +++ b/src/Orleans.Messaging/OutboxTransport.cs @@ -0,0 +1,21 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using Orleans.Runtime; + +namespace Orleans.Messaging; + +/// Calls a receiver's inbox for one outbox delivery attempt. +/// Tests replace the transport to inject delivery faults. +internal interface IOutboxTransport +{ + /// Delivers to the inbox of . + ValueTask DeliverAsync(GrainId receiver, InboxMessage message, CancellationToken cancellationToken); +} + +/// Delivers outbox messages through grain calls to the receivers' messaging extensions. +internal sealed class GrainOutboxTransport(IGrainFactory grainFactory) : IOutboxTransport +{ + public ValueTask DeliverAsync(GrainId receiver, InboxMessage message, CancellationToken cancellationToken) => + grainFactory.GetGrain(receiver).DeliverAsync(message, cancellationToken); +} diff --git a/src/Orleans.Messaging/README.md b/src/Orleans.Messaging/README.md new file mode 100644 index 00000000000..cf978e073df --- /dev/null +++ b/src/Orleans.Messaging/README.md @@ -0,0 +1,326 @@ +# Microsoft Orleans Messaging + +`Microsoft.Orleans.Messaging` provides grain-scoped durable inboxes and outboxes +built on Orleans Journaling and Durable Jobs. A grain sends a message by staging it +in its outbox; the message persists with the grain's other journaled state and is +delivered to the receiver's inbox, which processes it at least once and deduplicates +it by message ID. A message a grain sends to itself skips the outbox: its own inbox +accepts it in the same journal write. + +## Setup + +Configure Journaling and Durable Jobs storage, then call `AddMessaging` on the silo +builder or service collection. The optional `DurableMessagingOptions` callback configures +capacity, batch sizes, retries, deduplication, and dead-letter retention; options are +validated at startup. Register typed subjects with `AddMessageType(subject)`. + +Messaging grains derive from `DurableGrain` or implement `IMessagingGrain`, and +constructor-inject `IInbox` and `IOutbox`. During activation, the messaging grain-type +configurator installs the grain's messaging extension. Stateless worker grains are +rejected; reentrant and interleaving grains are supported, and the grain's code is +responsible for correctness when calls interleave. + +## Messages + +`Envelope` carries: + +- `MessageId`: an application-supplied `HierarchicalKey` wrapped in a `MessageId`, at + most 1,024 UTF-8 bytes. It identifies one command at the receiver. +- `Payload`: opaque bytes; an empty payload is valid. +- `Subject`: an optional ordinal name, at most 256 UTF-8 bytes, that selects a typed + handler. +- `Headers`: optional named byte arrays with nonempty keys and non-null values. +- `DeliverAt`: an optional time on the Durable Jobs clock. The receiver accepts the + message immediately and processes it once that clock reaches `DeliverAt`. A message + with `DeliverAt` counts toward `MaxCapacity` from its acceptance until it completes. + +`OutboxMessage` pairs an envelope with its receiver. `InboxMessage` carries the same +envelope and the logical send's `SentAt` time at the receiver. Keep published payload +and header arrays unchanged. + +`MessageId` wraps a `HierarchicalKey`. Create one with `new MessageId(key)` or +`MessageId.Create(segments)`, and read the key through `Key`. `HierarchicalKey` wraps +its canonical escaped path, and equality and hashing of both types compare that path +ordinally. Use `HierarchicalKey.Create`, `CreateChildKey`, and `Append` to build keys, +and `GetParent`, `IsParentOf`, `IsChildOf`, `IsAncestorOf`, and the segment enumerator +to navigate them. + +## Handle messages + +`IInbox.Register(IInboxHandler)` adds a handler. Each message goes to the first +registered handler whose `CanHandle(InboxMessage)` returns `true`. Selection reads +only the message; the selected handler finishes it in `HandleAsync`. Handlers can be +registered at any time; registering during construction or activation makes them +available to recovered messages. + +`InboxExtensions.Register` binds a `MessageType` subject to a typed method: + +```csharp +inbox.Register(reserve, HandleReserve); +inbox.Register(reserve, this, static (request, grain, context) => grain.HandleReserve(request, context)); +``` + +Overloads accept synchronous or asynchronous delegates, with or without a `TArg` +argument for static, allocation-free delegates. + +A handler finishes each message with `IInboxHandlerContext.Complete()` or +`Fail(reason)`: + +- `Complete()` stages the message's removal and completion record. +- `Fail(reason)` moves the message to the inbox dead letters and records its ID as + processed. +- A handler exception, or a return without either call, leaves the outcome unknown, + so the message retries with exponential backoff until `InboxRetryTimeout` (6 hours) + after its first failure, then moves to the dead letters. +- If the attempt staged journaled changes before failing, they can't commit with the + retry: the grain deactivates, and recovery from the journal discards them. A grain + whose state can't report its changes, such as journaled `IPersistentState`, recovers + this way after every failed attempt. +- Recovery after an activation stops without recording an attempt's outcome, through + such a failure or a crash, charges one attempt to each due message, so backoff and + `InboxRetryTimeout` still apply. A graceful deactivation journals a clean stop when + messages are pending, so it charges nothing. A crash can charge a message that hadn't + run, so attempts overcount by at most one per crash, and a charge alone never + dead-letters a message whose retries hadn't started. +- Calling both with conflicting outcomes deactivates the grain. + +The inbox persists each batch's outcomes, business changes, and outgoing messages in +one journal write. Perform asynchronous and fallible work with local values first, +then send, apply state changes and complete in one synchronous block with no await +after the first change. +## Send messages + +`IOutbox.Send(OutboxMessage)` stages a message. `OutboxExtensions` adds typed helpers: + +- `Send(type, messageId, receiver, body, deliverAt)` encodes and stages one + message, optionally scheduled; a batch overload stages several. +- `SendReply(type, context, destination, body)` stages a reply whose ID is the + received message ID's `result` child, sent to a destination the application + carries in its request. + +`MessageType.Create` and `OutboxMessage.ToInboxMessage(sentAt)` build messages for direct +inbox delivery. Sending equivalent content (destination, subject, payload, and +`DeliverAt`) with a pending ID coalesces with the original message; different content +throws `InvalidOperationException`. + +When the receiver is the sending grain, `Send` stages the message's acceptance in +the grain's own inbox instead of the outbox, applying the inbox's identity rules: an +ID that is pending with equivalent content or has a retained completion record +is a duplicate, and a pending ID with different content throws +`InvalidOperationException`. The acceptance persists in the caller's next journal +write, atomically with its other staged state, and the inbox processes the message +once that write completes. Because `Send` is synchronous, the grain's inbox admits the +message regardless of `MaxCapacity` and of handler registration; an unscheduled +self-sent message still occupies capacity for other senders until it completes, and +processing waits for a handler. + +## Journal payloads once + +When a payload already lives in the grain's own journaled state, an outbox source or +an inbox sink keeps that state as its only journaled copy. Register both during +construction or activation setup, before journal recovery. + +- `IOutbox.Register(IOutboxMessageSource)` adds a source, and + `IOutbox.Send(messageId)` stages a reference entry with the message ID and delivery + state. The first source whose `TryGetMessage` returns the message owns it; the + outbox calls `TryGetMessage` to validate the send and before each delivery attempt, + and the source returns the same `OutboxMessage` while it provides the message. Every + delivery rule above applies. `TryGetMessage` returns `false` for a message that + isn't the source's. A source withdraws a pending message with + `IOutbox.Withdraw(messageId)`, staged in the same journal write that releases its + payload: the outbox makes no further attempt and doesn't call `OnRetired`, and an + attempt already in flight can still deliver it. When the message retires, the outbox + calls `OnRetired` with an `OutboxMessageRetirement` whose `Status` is `Delivered` + (`Accepted` or `Duplicate`), `Rejected` (the receiver dead-lettered it) or + `DeadLettered` (the outbox dead-lettered it), whose `Reason` carries the dead + letter's reason, and whose `Code` carries the code of a `Rejected` message's + receiver, while it stages the removal, so the source's changes commit in the same + journal write. An outbox dead letter stores its own copy of the materialized + message, so the source can release its payload in `OnRetired`. After the send, a + source that throws, returns an invalid message, such as one with a different ID, or + no longer provides its message dead-letters that entry alone in the same journal + write as its `DeadLettered` retirement; the grain keeps processing its other + messages. That dead letter carries only the message ID, with an empty payload and a + default receiver, and its reason records the failure. A recovered entry that no + source provides is dead-lettered the same way, without an owner to notify. Sending a + pending reference again coalesces; an ID pending as an ordinary message, or one + which no source provides, throws `InvalidOperationException`, and an exception or + invalid message from the source during `Send` propagates without staging anything. + A source message addressed to the sending grain throws `ArgumentException`. A + recovered reference fails the activation when no source is registered. + `OnRetired` runs once per entry in the current activation. If the activation fails + before the retirement write commits, recovery restores the pending entry and the + source's prior journaled state, and retiring the recovered entry invokes the + callback again. Stage durable callback effects in the same journal write as + retirement; make effects outside that write idempotent across repeated calls. +- `IInbox.Register(IInboxMessageSink)` adds a sink. After validation and + deduplication, the inbox offers each message, including self-sent and scheduled + messages, to the sinks in order. The first sink whose `TryAccept` returns `true` + reports the `DeliveryResult`: + - `Accepted`: the sink staged the message in its own state. The inbox journals + only the processed record in the same write and returns `Accepted` once it is + durable. + - `Duplicate`: the sink already holds the message. + - `Backpressured`: the sink is at capacity, so the sender retries. A self-sent + message fails its `Send` instead. + - `DeadLettered`: the inbox dead-letters the message with the sink's reason and + optional application-defined `Code`, such as `orders/duplicate-order`, and the + sender's source receives both in a `Rejected` retirement. The processed record + keeps the reason and code, so a retained duplicate, such as + a retry whose first answer was lost, is rejected the same way. This is the only + permanent rejection. + + A sink exception, or any other status, leaves the outcome unknown: the inbox stages + nothing and the delivery fails, so the sender's outbox retries it with backoff + until `OutboxRetryTimeout`, and a self-sent message fails its `Send`. + Sink-owned messages never occupy capacity and have no inbox retry lifecycle; the + sink honors their `DeliverAt`. Declined messages follow the handler path once + `DeliverAt` passes. After its processed record is removed, a message at or below + the durable admission floor receives `Expired` before reaching the sink. +- `IInbox.Register(IInboxWorkSource)` adds a work source: a component, such as a + sink, that holds durable work in the grain's journal. `GetWakeupDelay` returns how + soon a deactivated grain must wake to resume that work, or `null` when none is + pending. Every journal write that leaves work pending schedules the messaging + wake-up before it is stored, and the wake-up reactivates the grain until the work + clears. The source's owner resumes its work on activation. + +## Delivery and processing + +Staged outgoing messages are delivered only after the sender's journal write +completes. Accepted inbox messages are processed only after the receiver's journal +write completes and, for a message with `DeliverAt`, once the Durable Jobs clock +reaches that time. Scheduled messages are delivered and accepted immediately, and a +scheduled message a grain sends to itself is accepted in the sending write; each +stays in the receiver's journal until it is due. Each receiver returns a +`DeliveryResult`: + +- `Accepted`: the message is durable in the inbox, or, for a message an inbox sink + accepted, in the sink's state together with its processed record. +- `Duplicate`: the message ID is pending or has a retained processed record. A + duplicate of a message that a sink rejected receives that rejection instead. +- `Backpressured`: the inbox holds `MaxCapacity` pending messages. The inbox admits + messages its grain sends to itself regardless of `MaxCapacity`. +- `HandlerNotFound`: the receiver has no registered handlers. +- `DeadLettered`: the receiver durably rejected the message and moved it to its + dead letters, with a reason and an optional `Code`. +- `Expired`: the message was sent before the receiver's deduplication window and + the receiver retains no outcome for it. This attempt stages no message effects; + an earlier attempt can have been accepted, so the delivery outcome is unknown. + +`Accepted`, `Duplicate`, and `DeadLettered` remove the outbox entry. `Expired` +keeps a sender dead letter and retires the entry with its earlier outcome unknown. +Any other outcome schedules a durable retry with exponential backoff, and +the outbox keeps up to `OutboxBatchSize` attempts in flight, so a slow receiver +delays only its own messages. Backpressure shares the send's absolute retry budget. +The outbox logs `HandlerNotFound` as an error. Other failures, including +calls which fail without an answer during an outage, dead-letter the message once +`OutboxRetryTimeout` (6 hours) elapses after it was sent. Retry deadlines and +backoff follow the Durable Jobs clock. Delivery order is unspecified. + +Each activation runs one messaging pump on its scheduler, processing inbox batches +and delivering outbox messages. One durable job per grain wakes the pump while either +queue or a registered work source holds work. Journaled state determines which +messages are due, so each wake-up processes exactly the ready work. A live activation +processes due work as soon as it is durable; the job covers a failed activation, so +work that is already due resumes within about a second of recovery. The job retries +indefinitely with backoff while activation fails, and every activation re-arms it +from recovered state, so committed work always keeps a wake-up. +## State + +The inbox and the outbox are event-sourced journal states named +`orleans.messaging.inbox` and `orleans.messaging.outbox`, and work with the JSON and +Orleans binary journal formats. They persist alongside the grain's other journaled +state. After the grain's state is deleted, both start empty and continue working. + +Completed message IDs become eligible for removal after `DeduplicationWindow` +from their latest observed `SentAt`. Acknowledged snapshots and recovery reclaim +eligible records once the admission floor covers their send time. Dead letters are +retained up to `DeadLetterRetentionPeriod` and `MaxRetainedDeadLetters` for each queue. +Dead-letter timestamps and retention follow the grain's `TimeProvider`. + +## Diagnostics + +`IMessagingDiagnostics` exposes `InboxDeadLetters` and `OutboxDeadLetters`, with each +record's message, reason, attempt count, and dead-letter time. +`RemoveInboxDeadLetter` and `RemoveOutboxDeadLetter` remove records, durably at the +grain's next journal write. + +The `Microsoft.Orleans` meter publishes counters for received, processed, sent, and +delivered messages tagged by grain type and status; histograms for processing and +delivery duration; and `orleans-messaging-inbox-depth` and +`orleans-messaging-outbox-depth` up-down counters for pending work. A message a grain +sends to itself counts as sent and as received by its inbox, and never contributes +to outbox delivery or depth. Reference sends count as sent; a reference entry whose +source fails to provide its message counts as delivered `source_error`. A message an +inbox sink accepts counts as received `accepted` and processed `success`; one which +the sink dead-letters counts as received and processed `dead_lettered`. + +`MessagingEvents`, the `Orleans.Messaging` diagnostic listener, reports each transition +synchronously on the activation which makes it: `InboxMessageReceiving` before the inbox +stages a delivered message, `OutboxRetryScheduled` when a failed delivery is rescheduled, +and `OutboxMessageRetired` when a message retires. Each event carries the activation's +`IGrainContext`. Tests use the events as barriers, and inject a delivery fault by throwing +from an `InboxMessageReceiving` observer, which fails the attempt like a sink exception. + +`DurableMessagingOptions` configures capacity, batch sizes, retries, deduplication, +and dead-letter retention. + +## Deduplication window + +The outbox journals each send's `SentAt` time; every delivery attempt and recovery +reuses it. `OutboxRetryTimeout`, six hours by default, bounds delivery attempts +after that time, including while the receiver is backpressured. + +The inbox retains each processed record for `DeduplicationWindow`, seven days by +default, after the latest observed `SentAt`, and then makes it eligible for removal. Before removing records, +the inbox journals an admission floor: its clock minus `DeduplicationWindow`, which +only advances. Pending and processed records determine known outcomes first. A +message the inbox no longer tracks whose `SentAt` is at or below the floor receives +`Expired` before any sink or handler runs. Recovery preserves the floor. +The inbox never accepts a delivered `SentAt` later than its clock plus +`DeduplicationWindow`. It rejects such a delivery with `ArgumentOutOfRangeException`, +staging nothing, when the message is untracked and above the admission floor, or the +time would extend its record, so the sender retries it within its retry budget. A +tracked message with an equal or earlier `SentAt` still receives its recorded +outcome, including after the receiver's clock steps backward, and an untracked +message at or below the floor still receives `Expired`. A grain's messages to itself +use its clock, or the first time above the floor after a backward clock jump. +Pending messages stay until they are processed, and completed records become eligible +for removal within twice the window of the receiver's clock, or just after the floor +when a self-send follows a backward jump. + +A resend with a later `SentAt` extends a tracked pending or completed record. +The receiver persists that extension before returning `Duplicate` or the original +durable rejection. A pending message still requires matching subject, payload, +and `DeliverAt`. + +A forward clock jump can expire undelivered work early; a backward jump can delay +cleanup and prolong retries. The journaled floor keeps removed messages expired in +both cases. Direct callers reuse a command's `SentAt` across retries. +`OutboxRetryTimeout` must be shorter than `DeduplicationWindow`, so the retry +budget ends within the receiver's retention window. + +## Recoverable terminal outcomes + +Applications which need to respond durably to expired or rejected sends register +an `IOutboxMessageSource` and stage those sends through `IOutbox.Send(messageId)`. +In `OnRetired`, stage the application's terminal outcome and any pending follow-up +work in journaled state, alongside releasing the source's payload. That work commits +atomically with outbox retirement and is recovered with the grain after a crash. +Stage a self-addressed inbox message in that callback to resume follow-up through +the messaging pump's durable wake-up and handler path after recovery. + +`Delivered` means the receiver durably accepted the message or recognized it as a +duplicate. `Rejected` carries the receiver's durable rejection. `DeadLettered` +means the outbox stopped delivering and the receiver's earlier outcome is unknown, +including when it received `Expired`. Reconcile uncertain outcomes using the +application's idempotency protocol before initiating a new send. Reuse the +original `SentAt` for retries of the same logical send. + +Dead-letter retention limits control diagnostic history. The application-owned +outcome and follow-up work have the lifetime the application gives them. +## Learn more + +See the [messaging guide](https://dotnet.github.io/orleans/docs/grains/messaging/) +for guarantees, recipes, idempotency, and operations. Compiled examples live in the +[documentation snippets](../../docs/site/src/content/docs/snippets/compiled/Grains/). diff --git a/src/api/Orleans.Journaling/Orleans.Journaling.cs b/src/api/Orleans.Journaling/Orleans.Journaling.cs index 396a1692cad..ae3d5f361d8 100644 --- a/src/api/Orleans.Journaling/Orleans.Journaling.cs +++ b/src/api/Orleans.Journaling/Orleans.Journaling.cs @@ -307,6 +307,7 @@ public partial interface IStateMachine { void OnRecoveryCompleted(); void OnWriteCompleted(); + System.Threading.Tasks.ValueTask PrepareWriteAsync(System.Threading.CancellationToken cancellationToken = default); void ReplayEntry(JournalEntry entry, JournalReplayContext context); void Reset(JournalStreamWriter writer); void WritePendingEntries(JournalStreamWriter writer); @@ -424,6 +425,12 @@ public sealed partial class JournaledStateManagerOptions public System.TimeSpan RetirementGracePeriod { get { throw null; } set { } } } + [GenerateSerializer] + public sealed partial class JournaledStatePreCommitException : System.Exception + { + public JournaledStatePreCommitException(System.Exception innerException) { } + } + public readonly ref partial struct JournalEntry { private readonly object _dummy; @@ -800,10 +807,36 @@ public void WriteField(ref global::Orleans.Serialization.Buffers. where TBufferWriter : System.Buffers.IBufferWriter { } } + [System.CodeDom.Compiler.GeneratedCode("OrleansCodeGen", "10.0.0.0")] + [System.ComponentModel.EditorBrowsable(System.ComponentModel.EditorBrowsableState.Never)] + [System.Diagnostics.CodeAnalysis.ExcludeFromCodeCoverage] + public sealed partial class Codec_JournaledStatePreCommitException : global::Orleans.Serialization.Codecs.IFieldCodec, global::Orleans.Serialization.Codecs.IFieldCodec + { + public Codec_JournaledStatePreCommitException(global::Orleans.Serialization.Serializers.IBaseCodec _baseTypeSerializer, global::Orleans.Serialization.Activators.IActivator _activator) { } + + public void Deserialize(ref global::Orleans.Serialization.Buffers.Reader reader, global::Orleans.Journaling.JournaledStatePreCommitException instance) { } + + public global::Orleans.Journaling.JournaledStatePreCommitException ReadValue(ref global::Orleans.Serialization.Buffers.Reader reader, global::Orleans.Serialization.WireProtocol.Field field) { throw null; } + + public void Serialize(ref global::Orleans.Serialization.Buffers.Writer writer, global::Orleans.Journaling.JournaledStatePreCommitException instance) + where TBufferWriter : System.Buffers.IBufferWriter { } + + public void WriteField(ref global::Orleans.Serialization.Buffers.Writer writer, uint fieldIdDelta, System.Type expectedType, global::Orleans.Journaling.JournaledStatePreCommitException value) + where TBufferWriter : System.Buffers.IBufferWriter { } + } + [System.CodeDom.Compiler.GeneratedCode("OrleansCodeGen", "10.0.0.0")] [System.ComponentModel.EditorBrowsable(System.ComponentModel.EditorBrowsableState.Never)] [System.Diagnostics.CodeAnalysis.ExcludeFromCodeCoverage] public sealed partial class Copier_DurableTaskCompletionSourceState : global::Orleans.Serialization.Cloning.ShallowCopier> { } + + [System.CodeDom.Compiler.GeneratedCode("OrleansCodeGen", "10.0.0.0")] + [System.ComponentModel.EditorBrowsable(System.ComponentModel.EditorBrowsableState.Never)] + [System.Diagnostics.CodeAnalysis.ExcludeFromCodeCoverage] + public sealed partial class Copier_JournaledStatePreCommitException : global::Orleans.Serialization.GeneratedCodeHelpers.OrleansGeneratedCodeHelper.ExceptionCopier + { + public Copier_JournaledStatePreCommitException(global::Orleans.Serialization.Serializers.ICodecProvider codecProvider) : base(default(Serialization.Serializers.ICodecProvider)!) { } + } } \ No newline at end of file diff --git a/src/api/Orleans.Messaging/Orleans.Messaging.cs b/src/api/Orleans.Messaging/Orleans.Messaging.cs new file mode 100644 index 00000000000..620150fff04 --- /dev/null +++ b/src/api/Orleans.Messaging/Orleans.Messaging.cs @@ -0,0 +1,652 @@ +//------------------------------------------------------------------------------ +// +// This code was generated by a tool. +// +// Changes to this file may cause incorrect behavior and will be lost if +// the code is regenerated. +// +//------------------------------------------------------------------------------ +namespace Orleans.Hosting +{ + public static partial class MessageTypeExtensions + { + public static Microsoft.Extensions.DependencyInjection.IServiceCollection AddMessageType(this Microsoft.Extensions.DependencyInjection.IServiceCollection services, string subject) { throw null; } + } + + public static partial class MessagingExtensions + { + public static Microsoft.Extensions.DependencyInjection.IServiceCollection AddMessaging(this Microsoft.Extensions.DependencyInjection.IServiceCollection services, System.Action? configureOptions = null) { throw null; } + + public static ISiloBuilder AddMessaging(this ISiloBuilder builder, System.Action? configureOptions = null) { throw null; } + } +} + +namespace Orleans.Messaging +{ + [GenerateSerializer] + [Alias("Orleans.Messaging.DeadLetter`1")] + public sealed partial class DeadLetter + { + [Id(3)] + public int AttemptCount { get { throw null; } init { } } + + [Id(4)] + public string? Code { get { throw null; } init { } } + + [Id(1)] + public System.DateTimeOffset DeadLetteredAt { get { throw null; } init { } } + + [Id(0)] + public required TMessage Message { get { throw null; } init { } } + + [Id(2)] + public required string Reason { get { throw null; } init { } } + } + + [GenerateSerializer] + [Alias("Orleans.Messaging.DeliveryResult")] + public readonly partial struct DeliveryResult + { + private readonly object _dummy; + private readonly int _dummyPrimitive; + [Id(2)] + public string? Code { get { throw null; } init { } } + + [Id(1)] + public string? Message { get { throw null; } init { } } + + [Id(0)] + public DeliveryStatus Status { get { throw null; } init { } } + + public static DeliveryResult Accepted() { throw null; } + + public static DeliveryResult Backpressured() { throw null; } + + public static DeliveryResult DeadLettered(string reason, string? code = null) { throw null; } + + public static DeliveryResult Duplicate() { throw null; } + + public static DeliveryResult Expired() { throw null; } + + public static DeliveryResult HandlerNotFound() { throw null; } + } + + public enum DeliveryStatus + { + Accepted = 1, + Duplicate = 2, + Backpressured = 3, + HandlerNotFound = 4, + DeadLettered = 5, + Expired = 6 + } + + [GenerateSerializer] + [Alias("Orleans.Messaging.Envelope")] + public readonly partial struct Envelope + { + private readonly object _dummy; + private readonly int _dummyPrimitive; + [Id(4)] + [System.Text.Json.Serialization.JsonIgnore(Condition = System.Text.Json.Serialization.JsonIgnoreCondition.WhenWritingNull)] + public System.DateTimeOffset? DeliverAt { get { throw null; } init { } } + + [Id(3)] + public System.Collections.Generic.Dictionary? Headers { get { throw null; } init { } } + + [Id(0)] + public required MessageId MessageId { get { throw null; } init { } } + + [Id(1)] + public required byte[] Payload { get { throw null; } init { } } + + [Id(2)] + public string? Subject { get { throw null; } init { } } + } + + [Immutable] + [Alias("Orleans.Messaging.HierarchicalKey")] + [System.Text.Json.Serialization.JsonConverter(typeof(HierarchicalKeyJsonConverter))] + public readonly partial struct HierarchicalKey : System.ISpanFormattable, System.IFormattable, System.IEquatable, System.IParsable, System.ISpanParsable + { + private readonly object _dummy; + private readonly int _dummyPrimitive; + public const char EscapeCharacter = '\\'; + public const char SegmentSeparator = '/'; + public bool IsDefault { get { throw null; } } + + public int Length { get { throw null; } } + + public readonly HierarchicalKey Append(HierarchicalKey suffix) { throw null; } + + public static HierarchicalKey Create(scoped params System.ReadOnlySpan values) { throw null; } + + public static HierarchicalKey Create(string value) { throw null; } + + public readonly HierarchicalKey CreateChildKey(string value) { throw null; } + + public readonly bool Equals(HierarchicalKey other) { throw null; } + + public override readonly bool Equals(object? obj) { throw null; } + + public readonly SegmentEnumerator GetEnumerator() { throw null; } + + public override readonly int GetHashCode() { throw null; } + + public readonly HierarchicalKey? GetParent() { throw null; } + + public readonly bool IsAncestorOf(HierarchicalKey other) { throw null; } + + public readonly bool IsChildOf(HierarchicalKey other) { throw null; } + + public readonly bool IsParentOf(HierarchicalKey other) { throw null; } + + public static bool operator ==(HierarchicalKey left, HierarchicalKey right) { throw null; } + + public static bool operator !=(HierarchicalKey left, HierarchicalKey right) { throw null; } + + static HierarchicalKey System.ISpanParsable.Parse(System.ReadOnlySpan s, System.IFormatProvider? provider) { throw null; } + + static HierarchicalKey System.IParsable.Parse(string s, System.IFormatProvider? provider) { throw null; } + + public override readonly string ToString() { throw null; } + + public readonly string ToString(string? format, System.IFormatProvider? formatProvider) { throw null; } + + public readonly bool TryFormat(System.Span destination, out int charsWritten, System.ReadOnlySpan format, System.IFormatProvider? provider) { throw null; } + + static bool System.ISpanParsable.TryParse(System.ReadOnlySpan s, System.IFormatProvider? provider, out HierarchicalKey result) { throw null; } + + static bool System.IParsable.TryParse(string? s, System.IFormatProvider? provider, out HierarchicalKey result) { throw null; } + + public sealed partial class HierarchicalKeyJsonConverter : System.Text.Json.Serialization.JsonConverter + { + public override HierarchicalKey Read(ref System.Text.Json.Utf8JsonReader reader, System.Type typeToConvert, System.Text.Json.JsonSerializerOptions options) { throw null; } + + public override HierarchicalKey ReadAsPropertyName(ref System.Text.Json.Utf8JsonReader reader, System.Type typeToConvert, System.Text.Json.JsonSerializerOptions options) { throw null; } + + public override void Write(System.Text.Json.Utf8JsonWriter writer, HierarchicalKey value, System.Text.Json.JsonSerializerOptions options) { } + + public override void WriteAsPropertyName(System.Text.Json.Utf8JsonWriter writer, HierarchicalKey value, System.Text.Json.JsonSerializerOptions options) { } + } + + public ref partial struct SegmentEnumerator + { + private object _dummy; + private int _dummyPrimitive; + public System.ReadOnlySpan Current { get { throw null; } } + + public bool MoveNext() { throw null; } + } + } + + public partial interface IInbox + { + IInbox Register(IInboxHandler handler); + IInbox Register(IInboxMessageSink sink); + IInbox Register(IInboxWorkSource source); + } + + public partial interface IInboxHandler + { + bool CanHandle(InboxMessage message); + System.Threading.Tasks.ValueTask HandleAsync(IInboxHandlerContext context, System.Threading.CancellationToken cancellationToken); + } + + public partial interface IInboxHandlerContext + { + InboxMessage Message { get; } + + void Complete(); + void Fail(string reason); + } + + public partial interface IInboxMessageSink + { + bool TryAccept(InboxMessage message, out DeliveryResult result); + } + + public partial interface IInboxWorkSource + { + System.TimeSpan? GetWakeupDelay(); + } + + public partial interface IMessagingDiagnostics + { + System.Collections.Generic.IReadOnlyList> InboxDeadLetters { get; } + + System.Collections.Generic.IReadOnlyList> OutboxDeadLetters { get; } + + bool RemoveInboxDeadLetter(MessageId messageId); + bool RemoveOutboxDeadLetter(MessageId messageId); + } + + [Alias("Orleans.Messaging.IMessagingExtension")] + public partial interface IMessagingExtension : Runtime.IGrainExtension, Runtime.IAddressable + { + [Alias("DeliverAsync")] + [Concurrency.AlwaysInterleave] + System.Threading.Tasks.ValueTask DeliverAsync(InboxMessage message, System.Threading.CancellationToken cancellationToken = default); + } + + public partial interface IMessagingGrain + { + } + + public static partial class InboxExtensions + { + public static IInbox Register(this IInbox inbox, MessageType messageType, System.Action handler) { throw null; } + + public static IInbox Register(this IInbox inbox, MessageType messageType, System.Func handler) { throw null; } + + public static IInbox Register(this IInbox inbox, MessageType messageType, TArg argument, System.Action handler) { throw null; } + + public static IInbox Register(this IInbox inbox, MessageType messageType, TArg argument, System.Func handler) { throw null; } + } + + [GenerateSerializer] + [Alias("Orleans.Messaging.InboxMessage")] + public readonly partial struct InboxMessage + { + private readonly int _dummyPrimitive; + [Id(0)] + public required Envelope Envelope { get { throw null; } init { } } + + [Id(1)] + public required System.DateTimeOffset SentAt { get { throw null; } init { } } + } + + public partial interface IOutbox + { + IOutbox Register(IOutboxMessageSource source); + void Send(MessageId messageId); + void Send(OutboxMessage message); + bool Withdraw(MessageId messageId); + } + + public partial interface IOutboxMessageSource + { + void OnRetired(MessageId messageId, OutboxMessageRetirement retirement); + bool TryGetMessage(MessageId messageId, out OutboxMessage message); + } + + [Immutable] + [Alias("Orleans.Messaging.MessageId")] + [System.Text.Json.Serialization.JsonConverter(typeof(MessageIdJsonConverter))] + public readonly partial struct MessageId : System.ISpanFormattable, System.IFormattable, System.IEquatable, System.IParsable, System.ISpanParsable + { + private readonly object _dummy; + private readonly int _dummyPrimitive; + public MessageId(HierarchicalKey key) { } + + public bool IsDefault { get { throw null; } } + + public HierarchicalKey Key { get { throw null; } } + + public static MessageId Create(scoped params System.ReadOnlySpan segments) { throw null; } + + public readonly bool Equals(MessageId other) { throw null; } + + public override readonly bool Equals(object? obj) { throw null; } + + public override readonly int GetHashCode() { throw null; } + + public static bool operator ==(MessageId left, MessageId right) { throw null; } + + public static bool operator !=(MessageId left, MessageId right) { throw null; } + + static MessageId System.ISpanParsable.Parse(System.ReadOnlySpan s, System.IFormatProvider? provider) { throw null; } + + static MessageId System.IParsable.Parse(string s, System.IFormatProvider? provider) { throw null; } + + public override readonly string ToString() { throw null; } + + public readonly string ToString(string? format, System.IFormatProvider? formatProvider) { throw null; } + + public readonly bool TryFormat(System.Span destination, out int charsWritten, System.ReadOnlySpan format, System.IFormatProvider? provider) { throw null; } + + static bool System.ISpanParsable.TryParse(System.ReadOnlySpan s, System.IFormatProvider? provider, out MessageId result) { throw null; } + + static bool System.IParsable.TryParse(string? s, System.IFormatProvider? provider, out MessageId result) { throw null; } + + public sealed partial class MessageIdJsonConverter : System.Text.Json.Serialization.JsonConverter + { + public override MessageId Read(ref System.Text.Json.Utf8JsonReader reader, System.Type typeToConvert, System.Text.Json.JsonSerializerOptions options) { throw null; } + + public override MessageId ReadAsPropertyName(ref System.Text.Json.Utf8JsonReader reader, System.Type typeToConvert, System.Text.Json.JsonSerializerOptions options) { throw null; } + + public override void Write(System.Text.Json.Utf8JsonWriter writer, MessageId value, System.Text.Json.JsonSerializerOptions options) { } + + public override void WriteAsPropertyName(System.Text.Json.Utf8JsonWriter writer, MessageId value, System.Text.Json.JsonSerializerOptions options) { } + } + } + + public sealed partial class MessageType + { + public MessageType(string subject, Serialization.Serializer serializer) { } + + public MessageType(string subject, Serialization.Serializer serializer) { } + + public string Subject { get { throw null; } } + + public OutboxMessage Create(MessageId messageId, Runtime.GrainId receiverId, T body, System.DateTimeOffset? deliverAt = null) { throw null; } + + public T Decode(Envelope envelope) { throw null; } + } + + public static partial class OutboxExtensions + { + public static void Send(this IOutbox outbox, MessageType messageType, MessageId messageId, Runtime.GrainId destination, T body, System.DateTimeOffset? deliverAt = null) { } + + public static void Send(this IOutbox outbox, MessageType messageType, MessageId messageId, Runtime.IAddressable destination, T body, System.DateTimeOffset? deliverAt = null) { } + + public static void Send(this IOutbox outbox, MessageType messageType, System.Collections.Generic.IEnumerable<(MessageId MessageId, Runtime.GrainId Destination, T Body)> messages) { } + + public static void SendReply(this IOutbox outbox, MessageType messageType, IInboxHandlerContext context, Runtime.GrainId destination, T body) { } + + public static void SendReply(this IOutbox outbox, MessageType messageType, IInboxHandlerContext context, Runtime.IAddressable destination, T body) { } + } + + [GenerateSerializer] + [Alias("Orleans.Messaging.OutboxMessage")] + public readonly partial struct OutboxMessage + { + [Id(0)] + public required Envelope Envelope { get { throw null; } init { } } + + [Id(1)] + public required Runtime.GrainId ReceiverId { get { throw null; } init { } } + + public readonly InboxMessage ToInboxMessage(System.DateTimeOffset sentAt) { throw null; } + } + + public readonly partial struct OutboxMessageRetirement + { + private readonly object _dummy; + private readonly int _dummyPrimitive; + public string? Code { get { throw null; } init { } } + + public string? Reason { get { throw null; } init { } } + + public OutboxMessageRetirementStatus Status { get { throw null; } init { } } + + public static OutboxMessageRetirement DeadLettered(string reason) { throw null; } + + public static OutboxMessageRetirement Delivered() { throw null; } + + public static OutboxMessageRetirement Rejected(string reason, string? code = null) { throw null; } + } + + public enum OutboxMessageRetirementStatus + { + Delivered = 1, + DeadLettered = 2, + Rejected = 3 + } +} + +namespace Orleans.Messaging.Configuration +{ + public partial class DurableMessagingOptions + { + public System.TimeSpan DeadLetterRetentionPeriod { get { throw null; } set { } } + + public System.TimeSpan DeduplicationWindow { get { throw null; } set { } } + + public int InboxBatchSize { get { throw null; } set { } } + + public System.TimeSpan InboxRetryTimeout { get { throw null; } set { } } + + public int MaxCapacity { get { throw null; } set { } } + + public int MaxRetainedDeadLetters { get { throw null; } set { } } + + public int OutboxBatchSize { get { throw null; } set { } } + + public System.TimeSpan OutboxRetryTimeout { get { throw null; } set { } } + + public System.TimeSpan RetryInitialInterval { get { throw null; } set { } } + + public void Validate() { } + } +} + +namespace Orleans.Messaging.Diagnostics +{ + public static partial class MessagingEvents + { + public const string ListenerName = "Orleans.Messaging"; + public static System.IObservable AllEvents { get { throw null; } } + + public sealed partial class InboxMessageReceiving : MessagingEvent + { + public readonly InboxMessage Message; + public InboxMessageReceiving(Runtime.IGrainContext grainContext, InboxMessage message) : base(default!) { } + } + + public abstract partial class MessagingEvent + { + public readonly Runtime.IGrainContext GrainContext; + protected MessagingEvent(Runtime.IGrainContext grainContext) { } + } + + public sealed partial class OutboxMessageRetired : MessagingEvent + { + public readonly MessageId MessageId; + public readonly OutboxMessageRetirement Retirement; + public OutboxMessageRetired(Runtime.IGrainContext grainContext, MessageId messageId, OutboxMessageRetirement retirement) : base(default!) { } + } + + public sealed partial class OutboxRetryScheduled : MessagingEvent + { + public readonly int AttemptCount; + public readonly string Error; + public readonly MessageId MessageId; + public readonly System.DateTimeOffset NextAttemptAt; + public readonly Runtime.GrainId ReceiverId; + public OutboxRetryScheduled(Runtime.IGrainContext grainContext, MessageId messageId, Runtime.GrainId receiverId, int attemptCount, System.DateTimeOffset nextAttemptAt, string error) : base(default!) { } + } + } +} + +namespace OrleansCodeGen.Orleans.Messaging +{ + [System.CodeDom.Compiler.GeneratedCode("OrleansCodeGen", "10.0.0.0")] + [System.ComponentModel.EditorBrowsable(System.ComponentModel.EditorBrowsableState.Never)] + [System.Diagnostics.CodeAnalysis.ExcludeFromCodeCoverage] + public sealed partial class Codec_DeadLetter : global::Orleans.Serialization.Codecs.IFieldCodec>, global::Orleans.Serialization.Codecs.IFieldCodec + { + public Codec_DeadLetter(global::Orleans.Serialization.Activators.IActivator> _activator, global::Orleans.Serialization.Serializers.ICodecProvider codecProvider) { } + + public void Deserialize(ref global::Orleans.Serialization.Buffers.Reader reader, global::Orleans.Messaging.DeadLetter instance) { } + + public global::Orleans.Messaging.DeadLetter ReadValue(ref global::Orleans.Serialization.Buffers.Reader reader, global::Orleans.Serialization.WireProtocol.Field field) { throw null; } + + public void Serialize(ref global::Orleans.Serialization.Buffers.Writer writer, global::Orleans.Messaging.DeadLetter instance) + where TBufferWriter : System.Buffers.IBufferWriter { } + + public void WriteField(ref global::Orleans.Serialization.Buffers.Writer writer, uint fieldIdDelta, System.Type expectedType, global::Orleans.Messaging.DeadLetter value) + where TBufferWriter : System.Buffers.IBufferWriter { } + } + + [System.CodeDom.Compiler.GeneratedCode("OrleansCodeGen", "10.0.0.0")] + [System.ComponentModel.EditorBrowsable(System.ComponentModel.EditorBrowsableState.Never)] + [System.Diagnostics.CodeAnalysis.ExcludeFromCodeCoverage] + public sealed partial class Codec_DeliveryResult : global::Orleans.Serialization.Codecs.IFieldCodec, global::Orleans.Serialization.Codecs.IFieldCodec, global::Orleans.Serialization.Serializers.IValueSerializer, global::Orleans.Serialization.Serializers.IValueSerializer + { + public Codec_DeliveryResult(global::Orleans.Serialization.Serializers.ICodecProvider codecProvider) { } + + public void Deserialize(ref global::Orleans.Serialization.Buffers.Reader reader, scoped ref global::Orleans.Messaging.DeliveryResult instance) { } + + public global::Orleans.Messaging.DeliveryResult ReadValue(ref global::Orleans.Serialization.Buffers.Reader reader, global::Orleans.Serialization.WireProtocol.Field field) { throw null; } + + public void Serialize(ref global::Orleans.Serialization.Buffers.Writer writer, scoped ref global::Orleans.Messaging.DeliveryResult instance) + where TBufferWriter : System.Buffers.IBufferWriter { } + + public void WriteField(ref global::Orleans.Serialization.Buffers.Writer writer, uint fieldIdDelta, System.Type expectedType, global::Orleans.Messaging.DeliveryResult value) + where TBufferWriter : System.Buffers.IBufferWriter { } + } + + [System.CodeDom.Compiler.GeneratedCode("OrleansCodeGen", "10.0.0.0")] + [System.ComponentModel.EditorBrowsable(System.ComponentModel.EditorBrowsableState.Never)] + [System.Diagnostics.CodeAnalysis.ExcludeFromCodeCoverage] + public sealed partial class Codec_Envelope : global::Orleans.Serialization.Codecs.IFieldCodec, global::Orleans.Serialization.Codecs.IFieldCodec, global::Orleans.Serialization.Serializers.IValueSerializer, global::Orleans.Serialization.Serializers.IValueSerializer + { + public Codec_Envelope(global::Orleans.Serialization.Activators.IActivator _activator, global::Orleans.Serialization.Serializers.ICodecProvider codecProvider) { } + + public void Deserialize(ref global::Orleans.Serialization.Buffers.Reader reader, scoped ref global::Orleans.Messaging.Envelope instance) { } + + public global::Orleans.Messaging.Envelope ReadValue(ref global::Orleans.Serialization.Buffers.Reader reader, global::Orleans.Serialization.WireProtocol.Field field) { throw null; } + + public void Serialize(ref global::Orleans.Serialization.Buffers.Writer writer, scoped ref global::Orleans.Messaging.Envelope instance) + where TBufferWriter : System.Buffers.IBufferWriter { } + + public void WriteField(ref global::Orleans.Serialization.Buffers.Writer writer, uint fieldIdDelta, System.Type expectedType, global::Orleans.Messaging.Envelope value) + where TBufferWriter : System.Buffers.IBufferWriter { } + } + + [System.CodeDom.Compiler.GeneratedCode("OrleansCodeGen", "10.0.0.0")] + [System.ComponentModel.EditorBrowsable(System.ComponentModel.EditorBrowsableState.Never)] + [System.Diagnostics.CodeAnalysis.ExcludeFromCodeCoverage] + public sealed partial class Codec_InboxMessage : global::Orleans.Serialization.Codecs.IFieldCodec, global::Orleans.Serialization.Codecs.IFieldCodec, global::Orleans.Serialization.Serializers.IValueSerializer, global::Orleans.Serialization.Serializers.IValueSerializer + { + public Codec_InboxMessage(global::Orleans.Serialization.Activators.IActivator _activator, global::Orleans.Serialization.Serializers.ICodecProvider codecProvider) { } + + public void Deserialize(ref global::Orleans.Serialization.Buffers.Reader reader, scoped ref global::Orleans.Messaging.InboxMessage instance) { } + + public global::Orleans.Messaging.InboxMessage ReadValue(ref global::Orleans.Serialization.Buffers.Reader reader, global::Orleans.Serialization.WireProtocol.Field field) { throw null; } + + public void Serialize(ref global::Orleans.Serialization.Buffers.Writer writer, scoped ref global::Orleans.Messaging.InboxMessage instance) + where TBufferWriter : System.Buffers.IBufferWriter { } + + public void WriteField(ref global::Orleans.Serialization.Buffers.Writer writer, uint fieldIdDelta, System.Type expectedType, global::Orleans.Messaging.InboxMessage value) + where TBufferWriter : System.Buffers.IBufferWriter { } + } + + [System.CodeDom.Compiler.GeneratedCode("OrleansCodeGen", "10.0.0.0")] + [System.ComponentModel.EditorBrowsable(System.ComponentModel.EditorBrowsableState.Never)] + [System.Diagnostics.CodeAnalysis.ExcludeFromCodeCoverage] + public sealed partial class Codec_Invokable_IMessagingExtension_GrainReference_Ext_93AB2A04 : global::Orleans.Serialization.Codecs.IFieldCodec, global::Orleans.Serialization.Codecs.IFieldCodec + { + public Codec_Invokable_IMessagingExtension_GrainReference_Ext_93AB2A04(global::Orleans.Serialization.Serializers.ICodecProvider codecProvider) { } + + public void Deserialize(ref global::Orleans.Serialization.Buffers.Reader reader, Invokable_IMessagingExtension_GrainReference_Ext_93AB2A04 instance) { } + + public Invokable_IMessagingExtension_GrainReference_Ext_93AB2A04 ReadValue(ref global::Orleans.Serialization.Buffers.Reader reader, global::Orleans.Serialization.WireProtocol.Field field) { throw null; } + + public void Serialize(ref global::Orleans.Serialization.Buffers.Writer writer, Invokable_IMessagingExtension_GrainReference_Ext_93AB2A04 instance) + where TBufferWriter : System.Buffers.IBufferWriter { } + + public void WriteField(ref global::Orleans.Serialization.Buffers.Writer writer, uint fieldIdDelta, System.Type expectedType, Invokable_IMessagingExtension_GrainReference_Ext_93AB2A04 value) + where TBufferWriter : System.Buffers.IBufferWriter { } + } + + [System.CodeDom.Compiler.GeneratedCode("OrleansCodeGen", "10.0.0.0")] + [System.ComponentModel.EditorBrowsable(System.ComponentModel.EditorBrowsableState.Never)] + [System.Diagnostics.CodeAnalysis.ExcludeFromCodeCoverage] + public sealed partial class Codec_OutboxMessage : global::Orleans.Serialization.Codecs.IFieldCodec, global::Orleans.Serialization.Codecs.IFieldCodec, global::Orleans.Serialization.Serializers.IValueSerializer, global::Orleans.Serialization.Serializers.IValueSerializer + { + public Codec_OutboxMessage(global::Orleans.Serialization.Activators.IActivator _activator, global::Orleans.Serialization.Serializers.ICodecProvider codecProvider) { } + + public void Deserialize(ref global::Orleans.Serialization.Buffers.Reader reader, scoped ref global::Orleans.Messaging.OutboxMessage instance) { } + + public global::Orleans.Messaging.OutboxMessage ReadValue(ref global::Orleans.Serialization.Buffers.Reader reader, global::Orleans.Serialization.WireProtocol.Field field) { throw null; } + + public void Serialize(ref global::Orleans.Serialization.Buffers.Writer writer, scoped ref global::Orleans.Messaging.OutboxMessage instance) + where TBufferWriter : System.Buffers.IBufferWriter { } + + public void WriteField(ref global::Orleans.Serialization.Buffers.Writer writer, uint fieldIdDelta, System.Type expectedType, global::Orleans.Messaging.OutboxMessage value) + where TBufferWriter : System.Buffers.IBufferWriter { } + } + + [System.CodeDom.Compiler.GeneratedCode("OrleansCodeGen", "10.0.0.0")] + [System.ComponentModel.EditorBrowsable(System.ComponentModel.EditorBrowsableState.Never)] + [System.Diagnostics.CodeAnalysis.ExcludeFromCodeCoverage] + public sealed partial class Copier_DeadLetter : global::Orleans.Serialization.Cloning.IDeepCopier>, global::Orleans.Serialization.Cloning.IDeepCopier + { + public Copier_DeadLetter(global::Orleans.Serialization.Activators.IActivator> _activator, global::Orleans.Serialization.Serializers.ICodecProvider codecProvider) { } + + public global::Orleans.Messaging.DeadLetter DeepCopy(global::Orleans.Messaging.DeadLetter original, global::Orleans.Serialization.Cloning.CopyContext context) { throw null; } + } + + [System.CodeDom.Compiler.GeneratedCode("OrleansCodeGen", "10.0.0.0")] + [System.ComponentModel.EditorBrowsable(System.ComponentModel.EditorBrowsableState.Never)] + [System.Diagnostics.CodeAnalysis.ExcludeFromCodeCoverage] + public sealed partial class Copier_Envelope : global::Orleans.Serialization.Cloning.IDeepCopier, global::Orleans.Serialization.Cloning.IDeepCopier + { + public Copier_Envelope(global::Orleans.Serialization.Activators.IActivator _activator, global::Orleans.Serialization.Serializers.ICodecProvider codecProvider) { } + + public global::Orleans.Messaging.Envelope DeepCopy(global::Orleans.Messaging.Envelope original, global::Orleans.Serialization.Cloning.CopyContext context) { throw null; } + } + + [System.CodeDom.Compiler.GeneratedCode("OrleansCodeGen", "10.0.0.0")] + [System.ComponentModel.EditorBrowsable(System.ComponentModel.EditorBrowsableState.Never)] + [System.Diagnostics.CodeAnalysis.ExcludeFromCodeCoverage] + public sealed partial class Copier_InboxMessage : global::Orleans.Serialization.Cloning.IDeepCopier, global::Orleans.Serialization.Cloning.IDeepCopier + { + public Copier_InboxMessage(global::Orleans.Serialization.Activators.IActivator _activator, global::Orleans.Serialization.Serializers.ICodecProvider codecProvider) { } + + public global::Orleans.Messaging.InboxMessage DeepCopy(global::Orleans.Messaging.InboxMessage original, global::Orleans.Serialization.Cloning.CopyContext context) { throw null; } + } + + [System.CodeDom.Compiler.GeneratedCode("OrleansCodeGen", "10.0.0.0")] + [System.ComponentModel.EditorBrowsable(System.ComponentModel.EditorBrowsableState.Never)] + [System.Diagnostics.CodeAnalysis.ExcludeFromCodeCoverage] + public sealed partial class Copier_Invokable_IMessagingExtension_GrainReference_Ext_93AB2A04 : global::Orleans.Serialization.Cloning.IDeepCopier, global::Orleans.Serialization.Cloning.IDeepCopier + { + public Copier_Invokable_IMessagingExtension_GrainReference_Ext_93AB2A04(global::Orleans.Serialization.Serializers.ICodecProvider codecProvider) { } + + public Invokable_IMessagingExtension_GrainReference_Ext_93AB2A04 DeepCopy(Invokable_IMessagingExtension_GrainReference_Ext_93AB2A04 original, global::Orleans.Serialization.Cloning.CopyContext context) { throw null; } + } + + [System.CodeDom.Compiler.GeneratedCode("OrleansCodeGen", "10.0.0.0")] + [System.ComponentModel.EditorBrowsable(System.ComponentModel.EditorBrowsableState.Never)] + [System.Diagnostics.CodeAnalysis.ExcludeFromCodeCoverage] + public sealed partial class Copier_OutboxMessage : global::Orleans.Serialization.Cloning.IDeepCopier, global::Orleans.Serialization.Cloning.IDeepCopier + { + public Copier_OutboxMessage(global::Orleans.Serialization.Activators.IActivator _activator, global::Orleans.Serialization.Serializers.ICodecProvider codecProvider) { } + + public global::Orleans.Messaging.OutboxMessage DeepCopy(global::Orleans.Messaging.OutboxMessage original, global::Orleans.Serialization.Cloning.CopyContext context) { throw null; } + } + + [System.CodeDom.Compiler.GeneratedCode("OrleansCodeGen", "10.0.0.0")] + [System.ComponentModel.EditorBrowsable(System.ComponentModel.EditorBrowsableState.Never)] + [System.Diagnostics.CodeAnalysis.ExcludeFromCodeCoverage] + [global::Orleans.CompoundTypeAlias(new[] { "inv", typeof(global::Orleans.Runtime.GrainReference), "Ext", typeof(global::Orleans.Messaging.IMessagingExtension), typeof(global::Orleans.Messaging.IMessagingExtension), "DeliverAsync" })] + public sealed partial class Invokable_IMessagingExtension_GrainReference_Ext_93AB2A04 : global::Orleans.Runtime.Request + { + public global::Orleans.Messaging.InboxMessage arg0; + public System.Threading.CancellationToken arg1; + public override bool IsCancellable { get { throw null; } } + + public override bool MayHaveGrainCancellationToken { get { throw null; } } + + public override void Dispose() { } + + public override string GetActivityName() { throw null; } + + public override object GetArgument(int index) { throw null; } + + public override int GetArgumentCount() { throw null; } + + public override System.Threading.CancellationToken GetCancellationToken() { throw null; } + + public override string GetInterfaceName() { throw null; } + + public override System.Type GetInterfaceType() { throw null; } + + public override System.Reflection.MethodInfo GetMethod() { throw null; } + + public override string GetMethodName() { throw null; } + + public override object GetTarget() { throw null; } + + protected override System.Threading.Tasks.ValueTask InvokeInner() { throw null; } + + public override void SetArgument(int index, object value) { } + + public override void SetTarget(global::Orleans.Serialization.Invocation.ITargetHolder holder) { } + + public override bool TryCancel() { throw null; } + } +} \ No newline at end of file diff --git a/test/Orleans.Journaling.Tests/StateManagerPreparationTests.cs b/test/Orleans.Journaling.Tests/StateManagerPreparationTests.cs new file mode 100644 index 00000000000..9f788961805 --- /dev/null +++ b/test/Orleans.Journaling.Tests/StateManagerPreparationTests.cs @@ -0,0 +1,349 @@ +using System.Threading.Channels; +using Microsoft.Extensions.DependencyInjection; +using Orleans.Serialization; +using Xunit; + +namespace Orleans.Journaling.Tests; + +public partial class StateManagerTests +{ + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task Preparation_RunsBeforeCaptureAndAcknowledgement(bool snapshot) + { + var storage = new CapturingStorage { IsCompactionRequested = snapshot }; + await using var manager = CreateTestSystem(storage).Manager; + List events = []; + var state = new PreparingValue("state", manager, CreateValueCodec(), events); + await manager.InitializeAsync(TestContext.Current.CancellationToken); + state["intent"] = 17; + + await manager.WriteStateAsync(TestContext.Current.CancellationToken); + + Assert.Equal(["prepare", "capture", "ack"], events); + Assert.Equal(17, state.CapturedValue); + Assert.Equal(1, state.WriteCompletedCount); + Assert.Single(snapshot ? storage.Replaces : storage.Appends); + } + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task Preparation_LaterAsynchronousStateRechecksEarlierInterleavedChanges(bool snapshot) + { + var storage = new CapturingStorage { IsCompactionRequested = snapshot }; + await using var manager = CreateTestSystem(storage).Manager; + var earlier = new PreparingValue("earlier", manager, CreateValueCodec()); + var later = new PreparingValue("later", manager, CreateValueCodec()); + var entered = NewSignal(); + var release = NewSignal(); + var prepared = false; + later.Preparation = async token => + { + if (!prepared) + { + entered.TrySetResult(); + await release.Task.WaitAsync(token); + later.PreparedValue = later.CurrentValue; + prepared = true; + } + }; + await manager.InitializeAsync(TestContext.Current.CancellationToken); + later["intent"] = 23; + var write = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + try + { + await WaitFor(entered.Task); + Assert.Equal(0, earlier.CaptureCount); + Assert.Empty(storage.Appends); + Assert.Empty(storage.Replaces); + earlier["intent"] = 41; + release.TrySetResult(); + await WaitFor(write); + Assert.Equal(41, earlier.CapturedValue); + Assert.Equal(41, earlier.PreparedValue); + Assert.Equal(2, earlier.PrepareCalls); + Assert.Equal(23, later.CapturedValue); + Assert.Equal(1, earlier.WriteCompletedCount); + + await using var recovered = CreateTestSystem(storage).Manager; + var replayed = new PreparingValue("earlier", recovered, CreateValueCodec()); + await recovered.InitializeAsync(TestContext.Current.CancellationToken); + Assert.Equal(41, replayed["intent"]); + } + finally + { + release.TrySetResult(); + } + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task Preparation_SynchronousReadyPassAndCaptureShareOneTurn(bool snapshot) + { + var scheduler = new QueuedTaskScheduler(); + await scheduler.Run(async () => + { + var storage = new CapturingStorage { IsCompactionRequested = snapshot }; + await using var manager = CreateTestSystem(storage).Manager; + var state = new PreparingValue("state", manager, CreateValueCodec()); + Task? mutation = null; + state.Preparation = _ => + { + state.PreparedValue = state.CurrentValue; + mutation ??= scheduler.Queue(() => state["intent"] = 29); + return ValueTask.CompletedTask; + }; + await manager.InitializeAsync(TestContext.Current.CancellationToken); + state["intent"] = 13; + + await manager.WriteStateAsync(TestContext.Current.CancellationToken); + await mutation!; + + Assert.Equal(13, state.CapturedValue); + Assert.Equal(29, state.CurrentValue); + await using var recovered = CreateTestSystem(storage).Manager; + var replayed = new PreparingValue("state", recovered, CreateValueCodec()); + await recovered.InitializeAsync(TestContext.Current.CancellationToken); + Assert.Equal(13, replayed["intent"]); + }); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task Preparation_FailureRetainsPendingStateForExplicitRetry(bool snapshot) + { + var storage = new CapturingStorage { IsCompactionRequested = snapshot }; + await using var manager = CreateTestSystem(storage).Manager; + var failure = new IOException("State prerequisite failed."); + var state = new PreparingValue("state", manager, CreateValueCodec()) + { + Preparation = _ => ValueTask.FromException(failure) + }; + await manager.InitializeAsync(TestContext.Current.CancellationToken); + state["intent"] = 31; + + var caught = await Assert.ThrowsAsync(() => + manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask()); + + Assert.Same(failure, caught.InnerException); + Assert.Equal(0, state.CaptureCount); + Assert.Equal(0, state.WriteCompletedCount); + Assert.Empty(storage.Appends); + Assert.Empty(storage.Replaces); + state.Preparation = null; + await manager.WriteStateAsync(TestContext.Current.CancellationToken); + Assert.Equal(31, state.CapturedValue); + Assert.Single(snapshot ? storage.Replaces : storage.Appends); + Assert.Empty(snapshot ? storage.Appends : storage.Replaces); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task Preparation_RunsForEveryStateIncludingThoseWithoutPendingChanges(bool snapshot) + { + var storage = new CapturingStorage { IsCompactionRequested = snapshot }; + await using var manager = CreateTestSystem(storage).Manager; + var changed = new PreparingValue("changed", manager, CreateValueCodec()); + var clean = new PreparingValue("clean", manager, CreateValueCodec()); + await manager.InitializeAsync(TestContext.Current.CancellationToken); + changed["intent"] = 5; + + await manager.WriteStateAsync(TestContext.Current.CancellationToken); + Assert.Equal(1, clean.PrepareCalls); + + // A state without pending changes can still veto a write whose other pending changes must not commit. + var veto = new InvalidOperationException("The pending changes are abandoned."); + clean.Preparation = _ => ValueTask.FromException(veto); + changed["intent"] = 6; + var caught = await Assert.ThrowsAsync(() => + manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask()); + Assert.Same(veto, caught.InnerException); + Assert.Equal(5, changed.CapturedValue); + Assert.Single(snapshot ? storage.Replaces : storage.Appends); + } + [Fact] + public async Task Preparation_CallerCancellationLeavesOwnedPreparationAndWriteRunning() + { + var storage = new CapturingStorage(); + await using var manager = CreateTestSystem(storage).Manager; + var state = new PreparingValue("state", manager, CreateValueCodec()); + var entered = NewSignal(); + var release = NewSignal(); + var prepared = false; + state.Preparation = async token => + { + if (!prepared) + { + entered.TrySetResult(); + await release.Task.WaitAsync(token); + state.PreparedValue = state.CurrentValue; + prepared = true; + } + }; + await manager.InitializeAsync(TestContext.Current.CancellationToken); + state["intent"] = 47; + using var caller = new CancellationTokenSource(); + var write = manager.WriteStateAsync(caller.Token).AsTask(); + try + { + await WaitFor(entered.Task); + caller.Cancel(); + await Assert.ThrowsAnyAsync(() => WaitFor(write)); + Assert.Equal(0, state.CaptureCount); + release.TrySetResult(); + await WaitFor(state.Acknowledged.Task); + Assert.Equal(47, state.CapturedValue); + Assert.Single(storage.Appends); + } + finally + { + release.TrySetResult(); + } + } + + [Fact] + public async Task Preparation_ShutdownDrainsPreparationAndPreventsCapture() + { + var storage = new CapturingStorage(); + var manager = CreateTestSystem(storage).Manager; + var state = new PreparingValue("state", manager, CreateValueCodec()); + var entered = NewSignal(); + var release = NewSignal(); + var observedCancellation = false; + state.Preparation = async token => + { + entered.TrySetResult(); + await release.Task; + observedCancellation = token.IsCancellationRequested; + state.PreparedValue = state.CurrentValue; + }; + await manager.InitializeAsync(TestContext.Current.CancellationToken); + state["intent"] = 53; + var write = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + try + { + await WaitFor(entered.Task); + var disposal = manager.DisposeAsync().AsTask(); + Assert.False(disposal.IsCompleted); + release.TrySetResult(); + await WaitFor(disposal); + await Assert.ThrowsAnyAsync(() => WaitFor(write)); + Assert.True(observedCancellation); + Assert.Equal(0, state.CaptureCount); + Assert.Equal(0, state.WriteCompletedCount); + Assert.Empty(storage.Appends); + } + finally + { + release.TrySetResult(); + await manager.DisposeAsync(); + } + } + + [Fact] + public void PreCommitFailure_RoundTripsWithOriginalCause() + { + Exception original = new JournaledStatePreCommitException(new IOException("Scheduling failed.")); + var serializer = ServiceProvider.GetRequiredService(); + + var copy = Assert.IsType(serializer.Deserialize(serializer.SerializeToArray(original))); + + Assert.Equal(original.Message, copy.Message); + Assert.Equal("Scheduling failed.", Assert.IsType(copy.InnerException).Message); + } + + private sealed class QueuedTaskScheduler : TaskScheduler + { + private readonly Channel _queue = Channel.CreateUnbounded(); + + public override int MaximumConcurrencyLevel => 1; + protected override IEnumerable? GetScheduledTasks() => null; + protected override void QueueTask(Task task) => Assert.True(_queue.Writer.TryWrite(task)); + protected override bool TryExecuteTaskInline(Task task, bool taskWasPreviouslyQueued) => false; + + public Task Queue(Action action) => + Task.Factory.StartNew(action, TestContext.Current.CancellationToken, TaskCreationOptions.DenyChildAttach, this); + + public async Task Run(Func action) + { + var task = Task.Factory.StartNew(action, TestContext.Current.CancellationToken, TaskCreationOptions.DenyChildAttach, this).Unwrap(); + while (!task.IsCompleted) + { + var next = await _queue.Reader.ReadAsync(TestContext.Current.CancellationToken).AsTask().WaitAsync(TimeSpan.FromSeconds(10)); + var previous = SynchronizationContext.Current; + SynchronizationContext.SetSynchronizationContext(null); + try + { + Assert.True(TryExecuteTask(next)); + } + finally + { + SynchronizationContext.SetSynchronizationContext(previous); + } + } + + await task; + } + } + private sealed class PreparingValue : IStateMachine, IDurableValueCommandHandler + { + private readonly IDurableValueCommandCodec _codec; + private readonly List? _events; + private JournalStreamWriter _writer; + + public PreparingValue(string name, IJournaledStateManager manager, IDurableValueCommandCodec codec, List? events = null) + { + _codec = codec; + _events = events; + manager.RegisterStateMachine(name, this); + } + public Func? Preparation { get; set; } + public int PrepareCalls { get; private set; } + public int PreparedValue { get; set; } + public int CapturedValue { get; private set; } + public int CaptureCount { get; private set; } + public int WriteCompletedCount { get; private set; } + private int _value; + public int CurrentValue => _value; + public int this[string key] + { + get => _value; + set { _codec.WriteSet(value, _writer); _value = value; } + } + void IDurableValueCommandHandler.ApplySet(int command) => _value = command; + public void ReplayEntry(JournalEntry entry, JournalReplayContext context) => + context.GetRequiredCommandCodec(entry.FormatKey, _codec).Apply(entry.Reader, this); + public void Reset(JournalStreamWriter writer) { _writer = writer; _value = 0; } + public void WritePendingEntries(JournalStreamWriter writer) => Capture(); + public void WriteSnapshot(JournalStreamWriter writer) { _codec.WriteSet(_value, writer); Capture(); } + public TaskCompletionSource Acknowledged { get; } = NewSignal(); + + public ValueTask PrepareWriteAsync(CancellationToken cancellationToken = default) + { + PrepareCalls++; + _events?.Add("prepare"); + if (Preparation is { } preparation) return preparation(cancellationToken); + PreparedValue = CurrentValue; + return ValueTask.CompletedTask; + } + + private void Capture() + { + _events?.Add("capture"); + CaptureCount++; + CapturedValue = CurrentValue; + Assert.Equal(PreparedValue, CapturedValue); + } + + public void OnWriteCompleted() + { + _events?.Add("ack"); + WriteCompletedCount++; + Acknowledged.TrySetResult(); + } + } +} diff --git a/test/Orleans.Journaling.Tests/StateManagerTests.cs b/test/Orleans.Journaling.Tests/StateManagerTests.cs index 43612e7c568..cf3a30b18f1 100644 --- a/test/Orleans.Journaling.Tests/StateManagerTests.cs +++ b/test/Orleans.Journaling.Tests/StateManagerTests.cs @@ -61,6 +61,49 @@ public async Task StateManager_RegisterState_Test() Assert.Equal(42, queue.Peek()); } + [Fact] + public async Task StagedChangeCount_AdvancesWhenStatesStageChangesAndNotWhenWritesCaptureThem() + { + var sut = CreateTestSystem(); + var manager = Assert.IsType(sut.Manager); + var codec = CodecProvider.GetCodec(); + var dictionary = new DurableDictionary("dict1", manager, new OrleansBinaryDurableDictionaryCommandCodec(CodecProvider.GetCodec(), codec, SessionPool)); + var value = new DurableValue("value1", manager, new OrleansBinaryDurableValueCommandCodec(codec, SessionPool)); + await sut.Lifecycle.OnStart(TestContext.Current.CancellationToken); + var start = manager.StagedChangeCount; + Assert.True(start >= 0); + + // A collection journals at mutation time; a value journals when a write captures it, so it counts at mutation. + dictionary.Add("key1", 1); + var afterDictionary = manager.StagedChangeCount; + Assert.True(afterDictionary > start); + value.Value = 7; + var afterValue = manager.StagedChangeCount; + Assert.True(afterValue > afterDictionary); + + await manager.WriteStateAsync(CancellationToken.None); + Assert.Equal(afterValue, manager.StagedChangeCount); + } + + [Fact] + public async Task StagedChangeCount_IsUnknownWhileAStateJournalsChangesItCannotObserve() + { + var sut = CreateTestSystem(); + var manager = Assert.IsType(sut.Manager); + IPersistentState> persistent = new JournaledPersistentState>( + "persistent", + manager, + new OrleansBinaryPersistentStateCommandCodec>(CodecProvider.GetCodec>(), SessionPool), + CodecProvider.GetActivator>()); + await sut.Lifecycle.OnStart(TestContext.Current.CancellationToken); + + // A persistent state's object is mutated in place and serialized whole at capture, so no reading can show that + // this change was staged; the count reports that it can't tell. + persistent.State.Add(1); + + Assert.True(manager.StagedChangeCount < 0); + } + [Fact] public async Task StateManager_Initialize_UsesStreamingStorageRead() { diff --git a/test/Orleans.Messaging.Tests/Contracts/ActivationValidationTests.cs b/test/Orleans.Messaging.Tests/Contracts/ActivationValidationTests.cs new file mode 100644 index 00000000000..3cf35285602 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Contracts/ActivationValidationTests.cs @@ -0,0 +1,99 @@ +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Orleans.Concurrency; +using Orleans.Journaling; +using Orleans.Messaging.Tests.Functional; +using Orleans.Messaging.Tests.Support; +using Orleans.Placement; +using Orleans.Runtime; +using Orleans.Serialization.Invocation; +using Xunit; + +namespace Orleans.Messaging.Tests.Contracts; + +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class ActivationValidationTests +{ + [Fact] + public void StatelessWorker_FailsWithSpecificDiagnostic() + { + var exception = Assert.Throws(() => + MessagingActivationValidator.Validate(typeof(StatelessGrain), new StatelessWorkerAttribute().PlacementStrategy)); + + Assert.Contains("one activation", exception.Message, StringComparison.Ordinal); + Assert.Contains(typeof(StatelessGrain).ToString(), exception.Message, StringComparison.Ordinal); + } + + [Theory] + [InlineData(typeof(ReentrantGrain))] + [InlineData(typeof(MayInterleaveGrain))] + [InlineData(typeof(InterleavableGrain))] + public void ReentrantAndInterleavingModels_AreAccepted(Type grainClass) => + MessagingActivationValidator.Validate(grainClass, new RandomPlacementAttribute().PlacementStrategy); + + [Fact] + public async Task StandardStateConstruction_MissingSelectedCodecFailsBeforeEnrollment() + { + const string selectedFormat = "orleans-binary"; + var builder = InboxStateManagerBoundaryTests.CreateBuilder(selectedFormat); + builder.Services.RemoveAllKeyed(typeof(IDurableValueCommandCodec<>), selectedFormat); + var id = new JournalId("missing-codec/" + Guid.NewGuid().ToString("N")); + builder.Services.AddScoped(sp => + sp.GetRequiredService().CreateStandalone(id)); + await using var provider = builder.Services.BuildServiceProvider(validateScopes: true); + await using var scope = provider.CreateAsyncScope(); + var owner = scope.ServiceProvider.GetRequiredService(); + var exception = Assert.Throws(() => + scope.ServiceProvider.GetRequiredKeyedService>("state")); + Assert.Contains(selectedFormat, exception.Message, StringComparison.Ordinal); + Assert.False(owner.TryGetStateMachine("state", out _)); + } + + [Fact] + public async Task CodecResolution_PreservesUnrelatedFailure() + { + var cause = new InvalidOperationException("codec resolution failure"); + const string selectedFormat = "orleans-binary"; + var builder = InboxStateManagerBoundaryTests.CreateBuilder(selectedFormat); + builder.Services.AddKeyedSingleton>(selectedFormat, (_, _) => throw cause); + var id = new JournalId("codec-resolution/" + Guid.NewGuid().ToString("N")); + builder.Services.AddScoped(sp => + sp.GetRequiredService().CreateStandalone(id)); + await using var provider = builder.Services.BuildServiceProvider(validateScopes: true); + await using var scope = provider.CreateAsyncScope(); + var owner = scope.ServiceProvider.GetRequiredService(); + + var exception = Assert.Throws(() => + scope.ServiceProvider.GetRequiredKeyedService>("state")); + + Assert.Same(cause, exception); + Assert.False(owner.TryGetStateMachine("state", out _)); + } + + [Reentrant] + private sealed class ReentrantGrain; + + [StatelessWorker] + private sealed class StatelessGrain; + + [MayInterleave(nameof(Interleave))] + private sealed class MayInterleaveGrain + { + public static bool Interleave(IInvokable request) => true; + } + + public interface IInterleavableBase + { + [AlwaysInterleave] + Task PingAsync(); + } + + public interface IInterleavableGrain : IGrain, IInterleavableBase; + + private sealed class InterleavableGrain : IInterleavableGrain + { + public Task PingAsync() => Task.CompletedTask; + } +} diff --git a/test/Orleans.Messaging.Tests/Contracts/DeliveryAndOptionsContractTests.cs b/test/Orleans.Messaging.Tests/Contracts/DeliveryAndOptionsContractTests.cs new file mode 100644 index 00000000000..33a818e7485 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Contracts/DeliveryAndOptionsContractTests.cs @@ -0,0 +1,164 @@ +using Microsoft.Extensions.DependencyInjection; +using Orleans.Messaging.Configuration; +using Orleans.Serialization; +using Xunit; + +namespace Orleans.Messaging.Tests.Contracts; + +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class DeliveryAndOptionsContractTests +{ + [Fact] + public void RetentionTimeArithmetic_HandlesMaximumDurationsWithoutOverflow() + { + var timestamp = DateTimeOffset.MaxValue - TimeSpan.FromTicks(1); + var fullDateTimeRange = TimeSpan.FromTicks(DateTimeOffset.MaxValue.UtcTicks - DateTimeOffset.MinValue.UtcTicks); + + Assert.False(MessagingTime.IsExpired(DateTimeOffset.MaxValue, timestamp, TimeSpan.MaxValue)); + Assert.True(MessagingTime.IsExpired(DateTimeOffset.MaxValue, DateTimeOffset.MinValue, fullDateTimeRange)); + Assert.False(MessagingTime.IsExpired(DateTimeOffset.MaxValue, DateTimeOffset.MinValue, TimeSpan.MaxValue)); + Assert.False(MessagingTime.IsExpired(DateTimeOffset.MinValue, DateTimeOffset.MaxValue, TimeSpan.FromTicks(1))); + Assert.Equal(DateTimeOffset.MaxValue, MessagingTime.AddClamped(timestamp, TimeSpan.MaxValue)); + } + + [Fact] + public void DeadLetterCompaction_SaturatesMaximumRetentionWithoutOverflow() + { + var entries = new Dictionary + { + ["oldest"] = DateTimeOffset.MinValue, + ["newest"] = DateTimeOffset.MaxValue + }; + + Assert.False(DeadLetterRetention.Compact(entries, DateTimeOffset.MaxValue, TimeSpan.MaxValue, int.MaxValue, static timestamp => timestamp)); + Assert.Equal(2, entries.Count); + + var fullDateTimeRange = TimeSpan.FromTicks(DateTimeOffset.MaxValue.UtcTicks - DateTimeOffset.MinValue.UtcTicks); + Assert.True(DeadLetterRetention.Compact(entries, DateTimeOffset.MaxValue, fullDateTimeRange, int.MaxValue, static timestamp => timestamp)); + Assert.DoesNotContain("oldest", entries); + Assert.Contains("newest", entries); + } + + [Fact] + public void DeliveryResult_EachFactory_PreservesStatusAndPayload() + { + var handlerMissing = DeliveryResult.HandlerNotFound(); + var deadLettered = DeliveryResult.DeadLettered("poison body"); + var coded = DeliveryResult.DeadLettered("duplicate order", "orders/duplicate"); + var expired = DeliveryResult.Expired(); + + Assert.Equal(DeliveryStatus.Accepted, DeliveryResult.Accepted().Status); + Assert.Equal(DeliveryStatus.Duplicate, DeliveryResult.Duplicate().Status); + Assert.Equal(DeliveryStatus.Backpressured, DeliveryResult.Backpressured().Status); + Assert.Equal(DeliveryStatus.HandlerNotFound, handlerMissing.Status); + Assert.Equal("No inbox handler is registered.", handlerMissing.Message); + Assert.Equal(DeliveryStatus.DeadLettered, deadLettered.Status); + Assert.Equal("poison body", deadLettered.Message); + Assert.Null(deadLettered.Code); + Assert.Equal("duplicate order", coded.Message); + Assert.Equal("orders/duplicate", coded.Code); + Assert.Equal(DeliveryStatus.Expired, expired.Status); + Assert.Equal("The message was sent before the receiver's deduplication window; its earlier delivery outcome is unknown.", expired.Message); + Assert.Null(expired.Code); + } + + [Fact] + public void DeliveryResult_Expired_RoundTripsStatusAndPayload() + { + using var services = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var serializer = services.GetRequiredService>(); + + var decoded = serializer.Deserialize(serializer.SerializeToArray(DeliveryResult.Expired())); + + Assert.Equal(DeliveryStatus.Expired, decoded.Status); + Assert.Equal("The message was sent before the receiver's deduplication window; its earlier delivery outcome is unknown.", decoded.Message); + Assert.Null(decoded.Code); + } + + [Fact] + public void DeliveryStatus_AllValues_HaveContiguousDistinctValues() + { + Assert.Equal( + [ + DeliveryStatus.Accepted, + DeliveryStatus.Duplicate, + DeliveryStatus.Backpressured, + DeliveryStatus.HandlerNotFound, + DeliveryStatus.DeadLettered, + DeliveryStatus.Expired + ], + Enum.GetValues()); + Assert.Equal([1, 2, 3, 4, 5, 6], Enum.GetValues().Select(static value => (int)value)); + Assert.False(Enum.IsDefined(default(DeliveryResult).Status)); + } + + [Fact] + public void Validate_DefaultOptions_SucceedsAndExposesDocumentedDefaults() + { + var options = new DurableMessagingOptions(); + + options.Validate(); + + Assert.Equal(1000, options.MaxCapacity); + Assert.Equal(TimeSpan.FromDays(7), options.DeduplicationWindow); + Assert.Equal(TimeSpan.FromHours(6), options.OutboxRetryTimeout); + Assert.Equal(TimeSpan.FromDays(30), options.DeadLetterRetentionPeriod); + Assert.Equal(TimeSpan.FromHours(6), options.InboxRetryTimeout); + Assert.Equal(1000, options.MaxRetainedDeadLetters); + Assert.Equal(32, options.InboxBatchSize); + Assert.Equal(32, options.OutboxBatchSize); + } + + [Fact] + public void Validate_EachCapacityRetryDeadLetterAndBatchBoundary_EnforcesContract() + { + var invalidCases = new (string Parameter, Action Mutate)[] + { + (nameof(DurableMessagingOptions.MaxCapacity), options => options.MaxCapacity = 0), + (nameof(DurableMessagingOptions.DeduplicationWindow), options => options.DeduplicationWindow = TimeSpan.Zero), + (nameof(DurableMessagingOptions.DeduplicationWindow), options => options.DeduplicationWindow = TimeSpan.FromTicks(-1)), + (nameof(DurableMessagingOptions.RetryInitialInterval), options => options.RetryInitialInterval = TimeSpan.Zero), + (nameof(DurableMessagingOptions.RetryInitialInterval), options => options.RetryInitialInterval = TimeSpan.MaxValue), + (nameof(DurableMessagingOptions.InboxRetryTimeout), options => options.InboxRetryTimeout = TimeSpan.FromTicks(-1)), + (nameof(DurableMessagingOptions.OutboxRetryTimeout), options => options.OutboxRetryTimeout = TimeSpan.Zero), + (nameof(DurableMessagingOptions.OutboxRetryTimeout), options => options.OutboxRetryTimeout = TimeSpan.FromTicks(-1)), + (nameof(DurableMessagingOptions.DeadLetterRetentionPeriod), options => options.DeadLetterRetentionPeriod = TimeSpan.Zero), + (nameof(DurableMessagingOptions.MaxRetainedDeadLetters), options => options.MaxRetainedDeadLetters = 0), + (nameof(DurableMessagingOptions.InboxBatchSize), options => options.InboxBatchSize = 0), + (nameof(DurableMessagingOptions.OutboxBatchSize), options => options.OutboxBatchSize = 0), + }; + + foreach (var (parameter, mutate) in invalidCases) + { + var options = new DurableMessagingOptions(); + mutate(options); + var exception = Assert.Throws(options.Validate); + Assert.Equal(parameter, exception.ParamName); + } + } + + [Theory] + [InlineData(1, 2, true)] + [InlineData(2, 2, false)] + [InlineData(3, 2, false)] + public void Validate_OutboxRetryTimeout_MustEndWithinTheDeduplicationWindow(long retryTimeoutTicks, long deduplicationWindowTicks, bool valid) + { + var options = new DurableMessagingOptions + { + DeduplicationWindow = TimeSpan.FromTicks(deduplicationWindowTicks), + OutboxRetryTimeout = TimeSpan.FromTicks(retryTimeoutTicks), + }; + + if (valid) + { + options.Validate(); + return; + } + + var exception = Assert.Throws(options.Validate); + Assert.Equal(nameof(DurableMessagingOptions.OutboxRetryTimeout), exception.ParamName); + Assert.Contains("less than DeduplicationWindow", exception.Message, StringComparison.Ordinal); + } +} diff --git a/test/Orleans.Messaging.Tests/Contracts/EnvelopeContractTests.cs b/test/Orleans.Messaging.Tests/Contracts/EnvelopeContractTests.cs new file mode 100644 index 00000000000..9b4c144112e --- /dev/null +++ b/test/Orleans.Messaging.Tests/Contracts/EnvelopeContractTests.cs @@ -0,0 +1,419 @@ +using System.Reflection; +using System.Text; +using Microsoft.Extensions.DependencyInjection; +using Orleans.Runtime; +using Orleans.Serialization; +using Xunit; + +namespace Orleans.Messaging.Tests.Contracts; + +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class EnvelopeContractTests +{ + private static MessageId Id => MessageId.Create("orders", "123"); + private static GrainId Receiver => GrainId.Create("receiver", "123"); + private static DateTimeOffset SentAt => new(2040, 1, 2, 3, 4, 5, TimeSpan.FromHours(2)); + + [Fact] + public void RequiredPayloadAndOptionalMetadata() + { + var envelope = Create([]); + Validate(envelope); + Validate(new InboxMessage { Envelope = envelope, SentAt = SentAt }); + Assert.Equal(Id, envelope.MessageId); + Assert.Empty(envelope.Payload); + Assert.Null(envelope.Subject); + Assert.Null(envelope.Headers); + using var services = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var serializer = services.GetRequiredService>(); + var decoded = serializer.Deserialize(serializer.SerializeToArray(envelope)); + Validate(decoded); + Assert.Empty(decoded.Payload); + Assert.Null(decoded.Subject); + Assert.Null(decoded.Headers); + Assert.Throws(() => Validate(new OutboxMessage { Envelope = envelope, ReceiverId = default })); + Assert.Throws(() => Validate(default(Envelope))); + Assert.Throws(() => Validate(envelope with { Payload = null! })); + } + + [Fact] + public void FirstClassSubjectAndEncodedSenderUseByteHeaders() + { + using var services = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var senderSerializer = services.GetRequiredService>(); + var sender = GrainId.Create("sender", "123"); + var envelope = Create([1]) with + { + Subject = "orders.v1", + Headers = new(StringComparer.Ordinal) + { + ["subject"] = "extra subject"u8.ToArray(), + ["sender"] = senderSerializer.SerializeToArray(sender), + ["custom"] = "first"u8.ToArray(), + ["CUSTOM"] = "second"u8.ToArray() + } + }; + Validate(envelope); + Assert.Equal("orders.v1", envelope.Subject); + Assert.Equal(sender, senderSerializer.Deserialize(envelope.Headers["sender"])); + Assert.Equal("extra subject"u8.ToArray(), envelope.Headers["subject"]); + Assert.Equal("first"u8.ToArray(), envelope.Headers["custom"]); + Assert.Equal("second"u8.ToArray(), envelope.Headers["CUSTOM"]); + Assert.False(envelope.Headers.TryAdd("custom", "replacement"u8.ToArray())); + Assert.Equal("first"u8.ToArray(), envelope.Headers["custom"]); + var serializer = services.GetRequiredService>(); + var decoded = serializer.Deserialize(serializer.SerializeToArray(envelope)); + Assert.NotNull(decoded.Headers); + Assert.Equal("orders.v1", decoded.Subject); + Assert.Equal(sender, senderSerializer.Deserialize(decoded.Headers["sender"])); + Assert.NotSame(envelope.Headers["sender"], decoded.Headers["sender"]); + Assert.Equal("extra subject"u8.ToArray(), decoded.Headers["subject"]); + } + + [Fact] + public void StructAssignmentAndLocalConversionSharePublishedObjects() + { + var envelope = Create([0, 255, 128]) with { Headers = new(StringComparer.Ordinal) { ["trace"] = "original"u8.ToArray() } }; + var assigned = envelope; + var outgoing = new OutboxMessage { Envelope = envelope, ReceiverId = Receiver }; + var incoming = outgoing.ToInboxMessage(SentAt); + Assert.Same(envelope.Payload, assigned.Payload); + Assert.Same(envelope.Headers, assigned.Headers); + Assert.Same(envelope.Payload, incoming.Envelope.Payload); + Assert.Same(envelope.Headers, incoming.Envelope.Headers); + Assert.Equal(Id, incoming.Envelope.MessageId); + Assert.Equal(SentAt, incoming.SentAt); + Assert.Equal(SentAt.Offset, incoming.SentAt.Offset); + Assert.Null(typeof(InboxMessage).GetProperty("ReceiverId")); + Assert.Null(typeof(Envelope).GetProperty("ReceiverId")); + } + + [Theory] + [InlineData(0)] + [InlineData(7)] + [InlineData(100000)] + public void OrdinaryCopyAndIndependentWireRoundTrip(int length) + { + using var sending = new ServiceCollection().AddSerializer().BuildServiceProvider(); + using var receiving = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var payload = Enumerable.Range(0, length).Select(i => (byte)i).ToArray(); + var envelope = Create(payload) with + { + Subject = "orders.v1", + Headers = new(StringComparer.Ordinal) { ["custom"] = "retained"u8.ToArray(), ["CUSTOM"] = "distinct"u8.ToArray(), ["empty"] = [] } + }; + var serializer = sending.GetRequiredService>(); + var wire = serializer.SerializeToArray(envelope); + var decoded = receiving.GetRequiredService>().Deserialize(wire); + var copied = sending.GetRequiredService().Copy(envelope); + foreach (var actual in new[] { decoded, copied }) + { + Validate(actual); + Assert.Equal(Id, actual.MessageId); + Assert.Equal(payload, actual.Payload); + Assert.Equal("orders.v1", actual.Subject); + Assert.NotNull(actual.Headers); + Assert.NotSame(envelope.Headers, actual.Headers); + Assert.Equal("retained"u8.ToArray(), actual.Headers["custom"]); + Assert.Equal("distinct"u8.ToArray(), actual.Headers["CUSTOM"]); + Assert.Empty(actual.Headers["empty"]); + Assert.NotSame(envelope.Headers["custom"], actual.Headers["custom"]); + actual.Headers["custom"][0] = 0; + Assert.Equal("retained"u8.ToArray(), envelope.Headers["custom"]); + if (length > 0) + { + Assert.NotSame(envelope.Payload, actual.Payload); + actual.Payload[0] = 42; + Assert.Equal((byte)0, envelope.Payload[0]); + } + } + Assert.Equal(wire, serializer.SerializeToArray(envelope)); + } + + [Fact] + public void DeliverAtRoundTripsThroughCreateAndIndependentWire() + { + using var sending = new ServiceCollection().AddSerializer().BuildServiceProvider(); + using var receiving = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var deliverAt = new DateTimeOffset(2040, 1, 2, 3, 4, 5, TimeSpan.FromHours(2)); + var type = new MessageType("orders.v1", sending.GetRequiredService>()); + var outgoing = type.Create(Id, Receiver, "scheduled", deliverAt); + Assert.Equal(deliverAt, outgoing.Envelope.DeliverAt); + Validate(outgoing); + + var wire = sending.GetRequiredService>().SerializeToArray(outgoing); + var decoded = receiving.GetRequiredService>().Deserialize(wire); + var copied = sending.GetRequiredService().Copy(outgoing); + foreach (var actual in new[] { decoded, copied }) + { + Assert.Equal(deliverAt, actual.Envelope.DeliverAt); + Assert.Equal(deliverAt.Offset, actual.Envelope.DeliverAt!.Value.Offset); + Assert.Equal(Receiver, actual.ReceiverId); + Assert.Equal("scheduled", type.Decode(actual.Envelope)); + Assert.True(Invoke("EnvelopeEquivalence", "AreEquivalent", outgoing, actual)); + } + } + + [Fact] + public void PayloadUsesOrdinaryExternallyBoundOrleansSerializer() + { + using var services = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var serializer = services.GetRequiredService>(); + var envelope = Create(serializer.SerializeToArray("typed application payload")) with { Subject = "typed.v1" }; + Assert.Equal("typed application payload", serializer.Deserialize(envelope.Payload)); + Assert.Equal("typed.v1", envelope.Subject); + Assert.Null(envelope.Headers); + } + + [Theory] + [InlineData("InboxMessage")] + [InlineData("OutboxMessage")] + [InlineData("InboxDeadLetter")] + [InlineData("OutboxDeadLetter")] + public void MessageAndDeadLetterCopyPreservesMetadataAndIsolation(string kind) + { + using var services = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var envelope = Create([0, 255]) with { Subject = "subject.v1", Headers = new() { ["custom"] = "value"u8.ToArray() } }; + object value = kind.StartsWith("Inbox", StringComparison.Ordinal) ? new InboxMessage { Envelope = envelope, SentAt = SentAt } + : new OutboxMessage { Envelope = envelope, ReceiverId = Receiver }; + if (kind.EndsWith("DeadLetter", StringComparison.Ordinal)) + { + var type = typeof(Envelope).Assembly.GetType("Orleans.Messaging." + kind, true)!; + var record = Activator.CreateInstance(type)!; + type.GetProperty("Message")!.SetValue(record, value); + type.GetProperty("DeadLetteredAt")!.SetValue(record, DateTimeOffset.UnixEpoch); + type.GetProperty("Reason")!.SetValue(record, "failure"); + type.GetProperty("AttemptCount")!.SetValue(record, 3); + value = record; + } + var serializer = services.GetRequiredService(); + foreach (var result in new[] { serializer.Deserialize(serializer.SerializeToArray(value)), services.GetRequiredService().Copy(value) }) + { + Assert.NotNull(result); + var message = kind.EndsWith("DeadLetter", StringComparison.Ordinal) ? result.GetType().GetProperty("Message")!.GetValue(result)! : result; + var actual = message is InboxMessage incoming ? incoming.Envelope : Assert.IsType(message).Envelope; + Assert.Equal(new byte[] { 0, 255 }, actual.Payload); + Assert.NotSame(envelope.Payload, actual.Payload); + Assert.Equal("subject.v1", actual.Subject); + Assert.NotNull(actual.Headers); + Assert.NotSame(envelope.Headers, actual.Headers); + Assert.Equal("value"u8.ToArray(), actual.Headers["custom"]); + Assert.NotSame(envelope.Headers!["custom"], actual.Headers["custom"]); + if (message is OutboxMessage outgoing) Assert.Equal(Receiver, outgoing.ReceiverId); + if (message is InboxMessage received) Assert.Equal(SentAt, received.SentAt); + if (kind.EndsWith("DeadLetter", StringComparison.Ordinal)) Assert.Equal(3, result.GetType().GetProperty("AttemptCount")!.GetValue(result)); + } + } + + [Fact] + public void PublicDeadLetterRoundTripsThroughOrleansSerialization() + { + using var services = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var letter = new DeadLetter + { + Message = new InboxMessage { Envelope = Create([1, 2]) with { Subject = "subject.v1" }, SentAt = SentAt }, + DeadLetteredAt = DateTimeOffset.UnixEpoch, + Reason = "failure", + AttemptCount = 3, + Code = "orders/invalid" + }; + var serializer = services.GetRequiredService>>(); + + var result = Assert.IsType>(serializer.Deserialize(serializer.SerializeToArray(letter))); + + Assert.Equal(letter.Message.Envelope.MessageId, result.Message.Envelope.MessageId); + Assert.Equal(new byte[] { 1, 2 }, result.Message.Envelope.Payload); + Assert.Equal("subject.v1", result.Message.Envelope.Subject); + Assert.Equal(DateTimeOffset.UnixEpoch, result.DeadLetteredAt); + Assert.Equal("failure", result.Reason); + Assert.Equal(3, result.AttemptCount); + Assert.Equal("orders/invalid", result.Code); + } + + [Fact] + public void SimpleEnvelopeSurfaceUsesGeneratedOrdinaryFields() + { + AssertIds(typeof(Envelope), ("MessageId", 0u), ("Payload", 1u), ("Subject", 2u), ("Headers", 3u), ("DeliverAt", 4u)); + AssertIds(typeof(InboxMessage), ("Envelope", 0u), ("SentAt", 1u)); + Assert.Equal(typeof(DateTimeOffset), typeof(InboxMessage).GetProperty("SentAt")!.PropertyType); + Assert.NotNull(typeof(InboxMessage).GetProperty("SentAt")!.GetCustomAttribute()); + AssertIds(typeof(OutboxMessage), ("Envelope", 0u), ("ReceiverId", 1u)); + Assert.Equal(typeof(Dictionary), typeof(Envelope).GetProperty("Headers")!.PropertyType); + Assert.Equal(typeof(byte[]), typeof(Envelope).GetProperty("Payload")!.PropertyType); + Assert.Null(typeof(Envelope).GetCustomAttribute()); + Assert.Single(typeof(Envelope).GetCustomAttributes()); + Assert.Equal("Orleans.Messaging.Envelope", typeof(Envelope).GetCustomAttribute()!.Alias); + Assert.Equal("Orleans.Messaging.IMessagingExtension", typeof(IMessagingExtension).GetCustomAttribute()!.Alias); + Assert.Equal(typeof(DateTimeOffset?), typeof(Envelope).GetProperty("DeliverAt")!.PropertyType); + Assert.Equal(new[] { "DeliverAt", "Headers", "MessageId", "Payload", "Subject" }, typeof(Envelope).GetProperties().Select(p => p.Name).Order(StringComparer.Ordinal)); + Assert.Null(typeof(Envelope).Assembly.GetType("Orleans.Messaging.PackedEnvelopeHeaders")); + Assert.Null(typeof(Envelope).Assembly.GetType("Orleans.Messaging.EnvelopeBuilder")); + Assert.Equal(typeof(void), typeof(IInboxHandlerContext).GetMethod("Fail")!.ReturnType); + Assert.Equal(typeof(string), Assert.Single(typeof(IInboxHandlerContext).GetMethod("Fail")!.GetParameters()).ParameterType); + Assert.Equal(new[] { "Accepted:1", "Backpressured:3", "DeadLettered:5", "Duplicate:2", "Expired:6", "HandlerNotFound:4" }, + Enum.GetNames().Select(name => $"{name}:{(int)Enum.Parse(name)}").Order(StringComparer.Ordinal)); + AssertIds(typeof(DeliveryResult), ("Status", 0u), ("Message", 1u), ("Code", 2u)); + using var services = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var outgoing = new OutboxMessage { Envelope = Create([7]), ReceiverId = Receiver }; + var other = outgoing with { ReceiverId = GrainId.Create("receiver", "other") }; + var serializer = services.GetRequiredService>(); + Assert.Equal(serializer.SerializeToArray(outgoing.ToInboxMessage(SentAt)), serializer.SerializeToArray(other.ToInboxMessage(SentAt))); + } + + [Theory] + [InlineData("same", true, true)] + [InlineData("headers", true, true)] + [InlineData("sender", true, true)] + [InlineData("receiver", false, true)] + [InlineData("id", false, false)] + [InlineData("child", false, false)] + [InlineData("subject", false, false)] + [InlineData("subject-case", false, false)] + [InlineData("no-subject", false, false)] + [InlineData("payload", false, false)] + [InlineData("payload-length", false, false)] + [InlineData("deliver-at-offset", true, true)] + [InlineData("deliver-at", false, false)] + [InlineData("no-deliver-at", false, false)] + public void PendingCommandEqualityIgnoresExtraHeadersAndSender(string change, bool outbox, bool inbox) + { + var deliverAt = new DateTimeOffset(2040, 1, 2, 3, 4, 5, TimeSpan.Zero); + var envelope = Create([0, 255, 128]) with { Subject = "subject.v1", Headers = new() { ["custom"] = "first"u8.ToArray() }, DeliverAt = deliverAt }; + var original = new OutboxMessage { Envelope = envelope, ReceiverId = Receiver }; + var repeated = new OutboxMessage + { + ReceiverId = change == "receiver" ? GrainId.Create("receiver", "other") : Receiver, + Envelope = envelope with + { + MessageId = change == "id" ? MessageId.Create("other") : change == "child" ? new MessageId(Id.Key.CreateChildKey("child")) : Id, + Payload = change == "payload" ? [0, 255, 129] : change == "payload-length" ? [0, 255] : [0, 255, 128], + Subject = change == "subject" ? "other" : change == "subject-case" ? "Subject.v1" : change == "no-subject" ? null : "subject.v1", + Headers = new() { ["custom"] = Encoding.UTF8.GetBytes(change == "headers" ? "second" : "first"), ["sender"] = change == "sender" ? [2] : [1] }, + DeliverAt = change switch + { + "deliver-at-offset" => deliverAt.ToOffset(TimeSpan.FromHours(2)), + "deliver-at" => deliverAt.AddTicks(1), + "no-deliver-at" => null, + _ => deliverAt + } + } + }; + Assert.Equal(outbox, Invoke("EnvelopeEquivalence", "AreEquivalent", original, repeated)); + Assert.Equal(inbox, Invoke("EnvelopeEquivalence", "AreSameCommand", original.ToInboxMessage(SentAt), repeated.ToInboxMessage(SentAt))); + Assert.Equal(inbox, Invoke("EnvelopeEquivalence", "AreSameCommand", original.Envelope, repeated.Envelope)); + Assert.Equal("first"u8.ToArray(), original.Envelope.Headers!["custom"]); + Assert.Equal(new byte[] { 0, 255, 128 }, original.Envelope.Payload); + } + + [Fact] + public void PendingRawCommandsAllowAbsentSubject() + { + var first = Create([]); + var repeated = first with { Headers = new() { ["custom"] = "different"u8.ToArray() } }; + Assert.True(Invoke("EnvelopeEquivalence", "AreSameCommand", first, repeated)); + Assert.Null(first.Subject); + Assert.Null(first.Headers); + } + + [Theory] + [InlineData("id", 1024, true)] + [InlineData("id", 1025, false)] + // Segment depth is bounded only by the encoded message ID size. + [InlineData("depth", 512, true)] + [InlineData("depth", 513, false)] + [InlineData("unicode-id", 512, true)] + [InlineData("unicode-id", 513, false)] + [InlineData("subject", 256, true)] + [InlineData("subject", 257, false)] + [InlineData("unicode-subject", 128, true)] + [InlineData("unicode-subject", 129, false)] + public void IdentityAndSubjectAdmissionBoundaries(string field, int length, bool valid) + { + var envelope = Create([]) with + { + MessageId = field switch + { + "id" => MessageId.Create(new string('x', length)), + "depth" => MessageId.Create(Enumerable.Repeat("x", length).ToArray()), + "unicode-id" => MessageId.Create(new string('\u00e9', length)), + _ => Id + }, + Subject = field.Contains("subject", StringComparison.Ordinal) ? new string(field == "subject" ? 's' : '\u00e9', length) : null + }; + if (valid) { Validate(envelope); Assert.Empty(envelope.Payload); } + else Assert.Throws(() => Validate(envelope)); + } + + [Theory] + [InlineData("empty-subject")] + [InlineData("invalid-subject")] + [InlineData("empty-header-key")] + [InlineData("null-header-value")] + [InlineData("lone-high-surrogate-header-key")] + [InlineData("lone-low-surrogate-header-key")] + [InlineData("reversed-surrogate-header-key")] + public void InvalidMetadataIsRejectedBeforeAdmission(string invalid) + { + var envelope = Create([1]) with + { + Subject = invalid == "empty-subject" ? "" : invalid == "invalid-subject" ? "\ud800" : null, + Headers = invalid switch + { + "empty-header-key" => new() { [""] = "value"u8.ToArray() }, + "null-header-value" => new() { ["key"] = null! }, + "lone-high-surrogate-header-key" => new() { ["trace-\ud83d"] = "value"u8.ToArray() }, + "lone-low-surrogate-header-key" => new() { ["\ude00-trace"] = "value"u8.ToArray() }, + "reversed-surrogate-header-key" => new() { ["trace-\ude00\ud83d"] = "value"u8.ToArray() }, + _ => null + } + }; + Assert.ThrowsAny(() => Validate(envelope)); + Assert.ThrowsAny(() => Validate(new InboxMessage { Envelope = envelope, SentAt = SentAt })); + Assert.ThrowsAny(() => Validate(new OutboxMessage { Envelope = envelope, ReceiverId = Receiver })); + Assert.Equal(new byte[] { 1 }, envelope.Payload); + } + + [Fact] + public void SurrogatePairHeaderKeysRoundTripThroughOrleansSerialization() + { + using var sending = new ServiceCollection().AddSerializer().BuildServiceProvider(); + using var receiving = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var envelope = Create([1]) with + { + MessageId = MessageId.Create("order-\U0001F600"), + Headers = new(StringComparer.Ordinal) { ["trace-\U0001F600"] = "first"u8.ToArray(), ["trace-\U0001F601"] = "second"u8.ToArray() } + }; + Validate(envelope); + + var decoded = receiving.GetRequiredService>().Deserialize(sending.GetRequiredService>().SerializeToArray(envelope)); + + Validate(decoded); + Assert.Equal(envelope.MessageId, decoded.MessageId); + Assert.NotNull(decoded.Headers); + Assert.Equal(["trace-\U0001F600", "trace-\U0001F601"], decoded.Headers.Keys.Order(StringComparer.Ordinal)); + Assert.Equal("first"u8.ToArray(), decoded.Headers["trace-\U0001F600"]); + Assert.Equal("second"u8.ToArray(), decoded.Headers["trace-\U0001F601"]); + } + + private static Envelope Create(byte[] payload) => new() { MessageId = Id, Payload = payload }; + private static void Validate(T message) => Invoke("EnvelopeValidation", "Validate", message!); + private static T Invoke(string type, string method, params object[] args) + { + var helper = typeof(Envelope).Assembly.GetType("Orleans.Messaging." + type, true)!; + try { return (T)helper.GetMethod(method, args.Select(a => a.GetType()).ToArray())!.Invoke(null, args)!; } + catch (TargetInvocationException exception) when (exception.InnerException is { } cause) + { + System.Runtime.ExceptionServices.ExceptionDispatchInfo.Capture(cause).Throw(); + throw; + } + } + private static void AssertIds(Type type, params (string Name, uint Id)[] expected) + { + var actual = type.GetMembers(BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic) + .SelectMany(member => member.GetCustomAttributes().Select(id => (member.Name, id.Id))) + .OrderBy(p => p.Id).ToArray(); + Assert.Equal(expected.OrderBy(p => p.Id), actual); + } +} diff --git a/test/Orleans.Messaging.Tests/Contracts/HandlerRoutingContractTests.cs b/test/Orleans.Messaging.Tests/Contracts/HandlerRoutingContractTests.cs new file mode 100644 index 00000000000..b9a8c2a828c --- /dev/null +++ b/test/Orleans.Messaging.Tests/Contracts/HandlerRoutingContractTests.cs @@ -0,0 +1,127 @@ +using Microsoft.Extensions.DependencyInjection; +using Orleans.Messaging.Tests.Support; +using Orleans.Runtime; +using Orleans.Serialization; +using Xunit; + +namespace Orleans.Messaging.Tests.Contracts; + +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class HandlerRoutingContractTests : IDisposable +{ + private readonly ServiceProvider _services = new ServiceCollection().AddSerializer().BuildServiceProvider(); + + public void Dispose() => _services.Dispose(); + + [Fact] + public async Task Dispatch_SameSubjectRegisteredTwice_FirstRegistrationWins() + { + var type = CreateType("orders/created"); + var calls = new List(); + var dispatcher = new TestInbox() + .Register(type, (body, context) => { calls.Add($"first:{body}"); context.Complete(); }) + .Register(type, (body, context) => { calls.Add($"second:{body}"); context.Complete(); }); + var context = CreateContext(type, "a"); + + await dispatcher.HandleAsync(context, CancellationToken.None); + + Assert.Equal(["first:a"], calls); + Assert.Equal(1, context.Completions); + } + + [Fact] + public async Task Dispatch_ConsultsCanHandleInRegistrationOrder() + { + var order = new List(); + var dispatcher = new TestInbox(); + dispatcher.Register(new ProbeHandler("declines", canHandle: false, order)); + dispatcher.Register(new ProbeHandler("accepts", canHandle: true, order)); + dispatcher.Register(new ProbeHandler("unreached", canHandle: true, order)); + + await dispatcher.HandleAsync(CreateContext(CreateType("any"), "body"), CancellationToken.None); + + Assert.Equal(["declines?", "accepts?", "accepts!"], order); + } + + [Theory] + [InlineData("unknown", "No handler is registered for message subject 'unknown'.")] + [InlineData(null, "No handler is registered for message subject ''.")] + public async Task Dispatch_NoMatchingHandler_ThrowsWithSubject(string? subject, string expectedMessage) + { + var dispatcher = new TestInbox().Register(CreateType("known"), (_, context) => context.Complete()); + var context = new FakeContext(new InboxMessage + { + Envelope = new Envelope + { + MessageId = MessageId.Create("unhandled"), + Payload = [], + Subject = subject + }, + SentAt = DateTimeOffset.UnixEpoch + }); + + var exception = await Assert.ThrowsAsync( + () => dispatcher.HandleAsync(context, CancellationToken.None).AsTask()); + + Assert.Equal(expectedMessage, exception.Message); + Assert.Equal(0, context.Completions); + } + + [Fact] + public void Dispatcher_IsEmptyUntilHandlerIsRegistered() + { + var dispatcher = new TestInbox(); + Assert.True(dispatcher.IsEmpty); + + dispatcher.Register(CreateType("subject"), (_, context) => context.Complete()); + + Assert.False(dispatcher.IsEmpty); + } + + [Fact] + public void HandlerNotFound_RoundTripsStatusAndDiagnostic() + { + var result = DeliveryResult.HandlerNotFound(); + var serializer = _services.GetRequiredService>(); + + var decoded = serializer.Deserialize(serializer.SerializeToArray(result)); + + Assert.Equal(DeliveryStatus.HandlerNotFound, decoded.Status); + Assert.Equal(result.Message, decoded.Message); + } + + private MessageType CreateType(string subject) => + new(subject, _services.GetRequiredService>()); + + private static FakeContext CreateContext(MessageType type, string body) => + new(new InboxMessage + { + Envelope = type.Create(MessageId.Create(Guid.NewGuid().ToString("N")), GrainId.Create("test", "receiver"), body).Envelope, + SentAt = DateTimeOffset.UnixEpoch + }); + + private sealed class FakeContext(InboxMessage message) : IInboxHandlerContext + { + public int Completions { get; private set; } + public InboxMessage Message { get; } = message; + public void Complete() => Completions++; + public void Fail(string reason) => throw new NotSupportedException(); + } + + private sealed class ProbeHandler(string name, bool canHandle, List order) : IInboxHandler + { + public bool CanHandle(InboxMessage message) + { + order.Add($"{name}?"); + return canHandle; + } + + public ValueTask HandleAsync(IInboxHandlerContext context, CancellationToken cancellationToken) + { + order.Add($"{name}!"); + return ValueTask.CompletedTask; + } + } +} diff --git a/test/Orleans.Messaging.Tests/Contracts/HierarchicalKeyPropertyTests.cs b/test/Orleans.Messaging.Tests/Contracts/HierarchicalKeyPropertyTests.cs new file mode 100644 index 00000000000..755a13be6dd --- /dev/null +++ b/test/Orleans.Messaging.Tests/Contracts/HierarchicalKeyPropertyTests.cs @@ -0,0 +1,159 @@ +using System.Text; +using System.Text.Json; +using CsCheck; +using Xunit; + +namespace Orleans.Messaging.Tests.Contracts; + +/// +/// Checks the identity laws on which deduplication relies against an independent model: a key is the list of its +/// literal segments. +/// +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class HierarchicalKeyPropertyTests +{ + // Characters which exercise escaping, multi-byte UTF-8, and surrogate pairs. + private static readonly Gen Atom = Gen.OneOfConst("a", "b", "/", "\\", "\\/", "é", "😀", " ", "\0"); + + private static readonly Gen Segment = Atom.Array[1, 6].Select(static parts => string.Concat(parts)); + + private static readonly Gen Segments = Segment.Array[1, 5]; + + private static readonly Gen LoneSurrogate = Gen.OneOf(Gen.Char['\uD800', '\uDBFF'], Gen.Char['\uDC00', '\uDFFF']); + + [Fact] + public void FormatAndParse_RoundTripEveryCreatedKey() => + Segments.Sample(static segments => + { + var key = HierarchicalKey.Create(segments); + Assert.Equal(key, HierarchicalKey.Parse(key.ToString())); + Assert.Equal(new MessageId(key), MessageId.Parse(key.ToString())); + }); + + [Fact] + public void Create_IsInjective() => + Gen.Select(Segments, Segments).Sample(static (left, right) => + { + var leftKey = HierarchicalKey.Create(left); + var rightKey = HierarchicalKey.Create(right); + Assert.Equal(left.AsSpan().SequenceEqual(right), leftKey == rightKey); + Assert.Equal(leftKey == rightKey, new MessageId(leftKey) == new MessageId(rightKey)); + if (leftKey == rightKey) Assert.Equal(leftKey.GetHashCode(), rightKey.GetHashCode()); + }); + + [Fact] + public void Segments_UnescapeToTheLiteralSegments() => + Segments.Sample(static segments => + { + var key = HierarchicalKey.Create(segments); + var actual = new List(); + foreach (var segment in key) actual.Add(Unescape(segment)); + Assert.Equal(segments, actual); + Assert.Equal(segments.Length, actual.Count); + }); + + [Fact] + public void Parse_AcceptsExactlyTheCanonicalForms() => + Gen.OneOfConst("a", "/", "\\", "\\/", "\\\\", "😀").Array[0, 8].Select(static parts => string.Concat(parts)).Sample(static text => + { + var parsed = HierarchicalKey.TryParse(text, null, out var key); + var canonical = TryModelParse(text, out var segments); + Assert.Equal(canonical, parsed); + if (parsed) Assert.Equal(HierarchicalKey.Create(segments.ToArray()), key); + }); + + [Fact] + public void Navigation_MatchesTheSegmentListModel() => + Gen.Select(Segments, Segments).Sample(static (left, right) => + { + var leftKey = HierarchicalKey.Create(left); + var rightKey = HierarchicalKey.Create(right); + var isPrefix = left.Length <= right.Length && right.AsSpan(0, left.Length).SequenceEqual(left); + Assert.Equal(isPrefix, leftKey.IsAncestorOf(rightKey)); + Assert.Equal(isPrefix && right.Length == left.Length + 1, leftKey.IsParentOf(rightKey)); + Assert.Equal(isPrefix && right.Length == left.Length + 1, rightKey.IsChildOf(leftKey)); + Assert.Equal(HierarchicalKey.Create([.. left, .. right]), leftKey.Append(rightKey)); + Assert.Equal(HierarchicalKey.Create([.. left, right[0]]), leftKey.CreateChildKey(right[0])); + Assert.Equal(left.Length == 1 ? null : HierarchicalKey.Create(left[..^1]), leftKey.GetParent()); + }); + + [Fact] + public void Create_RejectsUnpairedSurrogates() => + Gen.Select(Segment, LoneSurrogate, Gen.Int[0, 6]).Sample(static (segment, surrogate, position) => + { + var index = Math.Min(position, segment.Length); + if (index < segment.Length && char.IsLowSurrogate(segment[index])) index++; + var invalid = segment.Insert(index, surrogate.ToString()); + Assert.False(IsWellFormed(invalid)); + Assert.Throws(() => HierarchicalKey.Create(invalid)); + Assert.Throws(() => MessageId.Create("root", invalid)); + }); + + [Fact] + public void Json_RoundTripsKeysAndMessageIds() => + Segments.Sample(static segments => + { + var id = MessageId.Create(segments); + var json = JsonSerializer.Serialize(id); + Assert.Equal(JsonSerializer.Serialize(id.ToString()), json); + Assert.Equal(id, JsonSerializer.Deserialize(json)); + Assert.Equal(id.Key, JsonSerializer.Deserialize(json)); + }); + + private static bool IsWellFormed(string value) + + { + try + { + _ = new UTF8Encoding(false, true).GetBytes(value); + return true; + } + catch (EncoderFallbackException) + { + return false; + } + } + + // An independent decoder of the canonical form: '\' escapes exactly '\' or '/', and '/' separates nonempty segments. + private static bool TryModelParse(string text, out List segments) + { + segments = []; + var current = new StringBuilder(); + for (var i = 0; i < text.Length; i++) + { + var c = text[i]; + if (c == '\\') + { + if (++i == text.Length || text[i] is not ('\\' or '/')) return false; + current.Append(text[i]); + } + else if (c == '/') + { + if (current.Length == 0) return false; + segments.Add(current.ToString()); + current.Clear(); + } + else + { + current.Append(c); + } + } + + if (current.Length == 0) return false; + segments.Add(current.ToString()); + return true; + } + + private static string Unescape(ReadOnlySpan segment) + { + var builder = new StringBuilder(segment.Length); + for (var i = 0; i < segment.Length; i++) + { + builder.Append(segment[i] == '\\' ? segment[++i] : segment[i]); + } + + return builder.ToString(); + } +} diff --git a/test/Orleans.Messaging.Tests/Contracts/HierarchicalKeyTests.cs b/test/Orleans.Messaging.Tests/Contracts/HierarchicalKeyTests.cs new file mode 100644 index 00000000000..c19c7905421 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Contracts/HierarchicalKeyTests.cs @@ -0,0 +1,465 @@ +using System.Buffers; +using System.Reflection; +using System.Runtime.CompilerServices; +using System.Text.Json; +using Microsoft.Extensions.DependencyInjection; +using Orleans.Messaging.Tests.Support; +using Orleans.Serialization; +using Orleans.Serialization.Buffers; +using Orleans.Serialization.Codecs; +using Orleans.Serialization.Session; +using Orleans.Serialization.WireProtocol; +using Xunit; + +namespace Orleans.Messaging.Tests.Contracts; + +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public class HierarchicalKeyTests +{ + [Fact] + public void SerializationContract_IdentifiesCurrentType() + { + var type = typeof(HierarchicalKey); + Assert.Equal("Orleans.Messaging.HierarchicalKey", Assert.Single(type.GetCustomAttributes()).Alias); + Assert.True(type.IsValueType); + Assert.Single(type.GetCustomAttributes()); + Assert.Empty(type.GetCustomAttributes()); + var field = Assert.Single(type.GetFields(BindingFlags.Instance | BindingFlags.NonPublic)); + Assert.True(field.IsInitOnly); + Assert.Equal(typeof(string), field.FieldType); + } + + [Fact] + public void Json_ValueRoundTripsCanonicalEscapedPath() + { + var key = HierarchicalKey.Create("orders", "a/b", @"c\d"); + + var json = JsonSerializer.Serialize(key); + + Assert.Equal(JsonSerializer.Serialize(@"orders/a\/b/c\\d"), json); + var decoded = JsonSerializer.Deserialize(json); + Assert.Equal(key, decoded); + Assert.Equal(3, decoded.CountSegments()); + } + + [Fact] + public void Json_DefaultValueRoundTripsAsNull() + { + Assert.Equal("null", JsonSerializer.Serialize(default(HierarchicalKey))); + Assert.True(JsonSerializer.Deserialize("null").IsDefault); + } + + [Fact] + public void Json_DictionaryKeyRoundTripsCanonicalEscapedPath() + { + var key = HierarchicalKey.Create("orders", "a/b", @"c\d"); + var values = new Dictionary { [key] = 42, [HierarchicalKey.Create("other")] = 7 }; + + var json = JsonSerializer.Serialize(values); + var decoded = JsonSerializer.Deserialize>(json)!; + + Assert.Equal(JsonSerializer.Serialize(new Dictionary { [key.ToString()] = 42, ["other"] = 7 }), json); + Assert.Equal(2, decoded.Count); + Assert.Equal(42, decoded[key]); + Assert.Equal(7, decoded[HierarchicalKey.Create("other")]); + } + + [Fact] + public void Json_DefaultDictionaryKeyIsRejected() + { + var values = new Dictionary { [default] = 1 }; + + Assert.Throws(() => JsonSerializer.Serialize(values)); + } + + [Theory] + [InlineData("\"\"")] + [InlineData("\"a//b\"")] + [InlineData("\"/a\"")] + [InlineData("\"a/\"")] + [InlineData(@"""a\\x""")] + [InlineData(@"""a\\""")] + [InlineData("42")] + [InlineData("{}")] + public void Json_MalformedValueIsRejected(string json) + { + Assert.Throws(() => JsonSerializer.Deserialize(json)); + } + + [Theory] + [InlineData("{\"\":1}")] + [InlineData("{\"a//b\":1}")] + [InlineData(@"{""a\\x"":1}")] + public void Json_MalformedDictionaryKeyIsRejected(string json) + { + Assert.Throws(() => JsonSerializer.Deserialize>(json)); + } + + [Fact] + public void Default_IsDistinguishedUnsetValue() + { + var key = default(HierarchicalKey); + Assert.True(key.IsDefault); + Assert.Equal(0, key.Length); + Assert.Equal(0, key.CountSegments()); + Assert.Equal(0, key.GetHashCode()); + Assert.Equal(string.Empty, key.ToString()); + Assert.True(key == default); + Assert.False(key != default); + Assert.False(key.Equals(null)); + Assert.Null(key.GetParent()); + Assert.False(key.GetEnumerator().MoveNext()); + Assert.False(key.IsAncestorOf(default)); + Assert.False(key.IsParentOf(default)); + Assert.False(key.IsChildOf(default)); + Assert.False(HierarchicalKey.Create("a").IsAncestorOf(key)); + Assert.Throws(() => key.CreateChildKey("a")); + Assert.Throws(() => key.Append(HierarchicalKey.Create("a"))); + Assert.Throws(() => HierarchicalKey.Create("a").Append(default)); + Span empty = []; + Assert.True(key.TryFormat(empty, out var written, default, null)); + Assert.Equal(0, written); + } + + [Theory] + [InlineData("foo", "foo")] + [InlineData("foo/bar", @"foo\/bar")] + [InlineData(@"foo\bar", @"foo\\bar")] + [InlineData(@"foo\/bar/baz", @"foo\\\/bar\/baz")] + [InlineData("/", @"\/")] + [InlineData(@"\", @"\\")] + [InlineData("/a//b/", @"\/a\/\/b\/")] + [InlineData(" ", " ")] + public void Create_LiteralSegment_EscapesExactlyOnce(string literal, string canonical) + { + var key = HierarchicalKey.Create(literal); + Assert.Equal(canonical, key.ToString()); + Assert.Equal(canonical.Length, key.Length); + Assert.Equal(1, key.CountSegments()); + Assert.Null(key.GetParent()); + Assert.Equal(key, HierarchicalKey.Parse(canonical)); + Assert.Equal([canonical], Segments(key)); + Assert.Equal(key, HierarchicalKey.Create(new[] { literal })); + } + + [Fact] + public void Create_WithValues_CreatesRootFirstHierarchy() + { + var key = HierarchicalKey.Create("orders", "42", "payment"); + var root = HierarchicalKey.Create("orders"); + var parent = root.CreateChildKey("42"); + Assert.Equal("orders/42/payment", key.ToString()); + Assert.Equal(["orders", "42", "payment"], Segments(key)); + Assert.Equal(3, key.CountSegments()); + Assert.Equal(17, key.Length); + Assert.Equal(parent.CreateChildKey("payment"), key); + Assert.Equal(HierarchicalKey.Parse("orders/42/payment"), key); + Assert.Equal(parent, key.GetParent()); + Assert.Equal(root, key.GetParent()!.Value.GetParent()); + Assert.Null(key.GetParent()!.Value.GetParent()!.Value.GetParent()); + Assert.True(parent.IsParentOf(key)); + Assert.True(key.IsChildOf(parent)); + Assert.True(root.IsAncestorOf(key)); + Assert.False(root.IsParentOf(key)); + } + + [Fact] + public void Create_WithValues_ArrayAndSpanSlicesUseOnlyProvidedFragments() + { + string[] input = ["ignored", "orders", "42", "payment", "ignored"]; + var expected = HierarchicalKey.Parse("orders/42/payment"); + Assert.Equal(expected, HierarchicalKey.Create(input.AsSpan(1, 3))); + ReadOnlySpan slice = input.AsSpan(1, 3); + Assert.Equal(expected, HierarchicalKey.Create(slice)); + Assert.Equal(expected, HierarchicalKey.Create(["orders", "42", "payment"])); + } + + [Fact] + public void Create_WithValues_ArrayMutationDoesNotChangeKey() + { + string[] input = ["orders", "42", "payment"]; + var key = HierarchicalKey.Create(input); + var hash = key.GetHashCode(); + input[0] = "other"; + input[1] = null!; + Assert.Equal("orders/42/payment", key.ToString()); + Assert.Equal(hash, key.GetHashCode()); + Assert.Equal(HierarchicalKey.Parse("orders/42"), key.GetParent()); + } + + [Fact] + public void Create_WithValues_EmptyInputThrowsArgumentException() + { + Assert.Equal("values", Assert.Throws(() => HierarchicalKey.Create()).ParamName); + Assert.Throws(() => HierarchicalKey.Create(ReadOnlySpan.Empty)); + Assert.Throws(() => HierarchicalKey.Create(Array.Empty())); + } + + [Theory] + [InlineData(0)] + [InlineData(1)] + [InlineData(2)] + public void Create_WithValues_NullFragmentThrowsArgumentNullException(int position) + { + string[] input = ["a", "b", "c"]; + input[position] = null!; + Assert.Equal("value", Assert.Throws(() => HierarchicalKey.Create(input)).ParamName); + } + + [Theory] + [InlineData(0)] + [InlineData(1)] + [InlineData(2)] + public void Create_WithValues_EmptyLiteralSegmentRejects(int position) + { + string[] input = ["a", "b", "c"]; + input[position] = ""; + Assert.Equal("value", Assert.Throws(() => HierarchicalKey.Create(input)).ParamName); + } + + [Fact] + public void Create_LiteralSegment_RejectsNullAndEmpty() + { + Assert.Throws(() => HierarchicalKey.Create((string)null!)); + Assert.Throws(() => HierarchicalKey.Create("")); + var parent = HierarchicalKey.Create("root"); + Assert.Throws(() => parent.CreateChildKey(null!)); + Assert.Throws(() => parent.CreateChildKey("")); + } + + [Theory] + [InlineData("")] + [InlineData("/a")] + [InlineData("a/")] + [InlineData("a//b")] + [InlineData(@"a\")] + [InlineData(@"a\q")] + [InlineData(@"a\/b/")] + public void Parse_InvalidCanonicalPath_RejectsStringAndSpan(string path) + { + Assert.Throws(() => HierarchicalKey.Parse(path)); + Assert.Throws(() => HierarchicalKey.Parse(path.AsSpan())); + Assert.False(HierarchicalKey.TryParse(path, null, out var first)); + Assert.True(first.IsDefault); + Assert.False(HierarchicalKey.TryParse(path.AsSpan(), null, out var second)); + Assert.True(second.IsDefault); + } + + [Fact] + public void Parse_WithNullString_ThrowsArgumentNullException() + { + Assert.Throws(() => HierarchicalKey.Parse((string)null!)); + Assert.False(HierarchicalKey.TryParse(null, null, out var key)); + Assert.True(key.IsDefault); + } + + // Theory data replaces unpaired surrogates during discovery serialization, so cases use placeholders. + [Theory] + [InlineData("order-{high}")] + [InlineData("{low}-order")] + [InlineData("order-{low}{high}")] + [InlineData("{high}\U0001F600")] + public void Create_UnpairedSurrogate_ThrowsArgumentException(string template) + { + var literal = WithUnpairedSurrogates(template); + Assert.Equal("value", Assert.Throws(() => HierarchicalKey.Create(literal)).ParamName); + Assert.Equal("value", Assert.Throws(() => HierarchicalKey.Create("orders", literal)).ParamName); + Assert.Equal("value", Assert.Throws(() => HierarchicalKey.Create("orders").CreateChildKey(literal)).ParamName); + } + + [Theory] + [InlineData("order-{high}")] + [InlineData("{low}-order")] + [InlineData("order-{low}{high}")] + [InlineData("orders/{high}")] + [InlineData("orders/{low}")] + [InlineData("orders\\/{high}/x")] + public void Parse_UnpairedSurrogate_RejectsStringAndSpan(string template) + { + var path = WithUnpairedSurrogates(template); + Assert.Throws(() => HierarchicalKey.Parse(path)); + Assert.Throws(() => HierarchicalKey.Parse(path.AsSpan())); + Assert.False(HierarchicalKey.TryParse(path, null, out var first)); + Assert.True(first.IsDefault); + Assert.False(HierarchicalKey.TryParse(path.AsSpan(), null, out var second)); + Assert.True(second.IsDefault); + } + + [Fact] + public void SurrogatePairs_RoundTripThroughParseAndOrleansSerialization() + { + var key = HierarchicalKey.Create("order-\U0001F600", "\U0001F601/x"); + const string canonical = "order-\U0001F600/\U0001F601\\/x"; + Assert.Equal(canonical, key.ToString()); + Assert.Equal(2, key.CountSegments()); + Assert.Equal(key, HierarchicalKey.Parse(canonical)); + Assert.True(HierarchicalKey.TryParse(canonical.AsSpan(), null, out var parsed)); + Assert.Equal(key, parsed); + + using var first = new ServiceCollection().AddSerializer().BuildServiceProvider(); + using var second = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var decoded = second.GetRequiredService>().Deserialize(first.GetRequiredService>().SerializeToArray(key)); + Assert.Equal(canonical, decoded.ToString()); + Assert.NotEqual(decoded, HierarchicalKey.Create("order-\U0001F601", "\U0001F601/x")); + } + + [Theory] + [InlineData("a/b/c", 3)] + [InlineData(@"a\/b/c", 2)] + [InlineData(@"a\\/b/c", 3)] + [InlineData(@"a\\\/b/c", 2)] + public void Parse_CanonicalEscapes_PreserveSegmentBoundaries(string path, int count) + { + var key = HierarchicalKey.Parse(path); + Assert.Equal(path, key.ToString()); + Assert.Equal(count, key.CountSegments()); + Assert.Equal(count, Segments(key).Count); + Assert.True(HierarchicalKey.TryParse(path.AsSpan(), null, out var copy)); + Assert.True(key == copy); + Assert.Equal(key.GetHashCode(), copy.GetHashCode()); + var characters = path.ToCharArray(); + var fromCaller = HierarchicalKey.Parse(characters.AsSpan()); + characters[0] = 'z'; + Assert.Equal(key, fromCaller); + } + + [Fact] + public void Append_ComposesConstructedPaths_ChildAppendsOneLiteral() + { + var parent = HierarchicalKey.Parse(@"root/fo\/o"); + var suffix = HierarchicalKey.Parse(@"ba\\r/baz"); + var composed = parent.Append(suffix); + Assert.Equal(@"root/fo\/o/ba\\r/baz", composed.ToString()); + Assert.Equal(4, composed.CountSegments()); + Assert.Equal(["root", @"fo\/o", @"ba\\r", "baz"], Segments(composed)); + Assert.Equal(parent.Append(HierarchicalKey.Create(@"ba\r")), composed.GetParent()); + Assert.True(parent.IsAncestorOf(composed)); + Assert.False(parent.IsParentOf(composed)); + var literal = parent.CreateChildKey("ba/r/baz"); + Assert.Equal(@"root/fo\/o/ba\/r\/baz", literal.ToString()); + Assert.Equal(3, literal.CountSegments()); + Assert.Equal(parent, literal.GetParent()); + Assert.True(parent.IsParentOf(literal)); + Assert.NotEqual(composed, literal); + } + + [Theory] + [InlineData("foo", "foo/bar", true, true)] + [InlineData("foo", "foo/bar/baz", false, true)] + [InlineData("foo/bar", "foo/bar", false, true)] + [InlineData("foo", "foobar/child", false, false)] + [InlineData("foo", @"foo\/bar/child", false, false)] + [InlineData("foo/bar", "foo/baz", false, false)] + [InlineData("Foo", "foo/bar", false, false)] + public void Navigation_UsesExactOrdinalSegments(string root, string path, bool immediate, bool ancestor) + { + var parent = HierarchicalKey.Parse(root); + var child = HierarchicalKey.Parse(path); + Assert.Equal(immediate, parent.IsParentOf(child)); + Assert.Equal(immediate, child.IsChildOf(parent)); + Assert.Equal(ancestor, parent.IsAncestorOf(child)); + if (immediate) Assert.Equal(parent, child.GetParent()); + } + + [Fact] + public void Equality_HashOperatorsAndDictionary_UseCanonicalOrdinalIdentity() + { + var direct = HierarchicalKey.Parse(@"tenant/acme/orders\/42"); + var composed = HierarchicalKey.Create("tenant", "acme", "orders/42"); + Assert.True(direct == composed); + Assert.False(direct != composed); + Assert.True(direct.Equals((object)composed)); + Assert.Equal(direct.GetHashCode(), composed.GetHashCode()); + Assert.Equal(direct.ToString().GetHashCode(StringComparison.Ordinal), direct.GetHashCode()); + var other = HierarchicalKey.Create("Tenant", "acme", "orders/42"); + Assert.True(other != direct); + Assert.False(other == direct); + Assert.False(direct.Equals("tenant/acme/orders\\/42")); + var dictionary = new Dictionary { [direct] = 42, [other] = 81, [default] = 5 }; + Assert.Equal(42, dictionary[composed]); + Assert.Equal(81, dictionary[other]); + Assert.Equal(5, dictionary[default]); + Assert.Equal(3, dictionary.Count); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void SerializationAndCopy_RebuildCanonicalCacheAndPreserveValue(bool unset) + { + using var first = new ServiceCollection().AddSerializer().BuildServiceProvider(); + using var second = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var key = unset ? default : HierarchicalKey.Create("tenant", @"a\/b", "42"); + var serializer = first.GetRequiredService>(); + var bytes = serializer.SerializeToArray(key); + var decoded = second.GetRequiredService>().Deserialize(bytes); + var copy = first.GetRequiredService().Copy(key); + Assert.Equal(key, decoded); + Assert.Equal(key.IsDefault, decoded.IsDefault); + Assert.Equal(key.GetHashCode(), decoded.GetHashCode()); + Assert.Equal(key.CountSegments(), decoded.CountSegments()); + Assert.Equal(key, copy); + var data = Assert.Single(typeof(HierarchicalKey).GetFields(BindingFlags.Instance | BindingFlags.NonPublic)); + Assert.Same(data.GetValue(key), data.GetValue(copy)); + if (!unset) Assert.NotSame(data.GetValue(key), data.GetValue(decoded)); + Assert.Equal(bytes, serializer.SerializeToArray(decoded)); + var map = first.GetRequiredService>>(); + var dictionary = Assert.IsType>(map.Deserialize(map.SerializeToArray(new() { [key] = 42 }))); + Assert.Equal(42, dictionary[decoded]); + } + + [Theory] + [InlineData("")] + [InlineData("a//b")] + [InlineData(@"a\q")] + public void Codec_InvalidCanonicalWire_RejectsInsteadOfConstructingKey(string canonical) + { + using var services = new ServiceCollection().AddSerializer().BuildServiceProvider(); + using var session = services.GetRequiredService().GetSession(); + var output = new ArrayBufferWriter(); + var writer = Writer.Create(output, session); + ReferenceCodec.MarkValueField(session); + writer.WriteFieldHeader(0, typeof(HierarchicalKey), typeof(HierarchicalKey), WireType.TagDelimited); + services.GetRequiredService>().WriteField(ref writer, 0, typeof(string), canonical); + writer.WriteEndObject(); + writer.Commit(); + Assert.Throws(() => services.GetRequiredService>().Deserialize(output.WrittenSpan)); + } + + [Fact] + public void LongHierarchy_ConstructionNavigationAndFormatting_AreIterative() + { + var key = HierarchicalKey.Create(Enumerable.Repeat("x", 10000).ToArray()); + Assert.Equal(10000, key.CountSegments()); + Assert.Equal(19999, key.Length); + Assert.Equal(10000, Segments(key).Count); + Assert.Equal(9999, key.GetParent()!.Value.CountSegments()); + Assert.Equal(key, HierarchicalKey.Parse(key.ToString())); + Assert.True(HierarchicalKey.Create("x").IsAncestorOf(key)); + var span = new char[key.Length]; + Assert.True(key.TryFormat(span, out var written, default, null)); + Assert.Equal(key.Length, written); + Assert.Equal(key.ToString(), new string(span)); + Array.Fill(span, 'z'); + Assert.False(key.TryFormat(span.AsSpan(0, key.Length - 1), out written, default, null)); + Assert.Equal(0, written); + Assert.All(span, character => Assert.Equal('z', character)); + } + + [Fact] + public void EscapeCharacter_IsBackslash() => Assert.Equal('\\', HierarchicalKey.EscapeCharacter); + + [Fact] + public void SegmentSeparator_IsForwardSlash() => Assert.Equal('/', HierarchicalKey.SegmentSeparator); + + internal static string WithUnpairedSurrogates(string template) => template.Replace("{high}", "\ud83d").Replace("{low}", "\ude00"); + + private static List Segments(HierarchicalKey key) + { + var values = new List(); + foreach (var segment in key) values.Add(segment.ToString()); + return values; + } +} diff --git a/test/Orleans.Messaging.Tests/Contracts/MessageIdTests.cs b/test/Orleans.Messaging.Tests/Contracts/MessageIdTests.cs new file mode 100644 index 00000000000..ef6128f46c5 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Contracts/MessageIdTests.cs @@ -0,0 +1,180 @@ +using System.Buffers; +using System.Reflection; +using System.Text.Json; +using Microsoft.Extensions.DependencyInjection; +using Orleans.Serialization; +using Orleans.Serialization.Buffers; +using Orleans.Serialization.Codecs; +using Orleans.Serialization.Session; +using Orleans.Serialization.WireProtocol; +using Xunit; + +namespace Orleans.Messaging.Tests.Contracts; + +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public class MessageIdTests +{ + [Fact] + public void SerializationContract_IdentifiesCurrentType() + { + var type = typeof(MessageId); + Assert.Equal("Orleans.Messaging.MessageId", Assert.Single(type.GetCustomAttributes()).Alias); + Assert.True(type.IsValueType); + Assert.Empty(type.GetCustomAttributes()); + } + + [Fact] + public void Identity_WrapsKeyAndComparesByCanonicalPath() + { + var key = HierarchicalKey.Create("orders", "a/b", "reserve"); + var id = new MessageId(key); + + Assert.Equal(key, id.Key); + Assert.False(id.IsDefault); + Assert.Equal(id, MessageId.Create("orders", "a/b", "reserve")); + Assert.True(id == new MessageId(HierarchicalKey.Parse(key.ToString()))); + Assert.True(id != MessageId.Create("orders", "a/b")); + Assert.Equal(key.GetHashCode(), id.GetHashCode()); + Assert.False(id.Equals((object)key)); + Assert.True(default(MessageId).IsDefault); + Assert.Equal(default, new MessageId(default)); + } + + [Fact] + public void ParseAndFormat_UseCanonicalEscapedPath() + { + var id = MessageId.Create("orders", "a/b", @"c\d"); + const string canonical = @"orders/a\/b/c\\d"; + + Assert.Equal(canonical, id.ToString()); + Assert.Equal(canonical, $"{id}"); + Assert.Equal(id, MessageId.Parse(canonical)); + Assert.Equal(id, MessageId.Parse(canonical.AsSpan())); + Assert.True(MessageId.TryParse(canonical, null, out var parsed)); + Assert.Equal(id, parsed); + Assert.True(MessageId.TryParse(canonical.AsSpan(), null, out parsed)); + Assert.Equal(id, parsed); + var buffer = new char[canonical.Length]; + Assert.True(id.TryFormat(buffer, out var written, default, null)); + Assert.Equal(canonical, new string(buffer, 0, written)); + Assert.False(id.TryFormat(buffer.AsSpan(0, canonical.Length - 1), out written, default, null)); + Assert.Equal(0, written); + } + + [Theory] + [InlineData("")] + [InlineData("a//b")] + [InlineData(@"a\q")] + public void Parse_InvalidPath_IsRejected(string text) + { + Assert.False(MessageId.TryParse(text, null, out var result)); + Assert.True(result.IsDefault); + Assert.False(MessageId.TryParse(text.AsSpan(), null, out result)); + Assert.True(result.IsDefault); + Assert.Throws(() => MessageId.Parse(text)); + Assert.False(MessageId.TryParse(null, null, out result)); + Assert.True(result.IsDefault); + } + + [Fact] + public void Json_ValueAndDictionaryKeyRoundTripCanonicalEscapedPath() + { + var id = MessageId.Create("orders", "a/b", @"c\d"); + + var json = JsonSerializer.Serialize(id); + Assert.Equal(JsonSerializer.Serialize(@"orders/a\/b/c\\d"), json); + Assert.Equal(id, JsonSerializer.Deserialize(json)); + + var map = JsonSerializer.Serialize(new Dictionary { [id] = 42 }); + Assert.Equal(JsonSerializer.Serialize(new Dictionary { [id.ToString()] = 42 }), map); + Assert.Equal(42, JsonSerializer.Deserialize>(map)![id]); + } + + [Fact] + public void Json_DefaultValueRoundTripsAsNullAndIsRejectedAsDictionaryKey() + { + Assert.Equal("null", JsonSerializer.Serialize(default(MessageId))); + Assert.True(JsonSerializer.Deserialize("null").IsDefault); + Assert.Throws(() => JsonSerializer.Serialize(new Dictionary { [default] = 1 })); + } + + [Theory] + [InlineData("\"\"")] + [InlineData("\"a//b\"")] + [InlineData("42")] + public void Json_MalformedValueIsRejected(string json) => + Assert.Throws(() => JsonSerializer.Deserialize(json)); + + [Theory] + [InlineData("order-{high}")] + [InlineData("{low}-order")] + [InlineData("order-{low}{high}")] + public void UnpairedSurrogate_IsRejectedOnEveryConstructionPath(string template) + { + var text = HierarchicalKeyTests.WithUnpairedSurrogates(template); + Assert.Throws(() => MessageId.Create(text)); + Assert.Throws(() => MessageId.Create("orders", text)); + Assert.Throws(() => MessageId.Parse(text)); + Assert.Throws(() => MessageId.Parse(text.AsSpan())); + Assert.False(MessageId.TryParse(text, null, out var result)); + Assert.True(result.IsDefault); + Assert.False(MessageId.TryParse(text.AsSpan(), null, out result)); + Assert.True(result.IsDefault); + } + + [Fact] + public void SurrogatePairs_RoundTripThroughOrleansSerialization() + { + using var first = new ServiceCollection().AddSerializer().BuildServiceProvider(); + using var second = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var id = MessageId.Create("order-\U0001F600"); + + var decoded = second.GetRequiredService>().Deserialize(first.GetRequiredService>().SerializeToArray(id)); + + Assert.Equal(id, decoded); + Assert.Equal("order-\U0001F600", decoded.ToString()); + Assert.Equal(id, MessageId.Parse("order-\U0001F600")); + Assert.NotEqual(decoded, MessageId.Create("order-\U0001F601")); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void SerializationAndCopy_PreserveIdentity(bool unset) + { + using var first = new ServiceCollection().AddSerializer().BuildServiceProvider(); + using var second = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var id = unset ? default : MessageId.Create("tenant", @"a\/b", "42"); + var serializer = first.GetRequiredService>(); + var bytes = serializer.SerializeToArray(id); + var decoded = second.GetRequiredService>().Deserialize(bytes); + + Assert.Equal(id, decoded); + Assert.Equal(id.IsDefault, decoded.IsDefault); + Assert.Equal(id, first.GetRequiredService().Copy(id)); + Assert.Equal(bytes, serializer.SerializeToArray(decoded)); + var map = first.GetRequiredService>>(); + var dictionary = Assert.IsType>(map.Deserialize(map.SerializeToArray(new() { [id] = 42 }))); + Assert.Equal(42, dictionary[decoded]); + } + + [Theory] + [InlineData("")] + [InlineData("a//b")] + [InlineData(@"a\q")] + public void Codec_InvalidCanonicalWire_IsRejected(string canonical) + { + using var services = new ServiceCollection().AddSerializer().BuildServiceProvider(); + using var session = services.GetRequiredService().GetSession(); + var output = new ArrayBufferWriter(); + var writer = Writer.Create(output, session); + ReferenceCodec.MarkValueField(session); + writer.WriteFieldHeader(0, typeof(MessageId), typeof(MessageId), WireType.TagDelimited); + services.GetRequiredService>().WriteField(ref writer, 0, typeof(string), canonical); + writer.WriteEndObject(); + writer.Commit(); + Assert.Throws(() => services.GetRequiredService>().Deserialize(output.WrittenSpan)); + } +} diff --git a/test/Orleans.Messaging.Tests/Contracts/MessagingCommandSerializationTests.cs b/test/Orleans.Messaging.Tests/Contracts/MessagingCommandSerializationTests.cs new file mode 100644 index 00000000000..5364e2a832a --- /dev/null +++ b/test/Orleans.Messaging.Tests/Contracts/MessagingCommandSerializationTests.cs @@ -0,0 +1,173 @@ +using System.Text.Json; +using Microsoft.Extensions.DependencyInjection; +using Orleans.Runtime; +using Orleans.Serialization; +using Xunit; + +namespace Orleans.Messaging.Tests.Contracts; + +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class MessagingCommandSerializationTests +{ + private static readonly DateTimeOffset Now = new(2040, 1, 2, 3, 4, 5, TimeSpan.Zero); + private static readonly GrainId Receiver = GrainId.Create("receiver", "1"); + + public static TheoryData InboxCommands => new(Enumerable.Range(0, CreateInboxCommands().Length)); + public static TheoryData OutboxCommands => new(Enumerable.Range(0, CreateOutboxCommands().Length)); + + [Theory] + [MemberData(nameof(InboxCommands))] + public void InboxCommand_RoundTripsThroughJsonAndOrleansSerialization(int index) + { + var command = CreateInboxCommands()[index]; + var json = JsonSerializer.Serialize(command, InboxJsonContext.Default.InboxCommand); + var fromJson = JsonSerializer.Deserialize(json, InboxJsonContext.Default.InboxCommand)!; + using var services = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var serializer = services.GetRequiredService>(); + var fromBinary = serializer.Deserialize(serializer.SerializeToArray(command)); + + Assert.Equal(command.Kind, fromJson.Kind); + Assert.DoesNotContain("\"ProcessedAt\"", json, StringComparison.Ordinal); + Assert.Equal(json, JsonSerializer.Serialize(fromJson, InboxJsonContext.Default.InboxCommand)); + Assert.Equal(json, JsonSerializer.Serialize(fromBinary!, InboxJsonContext.Default.InboxCommand)); + } + + [Theory] + [MemberData(nameof(OutboxCommands))] + public void OutboxCommand_RoundTripsThroughJsonAndOrleansSerialization(int index) + { + var command = CreateOutboxCommands()[index]; + var json = JsonSerializer.Serialize(command, OutboxJsonContext.Default.OutboxCommand); + var fromJson = JsonSerializer.Deserialize(json, OutboxJsonContext.Default.OutboxCommand)!; + using var services = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var serializer = services.GetRequiredService>(); + var fromBinary = serializer.Deserialize(serializer.SerializeToArray(command)); + + Assert.Equal(command.Kind, fromJson.Kind); + Assert.Equal(json, JsonSerializer.Serialize(fromJson, OutboxJsonContext.Default.OutboxCommand)); + Assert.Equal(json, JsonSerializer.Serialize(fromBinary!, OutboxJsonContext.Default.OutboxCommand)); + } + + [Fact] + public void DeliverAt_RoundTripsThroughJsonAndOrleansSerializationAndIsOmittedWhenUnset() + { + var deliverAt = new DateTimeOffset(2040, 1, 2, 3, 4, 5, TimeSpan.FromHours(2)); + var scheduled = new InboxCommand { Kind = InboxCommandKind.Accept, Message = InboxMessage("scheduled", deliverAt) }; + var immediate = new InboxCommand { Kind = InboxCommandKind.Accept, Message = InboxMessage("immediate") }; + using var services = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var serializer = services.GetRequiredService>(); + + var json = JsonSerializer.Serialize(scheduled, InboxJsonContext.Default.InboxCommand); + var fromJson = JsonSerializer.Deserialize(json, InboxJsonContext.Default.InboxCommand)!.Message!.Value.Envelope.DeliverAt; + var fromBinary = serializer.Deserialize(serializer.SerializeToArray(scheduled))!.Message!.Value.Envelope.DeliverAt; + var immediateJson = JsonSerializer.Serialize(immediate, InboxJsonContext.Default.InboxCommand); + + Assert.Contains("\"DeliverAt\":", json, StringComparison.Ordinal); + Assert.Equal(deliverAt, fromJson); + Assert.Equal(deliverAt.Offset, fromJson!.Value.Offset); + Assert.Equal(deliverAt, fromBinary); + Assert.Equal(deliverAt.Offset, fromBinary!.Value.Offset); + Assert.DoesNotContain("DeliverAt", immediateJson, StringComparison.Ordinal); + Assert.Null(JsonSerializer.Deserialize(immediateJson, InboxJsonContext.Default.InboxCommand)!.Message!.Value.Envelope.DeliverAt); + Assert.Null(serializer.Deserialize(serializer.SerializeToArray(immediate))!.Message!.Value.Envelope.DeliverAt); + Assert.DoesNotContain("DeliverAt", + JsonSerializer.Serialize(new OutboxCommand { Kind = OutboxCommandKind.Enqueue, Message = OutboxMessage("immediate"), State = new() }, + OutboxJsonContext.Default.OutboxCommand), + StringComparison.Ordinal); + } + + [Fact] + public void ReferenceEnqueue_CarriesNoMessageInJsonAndOrleansSerialization() + { + var command = new OutboxCommand { Kind = OutboxCommandKind.Enqueue, Reference = Reference("reference"), State = new() }; + using var services = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var serializer = services.GetRequiredService>(); + + var json = JsonSerializer.Serialize(command, OutboxJsonContext.Default.OutboxCommand); + var fromJson = JsonSerializer.Deserialize(json, OutboxJsonContext.Default.OutboxCommand)!; + var fromBinary = serializer.Deserialize(serializer.SerializeToArray(command))!; + + Assert.DoesNotContain("Payload", json, StringComparison.Ordinal); + foreach (var recovered in new[] { fromJson, fromBinary }) + { + Assert.Null(recovered.Message); + Assert.Equal(MessageId.Create("orders", "reference"), recovered.Reference!.Value.MessageId); + } + } + + private static InboxMessage InboxMessage(string id, DateTimeOffset? deliverAt = null) => new() { Envelope = Envelope(id, deliverAt), SentAt = Now }; + + private static OutboxMessage OutboxMessage(string id, DateTimeOffset? deliverAt = null) => new() { Envelope = Envelope(id, deliverAt), ReceiverId = Receiver }; + + private static OutboxMessageReference Reference(string id) => new() { MessageId = MessageId.Create("orders", id) }; + + private static Envelope Envelope(string id, DateTimeOffset? deliverAt = null) => new() + { + MessageId = MessageId.Create("orders", id), + Payload = [1, 2, 3], + Subject = "orders.v1", + Headers = new(StringComparer.Ordinal) { ["trace"] = [4, 5] }, + DeliverAt = deliverAt + }; + + private static InboxCommand[] CreateInboxCommands() + { + var deadLetter = new InboxDeadLetter { Message = InboxMessage("dead"), DeadLetteredAt = Now, Reason = "failed", AttemptCount = 3, Code = "orders/invalid" }; + var rejection = new InboxRejection { Reason = "failed", Code = "orders/invalid" }; + return + [ + new() { Kind = InboxCommandKind.Accept, Message = InboxMessage("accept"), State = new() }, + new() { Kind = InboxCommandKind.Accept, Message = InboxMessage("scheduled", Now), State = new() }, + new() { Kind = InboxCommandKind.Retry, MessageId = MessageId.Create("orders", "retry"), State = new() { AttemptCount = 2, NextAttemptAt = Now, LastError = "boom" } }, + new() { Kind = InboxCommandKind.Complete, MessageId = MessageId.Create("orders", "done"), SentAt = Now, DeadLetter = deadLetter }, + new() { Kind = InboxCommandKind.Complete, MessageId = MessageId.Create("orders", "dead"), SentAt = Now, DeadLetter = deadLetter, Rejection = rejection }, + new() { Kind = InboxCommandKind.ExtendSentAt, MessageId = MessageId.Create("orders", "accept"), SentAt = Now.AddHours(1) }, + new() { Kind = InboxCommandKind.Complete, MessageId = MessageId.Create("orders", "done"), SentAt = Now.AddHours(1) }, + new() { Kind = InboxCommandKind.Complete, MessageId = MessageId.Create("orders", "dead"), SentAt = Now.AddHours(1), Rejection = rejection }, + new() { Kind = InboxCommandKind.RemoveDeadLetters, RemovedKeys = [MessageId.Create("orders", "dead")] }, + new() + { + Kind = InboxCommandKind.Snapshot, + Snapshot = new() + { + Messages = [new() { Message = InboxMessage("pending"), State = new() { AttemptCount = 1 } }, new() { Message = InboxMessage("scheduled", Now), State = new() }], + Processed = + [ + new() { MessageId = MessageId.Create("orders", "done"), SentAt = Now }, + new() { MessageId = MessageId.Create("orders", "dead"), SentAt = Now, Rejection = rejection } + ], + DeadLetters = [deadLetter] + } + } + ]; + } + + private static OutboxCommand[] CreateOutboxCommands() + { + var deadLetter = new OutboxDeadLetter { Message = OutboxMessage("dead"), DeadLetteredAt = Now, Reason = "rejected", AttemptCount = 3 }; + return + [ + new() { Kind = OutboxCommandKind.Enqueue, Message = OutboxMessage("enqueue"), State = new() }, + new() { Kind = OutboxCommandKind.Enqueue, Message = OutboxMessage("scheduled", Now), State = new() }, + new() { Kind = OutboxCommandKind.Enqueue, Reference = Reference("reference"), State = new() }, + new() { Kind = OutboxCommandKind.Delivery, MessageId = MessageId.Create("orders", "retry"), State = new() { AttemptCount = 2, NextAttemptAt = Now, LastError = "boom", SentAt = Now } }, + new() { Kind = OutboxCommandKind.Delivery, MessageId = MessageId.Create("orders", "dead"), DeadLetter = deadLetter }, + new() { Kind = OutboxCommandKind.RemoveDeadLetters, RemovedDeadLetters = [MessageId.Create("orders", "dead")] }, + new() + { + Kind = OutboxCommandKind.Snapshot, + Snapshot = new() + { + Messages = + [ + new() { Message = OutboxMessage("pending", Now), State = new() { AttemptCount = 1, SentAt = Now } }, + new() { Reference = Reference("pending-reference"), State = new() { AttemptCount = 2 } } + ], + DeadLetters = [deadLetter] + } + } + ]; + } +} diff --git a/test/Orleans.Messaging.Tests/Contracts/StandardJournaledStateTests.cs b/test/Orleans.Messaging.Tests/Contracts/StandardJournaledStateTests.cs new file mode 100644 index 00000000000..6af1bae3c74 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Contracts/StandardJournaledStateTests.cs @@ -0,0 +1,105 @@ +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; +using Orleans.Journaling; +using Orleans.Messaging.Tests.Functional; +using Orleans.Messaging.Tests.Support; +using Xunit; + +namespace Orleans.Messaging.Tests.Contracts; + +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class StandardJournaledStateTests +{ + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task Dictionary_WriteAcknowledgementPreservesLaterMutations(bool snapshot) + { + var id = new JournalId("standard-dictionary/" + Guid.NewGuid().ToString("N")); + var storage = new ControlledJournalStorageProvider(); + await using var provider = CreateProvider(storage, id); + await using var scope = provider.CreateAsyncScope(); + var owner = scope.ServiceProvider.GetRequiredService(); + var items = scope.ServiceProvider.GetRequiredKeyedService>("state"); + Assert.False(owner is IDurableStateManager); + Assert.True(owner.TryGetStateMachine("state", out var registered)); + Assert.Same(items, registered); + await owner.InitializeAsync(TestContext.Current.CancellationToken); + items.Add("first", 1); + if (snapshot) storage.RequestSnapshot(id); + using var blocked = storage.BlockWrite(id); + var first = owner.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + await blocked.WaitUntilEnteredAsync(); + items["later"] = 2; + Assert.False(first.IsCompleted); + blocked.Release(); + await first; + await using (var replay = provider.CreateAsyncScope()) + { + var recovered = replay.ServiceProvider.GetRequiredService(); + var persisted = replay.ServiceProvider.GetRequiredKeyedService>("state"); + await recovered.InitializeAsync(TestContext.Current.CancellationToken); + Assert.Equal(new KeyValuePair("first", 1), Assert.Single(persisted)); + } + Assert.Equal(2, items["later"]); + await owner.WriteStateAsync(TestContext.Current.CancellationToken); + await using var finalScope = provider.CreateAsyncScope(); + var finalOwner = finalScope.ServiceProvider.GetRequiredService(); + var final = finalScope.ServiceProvider.GetRequiredKeyedService>("state"); + await finalOwner.InitializeAsync(TestContext.Current.CancellationToken); + Assert.Equal(new[] { new KeyValuePair("first", 1), new KeyValuePair("later", 2) }, final.OrderBy(pair => pair.Key)); + Assert.Equal(2, storage.GetSuccessfulWriteCount(id)); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task Value_WriteAcknowledgementPreservesNewerPendingValue(bool snapshot) + { + var id = new JournalId("standard-value/" + Guid.NewGuid().ToString("N")); + var storage = new ControlledJournalStorageProvider(); + await using var provider = CreateProvider(storage, id); + await using var scope = provider.CreateAsyncScope(); + var owner = scope.ServiceProvider.GetRequiredService(); + var value = scope.ServiceProvider.GetRequiredKeyedService>("state"); + await owner.InitializeAsync(TestContext.Current.CancellationToken); + value.Value = 1; + if (snapshot) storage.RequestSnapshot(id); + using var blocked = storage.BlockWrite(id); + var first = owner.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + await blocked.WaitUntilEnteredAsync(); + value.Value = 2; + blocked.Release(); + await first; + await using (var replay = provider.CreateAsyncScope()) + { + var recovered = replay.ServiceProvider.GetRequiredService(); + var persisted = replay.ServiceProvider.GetRequiredKeyedService>("state"); + await recovered.InitializeAsync(TestContext.Current.CancellationToken); + Assert.Equal(1, persisted.Value); + } + Assert.Equal(2, value.Value); + await owner.WriteStateAsync(TestContext.Current.CancellationToken); + await using var finalScope = provider.CreateAsyncScope(); + var finalOwner = finalScope.ServiceProvider.GetRequiredService(); + var final = finalScope.ServiceProvider.GetRequiredKeyedService>("state"); + await finalOwner.InitializeAsync(TestContext.Current.CancellationToken); + Assert.Equal(2, final.Value); + Assert.Equal(2, storage.GetSuccessfulWriteCount(id)); + } + + private static ServiceProvider CreateProvider(ControlledJournalStorageProvider storage, JournalId id) + { + var builder = InboxStateManagerBoundaryTests.CreateBuilder("orleans-binary"); + builder.Services.AddSingleton(sp => + { + storage.Configure(sp.GetRequiredService>()); + return storage; + }); + builder.Services.AddScoped(sp => + sp.GetRequiredService().CreateStandalone(id)); + return builder.Services.BuildServiceProvider(validateScopes: true); + } +} diff --git a/test/Orleans.Messaging.Tests/Contracts/TypedMessageHelperTests.cs b/test/Orleans.Messaging.Tests/Contracts/TypedMessageHelperTests.cs new file mode 100644 index 00000000000..49085ebc987 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Contracts/TypedMessageHelperTests.cs @@ -0,0 +1,2158 @@ +using System.Buffers; +using System.Diagnostics.CodeAnalysis; +using System.Text; +using Microsoft.Extensions.DependencyInjection; +using NSubstitute; +using Orleans.Hosting; +using Orleans.Messaging.Tests.Support; +using Orleans.Runtime; +using Orleans.Serialization; +using Orleans.Serialization.Buffers; +using Orleans.Serialization.Codecs; +using Orleans.Serialization.Session; +using Orleans.Serialization.WireProtocol; +using Xunit; + +namespace Orleans.Messaging.Tests.Contracts; + +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class TypedMessageHelperTests +{ + private const string Subject = "orders.reserve"; + private static GrainId Sender => GrainId.Create("sender", "typed-helper"); + private static GrainId Receiver => GrainId.Create("receiver", "warehouse"); + private static MessageId MessageKey => MessageId.Create("orders", "42", "reserve"); + + [Fact] + public void MessageType_RejectsNullSerializer() + { + var exception = Assert.Throws( + () => new MessageType(Subject, (Serializer)null!)); + Assert.Equal("serializer", exception.ParamName); + + exception = Assert.Throws( + () => new MessageType(Subject, (Serializer)null!)); + Assert.Equal("serializer", exception.ParamName); + } + + [Fact] + public void MessageType_CreatedFromSerializer_RoundTripsBody() + { + using var services = CreateServices(); + var messageType = new MessageType(Subject, services.GetRequiredService()); + + var message = messageType.Create(MessageKey, Receiver, "payload"); + + Assert.Equal(Subject, messageType.Subject); + Assert.Equal(Subject, message.Envelope.Subject); + Assert.Equal("payload", messageType.Decode(message.Envelope)); + } + + [Fact] + public void MessageType_RejectsNullSubject() + { + using var services = CreateServices(); + var (serializer, probe) = CreateProbedSerializer(services); + var exception = Assert.Throws( + () => new MessageType(null!, serializer)); + Assert.Equal("subject", exception.ParamName); + Assert.Equal(0, probe.WriteCount); + Assert.Equal(0, probe.ReadCount); + Assert.Equal(Subject, new MessageType(Subject, serializer).Subject); + } + + [Theory] + [InlineData("empty")] + [InlineData("ascii")] + [InlineData("multibyte")] + public void MessageType_RejectsEmptyOrOversizedUtf8Subject(string variation) + { + using var services = CreateServices(); + var (serializer, probe) = CreateProbedSerializer(services); + var subject = InvalidSubject(variation); + + var exception = Assert.Throws(() => new MessageType(subject, serializer)); + Assert.Equal("subject", exception.ParamName); + Assert.Equal(0, probe.WriteCount); + Assert.Equal(0, probe.ReadCount); + Assert.Equal(Subject, new MessageType(Subject, serializer).Subject); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void MessageType_AcceptsSubjectAtUtf8ByteLimit(bool multibyte) + { + using var services = CreateServices(); + var serializer = services.GetRequiredService>(); + var subject = SubjectAtLimit(multibyte); + var messageType = new MessageType(subject, serializer); + var envelope = DirectEnvelope(serializer, subject, "boundary reserve €42"); + + Assert.Equal(256, Encoding.UTF8.GetByteCount(subject)); + Assert.Equal(subject, messageType.Subject); + Assert.Equal("boundary reserve €42", messageType.Decode(envelope)); + } + + [Theory] + [InlineData("reserve order 42: € / 東京")] + [InlineData("")] + [InlineData("first line\nsecond line")] + public void MessageType_DecodeReadsDirectSerializerPayload(string body) + { + using var sendingServices = CreateServices(); + using var receivingServices = CreateServices(); + var sendingSerializer = sendingServices.GetRequiredService>(); + var receivingSerializer = receivingServices.GetRequiredService>(); + var envelope = DirectEnvelope(sendingSerializer, Subject, body); + var expectedWire = sendingSerializer.SerializeToArray(body); + var messageType = new MessageType(Subject, receivingSerializer); + + Assert.Equal(body, messageType.Decode(envelope)); + Assert.Equal(Subject, messageType.Subject); + Assert.Equal(expectedWire, envelope.Payload.ToArray()); + Assert.Equal(body, receivingSerializer.Deserialize(envelope.Payload)); + } + + [Theory] + [InlineData("Orders.Reserve")] + [InlineData("orders.cancel")] + public void MessageType_DecodeRejectsWrongSubjectBeforeCodecRead(string wrongSubject) + { + using var services = CreateServices(); + var direct = services.GetRequiredService>(); + var (serializer, probe) = CreateProbedSerializer(services); + var sentinel = new InvalidDataException("read must not run"); + probe.ReadFailure = sentinel; + var messageType = new MessageType(Subject, serializer); + var envelope = DirectEnvelope(direct, wrongSubject, "reserve €42"); + var expectedWire = direct.SerializeToArray("reserve €42"); + + var exception = Assert.Throws(() => messageType.Decode(envelope)); + + Assert.Equal("envelope", exception.ParamName); + Assert.NotSame(sentinel, exception); + Assert.Equal(0, probe.ReadCount); + Assert.Equal(expectedWire, envelope.Payload.ToArray()); + Assert.Equal("reserve €42", direct.Deserialize(envelope.Payload)); + } + + [Fact] + public void MessageType_DecodeRejectsDirectSerializedNull() + { + using var services = CreateServices(); + var serializer = services.GetRequiredService>(); + var envelope = DirectEnvelope(serializer, Subject, null!); + var expectedWire = serializer.SerializeToArray(null); + var messageType = new MessageType(Subject, serializer); + Assert.Null(serializer.Deserialize(envelope.Payload)); + + var exception = Assert.Throws(() => messageType.Decode(envelope)); + + Assert.Equal("envelope", exception.ParamName); + Assert.Equal(Subject, messageType.Subject); + Assert.Equal(expectedWire, envelope.Payload.ToArray()); + Assert.Null(serializer.Deserialize(envelope.Payload)); + } + + [Fact] + public void AddMessageType_RejectsNullServiceCollection() + { + var exception = Assert.Throws( + () => MessageTypeExtensions.AddMessageType(null!, Subject)); + Assert.Equal("services", exception.ParamName); + } + + [Fact] + public void AddMessageType_RejectsNullSubjectWithoutChangingServices() + { + var services = new ServiceCollection(); + services.AddSerializer(); + var before = services.ToArray(); + + var exception = Assert.Throws( + () => services.AddMessageType(null!)); + + Assert.Equal("subject", exception.ParamName); + AssertUnchangedServices(before, services); + } + + [Theory] + [InlineData("empty")] + [InlineData("ascii")] + [InlineData("multibyte")] + public void AddMessageType_RejectsEmptyOrOversizedUtf8SubjectWithoutChangingServices(string variation) + { + var services = new ServiceCollection(); + services.AddSerializer(); + var before = services.ToArray(); + var subject = InvalidSubject(variation); + + var exception = Assert.Throws(() => services.AddMessageType(subject)); + + Assert.Equal("subject", exception.ParamName); + AssertUnchangedServices(before, services); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void AddMessageType_AcceptsSubjectAtUtf8ByteLimit(bool multibyte) + { + var subject = SubjectAtLimit(multibyte); + var services = new ServiceCollection(); + services.AddSerializer(); + Assert.Same(services, services.AddMessageType(subject)); + using var provider = services.BuildServiceProvider(); + var descriptor = provider.GetRequiredKeyedService>(subject); + var envelope = DirectEnvelope(provider.GetRequiredService>(), subject, "registered boundary"); + + Assert.Equal(256, Encoding.UTF8.GetByteCount(subject)); + Assert.Equal(subject, descriptor.Subject); + Assert.Equal("registered boundary", descriptor.Decode(envelope)); + Assert.Same(descriptor, provider.GetRequiredKeyedService>(subject)); + } + + [Fact] + public void AddMessageType_ReturnsOriginalCollectionAndRegistersExactKeyedSingleton() + { + var services = new ServiceCollection(); + var result = services.AddMessageType(Subject); + Assert.Same(services, result); + Assert.Equal(2, services.Count); + var registration = Assert.Single(services, descriptor => descriptor.IsKeyedService); + Assert.True(registration.IsKeyedService); + Assert.Equal(Subject, registration.ServiceKey); + Assert.Equal(typeof(MessageType), registration.ServiceType); + Assert.Equal(ServiceLifetime.Singleton, registration.Lifetime); + + // Successful registration before adding serializers proves lazy resolution. + services.AddSerializer(); + using var provider = services.BuildServiceProvider(); + using var scope = provider.CreateScope(); + var messageType = provider.GetRequiredKeyedService>(Subject); + Assert.Equal(Subject, messageType.Subject); + Assert.Same(messageType, provider.GetRequiredKeyedService>(Subject)); + Assert.Same(messageType, scope.ServiceProvider.GetRequiredKeyedService>(Subject)); + var envelope = DirectEnvelope(provider.GetRequiredService>(), Subject, "keyed reserve"); + Assert.Equal("keyed reserve", messageType.Decode(envelope)); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void AddMessageType_RegistersClosedSingletonSerializerFactoryBeforeKeyedBinding(bool valueType) + { + if (valueType) AssertClosedSerializerFactory(731); + else AssertClosedSerializerFactory("closed factory €42"); + } + + [Theory] + [InlineData(false, false)] + [InlineData(false, true)] + [InlineData(true, false)] + [InlineData(true, true)] + public void AddMessageType_PreservesExistingCustomClosedSerializerAndUsesItForBinding(bool valueType, bool factory) + { + if (valueType) AssertCustomClosedSerializerPreserved(731, factory); + else AssertCustomClosedSerializerPreserved("custom closed €42", factory); + } + + [Fact] + public void AddMessageType_AllowsMultipleSubjectsForSamePayloadType() + { + var services = new ServiceCollection(); + services.AddSerializer(); + services.AddMessageType(Subject).AddMessageType("orders.cancel"); + using var provider = services.BuildServiceProvider(); + using var scope = provider.CreateScope(); + var reserve = provider.GetRequiredKeyedService>(Subject); + var cancel = provider.GetRequiredKeyedService>("orders.cancel"); + + Assert.Equal(Subject, reserve.Subject); + Assert.Equal("orders.cancel", cancel.Subject); + Assert.NotSame(reserve, cancel); + Assert.Same(reserve, provider.GetRequiredKeyedService>(Subject)); + Assert.Same(cancel, provider.GetRequiredKeyedService>("orders.cancel")); + Assert.Same(reserve, scope.ServiceProvider.GetRequiredKeyedService>(Subject)); + Assert.Same(cancel, scope.ServiceProvider.GetRequiredKeyedService>("orders.cancel")); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void AddMessageType_RejectsSameAndCrossTypeDuplicateSubjectsWithoutChangingServices(bool crossType) + { + var services = new ServiceCollection(); + services.AddSerializer(); + services.AddMessageType(Subject); + var before = services.ToArray(); + + Assert.Throws(() => + { + if (crossType) + { + services.AddMessageType(Subject); + } + else + { + services.AddMessageType(Subject); + } + }); + + AssertUnchangedServices(before, services); + using var provider = services.BuildServiceProvider(); + var original = provider.GetRequiredKeyedService>(Subject); + Assert.Equal(Subject, original.Subject); + var envelope = DirectEnvelope(provider.GetRequiredService>(), Subject, "original binding"); + Assert.Equal("original binding", original.Decode(envelope)); + Assert.Null(provider.GetKeyedService>(Subject)); + } + + [Fact] + public void AddMessageType_AllowsCaseDistinctSubjects() + { + var services = new ServiceCollection(); + services.AddSerializer(); + services.AddMessageType(Subject).AddMessageType("Orders.Reserve"); + using var provider = services.BuildServiceProvider(); + var lower = provider.GetRequiredKeyedService>(Subject); + var upper = provider.GetRequiredKeyedService>("Orders.Reserve"); + + Assert.Equal(Subject, lower.Subject); + Assert.Equal("Orders.Reserve", upper.Subject); + Assert.NotSame(lower, upper); + var lowerEnvelope = DirectEnvelope(provider.GetRequiredService>(), Subject, "lower"); + var upperEnvelope = DirectEnvelope(provider.GetRequiredService>(), "Orders.Reserve", "upper"); + Assert.Equal("lower", lower.Decode(lowerEnvelope)); + Assert.Equal("upper", upper.Decode(upperEnvelope)); + Assert.Throws(() => lower.Decode(upperEnvelope)); + } + + [Theory] + [InlineData("receiver")] + [InlineData("messageId")] + [InlineData("key-ascii-bytes")] + [InlineData("key-multibyte-bytes")] + public void MessageType_CreateRejectsInvalidIdentitiesBeforeCodecWrite(string invalidIdentity) + { + using var services = CreateServices(); + var (serializer, probe) = CreateProbedSerializer(services); + var messageType = new MessageType(Subject, serializer); + var receiver = invalidIdentity == "receiver" ? default : Receiver; + var key = invalidIdentity switch + { + "messageId" => default, + "key-ascii-bytes" => MessageId.Create(new string('k', 1025)), + "key-multibyte-bytes" => MessageId.Create(new string('\u20ac', 341) + "ab"), + _ => MessageKey + }; + OutboxMessage? result = null; + var exception = Assert.Throws(() => + { + result = messageType.Create(key, receiver, "must not encode"); + }); + Assert.Equal(invalidIdentity switch + { + "receiver" => "receiverId", + _ => "messageId" + }, exception.ParamName); + Assert.Null(result); + Assert.Equal(0, probe.WriteCount); + } + + [Fact] + public void MessageType_CreateUsesExplicitIdentitiesAndSuppliedHierarchicalKey() + { + using var services = CreateServices(); + var serializer = services.GetRequiredService>(); + var messageType = new MessageType(Subject, serializer); + var key = MessageId.Create("orders", "42", "reserve"); + var first = messageType.Create(key, Receiver, "reserve first"); + var otherReceiver = GrainId.Create("receiver", "other-warehouse"); + var second = messageType.Create(key, otherReceiver, "reserve second"); + + AssertEnvelopeIdentity(first, key, Receiver, Subject); + AssertEnvelopeIdentity(second, key, otherReceiver, Subject); + Assert.Equal("orders/42/reserve", first.Envelope.MessageId.ToString()); + Assert.Equal("orders/42/reserve", second.Envelope.MessageId.ToString()); + Assert.Equal(3, first.Envelope.MessageId.Key.CountSegments()); + Assert.Equal(3, second.Envelope.MessageId.Key.CountSegments()); + Assert.Equal("reserve first", serializer.Deserialize(first.Envelope.Payload)); + Assert.Equal("reserve second", serializer.Deserialize(second.Envelope.Payload)); + } + + [Theory] + [InlineData("segments")] + [InlineData("ascii-bytes")] + [InlineData("multibyte-bytes")] + public void MessageType_CreateAcceptsMessageKeyAtAdmissionLimits(string variation) + { + using var services = CreateServices(); + var (serializer, probe) = CreateProbedSerializer(services); + var messageType = new MessageType(Subject, serializer); + var key = variation switch + { + "segments" => MessageId.Create(Enumerable.Repeat("s", 32).ToArray()), + "ascii-bytes" => MessageId.Create(new string('k', 1024)), + "multibyte-bytes" => MessageId.Create(new string('\u20ac', 341) + "a"), + _ => throw new ArgumentOutOfRangeException(nameof(variation)) + }; + + var envelope = messageType.Create(key, Receiver, "admitted boundary"); + + Assert.Equal(1, probe.WriteCount); + AssertEnvelopeIdentity(envelope, key, Receiver, Subject); + if (variation == "segments") + { + Assert.Equal(32, envelope.Envelope.MessageId.Key.CountSegments()); + } + else + { + Assert.Equal(1024, Encoding.UTF8.GetByteCount(envelope.Envelope.MessageId.ToString())); + } + Assert.Equal("admitted boundary", services.GetRequiredService>().Deserialize(envelope.Envelope.Payload)); + } + + [Theory] + [InlineData("string")] + [InlineData("empty-string")] + [InlineData("int")] + [InlineData("zero-int")] + [InlineData("negative-int")] + [InlineData("bytes")] + [InlineData("empty-bytes")] + public void MessageType_CreateMatchesDirectSerializerWireFormat(string contract) + { + using var services = CreateServices(); + using var receivingServices = CreateServices(); + switch (contract) + { + case "string": + AssertCreateWire(services, receivingServices, "reserve €42 / 東京"); + break; + case "empty-string": + AssertCreateWire(services, receivingServices, ""); + break; + case "int": + AssertCreateWire(services, receivingServices, 42_017); + break; + case "zero-int": + AssertCreateWire(services, receivingServices, 0); + break; + case "negative-int": + AssertCreateWire(services, receivingServices, -709); + break; + case "bytes": + AssertCreateWire(services, receivingServices, new byte[] { 0, 255, 128, 17, 42 }); + break; + case "empty-bytes": + AssertCreateWire(services, receivingServices, Array.Empty()); + break; + default: + throw new ArgumentOutOfRangeException(nameof(contract)); + } + + } + + [Fact] + public void MessageType_CreateAllocatesIndependentArraysAcrossSubsequentCalls() + { + using var services = CreateServices(); + var serializer = services.GetRequiredService>(); + var messageType = new MessageType(Subject, serializer); + var firstBody = new byte[] { 9, 8, 7, 255 }; + var largeBody = Enumerable.Range(0, 1_048_576).Select(i => (byte)(i * 31 + 7)).ToArray(); + var lastBody = new byte[] { 42, 0, 128, 255, 19, 21 }; + var largeWire = serializer.SerializeToArray(largeBody); + var lastWire = serializer.SerializeToArray(lastBody); + var first = messageType.Create(MessageKey, Receiver, firstBody); + var repeated = messageType.Create(MessageKey, Receiver, firstBody); + var large = messageType.Create(MessageKey, Receiver, largeBody); + var last = messageType.Create(MessageKey, Receiver, lastBody); + + Assert.NotNull(first.Envelope.Payload); + Assert.NotSame(firstBody, first.Envelope.Payload); + Assert.NotSame(first.Envelope.Payload, repeated.Envelope.Payload); + Assert.NotSame(first.Envelope.Payload, large.Envelope.Payload); + Assert.NotSame(large.Envelope.Payload, last.Envelope.Payload); + Assert.Equal(serializer.SerializeToArray(firstBody), first.Envelope.Payload.ToArray()); + Assert.Equal(first.Envelope.Payload.ToArray(), repeated.Envelope.Payload.ToArray()); + firstBody[0] = 99; + Assert.Equal(new byte[] { 9, 8, 7, 255 }, serializer.Deserialize(first.Envelope.Payload)); + Assert.Equal(new byte[] { 9, 8, 7, 255 }, serializer.Deserialize(repeated.Envelope.Payload)); + Assert.Equal(largeWire, large.Envelope.Payload.ToArray()); + Assert.Equal(lastWire, last.Envelope.Payload.ToArray()); + Assert.Equal(largeBody, serializer.Deserialize(large.Envelope.Payload)); + Assert.Equal(lastBody, serializer.Deserialize(last.Envelope.Payload)); + AssertEnvelopeIdentity(large, MessageKey, Receiver, Subject); + AssertEnvelopeIdentity(last, MessageKey, Receiver, Subject); + } + + [Fact] + public void MessageType_EncodingFailurePreservesOutstandingBytesAndPristineRetry() + { + using var services = CreateServices(); + var pristine = services.GetRequiredService>(); + var (serializer, probe) = CreateProbedSerializer(services); + var messageType = new MessageType(Subject, serializer); + var outstanding = messageType.Create(MessageKey, Receiver, "outstanding € order"); + var outstandingWire = pristine.SerializeToArray("outstanding € order"); + var sentinel = new InvalidDataException("committed partial codec failure"); + probe.NextWriteFailure = sentinel; + OutboxMessage? failedResult = null; + var exception = Assert.Throws(() => + { + failedResult = messageType.Create(MessageKey, Receiver, "failing preparation"); + }); + Assert.Same(sentinel, exception); + Assert.Null(failedResult); + Assert.Equal(2, probe.WriteCount); + Assert.Equal(1, probe.CommittedFailureCount); + Assert.Equal((byte)0x7e, probe.CommittedFailureByte); + + var recovered = messageType.Create(MessageKey, Receiver, "recovered 東京 reserve"); + Assert.Equal(3, probe.WriteCount); + Assert.Equal(pristine.SerializeToArray("recovered 東京 reserve"), recovered.Envelope.Payload.ToArray()); + Assert.Equal("recovered 東京 reserve", pristine.Deserialize(recovered.Envelope.Payload)); + Assert.Equal(outstandingWire, outstanding.Envelope.Payload.ToArray()); + Assert.Equal("outstanding € order", pristine.Deserialize(outstanding.Envelope.Payload)); + Assert.NotSame(outstanding.Envelope.Payload, recovered.Envelope.Payload); + AssertEnvelopeIdentity(recovered, MessageKey, Receiver, Subject); + } + + [Fact] + public void MessageType_EnvelopeCopySharesStableArrayAcrossSubsequentCreate() + { + using var services = CreateServices(); + var (serializer, probe) = CreateProbedSerializer(services); + var messageType = new MessageType(Subject, serializer); + var original = messageType.Create(MessageKey, Receiver, "surviving reserve"); + var surviving = original; + Assert.Same(original.Envelope.Payload, surviving.Envelope.Payload); + var expected = original.Envelope.Payload.ToArray(); + original = default; + var subsequent = messageType.Create(new MessageId(MessageKey.Key.CreateChildKey("next")), Receiver, "new reserve"); + + Assert.Equal(2, probe.WriteCount); + Assert.NotSame(surviving.Envelope.Payload, subsequent.Envelope.Payload); + Assert.Equal(expected, surviving.Envelope.Payload.ToArray()); + Assert.Equal(MessageKey, surviving.Envelope.MessageId); + Assert.Equal(new MessageId(MessageKey.Key.CreateChildKey("next")), subsequent.Envelope.MessageId); + Assert.Equal("surviving reserve", services.GetRequiredService>().Deserialize(surviving.Envelope.Payload)); + Assert.Equal("new reserve", services.GetRequiredService>().Deserialize(subsequent.Envelope.Payload)); + } + + [Fact] + public void MessageType_CreateRejectsNullBodyBeforeCodecWrite() + { + using var services = CreateServices(); + var (serializer, probe) = CreateProbedSerializer(services); + var messageType = new MessageType(Subject, serializer); + + var exception = Assert.Throws( + () => messageType.Create(MessageKey, Receiver, null!)); + + Assert.Equal("body", exception.ParamName); + Assert.Equal(0, probe.WriteCount); + var control = messageType.Create(MessageKey, Receiver, "nonnull recovery"); + Assert.Equal(1, probe.WriteCount); + Assert.Equal("nonnull recovery", services.GetRequiredService>().Deserialize(control.Envelope.Payload)); + } + + [Fact] + public void MessageType_CreateLeavesResourceBodyAndDecodedValueApplicationOwned() + { + using var services = CreateServices(); + var codec = new ResourceCodec(); + var serializer = new Serializer(codec, services.GetRequiredService()); + var type = new MessageType(Subject, serializer); + using var body = new OwnedResource("resource €42"); + var original = type.Create(MessageKey, Receiver, body); + var retained = original; + Assert.Same(original.Envelope.Payload, retained.Envelope.Payload); + + Assert.Equal(0, body.DisposeCount); + Assert.Equal(MessageKey, retained.Envelope.MessageId); + Assert.Equal(services.GetRequiredService>().SerializeToArray(body.Name), retained.Envelope.Payload.ToArray()); + using var decoded = type.Decode(retained.Envelope); + Assert.NotSame(body, decoded); + Assert.Same(codec.LastDecoded, decoded); + Assert.Equal("resource €42", decoded.Name); + Assert.Equal(0, decoded.DisposeCount); + Assert.Equal(1, codec.ReadCount); + } + + [Fact] + public void InboxDispatcher_RegisterRejectsNullMessageType() + { + var dispatcher = new TestInbox(); + var calls = 0; + var exception = Assert.Throws(() => dispatcher.Register(null!, (_, _, _) => + { + calls++; + return ValueTask.CompletedTask; + })); + + Assert.Equal("messageType", exception.ParamName); + Assert.Equal(0, calls); + } + + [Fact] + public void InboxDispatcher_RegisterRejectsNullHandler() + { + using var services = CreateServices(); + var (serializer, probe) = CreateProbedSerializer(services); + var messageType = new MessageType(Subject, serializer); + var dispatcher = new TestInbox(); + + Func? handler = null; + var exception = Assert.Throws(() => dispatcher.Register(messageType, handler!)); + + Assert.Equal("handler", exception.ParamName); + Assert.Equal(0, probe.ReadCount); + dispatcher.Register(messageType, (_, _, _) => ValueTask.CompletedTask); + } + + [Fact] + public async Task InboxDispatcher_HandleAsyncRoutesMultipleExactSubjectsAndPayloadTypes() + { + using var services = CreateServices(); + var strings = services.GetRequiredService>(); + var integers = services.GetRequiredService>(); + var dispatcher = new TestInbox(); + var lowerValues = new List(); + var upperValues = new List(); + var integerValues = new List(); + dispatcher.Register(new MessageType(Subject, strings), (body, _, _) => + { + lowerValues.Add(body); + return ValueTask.CompletedTask; + }); + dispatcher.Register(new MessageType("Orders.Reserve", strings), (body, _, _) => + { + upperValues.Add(body); + return ValueTask.CompletedTask; + }); + dispatcher.Register(new MessageType("orders.quantity", integers), (body, _, _) => + { + integerValues.Add(body); + return ValueTask.CompletedTask; + }); + var lowerEnvelope = DirectEnvelope(strings, Subject, "lower €42"); + var upperEnvelope = DirectEnvelope(strings, "Orders.Reserve", "upper 東京"); + var integerEnvelope = DirectEnvelope(integers, "orders.quantity", 42_017); + var lowerContext = new CountingInboxHandlerContext(lowerEnvelope); + var upperContext = new CountingInboxHandlerContext(upperEnvelope); + var integerContext = new CountingInboxHandlerContext(integerEnvelope); + + await dispatcher.HandleAsync(upperContext, CancellationToken.None); + Assert.Equal(new[] { "upper 東京" }, upperValues); + Assert.Empty(lowerValues); + Assert.Empty(integerValues); + await dispatcher.HandleAsync(integerContext, CancellationToken.None); + Assert.Equal(new[] { 42_017 }, integerValues); + Assert.Equal(new[] { "upper 東京" }, upperValues); + Assert.Empty(lowerValues); + await dispatcher.HandleAsync(lowerContext, CancellationToken.None); + Assert.Equal(new[] { "lower €42" }, lowerValues); + Assert.Equal(new[] { "upper 東京" }, upperValues); + Assert.Equal(new[] { 42_017 }, integerValues); + Assert.Equal(0, lowerContext.CompletionCount); + Assert.Equal(0, upperContext.CompletionCount); + Assert.Equal(0, integerContext.CompletionCount); + } + + [Fact] + public async Task InboxDispatcher_HandleAsyncRejectsUnknownSubjectBeforeDecodeOrHandler() + { + using var services = CreateServices(); + var (serializer, probe) = CreateProbedSerializer(services); + probe.ReadFailure = new InvalidDataException("unknown subject must not decode"); + var dispatcher = new TestInbox(); + var calls = 0; + dispatcher.Register(new MessageType(Subject, serializer), (_, _, _) => + { + calls++; + return ValueTask.CompletedTask; + }); + var direct = services.GetRequiredService>(); + var envelope = DirectEnvelope(direct, "orders.unknown", "valid reserve €42"); + var context = new CountingInboxHandlerContext(envelope); + + await Assert.ThrowsAsync( + async () => await dispatcher.HandleAsync(context, CancellationToken.None)); + + Assert.Equal(0, probe.ReadCount); + Assert.Equal(0, calls); + Assert.Equal(0, context.CompletionCount); + Assert.Equal("valid reserve €42", direct.Deserialize(envelope.Payload)); + } + + [Fact] + public async Task InboxDispatcher_HandleAsyncRejectsMalformedKnownPayloadBeforeHandler() + { + using var services = CreateServices(); + var serializer = services.GetRequiredService>(); + var wire = serializer.SerializeToArray("nonempty truncated €42"); + var envelope = WireEnvelope(Subject, wire[..^1]); + var context = new CountingInboxHandlerContext(envelope); + var dispatcher = new TestInbox(); + var calls = 0; + dispatcher.Register(new MessageType(Subject, serializer), (_, _, _) => + { + calls++; + return ValueTask.CompletedTask; + }); + // StringCodec.ReadRaw -> Reader.EnsureAvailable rejects the declared + // length exceeding remaining bytes, independently of subject routing. + Assert.Throws(() => serializer.Deserialize(envelope.Payload)); + + await Assert.ThrowsAsync( + async () => await dispatcher.HandleAsync(context, CancellationToken.None)); + + Assert.Equal(Subject, GetSubject(envelope)); + Assert.Equal(wire[..^1], envelope.Payload.ToArray()); + Assert.Equal(0, calls); + Assert.Equal(0, context.CompletionCount); + } + + [Fact] + public async Task InboxDispatcher_HandleAsyncPassesExactContextAndCancellationToken() + { + using var services = CreateServices(); + var serializer = services.GetRequiredService>(); + var envelope = DirectEnvelope(serializer, Subject, "context €42"); + var context = new CountingInboxHandlerContext(envelope); + using var cancellation = new CancellationTokenSource(); + var dispatcher = new TestInbox(); + string? capturedBody = null; + IInboxHandlerContext? capturedContext = null; + CancellationToken capturedToken = default; + var calls = 0; + dispatcher.Register(new MessageType(Subject, serializer), (body, caller, token) => + { + capturedBody = body; + capturedContext = caller; + capturedToken = token; + calls++; + return ValueTask.CompletedTask; + }); + + await dispatcher.HandleAsync(context, cancellation.Token); + + Assert.Equal("context €42", capturedBody); + Assert.Same(context, capturedContext); + Assert.Equal(cancellation.Token, capturedToken); + Assert.True(capturedToken.CanBeCanceled); + Assert.False(capturedToken.IsCancellationRequested); + Assert.Equal(1, calls); + Assert.Equal(0, context.CompletionCount); + } + + [Fact] + public async Task InboxDispatcher_HandleAsyncPropagatesSynchronousValueTaskCompletionWithoutAutomaticComplete() + { + using var services = CreateServices(); + var serializer = services.GetRequiredService>(); + var envelope = DirectEnvelope(serializer, Subject, "synchronous reserve"); + var context = new CountingInboxHandlerContext(envelope); + var dispatcher = new TestInbox(); + var calls = 0; + string? captured = null; + dispatcher.Register(new MessageType(Subject, serializer), (body, _, _) => + { + calls++; + captured = body; + return ValueTask.CompletedTask; + }); + + var dispatch = dispatcher.HandleAsync(context, CancellationToken.None); + Assert.True(dispatch.IsCompletedSuccessfully); + await dispatch; + + Assert.Equal("synchronous reserve", captured); + Assert.Equal(1, calls); + Assert.Equal(0, context.CompletionCount); + } + + [Fact] + public async Task InboxDispatcher_HandleAsyncPropagatesSynchronousFaultedValueTask() + { + using var services = CreateServices(); + var serializer = services.GetRequiredService>(); + var envelope = DirectEnvelope(serializer, Subject, "faulted task reserve"); + var context = new CountingInboxHandlerContext(envelope); + var dispatcher = new TestInbox(); + var sentinel = new InvalidDataException("synchronous faulted ValueTask"); + var calls = 0; + string? captured = null; + dispatcher.Register(new MessageType(Subject, serializer), (body, _, _) => + { + calls++; + captured = body; + return ValueTask.FromException(sentinel); + }); + + var dispatch = dispatcher.HandleAsync(context, CancellationToken.None); + Assert.True(dispatch.IsCompleted); + Assert.False(dispatch.IsCompletedSuccessfully); + var exception = await Assert.ThrowsAsync(async () => await dispatch); + + Assert.Same(sentinel, exception); + Assert.Equal("faulted task reserve", captured); + Assert.Equal(1, calls); + Assert.Equal(0, context.CompletionCount); + } + + [Fact] + public async Task InboxDispatcher_HandleAsyncPropagatesSynchronousDelegateException() + { + using var services = CreateServices(); + var serializer = services.GetRequiredService>(); + var envelope = DirectEnvelope(serializer, Subject, "throwing delegate reserve"); + var context = new CountingInboxHandlerContext(envelope); + var dispatcher = new TestInbox(); + var sentinel = new InvalidDataException("delegate throws before returning"); + var calls = 0; + string? captured = null; + dispatcher.Register(new MessageType(Subject, serializer), (body, _, _) => + { + calls++; + captured = body; + throw sentinel; + }); + + var exception = await Assert.ThrowsAsync( + async () => await dispatcher.HandleAsync(context, CancellationToken.None)); + + Assert.Same(sentinel, exception); + Assert.Equal("throwing delegate reserve", captured); + Assert.Equal(1, calls); + Assert.Equal(0, context.CompletionCount); + } + + [Fact] + public async Task InboxDispatcher_HandleAsyncPropagatesDeferredValueTaskCompletion() + { + using var services = CreateServices(); + var serializer = services.GetRequiredService>(); + var envelope = DirectEnvelope(serializer, Subject, "deferred €42"); + var context = new CountingInboxHandlerContext(envelope); + using var cancellation = new CancellationTokenSource(); + var completion = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var dispatcher = new TestInbox(); + var calls = 0; + string? capturedBody = null; + IInboxHandlerContext? capturedContext = null; + CancellationToken capturedToken = default; + dispatcher.Register(new MessageType(Subject, serializer), (body, caller, token) => + { + calls++; + capturedBody = body; + capturedContext = caller; + capturedToken = token; + return new ValueTask(completion.Task); + }); + + var dispatch = dispatcher.HandleAsync(context, cancellation.Token); + // Record the controlled incomplete state, then release and observe once + // before assertions, so even assertion failure cannot strand a dispatch. + var completedBeforeRelease = dispatch.IsCompleted; + var completionsBeforeRelease = context.CompletionCount; + completion.SetResult(); + await dispatch; + + Assert.False(completedBeforeRelease); + Assert.Equal(0, completionsBeforeRelease); + Assert.Equal("deferred €42", capturedBody); + Assert.Same(context, capturedContext); + Assert.Equal(cancellation.Token, capturedToken); + Assert.Equal(1, calls); + Assert.Equal(0, context.CompletionCount); + Assert.Equal("deferred €42", serializer.Deserialize(envelope.Payload)); + } + + [Fact] + public async Task InboxDispatcher_HandleAsyncPropagatesDeferredValueTaskFault() + { + using var services = CreateServices(); + var serializer = services.GetRequiredService>(); + var envelope = DirectEnvelope(serializer, Subject, "deferred fault €42"); + var context = new CountingInboxHandlerContext(envelope); + using var cancellation = new CancellationTokenSource(); + var completion = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var sentinel = new InvalidDataException("controlled deferred fault"); + var dispatcher = new TestInbox(); + var calls = 0; + string? capturedBody = null; + IInboxHandlerContext? capturedContext = null; + CancellationToken capturedToken = default; + dispatcher.Register(new MessageType(Subject, serializer), (body, caller, token) => + { + calls++; + capturedBody = body; + capturedContext = caller; + capturedToken = token; + return new ValueTask(completion.Task); + }); + + var dispatch = dispatcher.HandleAsync(context, cancellation.Token); + var completedBeforeRelease = dispatch.IsCompleted; + var completionsBeforeRelease = context.CompletionCount; + completion.SetException(sentinel); + var exception = await Assert.ThrowsAsync(async () => await dispatch); + + Assert.False(completedBeforeRelease); + Assert.Equal(0, completionsBeforeRelease); + Assert.Same(sentinel, exception); + Assert.Equal("deferred fault €42", capturedBody); + Assert.Same(context, capturedContext); + Assert.Equal(cancellation.Token, capturedToken); + Assert.Equal(1, calls); + Assert.Equal(0, context.CompletionCount); + Assert.Equal("deferred fault €42", serializer.Deserialize(envelope.Payload)); + } + + [Fact] + public async Task InboxDispatcher_HandleAsyncStagesOnlyExplicitDelegateCompletion() + { + using var services = CreateServices(); + var serializer = services.GetRequiredService>(); + var envelope = DirectEnvelope(serializer, Subject, "explicit completion"); + var context = new CountingInboxHandlerContext(envelope); + var dispatcher = new TestInbox(); + var calls = 0; + string? capturedBody = null; + IInboxHandlerContext? capturedContext = null; + dispatcher.Register(new MessageType(Subject, serializer), (body, caller, _) => + { + calls++; + capturedBody = body; + capturedContext = caller; + caller.Complete(); + return ValueTask.CompletedTask; + }); + + await dispatcher.HandleAsync(context, CancellationToken.None); + + Assert.Equal("explicit completion", capturedBody); + Assert.Same(context, capturedContext); + Assert.Equal(1, calls); + Assert.Equal(1, context.CompletionCount); + Assert.Equal("explicit completion", serializer.Deserialize(envelope.Payload)); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task InboxDispatcher_HandleAsyncHonorsPreCanceledTokenWithoutDecodeHandlerOrComplete(bool unknownSubject) + { + using var services = CreateServices(); + var (serializer, probe) = CreateProbedSerializer(services); + probe.ReadFailure = new InvalidDataException("canceled attempts must not decode"); + var direct = services.GetRequiredService>(); + var envelope = DirectEnvelope(direct, unknownSubject ? "orders.unknown" : Subject, "valid canceled reserve"); + var context = new CountingInboxHandlerContext(envelope); + using var cancellation = new CancellationTokenSource(); + cancellation.Cancel(); + var calls = 0; + var dispatcher = new TestInbox(); + dispatcher.Register(new MessageType(Subject, serializer), (_, _, _) => + { + calls++; + return ValueTask.CompletedTask; + }); + + var exception = await Assert.ThrowsAsync( + async () => await dispatcher.HandleAsync(context, cancellation.Token)); + + Assert.Equal(cancellation.Token, exception.CancellationToken); + Assert.Equal(0, probe.ReadCount); + Assert.Equal(0, calls); + Assert.Equal(0, context.CompletionCount); + Assert.Equal("valid canceled reserve", direct.Deserialize(envelope.Payload)); + } + + [Fact] + public async Task InboxDispatcher_HandleAsyncLeavesDecodedResourceApplicationOwned() + { + using var services = CreateServices(); + var sessions = services.GetRequiredService(); + var codec = new ResourceCodec(); + var serializer = new Serializer(codec, sessions); + using var original = new OwnedResource("owned reserve €42"); + var envelope = DirectEnvelope(serializer, Subject, original); + var context = new CountingInboxHandlerContext(envelope); + var dispatcher = new TestInbox(); + OwnedResource? captured = null; + var calls = 0; + dispatcher.Register(new MessageType(Subject, serializer), (body, _, _) => + { + calls++; + captured = body; + return ValueTask.CompletedTask; + }); + try + { + await dispatcher.HandleAsync(context, CancellationToken.None); + + var decoded = Assert.IsType(captured); + Assert.Same(codec.LastDecoded, decoded); + Assert.NotSame(original, decoded); + Assert.Equal("owned reserve €42", decoded.Name); + Assert.Equal(1, calls); + Assert.Equal(1, codec.ReadCount); + Assert.Equal(0, context.CompletionCount); + Assert.Equal(0, original.DisposeCount); + Assert.Equal(0, decoded.DisposeCount); + envelope = default; + Assert.Equal("owned reserve €42", decoded.Name); + Assert.Equal(0, decoded.DisposeCount); + decoded.Dispose(); + Assert.Equal(1, decoded.DisposeCount); + Assert.Equal(0, original.DisposeCount); + } + finally + { + if (codec.LastDecoded is { DisposeCount: 0 } resource) + { + resource.Dispose(); + } + } + } + + [Fact] + public async Task InboxDispatcher_CancellationDuringDecodeDeliversDecodedResourceToHandler() + { + using var services = CreateServices(); + var codec = new ResourceCodec(); + var serializer = new Serializer(codec, services.GetRequiredService()); + using var original = new OwnedResource("canceled resource €42"); + var envelope = DirectEnvelope(serializer, Subject, original); + var context = new CountingInboxHandlerContext(envelope); + using var cancellation = new CancellationTokenSource(); + codec.OnRead = cancellation.Cancel; + OwnedResource? captured = null; + var observedToken = CancellationToken.None; + var dispatcher = new TestInbox(); + dispatcher.Register(new MessageType(Subject, serializer), (body, _, token) => + { + captured = body; + observedToken = token; + return ValueTask.CompletedTask; + }); + try + { + await dispatcher.HandleAsync(context, cancellation.Token); + + var decoded = Assert.IsType(captured); + Assert.Same(codec.LastDecoded, decoded); + Assert.Equal(1, codec.ReadCount); + Assert.Equal(cancellation.Token, observedToken); + Assert.True(observedToken.IsCancellationRequested); + Assert.Equal(0, decoded.DisposeCount); + Assert.Equal(0, context.CompletionCount); + } + finally + { + codec.LastDecoded?.Dispose(); + } + } + + [Fact] + public async Task InboxDispatcher_HandleAsyncRejectsSerializedNullBeforeHandler() + { + using var services = CreateServices(); + var serializer = services.GetRequiredService>(); + var envelope = DirectEnvelope(serializer, Subject, null!); + var expectedWire = serializer.SerializeToArray(null); + var context = new CountingInboxHandlerContext(envelope); + var dispatcher = new TestInbox(); + var calls = 0; + dispatcher.Register(new MessageType(Subject, serializer), (_, caller, _) => + { + calls++; + caller.Complete(); + return ValueTask.CompletedTask; + }); + Assert.Null(serializer.Deserialize(envelope.Payload)); + + var exception = await Assert.ThrowsAsync( + async () => await dispatcher.HandleAsync(context, CancellationToken.None)); + + Assert.Equal("envelope", exception.ParamName); + Assert.Equal(0, calls); + Assert.Equal(0, context.CompletionCount); + Assert.Equal(expectedWire, envelope.Payload.ToArray()); + Assert.Null(serializer.Deserialize(envelope.Payload)); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task RegisterAction_InvokesDecodedBodySynchronouslyWithExplicitCompletion(bool complete) + { + using var services = CreateServices(); + var serializer = services.GetRequiredService>(); + var type = new MessageType(Subject, serializer); + var envelope = DirectEnvelope(serializer, Subject, "synchronous"); + var context = new CountingInboxHandlerContext(envelope); + var dispatcher = new TestInbox(); + var calls = 0; + dispatcher.Register(type, (body, caller) => + { + Assert.Equal("synchronous", body); + Assert.Same(context, caller); + calls++; + if (complete) caller.Complete(); + }); + + var outcome = dispatcher.HandleAsync(context, CancellationToken.None); + + Assert.True(outcome.IsCompletedSuccessfully); + Assert.Equal(1, calls); + Assert.Equal(complete ? 1 : 0, context.CompletionCount); + await outcome; + } + + [Fact] + public void RegisterAction_RejectsNullArguments() + { + using var services = CreateServices(); + var type = new MessageType(Subject, services.GetRequiredService>()); + var dispatcher = new TestInbox(); + var missingType = Assert.Throws(() => + dispatcher.Register(null!, (_, _) => { })); + Action? handler = null; + var missingHandler = Assert.Throws(() => dispatcher.Register(type, handler!)); + + Assert.Equal("messageType", missingType.ParamName); + Assert.Equal("handler", missingHandler.ParamName); + dispatcher.Register(type, (_, _) => { }); + } + + [Fact] + public void RegisterAction_PropagatesOriginalDelegateFailure() + { + using var services = CreateServices(); + var serializer = services.GetRequiredService>(); + var envelope = DirectEnvelope(serializer, Subject, "failure"); + var context = new CountingInboxHandlerContext(envelope); + var sentinel = new IOException("synchronous handler failed"); + var dispatcher = new TestInbox().Register( + new MessageType(Subject, serializer), (_, _) => throw sentinel); + + var exception = Assert.Throws(() => dispatcher.HandleAsync(context, CancellationToken.None)); + + Assert.Same(sentinel, exception); + Assert.Equal(0, context.CompletionCount); + } + + [Fact] + public void RegisterAction_CancellationBeforeEntryLeavesHandlerAndCompletionUntouched() + { + using var services = CreateServices(); + var direct = services.GetRequiredService>(); + var (serializer, probe) = CreateProbedSerializer(services); + using var cancellation = new CancellationTokenSource(); + cancellation.Cancel(); + var envelope = DirectEnvelope(direct, Subject, "cancellation"); + var context = new CountingInboxHandlerContext(envelope); + var calls = 0; + var dispatcher = new TestInbox().Register( + new MessageType(Subject, serializer), (_, _) => calls++); + + var exception = Assert.Throws(() => + dispatcher.HandleAsync(context, cancellation.Token)); + + Assert.Equal(cancellation.Token, exception.CancellationToken); + Assert.Equal(0, probe.ReadCount); + Assert.Equal(0, calls); + Assert.Equal(0, context.CompletionCount); + } + + [Fact] + public async Task RegisterAction_DuplicateSubjectDispatchesToFirstRegistration() + { + using var services = CreateServices(); + var serializer = services.GetRequiredService>(); + var type = new MessageType(Subject, serializer); + var originalCalls = 0; + var replacementCalls = 0; + var dispatcher = new TestInbox().Register(type, (_, _) => originalCalls++); + dispatcher.Register(type, (_, _) => replacementCalls++); + var envelope = DirectEnvelope(serializer, Subject, "original"); + var context = new CountingInboxHandlerContext(envelope); + + await dispatcher.HandleAsync(context, CancellationToken.None); + + Assert.Equal(1, originalCalls); + Assert.Equal(0, replacementCalls); + Assert.Equal(0, context.CompletionCount); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task RegisterStateAction_StaticDelegateReceivesExactReferenceStateBodyAndContext(bool complete) + { + using var services = CreateServices(); + var serializer = services.GetRequiredService>(); + var type = new MessageType(Subject, serializer); + var envelope = type.Create(MessageKey, Receiver, "stateful €42"); + var context = new CountingInboxHandlerContext(envelope); + var state = new HandlerState { Complete = complete }; + Action handler = ObserveState; + Assert.Null(handler.Target); // Named static method, not a captured test closure. + var dispatcher = new TestInbox(); + dispatcher.Register(type, state, handler); + + var outcome = dispatcher.HandleAsync(context, CancellationToken.None); + + Assert.True(outcome.IsCompletedSuccessfully); + await outcome; + Assert.Equal("stateful €42", state.Body); + Assert.Same(context, state.Context); + Assert.Same(state, state.Argument); + Assert.Equal(1, state.Calls); + Assert.Equal(complete ? 1 : 0, context.CompletionCount); + } + + [Fact] + public async Task RegisterState_MixedStaticReferenceAndStructArgumentsRouteOrdinalSubjects() + { + using var services = CreateServices(); + var serializer = services.GetRequiredService>(); + var syncType = new MessageType(Subject, serializer); + var asyncType = new MessageType("Orders.Reserve", serializer); + var syncState = new HandlerState { Complete = true }; + var asyncState = new HandlerState(); + var argument = new StructHandlerState(asyncState, 731); + Action sync = ObserveState; + Func asyncHandler = ObserveStructState; + Assert.Null(sync.Target); + Assert.Null(asyncHandler.Target); + var dispatcher = new TestInbox().Register(syncType, syncState, sync).Register(asyncType, argument, asyncHandler); + var syncEnvelope = syncType.Create(MessageKey, Receiver, "lower"); + var asyncEnvelope = asyncType.Create(MessageKey, Receiver, "upper"); + var syncContext = new CountingInboxHandlerContext(syncEnvelope); + var asyncContext = new CountingInboxHandlerContext(asyncEnvelope); + using var cancellation = new CancellationTokenSource(); + + await dispatcher.HandleAsync(syncContext, CancellationToken.None); + await dispatcher.HandleAsync(asyncContext, cancellation.Token); + + Assert.Equal("lower", syncState.Body); + Assert.Equal("upper", asyncState.Body); + Assert.Same(syncContext, syncState.Context); + Assert.Same(asyncContext, asyncState.Context); + Assert.Equal(731, asyncState.Marker); + Assert.Equal(cancellation.Token, asyncState.Token); + Assert.Equal(1, syncState.Calls); + Assert.Equal(1, asyncState.Calls); + Assert.Equal(1, syncContext.CompletionCount); + Assert.Equal(0, asyncContext.CompletionCount); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task RegisterState_StaticStructArgumentIsStoredByValueAndInvokedExactly(bool asynchronous) + { + using var services = CreateServices(); + var serializer = services.GetRequiredService>(); + var type = new MessageType(Subject, serializer); + var envelope = type.Create(MessageKey, Receiver, "struct state"); + var context = new CountingInboxHandlerContext(envelope); + using var cancellation = new CancellationTokenSource(); + var observation = new HandlerState { Complete = true }; + var argument = new StructHandlerState(observation, 731); + var dispatcher = new TestInbox(); + if (asynchronous) + { + Func handler = ObserveStructState; + Assert.Null(handler.Target); + dispatcher.Register(type, argument, handler); + } + else + { + Action handler = ObserveStructAction; + Assert.Null(handler.Target); + dispatcher.Register(type, argument, handler); + } + argument = argument with { Marker = 999 }; + + await dispatcher.HandleAsync(context, cancellation.Token); + + Assert.Equal(999, argument.Marker); + Assert.Equal(731, observation.Marker); + Assert.Equal("struct state", observation.Body); + Assert.Same(context, observation.Context); + Assert.Same(observation, observation.Argument); + Assert.Equal(asynchronous ? cancellation.Token : default, observation.Token); + Assert.Equal(1, observation.Calls); + Assert.Equal(1, context.CompletionCount); + } + + [Theory] + [InlineData("success")] + [InlineData("fault")] + [InlineData("cancel")] + public async Task RegisterStateAsync_ReturnsActualDeferredValueTaskAndExactOutcome(string outcomeKind) + { + using var services = CreateServices(); + var serializer = services.GetRequiredService>(); + var type = new MessageType(Subject, serializer); + var envelope = type.Create(MessageKey, Receiver, "deferred state"); + var context = new CountingInboxHandlerContext(envelope); + using var cancellation = new CancellationTokenSource(); + var gate = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var state = new HandlerState { Outcome = new ValueTask(gate.Task) }; + Func handler = ObserveAsyncState; + Assert.Null(handler.Target); + var dispatcher = new TestInbox().Register(type, state, handler); + + var outcome = dispatcher.HandleAsync(context, cancellation.Token); + + Assert.Equal(state.Outcome, outcome); // No async adapter replacing the actual ValueTask. + Assert.False(outcome.IsCompleted); + Assert.Equal("deferred state", state.Body); + Assert.Same(state, state.Argument); + Assert.Same(context, state.Context); + Assert.Equal(cancellation.Token, state.Token); + Assert.Equal(1, state.Calls); + Assert.Equal(0, context.CompletionCount); + var sentinel = new IOException("stateful deferred sentinel"); + if (outcomeKind == "fault") + { + gate.SetException(sentinel); + Assert.Same(sentinel, await Assert.ThrowsAsync(async () => await outcome)); + } + else if (outcomeKind == "cancel") + { + cancellation.Cancel(); + gate.SetCanceled(cancellation.Token); + var exception = await Assert.ThrowsAnyAsync(async () => await outcome); + Assert.Equal(cancellation.Token, exception.CancellationToken); + } + else + { + // Cancellation after a delegate's successful work must not change its outcome. + cancellation.Cancel(); + gate.SetResult(); + await outcome; + Assert.True(outcome.IsCompletedSuccessfully); + } + Assert.Equal(0, context.CompletionCount); + Assert.Equal(1, state.Calls); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task RegisterState_PropagatesOriginalSynchronousFailureWithoutCompleting(bool faultedValueTask) + { + using var services = CreateServices(); + var serializer = services.GetRequiredService>(); + var type = new MessageType(Subject, serializer); + var envelope = type.Create(MessageKey, Receiver, "sentinel"); + var context = new CountingInboxHandlerContext(envelope); + var sentinel = new IOException("stateful sync sentinel"); + var state = new HandlerState(); + var dispatcher = new TestInbox(); + if (faultedValueTask) + { + state.Outcome = ValueTask.FromException(sentinel); + dispatcher.Register(type, state, ObserveAsyncState); + var outcome = dispatcher.HandleAsync(context, CancellationToken.None); + Assert.Equal(state.Outcome, outcome); + Assert.Same(sentinel, await Assert.ThrowsAsync(async () => await outcome)); + } + else + { + state.Failure = sentinel; + dispatcher.Register(type, state, ObserveState); + Assert.Same(sentinel, Assert.Throws(() => dispatcher.HandleAsync(context, CancellationToken.None))); + } + Assert.Equal("sentinel", state.Body); + Assert.Same(context, state.Context); + Assert.Equal(1, state.Calls); + Assert.Equal(0, context.CompletionCount); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void RegisterState_CancellationBeforeDecodePreventsAllDelegateEffects(bool asynchronous) + { + using var services = CreateServices(); + var direct = services.GetRequiredService>(); + var (serializer, probe) = CreateProbedSerializer(services); + using var cancellation = new CancellationTokenSource(); + cancellation.Cancel(); + var envelope = DirectEnvelope(direct, Subject, "canceled state"); + var context = new CountingInboxHandlerContext(envelope); + var state = new HandlerState { Complete = true }; + var type = new MessageType(Subject, serializer); + var dispatcher = new TestInbox(); + if (asynchronous) dispatcher.Register(type, state, ObserveAsyncState); + else dispatcher.Register(type, state, ObserveState); + + var exception = Assert.Throws(() => dispatcher.HandleAsync(context, cancellation.Token)); + + Assert.Equal(cancellation.Token, exception.CancellationToken); + Assert.Equal(0, probe.ReadCount); + Assert.Equal(0, state.Calls); + Assert.Null(state.Body); + Assert.Null(state.Context); + Assert.Equal(0, context.CompletionCount); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task RegisterState_CancellationInsideSuccessfulDelegatePreservesExplicitCompletion(bool asynchronous) + { + using var services = CreateServices(); + var serializer = services.GetRequiredService>(); + var type = new MessageType(Subject, serializer); + var envelope = type.Create(MessageKey, Receiver, "completed despite cancel"); + var context = new CountingInboxHandlerContext(envelope); + using var cancellation = new CancellationTokenSource(); + var state = new HandlerState { Complete = true, CancelOnInvoke = cancellation }; + var dispatcher = new TestInbox(); + if (asynchronous) dispatcher.Register(type, state, ObserveAsyncState); + else dispatcher.Register(type, state, ObserveState); + + var outcome = dispatcher.HandleAsync(context, cancellation.Token); + await outcome; + + Assert.True(cancellation.IsCancellationRequested); + Assert.True(outcome.IsCompletedSuccessfully); + Assert.Equal("completed despite cancel", state.Body); + Assert.Same(context, state.Context); + Assert.Equal(1, state.Calls); + Assert.Equal(1, context.CompletionCount); + } + + [Theory] + [InlineData(false, "unknown")] + [InlineData(true, "unknown")] + [InlineData(false, "malformed")] + [InlineData(true, "malformed")] + [InlineData(false, "codec-failure")] + [InlineData(true, "codec-failure")] + [InlineData(false, "null")] + [InlineData(true, "null")] + public void RegisterState_UnknownMalformedOrNullPayloadRejectsBeforeEffects(bool asynchronous, string variation) + { + using var services = CreateServices(); + var direct = services.GetRequiredService>(); + var (serializer, probe) = CreateProbedSerializer(services); + var sentinel = new InvalidDataException("malformed stateful payload"); + if (variation == "codec-failure") probe.ReadFailure = sentinel; + var wire = direct.SerializeToArray("nonempty truncated €42"); + var envelope = variation == "malformed" + ? WireEnvelope(Subject, wire[..^1]) + : DirectEnvelope(direct, variation == "unknown" ? "Orders.Reserve" : Subject, + variation == "null" ? null! : "payload"); + var context = new CountingInboxHandlerContext(envelope); + var type = new MessageType(Subject, serializer); + var state = new HandlerState { Complete = true }; + var dispatcher = new TestInbox(); + if (asynchronous) dispatcher.Register(type, state, ObserveAsyncState); + else dispatcher.Register(type, state, ObserveState); + + if (variation == "unknown") + Assert.Throws(() => dispatcher.HandleAsync(context, CancellationToken.None)); + else if (variation == "codec-failure") + Assert.Same(sentinel, Assert.Throws(() => dispatcher.HandleAsync(context, CancellationToken.None))); + else if (variation == "malformed") + { + Assert.Throws(() => dispatcher.HandleAsync(context, CancellationToken.None)); + Assert.Equal(wire[..^1], envelope.Payload.ToArray()); + } + else + Assert.Equal("envelope", Assert.Throws(() => + dispatcher.HandleAsync(context, CancellationToken.None)).ParamName); + + Assert.Equal(variation == "unknown" ? 0 : 1, probe.ReadCount); + Assert.Equal(0, state.Calls); + Assert.Null(state.Context); + Assert.Equal(0, context.CompletionCount); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task RegisterState_DuplicateSameOrCrossTypeDispatchesToFirstRegistration(bool crossType) + { + using var services = CreateServices(); + var serializer = services.GetRequiredService>(); + var type = new MessageType(Subject, serializer); + var original = new HandlerState(); + var replacement = new HandlerState { Complete = true }; + var dispatcher = new TestInbox().Register(type, original, ObserveState); + if (crossType) + dispatcher.Register( + new MessageType(Subject, services.GetRequiredService>()), + replacement, ObserveNumberState); + else + dispatcher.Register(type, replacement, ObserveAsyncState); + var envelope = type.Create(MessageKey, Receiver, "original state"); + var context = new CountingInboxHandlerContext(envelope); + + await dispatcher.HandleAsync(context, CancellationToken.None); + + Assert.Equal("original state", original.Body); + Assert.Same(original, original.Argument); + Assert.Equal(1, original.Calls); + Assert.Equal(0, replacement.Calls); + Assert.Equal(0, context.CompletionCount); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void RegisterState_RejectsNullTypeAndDelegateWithoutReservingSubject(bool asynchronous) + { + using var services = CreateServices(); + var type = new MessageType(Subject, services.GetRequiredService>()); + var state = new HandlerState(); + var dispatcher = new TestInbox(); + if (asynchronous) + { + Assert.Equal("messageType", Assert.Throws(() => + dispatcher.Register(null!, state, ObserveAsyncState)).ParamName); + Assert.Equal("handler", Assert.Throws(() => dispatcher.Register(type, state, + (Func)null!)).ParamName); + dispatcher.Register(type, state, ObserveAsyncState); + } + else + { + Assert.Equal("messageType", Assert.Throws(() => + dispatcher.Register(null!, state, ObserveState)).ParamName); + Assert.Equal("handler", Assert.Throws(() => dispatcher.Register(type, state, + (Action)null!)).ParamName); + dispatcher.Register(type, state, ObserveState); + } + var envelope = type.Create(MessageKey, Receiver, "positive control"); + var context = new CountingInboxHandlerContext(envelope); + Assert.True(dispatcher.HandleAsync(context, CancellationToken.None).IsCompletedSuccessfully); + Assert.Equal("positive control", state.Body); + Assert.Equal(1, state.Calls); + Assert.Equal(0, context.CompletionCount); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void RegisterState_StructArgumentDispatchDoesNotAddPerInvocationBoxingAllocations(bool asynchronous) + { + using var services = CreateServices(); + var serializer = services.GetRequiredService>(); + var type = new MessageType(Subject, serializer); + var envelope = type.Create(MessageKey, Receiver, "allocation control"); + var context = new CountingInboxHandlerContext(envelope); + var baselineState = new HandlerState(); + var structState = new HandlerState(); + var baseline = new TestInbox(); + var withStruct = new TestInbox(); + if (asynchronous) + { + baseline.Register(type, baselineState, ObserveAsyncState); + withStruct.Register(type, new StructHandlerState(structState, 731), ObserveStructState); + } + else + { + baseline.Register(type, baselineState, ObserveState); + withStruct.Register(type, new StructHandlerState(structState, 731), ObserveStructAction); + } + // Same serializer/body/context, warmed routes and session pool. No assertions, + // async continuations, timing, or boxed arguments inside the measured loops. + MeasureDispatchAllocations(baseline, context, 64); + MeasureDispatchAllocations(withStruct, context, 64); + var referenceBytes = MeasureDispatchAllocations(baseline, context, 256); + var structBytes = MeasureDispatchAllocations(withStruct, context, 256); + + Assert.True(structBytes <= referenceBytes, $"Struct dispatch allocated {structBytes}, reference dispatch {referenceBytes}."); + Assert.Equal(320, baselineState.Calls); + Assert.Equal(320, structState.Calls); + Assert.Equal(731, structState.Marker); + Assert.Equal("allocation control", structState.Body); + Assert.Same(context, structState.Context); + Assert.Equal(0, context.CompletionCount); + } + + private static ServiceProvider CreateServices() => + new ServiceCollection().AddSerializer().BuildServiceProvider(); + + private static void AssertClosedSerializerFactory(T body) + { + var services = new ServiceCollection(); + services.AddMessageType(Subject).AddMessageType("orders.cancel"); + var closed = Assert.Single(services, descriptor => + !descriptor.IsKeyedService && descriptor.ServiceType == typeof(Serializer)); + Assert.Equal(ServiceLifetime.Singleton, closed.Lifetime); + Assert.Null(closed.ImplementationType); + Assert.Null(closed.ImplementationInstance); + var factory = closed.ImplementationFactory; + Assert.NotNull(factory); + var bindings = services.Where(descriptor => descriptor.IsKeyedService).ToArray(); + Assert.Equal(2, bindings.Length); + Assert.All(bindings, descriptor => Assert.True(services.IndexOf(closed) < services.IndexOf(descriptor))); + // Factory registration succeeds before serializer infrastructure exists: + // it must be lazy and must use the closed contract, including struct T. + services.AddSerializer(); + using var provider = services.BuildServiceProvider(); + using var scope = provider.CreateScope(); + var serializer = provider.GetRequiredService>(); + Assert.Same(serializer, scope.ServiceProvider.GetRequiredService>()); + var factoryProvider = Substitute.For(); + factoryProvider.GetService(typeof(Serializer)).Returns(provider.GetRequiredService()); + factoryProvider.ClearReceivedCalls(); + var createdByFactory = Assert.IsType>(factory(factoryProvider)); + factoryProvider.Received(1).GetService(typeof(Serializer)); + factoryProvider.DidNotReceive().GetService(typeof(Serializer)); + Assert.Equal(serializer.SerializeToArray(body), createdByFactory.SerializeToArray(body)); + var first = provider.GetRequiredKeyedService>(Subject); + var second = provider.GetRequiredKeyedService>("orders.cancel"); + var envelope = first.Create(MessageKey, Receiver, body); + var other = second.Create(MessageKey, Receiver, body); + Assert.Equal(serializer.SerializeToArray(body), envelope.Envelope.Payload.ToArray()); + Assert.Equal(body, first.Decode(envelope.Envelope)); + Assert.Equal(body, second.Decode(other.Envelope)); + AssertEnvelopeIdentity(envelope, MessageKey, Receiver, Subject); + AssertEnvelopeIdentity(other, MessageKey, Receiver, "orders.cancel"); + Assert.NotSame(first, second); + } + + private static void AssertCustomClosedSerializerPreserved(T body, bool factory) + { + using var codecServices = CreateServices(); + var (custom, probe) = CreateProbedSerializer(codecServices); + var services = new ServiceCollection(); + var factoryCalls = 0; + if (factory) + services.AddSingleton>(_ => + { + factoryCalls++; + return custom; + }); + else services.AddSingleton(custom); + var original = Assert.Single(services); + + services.AddMessageType(Subject).AddMessageType("orders.cancel"); + + Assert.Same(original, Assert.Single(services, descriptor => + !descriptor.IsKeyedService && descriptor.ServiceType == typeof(Serializer))); + Assert.Equal(0, factoryCalls); + // No untyped Serializer service is installed here. Replacing this closed + // custom registration with the default factory would fail resolution. + using var provider = services.BuildServiceProvider(); + using var scope = provider.CreateScope(); + Assert.Same(custom, provider.GetRequiredService>()); + Assert.Same(custom, scope.ServiceProvider.GetRequiredService>()); + var first = provider.GetRequiredKeyedService>(Subject); + var second = provider.GetRequiredKeyedService>("orders.cancel"); + var envelope = first.Create(MessageKey, Receiver, body); + var other = second.Create(MessageKey, Receiver, body); + Assert.Equal(2, probe.WriteCount); + Assert.Equal(codecServices.GetRequiredService>().SerializeToArray(body), envelope.Envelope.Payload.ToArray()); + Assert.Equal(body, first.Decode(envelope.Envelope)); + Assert.Equal(body, second.Decode(other.Envelope)); + Assert.Equal(2, probe.ReadCount); + Assert.Equal(factory ? 1 : 0, factoryCalls); + AssertEnvelopeIdentity(envelope, MessageKey, Receiver, Subject); + AssertEnvelopeIdentity(other, MessageKey, Receiver, "orders.cancel"); + } + + private static void ObserveState(string body, HandlerState argument, IInboxHandlerContext context) + { + argument.Calls++; + argument.Body = body; + argument.Context = context; + argument.Argument = argument; + if (argument.Failure is { } failure) throw failure; + argument.CancelOnInvoke?.Cancel(); + if (argument.Complete) context.Complete(); + } + + private static ValueTask ObserveAsyncState(string body, HandlerState argument, IInboxHandlerContext context, CancellationToken token) + { + argument.Token = token; + ObserveState(body, argument, context); + return argument.Outcome; + } + + private static ValueTask ObserveStructState(string body, StructHandlerState argument, IInboxHandlerContext context, CancellationToken token) + { + argument.Observation.Marker = argument.Marker; + return ObserveAsyncState(body, argument.Observation, context, token); + } + + private static void ObserveStructAction(string body, StructHandlerState argument, IInboxHandlerContext context) + { + argument.Observation.Marker = argument.Marker; + ObserveState(body, argument.Observation, context); + } + + private static ValueTask ObserveNumberState(int body, HandlerState argument, IInboxHandlerContext context, CancellationToken token) + { + argument.Marker = body; + return ObserveAsyncState("number", argument, context, token); + } + + private static long MeasureDispatchAllocations(IInbox dispatcher, IInboxHandlerContext context, int count) + { + var before = GC.GetAllocatedBytesForCurrentThread(); + for (var i = 0; i < count; i++) + dispatcher.HandleAsync(context, CancellationToken.None).GetAwaiter().GetResult(); + return GC.GetAllocatedBytesForCurrentThread() - before; + } + + private readonly record struct StructHandlerState(HandlerState Observation, int Marker); + + private sealed class HandlerState + { + public int Calls { get; set; } + public string? Body { get; set; } + public object? Argument { get; set; } + public IInboxHandlerContext? Context { get; set; } + public CancellationToken Token { get; set; } + public int Marker { get; set; } + public bool Complete { get; set; } + public CancellationTokenSource? CancelOnInvoke { get; set; } + public Exception? Failure { get; set; } + public ValueTask Outcome { get; set; } + } + + private static string InvalidSubject(string variation) => variation switch + { + "empty" => "", + "ascii" => new string('s', 257), + "multibyte" => new string('\u20ac', 85) + "ab", + _ => throw new ArgumentOutOfRangeException(nameof(variation)) + }; + + private static string SubjectAtLimit(bool multibyte) => + multibyte ? new string('\u20ac', 85) + "a" : new string('s', 256); + + private static (Serializer Serializer, ProbeCodec Probe) CreateProbedSerializer(IServiceProvider services) + { + var sessions = services.GetRequiredService(); + var probe = new ProbeCodec(sessions.CodecProvider.GetCodec()); + return (new Serializer(probe, sessions), probe); + } + + private static Envelope DirectEnvelope(Serializer serializer, string subject, [AllowNull] T body) => + Envelope(subject, serializer.SerializeToArray(body)); + + private static Envelope WireEnvelope(string subject, byte[] wire) => + Envelope(subject, wire); + + private static Envelope Envelope(string subject, byte[] payload) + { + return new Envelope + { + MessageId = MessageKey, + Payload = payload, + Subject = subject + }; + } + + private static void AssertEnvelopeIdentity( + OutboxMessage message, MessageId key, GrainId receiver, string subject) + { + Assert.Equal(key, message.Envelope.MessageId); + Assert.Equal(receiver, message.ReceiverId); + Assert.Equal(subject, GetSubject(message.Envelope)); + } + + private static string GetSubject(Envelope envelope) + { + return Assert.IsType(envelope.Subject); + } + + private static void AssertCreateWire( + IServiceProvider sendingServices, IServiceProvider receivingServices, T body) + { + var sending = sendingServices.GetRequiredService>(); + var receiving = receivingServices.GetRequiredService>(); + var messageType = new MessageType(Subject, sending); + var key = MessageKey; + var expectedWire = sending.SerializeToArray(body); + var envelope = messageType.Create(key, Receiver, body); + Assert.Equal(expectedWire, envelope.Envelope.Payload.ToArray()); + Assert.Equal(body, receiving.Deserialize(envelope.Envelope.Payload)); + AssertEnvelopeIdentity(envelope, key, Receiver, Subject); + } + + private static void AssertUnchangedServices(ServiceDescriptor[] before, IServiceCollection after) + { + Assert.Equal(before.Length, after.Count); + for (var i = 0; i < before.Length; i++) + { + Assert.Same(before[i], after[i]); + } + } + + private sealed class CountingInboxHandlerContext(InboxMessage message) : IInboxHandlerContext + { + public CountingInboxHandlerContext(Envelope envelope) : this(new InboxMessage { Envelope = envelope, SentAt = DateTimeOffset.UnixEpoch }) { } + public CountingInboxHandlerContext(OutboxMessage outgoing) : this(outgoing.ToInboxMessage(DateTimeOffset.UnixEpoch)) { } + + public InboxMessage Message { get; } = message; + public int CompletionCount { get; private set; } + public void Complete() => CompletionCount++; + public void Fail(string reason) => throw new InvalidOperationException("Unexpected dead letter."); + } + + private sealed class ProbeCodec(IFieldCodec inner) : IFieldCodec + { + public int WriteCount { get; private set; } + public int ReadCount { get; private set; } + public int CommittedFailureCount { get; private set; } + public byte? CommittedFailureByte { get; private set; } + public Exception? NextWriteFailure { get; set; } + public Exception? ReadFailure { get; set; } + public Action? OnRead { get; set; } + + public void WriteField( + ref Writer writer, uint fieldIdDelta, [AllowNull] Type expectedType, [AllowNull] T value) + where TBufferWriter : IBufferWriter + { + WriteCount++; + if (NextWriteFailure is { } failure) + { + NextWriteFailure = null; + writer.WriteByte(0x7e); + writer.Commit(); + CommittedFailureByte = 0x7e; + CommittedFailureCount++; + throw failure; + } + + inner.WriteField(ref writer, fieldIdDelta, expectedType, value); + } + + [return: MaybeNull] + public T ReadValue(ref Reader reader, Field field) + { + ReadCount++; + if (ReadFailure is { } failure) throw failure; + var value = inner.ReadValue(ref reader, field); + OnRead?.Invoke(); + return value; + } + } + + private sealed class OwnedResource(string name) : IDisposable + { + public string Name { get; } = name; + public int DisposeCount { get; private set; } + public void Dispose() => DisposeCount++; + } + + private sealed class ResourceCodec : IFieldCodec + { + public OwnedResource? LastDecoded { get; private set; } + public int ReadCount { get; private set; } + public Action? OnRead { get; set; } + + public void WriteField( + ref Writer writer, uint fieldIdDelta, [AllowNull] Type expectedType, + [AllowNull] OwnedResource value) where TBufferWriter : IBufferWriter + { + ArgumentNullException.ThrowIfNull(value); + // Manual local wire contract: the resource's name uses genuine string + // encoding. No global registration or generated disposal/copy behavior. + StringCodec.WriteField(ref writer, fieldIdDelta, value.Name); + } + + [return: MaybeNull] + public OwnedResource ReadValue(ref Reader reader, Field field) + { + ReadCount++; + var name = StringCodec.ReadValue(ref reader, field) + ?? throw new InvalidDataException("Resource name must not be null."); + LastDecoded = new OwnedResource(name); + OnRead?.Invoke(); + return LastDecoded; + } + } + + [Fact] + public void MessageType_DecodeRejectsAbsentSubjectBeforeCodecRead() + { + using var services = CreateServices(); + var direct = services.GetRequiredService>(); + var (serializer, probe) = CreateProbedSerializer(services); + probe.ReadFailure = new InvalidDataException("payload codec must not run"); + var type = new MessageType(Subject, serializer); + var envelope = new Envelope { MessageId = MessageKey, Payload = direct.SerializeToArray("unrouted €42") }; + + Assert.Equal("envelope", Assert.Throws(() => type.Decode(envelope)).ParamName); + Assert.Equal(0, probe.ReadCount); + Assert.Equal(0, probe.WriteCount); + Assert.Equal("unrouted €42", direct.Deserialize(envelope.Payload)); + Assert.Equal(MessageKey, envelope.MessageId); + Assert.Null(envelope.Subject); + Assert.Null(envelope.Headers); + } + + [Theory] + [InlineData(false, false)] + [InlineData(false, true)] + [InlineData(true, false)] + [InlineData(true, true)] + public void InboxDispatcher_SubjectHeaderFailurePrecedesCodecHandlerAndOutboxMutation( + bool absent, bool asynchronous) + { + using var services = CreateServices(); + var direct = services.GetRequiredService>(); + var (serializer, probe) = CreateProbedSerializer(services); + probe.ReadFailure = new InvalidDataException("codec must not run"); + var type = new MessageType(Subject, serializer); + var envelope = new Envelope + { + MessageId = MessageKey, + Payload = direct.SerializeToArray("untouched"), + Subject = absent ? null : "Orders.Reserve" + }; + var context = new CountingInboxHandlerContext(envelope); + var outbox = Substitute.For(); + var stock = 7; + var calls = 0; + var dispatcher = new TestInbox(); + void Mutate(string body, IInboxHandlerContext caller) + { + calls++; + stock--; + outbox.Send(type, new MessageId(MessageKey.Key.CreateChildKey("result")), Receiver, body); + caller.Complete(); + } + if (asynchronous) + dispatcher.Register(type, (body, caller, _) => + { + Mutate(body, caller); + return ValueTask.CompletedTask; + }); + else dispatcher.Register(type, Mutate); + + var exception = Assert.Throws(() => + dispatcher.HandleAsync(context, CancellationToken.None)); + + // Handlers match subjects exactly, so an absent subject matches no typed handler. + Assert.Contains("No handler is registered", exception.Message, StringComparison.Ordinal); + Assert.Equal(0, probe.ReadCount); + Assert.Equal(0, probe.WriteCount); + Assert.Equal(0, calls); + Assert.Equal(7, stock); + Assert.Equal(0, context.CompletionCount); + outbox.DidNotReceive().Send(Arg.Any()); + Assert.Equal("untouched", direct.Deserialize(context.Message.Envelope.Payload)); + Assert.Equal(MessageKey, context.Message.Envelope.MessageId); + if (absent) + { + Action validate = EnvelopeValidation.Validate; + foreach (var invalidSubject in new[] { "", new string('s', 257), new string('\u20ac', 85) + "ab" }) + { + var invalid = new Envelope + { + MessageId = MessageKey, + Payload = envelope.Payload, + Subject = invalidSubject + }; + Assert.Equal("subject", Assert.Throws(() => + validate(invalid)).ParamName); + Assert.Same(envelope.Payload, invalid.Payload); + } + var invalidUtf16 = new Envelope + { + MessageId = MessageKey, + Payload = envelope.Payload, + Subject = "\ud800" + }; + var utf16Exception = Assert.Throws(() => validate(invalidUtf16)); + Assert.IsType(utf16Exception); + Assert.Equal("subject", utf16Exception.ParamName); + Assert.Contains("unpaired surrogate", utf16Exception.Message, StringComparison.Ordinal); + var missingPayload = new Envelope { MessageId = MessageKey, Payload = null!, Subject = Subject }; + Assert.Equal("envelope", Assert.Throws(() => + validate(missingPayload)).ParamName); + var valid = new Envelope { MessageId = MessageKey, Payload = envelope.Payload, Subject = Subject }; + validate(valid); + Assert.Equal(Subject, GetSubject(valid)); + Assert.Equal("untouched", direct.Deserialize(valid.Payload)); + Assert.Equal(0, probe.ReadCount); + Assert.Equal(0, calls); + Assert.Equal(7, stock); + Assert.Equal(0, context.CompletionCount); + outbox.DidNotReceive().Send(Arg.Any()); + } + } + + [Fact] + public void MessageType_CreateSharesEnvelopeObjectsAndDecodesAllMessageRoles() + { + using var services = CreateServices(); + var serializer = services.GetRequiredService>(); + var type = new MessageType(Subject, serializer); + + var outgoing = type.Create(MessageKey, Receiver, "shared payload € / 東京"); + var incoming = outgoing.ToInboxMessage(DateTimeOffset.UnixEpoch); + var envelope = outgoing.Envelope; + + Assert.Equal(MessageKey, envelope.MessageId); + Assert.Equal(Receiver, outgoing.ReceiverId); + Assert.Equal(Subject, GetSubject(envelope)); + Assert.Null(envelope.Headers); + Assert.Equal(serializer.SerializeToArray("shared payload € / 東京"), envelope.Payload); + Assert.Same(envelope.Payload, incoming.Envelope.Payload); + Assert.Same(envelope.Subject, incoming.Envelope.Subject); + Assert.Null(incoming.Envelope.Headers); + Assert.Equal("shared payload € / 東京", type.Decode(envelope)); + Assert.Equal("shared payload € / 東京", type.Decode(incoming.Envelope)); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void Envelope_IdAndPayloadOnlyPermitEmptyPayloadAndOptionalMetadata(bool emptyPayload) + { + byte[] bytes = emptyPayload ? [] : [0, 17, 255]; + var envelope = new Envelope { MessageId = MessageKey, Payload = bytes }; + + Assert.Equal(MessageKey, envelope.MessageId); + Assert.Equal(bytes, envelope.Payload.ToArray()); + Assert.Same(bytes, envelope.Payload); + Assert.Null(envelope.Subject); + Assert.Null(envelope.Headers); + } + + [Fact] + public void InboxDispatcher_SubjectHeaderLookupDoesNotAllocateDecodedSubjectStrings() + { + using var services = CreateServices(); + var serializer = services.GetRequiredService>(); + var type = new MessageType(SubjectAtLimit(multibyte: true), serializer); + var outgoing = type.Create(MessageKey, Receiver, "allocation control"); + var context = new CountingInboxHandlerContext(outgoing); + var state = new HandlerState(); + var dispatcher = new TestInbox().Register(type, state, ObserveState); + for (var i = 0; i < 64; i++) type.Decode(outgoing.Envelope); + MeasureDispatchAllocations(dispatcher, context, 64); + var before = GC.GetAllocatedBytesForCurrentThread(); + for (var i = 0; i < 256; i++) type.Decode(outgoing.Envelope); + var decodeBytes = GC.GetAllocatedBytesForCurrentThread() - before; + + var dispatchBytes = MeasureDispatchAllocations(dispatcher, context, 256); + + Assert.True(dispatchBytes <= decodeBytes, + $"Subject dispatch allocated {dispatchBytes}, direct decode allocated {decodeBytes}."); + Assert.Equal(320, state.Calls); + Assert.Equal("allocation control", state.Body); + Assert.Same(context, state.Context); + Assert.Equal(0, context.CompletionCount); + Assert.Equal(SubjectAtLimit(multibyte: true), GetSubject(outgoing.Envelope)); + Assert.Equal(MessageKey, outgoing.Envelope.MessageId); + Assert.Equal("allocation control", type.Decode(context.Message.Envelope)); + } + + [Fact] + public void AddMessageType_GrainIdPayloadCollapsesSharedSerializerRegistration() + { + var services = new ServiceCollection(); + + Assert.Same(services, services.AddMessageType(Subject)); + + Assert.Equal(2, services.Count); + var closed = services[0]; + Assert.False(closed.IsKeyedService); + Assert.Equal(typeof(Serializer), closed.ServiceType); + Assert.Equal(ServiceLifetime.Singleton, closed.Lifetime); + Assert.Null(closed.ImplementationInstance); + Assert.Null(closed.ImplementationType); + Assert.NotNull(closed.ImplementationFactory); + var binding = services[1]; + Assert.True(binding.IsKeyedService); + Assert.Equal(typeof(MessageType), binding.ServiceType); + Assert.Equal(Subject, binding.ServiceKey); + Assert.Equal(ServiceLifetime.Singleton, binding.Lifetime); + var bindingFactory = binding.KeyedImplementationFactory; + Assert.NotNull(bindingFactory); + + // Registration must be lazy: the serializer infrastructure is added later. + services.AddSerializer(); + using var provider = services.BuildServiceProvider(); + using var scope = provider.CreateScope(); + var serializer = provider.GetRequiredService>(); + var messageType = provider.GetRequiredKeyedService>(Subject); + Assert.Equal(Subject, messageType.Subject); + Assert.Same(serializer, scope.ServiceProvider.GetRequiredService>()); + Assert.Same(messageType, scope.ServiceProvider.GetRequiredKeyedService>(Subject)); + Assert.Same(messageType, provider.GetRequiredKeyedService>(Subject)); + var factoryProvider = Substitute.For(); + factoryProvider.GetService(typeof(Serializer)).Returns(serializer); + factoryProvider.ClearReceivedCalls(); + + var fromFactory = Assert.IsType>(bindingFactory(factoryProvider, Subject)); + + Assert.Equal(Subject, fromFactory.Subject); + factoryProvider.Received(1).GetService(typeof(Serializer)); + factoryProvider.DidNotReceive().GetService(typeof(Serializer)); + + // An existing closed serializer serves both roles without replacement or + // an untyped Serializer dependency, and its factory runs only once. + using var codecServices = CreateServices(); + var (custom, probe) = CreateProbedSerializer(codecServices); + var customServices = new ServiceCollection(); + var factoryCalls = 0; + customServices.AddSingleton>(_ => + { + factoryCalls++; + return custom; + }); + var original = Assert.Single(customServices); + + Assert.Same(customServices, customServices.AddMessageType(Subject)); + + Assert.Equal(2, customServices.Count); + Assert.Same(original, customServices[0]); + Assert.Same(original, Assert.Single(customServices, descriptor => + !descriptor.IsKeyedService && descriptor.ServiceType == typeof(Serializer))); + Assert.Equal(0, factoryCalls); + using var customProvider = customServices.BuildServiceProvider(); + using var customScope = customProvider.CreateScope(); + Assert.Null(customProvider.GetService()); + var customBinding = customProvider.GetRequiredKeyedService>(Subject); + Assert.Equal(Subject, customBinding.Subject); + Assert.Same(customBinding, customScope.ServiceProvider.GetRequiredKeyedService>(Subject)); + Assert.Same(custom, customProvider.GetRequiredService>()); + Assert.Same(custom, customScope.ServiceProvider.GetRequiredService>()); + Assert.Equal(1, factoryCalls); + Assert.Equal(0, probe.WriteCount); + Assert.Equal(0, probe.ReadCount); + + var bytes = custom.SerializeToArray(Receiver); + + Assert.Equal(codecServices.GetRequiredService>().SerializeToArray(Receiver), bytes); + Assert.Equal(Receiver, custom.Deserialize(bytes)); + Assert.Equal(1, probe.WriteCount); + Assert.Equal(1, probe.ReadCount); + Assert.Equal(1, factoryCalls); + + var outgoing = customBinding.Create(MessageKey, Receiver, Receiver); + Assert.Equal(2, probe.WriteCount); + Assert.Equal(Receiver, outgoing.ReceiverId); + Assert.Equal(MessageKey, outgoing.Envelope.MessageId); + Assert.Equal(Receiver, customBinding.Decode(outgoing.Envelope)); + Assert.Null(outgoing.Envelope.Headers); + Assert.Equal(2, probe.ReadCount); + Assert.Equal(1, factoryCalls); + } +} diff --git a/test/Orleans.Messaging.Tests/Contracts/TypedOutboxExtensionsTests.cs b/test/Orleans.Messaging.Tests/Contracts/TypedOutboxExtensionsTests.cs new file mode 100644 index 00000000000..fe79fe7f23b --- /dev/null +++ b/test/Orleans.Messaging.Tests/Contracts/TypedOutboxExtensionsTests.cs @@ -0,0 +1,617 @@ +using System.Buffers; +using System.Diagnostics.CodeAnalysis; +using Microsoft.Extensions.DependencyInjection; +using NSubstitute; +using Orleans.Messaging.Tests.Support; +using Orleans.Runtime; +using Orleans.Serialization; +using Orleans.Serialization.Buffers; +using Orleans.Serialization.Codecs; +using Orleans.Serialization.Session; +using Orleans.Serialization.WireProtocol; +using Xunit; + +namespace Orleans.Messaging.Tests.Contracts; + +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class TypedOutboxExtensionsTests +{ + private const string Subject = "orders.result"; + private static GrainId Owner => GrainId.Create("warehouse", "reply-owner"); + private static GrainId ReceivedSender => GrainId.Create("customer", "original-command-sender"); + private static GrainId Destination => GrainId.Create("audit", "explicit-destination"); + private static MessageId CommandId => MessageId.Create("orders", "42", "reserve"); + + [Fact] + public void SendBatch_EnumeratesOnceAndEncodesEveryMessageBeforeStaging() + { + using var services = CreateServices(); + var type = new MessageType(Subject, services.GetRequiredService>()); + var outbox = new StoringOutbox(); + var enumerations = 0; + IEnumerable<(MessageId, GrainId, string)> Messages() + { + Assert.Equal(1, ++enumerations); + yield return (CommandId, Destination, "first"); + Assert.Equal(0, outbox.RawSendCalls); + yield return (new MessageId(CommandId.Key.CreateChildKey("second")), ReceivedSender, "second"); + Assert.Equal(0, outbox.RawSendCalls); + } + + outbox.Send(type, Messages()); + + Assert.Equal(1, enumerations); + Assert.Equal(2, outbox.RawSendCalls); + Assert.Equal(new[] { CommandId, new MessageId(CommandId.Key.CreateChildKey("second")) }, outbox.Messages.Select(value => value.Envelope.MessageId)); + Assert.Equal(new[] { Destination, ReceivedSender }, outbox.Messages.Select(value => value.ReceiverId)); + Assert.All(outbox.Messages, value => Assert.Equal(Subject, GetSubject(value.Envelope))); + Assert.Equal(new[] { "first", "second" }, outbox.Messages.Select(message => type.Decode(message.Envelope))); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void SendBatch_PreparationFailureStagesNothingAndPreservesOriginalError(bool encodingFailure) + { + using var services = CreateServices(); + var sessions = services.GetRequiredService(); + var codec = new FailingCodec(sessions.CodecProvider.GetCodec()); + var type = new MessageType(Subject, new Serializer(codec, sessions)); + var outbox = new StoringOutbox(); + var sentinel = new InvalidDataException("later preparation failed"); + IEnumerable<(MessageId, GrainId, string)> Messages() + { + yield return (CommandId, Destination, "first"); + if (!encodingFailure) throw sentinel; + codec.NextFailure = sentinel; + yield return (new MessageId(CommandId.Key.CreateChildKey("second")), Destination, "second"); + } + + Assert.Same(sentinel, Assert.Throws(() => outbox.Send(type, Messages()))); + Assert.Equal(0, outbox.RawSendCalls); + Assert.Empty(outbox.Messages); + Assert.Equal(encodingFailure ? 2 : 1, codec.Writes); + } + + [Theory] + [InlineData("outbox")] + [InlineData("messageType")] + [InlineData("messages")] + public void SendBatch_RejectsNullArgumentsBeforeEncodingOrStaging(string variation) + { + using var services = CreateServices(); + var actual = new StoringOutbox(); + var type = new MessageType(Subject, services.GetRequiredService>()); + IOutbox outbox = actual; + IEnumerable<(MessageId, GrainId, string)> messages = [(CommandId, Destination, "body")]; + switch (variation) + { + case "outbox": outbox = null!; break; + case "messageType": type = null!; break; + case "messages": messages = null!; break; + } + + Assert.Equal(variation, Assert.Throws(() => outbox.Send(type, messages)).ParamName); + Assert.Equal(0, actual.RawSendCalls); + Assert.Empty(actual.Messages); + } + + [Fact] + public void SendBatch_EmptyBatchStagesNothing() + { + using var services = CreateServices(); + var outbox = new StoringOutbox(); + var type = new MessageType(Subject, services.GetRequiredService>()); + + outbox.Send(type, []); + + Assert.Equal(0, outbox.RawSendCalls); + Assert.Empty(outbox.Messages); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void SendBatch_RawFailurePreservesPublishedArrayAndOriginalError(bool storeBeforeFailure) + { + using var services = CreateServices(); + var serializer = services.GetRequiredService>(); + var type = new MessageType(Subject, serializer); + var body = new byte[3_145_728]; + body[body.Length / 2] = 17; + var sentinel = new IOException("raw staging failed"); + var outbox = new StoringOutbox() { Failure = sentinel, StoreBeforeFailure = storeBeforeFailure }; + (MessageId, GrainId, byte[])[] messages = + [ + (CommandId, Destination, body), + (new MessageId(CommandId.Key.CreateChildKey("second")), Destination, body) + ]; + + Assert.Same(sentinel, Assert.Throws(() => outbox.Send(type, messages))); + Assert.Equal(1, outbox.RawSendCalls); + var published = outbox.LastPublished!.Value; + Assert.NotNull(outbox.WireSeen); + Assert.Same(published.Envelope.Payload, outbox.WireSeen); + Assert.Equal(serializer.SerializeToArray(body), published.Envelope.Payload.ToArray()); + Assert.Equal(body, serializer.Deserialize(published.Envelope.Payload)); + if (storeBeforeFailure) + { + Assert.Same(published.Envelope.Payload, Assert.Single(outbox.Messages).Envelope.Payload); + outbox.ClearMessages(); + } + Assert.Empty(outbox.Messages); + Assert.Equal(body, serializer.Deserialize(published.Envelope.Payload)); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void Send_SerializesExactEnvelopeWithExplicitDestination(bool addressable) + { + using var services = CreateServices(); + var serializer = services.GetRequiredService>(); + var type = new MessageType(Subject, serializer); + var outbox = new StoringOutbox(); + IAddressable reference = new AddressableDestination(Destination); + + if (addressable) outbox.Send(type, CommandId, reference, "typed reserve € / 東京"); + else outbox.Send(type, CommandId, Destination, "typed reserve € / 東京"); + + var staged = Assert.Single(outbox.Messages); + Assert.Equal(1, outbox.RawSendCalls); + Assert.Equal(1, outbox.Count); + Assert.Equal(CommandId, staged.Envelope.MessageId); + Assert.Equal(Destination, staged.ReceiverId); + Assert.Equal(Subject, GetSubject(staged.Envelope)); + Assert.Equal(serializer.SerializeToArray("typed reserve € / 東京"), staged.Envelope.Payload.ToArray()); + Assert.Equal("typed reserve € / 東京", serializer.Deserialize(staged.Envelope.Payload)); + AssertEnvelopeFields(staged.Envelope); + Assert.True(outbox.TryGetMessage(CommandId, out var found)); + Assert.Same(staged.Envelope.Payload, found.Envelope.Payload); + Assert.Null(staged.Envelope.DeliverAt); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void Send_CarriesOptionalDeliverAt(bool addressable) + { + using var services = CreateServices(); + var type = new MessageType(Subject, services.GetRequiredService>()); + var outbox = new StoringOutbox(); + var deliverAt = new DateTimeOffset(2040, 1, 2, 3, 4, 5, TimeSpan.FromHours(1)); + + if (addressable) outbox.Send(type, CommandId, new AddressableDestination(Destination), "scheduled", deliverAt); + else outbox.Send(type, CommandId, Destination, "scheduled", deliverAt); + + var staged = Assert.Single(outbox.Messages); + Assert.Equal(deliverAt, staged.Envelope.DeliverAt); + Assert.Equal(deliverAt.Offset, staged.Envelope.DeliverAt!.Value.Offset); + Assert.Equal(Destination, staged.ReceiverId); + Assert.Equal("scheduled", type.Decode(staged.Envelope)); + Assert.Equal(deliverAt, type.Create(CommandId, Destination, "scheduled", deliverAt).Envelope.DeliverAt); + Assert.Null(type.Create(CommandId, Destination, "scheduled").Envelope.DeliverAt); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void SendReply_UsesResultChildAndExplicitDestination(bool addressable) + { + using var services = CreateServices(); + var serializer = services.GetRequiredService>(); + var type = new MessageType(Subject, serializer); + var received = new MessageType("orders.reserve", serializer) + .Create(CommandId, Owner, "incoming reserve"); + var context = new CountingContext(received.ToInboxMessage(DateTimeOffset.UnixEpoch)); + var outbox = new StoringOutbox(); + IAddressable reference = new AddressableDestination(Destination); + + if (addressable) outbox.SendReply(type, context, reference, "approved €42"); + else outbox.SendReply(type, context, Destination, "approved €42"); + + var staged = Assert.Single(outbox.Messages); + Assert.Equal(new MessageId(CommandId.Key.CreateChildKey("result")), staged.Envelope.MessageId); + Assert.Equal("orders/42/reserve/result", staged.Envelope.MessageId.ToString()); + Assert.Equal(4, staged.Envelope.MessageId.Key.CountSegments()); + Assert.Equal(Destination, staged.ReceiverId); + Assert.NotEqual(ReceivedSender, staged.ReceiverId); + Assert.Equal(Subject, GetSubject(staged.Envelope)); + Assert.Equal(serializer.SerializeToArray("approved €42"), staged.Envelope.Payload.ToArray()); + Assert.Equal("approved €42", serializer.Deserialize(staged.Envelope.Payload)); + AssertEnvelopeFields(staged.Envelope); + Assert.Equal(1, outbox.RawSendCalls); + Assert.Equal(0, context.CompletionCount); + Assert.Equal("incoming reserve", serializer.Deserialize(received.Envelope.Payload)); + Assert.Equal(CommandId, received.Envelope.MessageId); + } + + [Theory] + [InlineData(false, "segments")] + [InlineData(true, "segments")] + [InlineData(false, "ascii-bytes")] + [InlineData(true, "ascii-bytes")] + [InlineData(false, "multibyte-bytes")] + [InlineData(true, "multibyte-bytes")] + public void SendReply_ResultChildAtAdmissionLimitEncodesAndStages(bool addressable, string variation) + { + using var services = CreateServices(); + var serializer = services.GetRequiredService>(); + var type = new MessageType(Subject, serializer); + var key = variation switch + { + "segments" => MessageId.Create(Enumerable.Repeat("s", 31).ToArray()), + "ascii-bytes" => MessageId.Create(new string('k', 1017)), + "multibyte-bytes" => MessageId.Create(new string('\u20ac', 339)), + _ => throw new ArgumentOutOfRangeException(nameof(variation)) + }; + var context = Context(key); + var outbox = new StoringOutbox(); + + Invoke(outbox, type, context, key, Destination, "boundary result", reply: true, addressable: addressable); + + var staged = Assert.Single(outbox.Messages); + Assert.Equal(new MessageId(key.Key.CreateChildKey("result")), staged.Envelope.MessageId); + if (variation == "segments") Assert.Equal(32, staged.Envelope.MessageId.Key.CountSegments()); + else Assert.Equal(1024, System.Text.Encoding.UTF8.GetByteCount(staged.Envelope.MessageId.ToString())); + Assert.Equal(Destination, staged.ReceiverId); + Assert.Equal(serializer.SerializeToArray("boundary result"), staged.Envelope.Payload.ToArray()); + Assert.Equal(1, outbox.RawSendCalls); + Assert.Equal(0, context.CompletionCount); + } + + [Theory] + [InlineData(false, false)] + [InlineData(false, true)] + [InlineData(true, false)] + [InlineData(true, true)] + public void TypedSend_StoresPublishedArrayAndPreservesBytesAfterRemoval(bool reply, bool addressable) + { + using var services = CreateServices(); + var serializer = services.GetRequiredService>(); + var type = new MessageType(Subject, serializer); + var body = Enumerable.Range(0, 3_145_728).Select(i => (byte)(i * 31 + 7)).ToArray(); + var wire = serializer.SerializeToArray(body); + var outbox = new StoringOutbox(); + var context = Context(); + + Invoke(outbox, type, context, CommandId, Destination, body, reply, addressable); + + var staged = Assert.Single(outbox.Messages); + var published = outbox.LastPublished!.Value; + Assert.NotNull(outbox.WireSeen); + Assert.Equal(wire, staged.Envelope.Payload.ToArray()); + Assert.Equal(body, serializer.Deserialize(staged.Envelope.Payload)); + Assert.Same(published.Envelope.Payload, staged.Envelope.Payload); + Assert.Same(published.Envelope.Payload, outbox.WireSeen); + Assert.NotSame(body, published.Envelope.Payload); + Assert.Equal(body, serializer.Deserialize(published.Envelope.Payload)); + Assert.Equal(reply ? new MessageId(CommandId.Key.CreateChildKey("result")) : CommandId, staged.Envelope.MessageId); + Assert.Equal(Destination, staged.ReceiverId); + Assert.Equal(0, context.CompletionCount); + outbox.ClearMessages(); + + Assert.Empty(outbox.Messages); + Assert.Equal(0, outbox.Count); + Assert.Equal(wire, published.Envelope.Payload.ToArray()); + Assert.Equal(body, serializer.Deserialize(published.Envelope.Payload)); + Assert.Equal(1, outbox.RawSendCalls); + Invoke(outbox, type, context, CommandId, Destination, body, reply, addressable); + var subsequent = Assert.Single(outbox.Messages); + Assert.NotSame(published.Envelope.Payload, subsequent.Envelope.Payload); + Assert.Equal(wire, subsequent.Envelope.Payload.ToArray()); + Assert.Equal(wire, published.Envelope.Payload.ToArray()); + Assert.Equal(2, outbox.RawSendCalls); + Assert.Equal(0, context.CompletionCount); + } + + [Theory] + [InlineData(false, false, false)] + [InlineData(false, true, false)] + [InlineData(true, false, false)] + [InlineData(true, true, false)] + [InlineData(false, false, true)] + [InlineData(false, true, true)] + [InlineData(true, false, true)] + [InlineData(true, true, true)] + public void TypedSend_RawSendFailurePreservesExactExceptionAndPublishedArray(bool reply, bool addressable, bool storeBeforeFailure) + { + using var services = CreateServices(); + var serializer = services.GetRequiredService>(); + var type = new MessageType(Subject, serializer); + var body = Enumerable.Range(0, 3_145_728).Select(i => (byte)(i * 17 + 3)).ToArray(); + var sentinel = new IOException("rawSend sentinel"); + var outbox = new StoringOutbox() { Failure = sentinel, StoreBeforeFailure = storeBeforeFailure }; + var context = Context(); + + var exception = Assert.Throws(() => + Invoke(outbox, type, context, CommandId, Destination, body, reply, addressable)); + + Assert.Same(sentinel, exception); + Assert.Equal(1, outbox.RawSendCalls); + Assert.NotNull(outbox.WireSeen); + Assert.Equal(serializer.SerializeToArray(body), outbox.WireSeen.ToArray()); + var published = outbox.LastPublished!.Value; + Assert.Equal(Destination, published.ReceiverId); + Assert.Equal(reply ? new MessageId(CommandId.Key.CreateChildKey("result")) : CommandId, published.Envelope.MessageId); + Assert.Equal(Subject, GetSubject(published.Envelope)); + Assert.Same(outbox.WireSeen, published.Envelope.Payload); + if (storeBeforeFailure) + { + var stored = Assert.Single(outbox.Messages); + Assert.Same(published.Envelope.Payload, stored.Envelope.Payload); + Assert.Equal(body, serializer.Deserialize(stored.Envelope.Payload)); + outbox.ClearMessages(); + } + Assert.Equal(body, serializer.Deserialize(published.Envelope.Payload)); + Assert.Empty(outbox.Messages); + Assert.Equal(0, outbox.Count); + Assert.Equal(0, context.CompletionCount); + } + + [Theory] + [InlineData(false, false)] + [InlineData(false, true)] + [InlineData(true, false)] + [InlineData(true, true)] + public void TypedSend_PartialEncodingFailureNeverStagesAndNextCallHasPristineWire(bool reply, bool addressable) + { + using var services = CreateServices(); + var direct = services.GetRequiredService>(); + var sessions = services.GetRequiredService(); + var codec = new FailingCodec(sessions.CodecProvider.GetCodec()); + var type = new MessageType(Subject, new Serializer(codec, sessions)); + var outbox = new StoringOutbox(); + var context = Context(); + var sentinel = new InvalidDataException("committed encoder sentinel"); + codec.NextFailure = sentinel; + + var exception = Assert.Throws(() => + Invoke(outbox, type, context, CommandId, Destination, "failing", reply, addressable)); + + Assert.Same(sentinel, exception); + Assert.Equal(1, codec.Writes); + Assert.Equal(1, codec.CommittedFailures); + Assert.Equal(0, outbox.RawSendCalls); + Assert.Null(outbox.LastPublished); + Assert.Null(outbox.WireSeen); + Assert.Empty(outbox.Messages); + Assert.Equal(0, context.CompletionCount); + Invoke(outbox, type, context, CommandId, Destination, "recovered €42", reply, addressable); + var staged = Assert.Single(outbox.Messages); + Assert.Equal(direct.SerializeToArray("recovered €42"), staged.Envelope.Payload.ToArray()); + Assert.Equal("recovered €42", direct.Deserialize(staged.Envelope.Payload)); + Assert.Equal(reply ? new MessageId(CommandId.Key.CreateChildKey("result")) : CommandId, staged.Envelope.MessageId); + Assert.Equal(2, codec.Writes); + Assert.Equal(1, outbox.RawSendCalls); + Assert.Equal(0, context.CompletionCount); + } + + public static IEnumerable ValidationCases() + { + foreach (var reply in new[] { false, true }) + foreach (var addressable in new[] { false, true }) + { + foreach (var variation in new[] { "outbox", "messageType", "body", "bytes", "multibyte-bytes" }) + yield return [reply, addressable, variation]; + if (reply) + { + yield return [reply, addressable, "context"]; + yield return [reply, addressable, "unset-parent"]; + } + else yield return [reply, addressable, "messageId"]; + yield return [reply, addressable, addressable ? "null-reference" : "destination"]; + if (addressable) yield return [reply, addressable, "invalid-reference"]; + } + } + + [Theory] + [MemberData(nameof(ValidationCases))] + public void TypedSend_ValidationFailureNeverEncodesOrStages(bool reply, bool addressable, string variation) + { + using var services = CreateServices(); + var sessions = services.GetRequiredService(); + var codec = new FailingCodec(sessions.CodecProvider.GetCodec()); + var type = new MessageType(Subject, new Serializer(codec, sessions)); + var realOutbox = new StoringOutbox(); + IOutbox outbox = variation == "outbox" ? null! : realOutbox; + var key = variation switch + { + "messageId" or "unset-parent" => default, + // Reply adds seven UTF8 bytes ("/result"). + "bytes" => MessageId.Create(new string('k', reply ? 1018 : 1025)), + "multibyte-bytes" => MessageId.Create(new string('\u20ac', reply ? 339 : 341) + (reply ? "a" : "ab")), + _ => CommandId + }; + var context = Context(reply ? key : null); + var destination = variation is "destination" or "invalid-reference" ? default : Destination; + var body = variation == "body" ? null! : "never encoded"; + if (variation == "messageType") type = null!; + if (variation == "context") context = null!; + + var exception = Record.Exception(() => + { + if (addressable) + { + IAddressable reference = variation == "null-reference" ? null! : new AddressableDestination(destination); + if (reply) outbox.SendReply(type, context, reference, body); + else outbox.Send(type, key, reference, body); + } + else if (reply) outbox.SendReply(type, context, destination, body); + else outbox.Send(type, key, destination, body); + }); + + if (variation == "unset-parent") + { + Assert.IsType(exception); + } + else + { + var argumentError = Assert.IsAssignableFrom(exception); + Assert.Equal(variation switch + { + "null-reference" => "destination", + "invalid-reference" => "grain", + "destination" => "receiverId", + "bytes" or "multibyte-bytes" or "messageId" => "messageId", + _ => variation + }, argumentError.ParamName); + } + Assert.Equal(0, codec.Writes); + Assert.Equal(0, realOutbox.RawSendCalls); + Assert.Null(realOutbox.LastPublished); + Assert.Null(realOutbox.WireSeen); + Assert.Empty(realOutbox.Messages); + Assert.Equal(0, realOutbox.Count); + if (context is not null) Assert.Equal(0, context.CompletionCount); + // Positive control: same codec still encodes and stages after rejection. + var positive = new StoringOutbox(); + Invoke(positive, + new MessageType(Subject, new Serializer(codec, sessions)), + Context(), CommandId, Destination, "valid control", reply, addressable); + Assert.Equal(1, codec.Writes); + Assert.Equal(1, positive.RawSendCalls); + var staged = Assert.Single(positive.Messages); + Assert.Equal(services.GetRequiredService>().SerializeToArray("valid control"), staged.Envelope.Payload.ToArray()); + Assert.Equal(Destination, staged.ReceiverId); + } + + private static ServiceProvider CreateServices() => + new ServiceCollection().AddSerializer().BuildServiceProvider(); + + private static CountingContext Context(MessageId? key = null) + { + var messageId = key ?? CommandId; + if (messageId == default) return new(new InboxMessage { Envelope = default, SentAt = DateTimeOffset.UnixEpoch }); + using var services = CreateServices(); + return new(new InboxMessage + { + Envelope = new Envelope + { + MessageId = messageId, + Payload = [], + Subject = "orders.reserve", + Headers = new Dictionary(StringComparer.Ordinal) + { + ["sender"] = services.GetRequiredService>().SerializeToArray(ReceivedSender) + } + }, + SentAt = DateTimeOffset.UnixEpoch + }); + } + + private static void Invoke(IOutbox outbox, MessageType type, CountingContext context, + MessageId key, GrainId destination, T body, bool reply, bool addressable) + { + if (addressable) + { + IAddressable reference = new AddressableDestination(destination); + if (reply) outbox.SendReply(type, context, reference, body); + else outbox.Send(type, key, reference, body); + } + else if (reply) outbox.SendReply(type, context, destination, body); + else outbox.Send(type, key, destination, body); + } + + private sealed class AddressableDestination : IAddressable, IGrainBase + { + public AddressableDestination(GrainId id) + { + GrainContext = Substitute.For(); + GrainContext.GrainId.Returns(id); + } + public IGrainContext GrainContext { get; } + } + + private sealed class CountingContext(InboxMessage message) : IInboxHandlerContext + { + public InboxMessage Message { get; } = message; + public int CompletionCount { get; private set; } + public void Complete() => CompletionCount++; + public void Fail(string reason) => throw new InvalidOperationException("Unexpected dead letter."); + } + + private sealed class StoringOutbox : IOutbox + { + private readonly List _messages = []; + public int Count => _messages.Count; + public IEnumerable Messages => _messages; + public int RawSendCalls { get; private set; } + public OutboxMessage? LastPublished { get; private set; } + public byte[]? WireSeen { get; private set; } + public Exception? Failure { get; set; } + public bool StoreBeforeFailure { get; set; } + + public void Send(OutboxMessage message) + { + RawSendCalls++; + LastPublished = message; + WireSeen = message.Envelope.Payload; + if (Failure is { } failure) + { + if (StoreBeforeFailure) _messages.Add(message); + throw failure; + } + _messages.Add(message); + } + + public void Send(MessageId messageId) => throw new NotSupportedException(); + + public bool Withdraw(MessageId messageId) => throw new NotSupportedException(); + + public IOutbox Register(IOutboxMessageSource source) => throw new NotSupportedException(); + + public bool TryGetMessage(MessageId messageId, [MaybeNullWhen(false)] out OutboxMessage envelope) + { + foreach (var message in _messages) + if (message.Envelope.MessageId == messageId) + { + envelope = message; + return true; + } + envelope = default; + return false; + } + + public void ClearMessages() => _messages.Clear(); + } + + private static string GetSubject(Envelope envelope) + { + return Assert.IsType(envelope.Subject); + } + + + private static void AssertEnvelopeFields(Envelope envelope) + { + Assert.Equal(Subject, envelope.Subject); + Assert.Null(envelope.Headers); + } + + private sealed class FailingCodec(IFieldCodec inner) : IFieldCodec + { + public int Writes { get; private set; } + public int CommittedFailures { get; private set; } + public Exception? NextFailure { get; set; } + + public void WriteField(ref Writer writer, uint fieldIdDelta, + [AllowNull] Type expectedType, [AllowNull] string value) where TBufferWriter : IBufferWriter + { + Writes++; + if (NextFailure is { } failure) + { + NextFailure = null; + writer.WriteByte(0x7e); + writer.Commit(); + CommittedFailures++; + throw failure; + } + inner.WriteField(ref writer, fieldIdDelta, expectedType, value); + } + + [return: MaybeNull] + public string ReadValue(ref Reader reader, Field field) => inner.ReadValue(ref reader, field); + } +} diff --git a/test/Orleans.Messaging.Tests/Contracts/WireFormatTests.cs b/test/Orleans.Messaging.Tests/Contracts/WireFormatTests.cs new file mode 100644 index 00000000000..b75ffc971d0 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Contracts/WireFormatTests.cs @@ -0,0 +1,106 @@ +using Microsoft.Extensions.DependencyInjection; +using Orleans.Runtime; +using Orleans.Serialization; +using Xunit; + +namespace Orleans.Messaging.Tests.Contracts; + +/// +/// Pins the serialized bytes of every messaging wire type. Messages cross silos of different versions and persist in +/// journals, so any change to these bytes is a compatibility decision which must update the golden values deliberately. +/// +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class WireFormatTests : IDisposable +{ + private static readonly GrainId Receiver = GrainId.Create("receiver", "r1"); + private static readonly MessageId Id = MessageId.Create("orders", "4/2", "reserve"); + private static readonly DateTimeOffset DeliverAt = new(2026, 10, 19, 12, 30, 0, TimeSpan.FromHours(2)); + private readonly ServiceProvider _services = new ServiceCollection().AddSerializer().BuildServiceProvider(); + + public static TheoryData Golden => new() + { + { "MessageId", "2040276F72646572732F345C2F322F72657365727665E0" }, + { "HierarchicalKey", "2040276F72646572732F345C2F322F72657365727665E0" }, + { "Envelope.Minimal", "202040276F72646572732F345C2F322F72657365727665E04101C101C101C101E0" }, + { "Envelope.Full", "202040276F72646572732F345C2F322F72657365727665E041070102034129696E76656E746F72792E726573657276652E7631210103410B747261636540050908E02180005444B1DC2DDF088100D088C310000000E0E0" }, + { "InboxMessage", "20202040276F72646572732F345C2F322F72657365727665E041070102034129696E76656E746F72792E726573657276652E7631210103410B747261636540050908E02180005444B1DC2DDF088100D088C310000000E0E021800094DA86132DDF088100D088C310000000E0E0" }, + { "OutboxMessage", "20202040276F72646572732F345C2F322F72657365727665E041070102034129696E76656E746F72792E726573657276652E7631210103410B747261636540050908E02180005444B1DC2DDF088100D088C310000000E0E021202060697CAED241117265636569766572E0E021605D5ABF1541057231E0E0E0" }, + { "DeliveryResult.Accepted", "20080305C101C101E0" }, + { "DeliveryResult.Duplicate", "20080309C101C101E0" }, + { "DeliveryResult.Backpressured", "2008030DC101C101E0" }, + { "DeliveryResult.HandlerNotFound", "20080311413F4E6F20696E626F782068616E646C657220697320726567697374657265642EC101E0" }, + { "DeliveryResult.DeadLettered", "2008031541196F7574206F662073746F636B41196F72646572732F73746F636BE0" }, + { "DeliveryResult.Expired", "2008031941D3546865206D657373616765207761732073656E74206265666F72652074686520726563656976657227732064656475706C69636174696F6E2077696E646F773B20697473206561726C6965722064656C6976657279206F7574636F6D6520697320756E6B6E6F776E2EC101E0" }, + }; + + [Theory] + [MemberData(nameof(Golden))] + public void Encoding_MatchesGoldenBytesAndDecodesFromThem(string name, string hex) + { + switch (name) + { + case "MessageId": Check(Id, hex); break; + case "HierarchicalKey": Check(Id.Key, hex); break; + case "Envelope.Minimal": Check(new Envelope { MessageId = Id, Payload = [] }, hex, AssertEnvelope); break; + case "Envelope.Full": Check(FullEnvelope(), hex, AssertEnvelope); break; + case "InboxMessage": + Check(new InboxMessage { Envelope = FullEnvelope(), SentAt = DeliverAt.AddDays(-1) }, hex, AssertInboxMessage); + break; + case "OutboxMessage": + Check(new OutboxMessage { Envelope = FullEnvelope(), ReceiverId = Receiver }, hex, (e, a) => + { + AssertEnvelope(e.Envelope, a.Envelope); + Assert.Equal(e.ReceiverId, a.ReceiverId); + }); + break; + case "DeliveryResult.Accepted": Check(DeliveryResult.Accepted(), hex); break; + case "DeliveryResult.Duplicate": Check(DeliveryResult.Duplicate(), hex); break; + case "DeliveryResult.Backpressured": Check(DeliveryResult.Backpressured(), hex); break; + case "DeliveryResult.HandlerNotFound": Check(DeliveryResult.HandlerNotFound(), hex); break; + case "DeliveryResult.DeadLettered": Check(DeliveryResult.DeadLettered("out of stock", "orders/stock"), hex); break; + case "DeliveryResult.Expired": Check(DeliveryResult.Expired(), hex); break; + default: throw new ArgumentOutOfRangeException(nameof(name), name, null); + } + } + + public void Dispose() => _services.Dispose(); + + private static Envelope FullEnvelope() => new() + { + MessageId = Id, + Payload = [1, 2, 3], + Subject = "inventory.reserve.v1", + Headers = new() { ["trace"] = [9, 8] }, + DeliverAt = DeliverAt + }; + + private void Check(T value, string hex, Action? assertEqual = null) + { + var serializer = _services.GetRequiredService>(); + var actual = Convert.ToHexString(serializer.SerializeToArray(value)); + Assert.Equal(hex, actual); + var decoded = serializer.Deserialize(Convert.FromHexString(hex)); + if (assertEqual is null) Assert.Equal(value, decoded); + else assertEqual(value, decoded!); + } + + private static void AssertInboxMessage(InboxMessage expected, InboxMessage actual) + { + AssertEnvelope(expected.Envelope, actual.Envelope); + Assert.Equal(expected.SentAt, actual.SentAt); + Assert.Equal(expected.SentAt.Offset, actual.SentAt.Offset); + } + + private static void AssertEnvelope(Envelope expected, Envelope actual) + { + Assert.Equal(expected.MessageId, actual.MessageId); + Assert.Equal(expected.Payload, actual.Payload); + Assert.Equal(expected.Subject, actual.Subject); + Assert.Equal(expected.DeliverAt, actual.DeliverAt); + Assert.Equal(expected.DeliverAt?.Offset, actual.DeliverAt?.Offset); + Assert.Equal(expected.Headers?.OrderBy(static h => h.Key, StringComparer.Ordinal).Select(static h => (h.Key, Convert.ToHexString(h.Value))), + actual.Headers?.OrderBy(static h => h.Key, StringComparer.Ordinal).Select(static h => (h.Key, Convert.ToHexString(h.Value)))); + } +} diff --git a/test/Orleans.Messaging.Tests/Functional/ByteMessagingPayloadTests.cs b/test/Orleans.Messaging.Tests/Functional/ByteMessagingPayloadTests.cs new file mode 100644 index 00000000000..2b50fd3307a --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/ByteMessagingPayloadTests.cs @@ -0,0 +1,109 @@ +using Microsoft.Extensions.DependencyInjection; +using Orleans.Messaging.Tests.Support; +using Orleans.Runtime; +using Xunit; + +namespace Orleans.Messaging.Tests.Functional; + +[Collection(MessagingClusterCollection.Name)] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class ByteMessagingPayloadTests : MessagingBehaviorTestBase +{ + [Fact] + public async Task RpcAcceptanceAndDuplicate_CopyPayloadAndPreserveCallerBytesAcrossReplay() + { + var receiver = NewGrain(); + _ = await receiver.GetSnapshotAsync(); + var context = Fixture.GetGrainContext(receiver); + var grain = Assert.IsType(context.GrainInstance); + var envelope = CreateEnvelope(receiver, NewMessage(302, "rpc-bytes")); + var expected = envelope.Envelope.Payload.ToArray(); + using var handler = new PayloadProbe(expected); + await OnTurnAsync(context, () => grain.HandlerOverride = handler); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + await handler.Entered.Task.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + Assert.NotSame(envelope.Envelope.Payload, handler.Payload); + envelope.Envelope.Payload[0] ^= 0xff; + Assert.Equal(expected, handler.Payload.ToArray()); + envelope.Envelope.Payload[0] ^= 0xff; + Assert.Equal(expected, envelope.Envelope.Payload.ToArray()); + handler.Release.TrySetResult(); + await handler.Completed.Task.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + await Fixture.SnapshotProbe.WaitAsync(receiver.GetGrainId(), + snapshot => snapshot.InboxCount == 0 && snapshot.ProcessedMessageCount == 1); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(receiver, envelope)).Status); + await receiver.RequestDeactivationAsync(); + await context.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + Assert.Equal(expected, envelope.Envelope.Payload.ToArray()); + _ = await receiver.GetSnapshotAsync(); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(receiver, envelope)).Status); + } + + [Fact] + public async Task RpcRejection_PreservesCallerBytesWithoutStagingState() + { + var receiver = NewGrain(); + await receiver.ConfigureHandlerAsync(false); + var original = Fixture.GetGrainContext(receiver); + await receiver.RequestDeactivationAsync(); + await original.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + _ = await receiver.GetSnapshotAsync(); + var context = Fixture.GetGrainContext(receiver); + var envelope = CreateEnvelope(receiver, NewMessage(301, "rejected-bytes")); + var expected = envelope.Envelope.Payload.ToArray(); + Assert.Equal(DeliveryStatus.HandlerNotFound, (await DeliverAsync(receiver, envelope)).Status); + var rejected = await receiver.GetSnapshotAsync(); + Assert.Equal(0, rejected.InboxCount); + Assert.Equal(0, rejected.ProcessedMessageCount); + Assert.Empty(rejected.Effects); + await receiver.RequestDeactivationAsync(); + await context.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + Assert.Equal(expected, envelope.Envelope.Payload.ToArray()); + } + + private static Task OnTurnAsync(IGrainContext context, Action action) + { + var completed = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + context.Scheduler.QueueAction(() => + { + try { action(); completed.SetResult(); } + catch (Exception exception) { completed.SetException(exception); } + }); + return completed.Task; + } + + private sealed class PayloadProbe(byte[] expected) : IInboxHandler, IDisposable + { + public byte[] Payload { get; private set; } = null!; + public TaskCompletionSource Entered { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + public TaskCompletionSource Release { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + public TaskCompletionSource Completed { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public bool CanHandle(InboxMessage message) => true; + + + public async ValueTask HandleAsync(IInboxHandlerContext context, CancellationToken cancellationToken) + { + try + { + Payload = context.Message.Envelope.Payload; + Entered.TrySetResult(); + await Release.Task.WaitAsync(cancellationToken); + Assert.Equal(expected, Payload.ToArray()); + context.Complete(); + Assert.Same(Payload, context.Message.Envelope.Payload); + Assert.Equal(expected, context.Message.Envelope.Payload.ToArray()); + Completed.TrySetResult(); + } + catch (Exception exception) + { + Completed.TrySetException(exception); + throw; + } + } + + public void Dispose() => Release.TrySetResult(); + } +} diff --git a/test/Orleans.Messaging.Tests/Functional/DedupeExpiryBehaviorTests.cs b/test/Orleans.Messaging.Tests/Functional/DedupeExpiryBehaviorTests.cs new file mode 100644 index 00000000000..85235b92294 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/DedupeExpiryBehaviorTests.cs @@ -0,0 +1,113 @@ +using Microsoft.Extensions.DependencyInjection; +using Orleans.Journaling; +using Orleans.Messaging.Tests.Support; +using Orleans.Runtime; +using Orleans.Serialization.Session; +using Xunit; + +namespace Orleans.Messaging.Tests.Functional; + +[CollectionDefinition(Name, DisableParallelization = true)] +public sealed class DedupeExpiryClusterCollection : ICollectionFixture +{ + public const string Name = "Messaging dedupe expiry cluster"; +} + +[Collection(DedupeExpiryClusterCollection.Name)] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class DedupeExpiryBehaviorTests(DedupeExpiryClusterFixture fixture) +{ + [Fact] + public async Task TrackedSentAt_RemainsDuplicateUntilSnapshotCommitsItsRetentionFloor() + { + var receiver = fixture.Client.GetGrain(Guid.NewGuid()); + var original = new TestMessage(TestApplicationProtocol.NewMessageId(), 15, "expires"); + var first = CreateEnvelope(receiver, original); + + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, first)).Status); + await fixture.WaitForEffectCountAsync(receiver, 1); + await WaitForIdleInboxAsync(receiver); + fixture.Clock.Advance(TimeSpan.FromMinutes(10) - TimeSpan.FromTicks(1)); + + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(receiver, first)).Status); + Assert.Equal(1, Assert.Single((await receiver.GetSnapshotAsync()).Effects).Count); + + fixture.Clock.Advance(TimeSpan.FromTicks(1)); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(receiver, first)).Status); + var state = await receiver.GetSnapshotAsync(); + + Assert.Equal(1, state.Effects.Single(effect => effect.LogicalId == original.LogicalId).Count); + fixture.Storage.RequestSnapshot(JournalId.FromGrainId(receiver.GetGrainId())); + await receiver.RetryWriteStateAsync(); + Assert.Equal(DeliveryStatus.Expired, (await DeliverAsync(receiver, first)).Status); + Assert.Equal(1, Assert.Single((await receiver.GetSnapshotAsync()).Effects).Count); + } + + [Fact] + public async Task FailedSnapshot_PreservesSentAtEvidenceUntilTheRetentionFloorIsDurable() + { + var receiver = fixture.Client.GetGrain(Guid.NewGuid()); + var envelope = CreateEnvelope( + receiver, + new TestMessage(TestApplicationProtocol.NewMessageId(), 16, "failed-expiry-replacement")); + + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + await fixture.WaitForEffectCountAsync(receiver, 1); + await WaitForIdleInboxAsync(receiver); + fixture.Clock.Advance(TimeSpan.FromMinutes(10)); + var oldContext = fixture.GetGrainContext(receiver); + var journal = JournalId.FromGrainId(receiver.GetGrainId()); + fixture.Storage.RequestSnapshot(journal); + fixture.Storage.FailWrite(journal); + + await Assert.ThrowsAsync(() => receiver.RetryWriteStateAsync()); + + await oldContext.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + var failed = await receiver.GetSnapshotAsync(); + Assert.Equal(0, failed.InboxCount); + Assert.Equal(1, Assert.Single(failed.Effects).Count); + var replayed = Assert.IsType(fixture.GetGrainContext(receiver).GrainInstance).ReplayedSnapshot; + Assert.Equal(1, Assert.IsType(replayed).ProcessedMessageCount); + // The failed snapshot did not durably advance the floor, so journal replay retains the accepted send time. + Assert.Equal(1, failed.ProcessedMessageCount); + + await receiver.RequestDeactivationAsync(); + var recovered = await receiver.GetSnapshotAsync(); + Assert.NotEqual(failed.ActivationId, recovered.ActivationId); + Assert.Equal(0, recovered.InboxCount); + Assert.Equal(1, recovered.ProcessedMessageCount); + Assert.Equal(1, Assert.Single(recovered.Effects).Count); + + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(receiver, envelope)).Status); + var retried = await receiver.GetSnapshotAsync(); + Assert.Equal(1, Assert.Single(retried.Effects).Count); + } + + private Task WaitForIdleInboxAsync(IMessagingTestGrain receiver) => + fixture.SnapshotProbe.WaitAsync( + receiver.GetGrainId(), + static snapshot => snapshot.InboxCount == 0); + + private static async Task DeliverAsync( + IMessagingTestGrain receiver, + InboxMessage envelope) => + await receiver.AsReference().DeliverAsync(envelope); + + private InboxMessage CreateEnvelope( + IMessagingTestGrain receiver, + TestMessage message, + MessageId? messageId = null) + { + var sessions = fixture.Client.ServiceProvider.GetRequiredService(); + var sender = GrainId.Create("expiry-test-sender", "stable"); + var built = TestApplicationProtocol.CreateInbox(sessions, sender, "messages/expiry", message, fixture.JobNow); + if (messageId is { } id) + { + built = TestApplicationProtocol.With(built, sessions, messageId: id); + } + + return built; + } +} diff --git a/test/Orleans.Messaging.Tests/Functional/DeliverySimulationTests.cs b/test/Orleans.Messaging.Tests/Functional/DeliverySimulationTests.cs new file mode 100644 index 00000000000..bbc8719db24 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/DeliverySimulationTests.cs @@ -0,0 +1,109 @@ +using CsCheck; +using Microsoft.Extensions.DependencyInjection; +using Orleans.Messaging.Tests.Support; +using Xunit; + +namespace Orleans.Messaging.Tests.Functional; + +/// +/// Runs generated schedules of sends, delivery faults, crashes, and time against one sender and one receiver, and checks +/// the end-to-end delivery contract: no fault ever applies a message's effect twice, and once faults stop, every message +/// is processed exactly once in effect or dead-lettered at the sender when its absolute retry budget expires. +/// A fenced terminal outcome is unknown: the receiver may already have applied the message before an acknowledgement was lost. +/// +[Collection(MessagingClusterCollection.Name)] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class DeliverySimulationTests : MessagingBehaviorTestBase +{ + private const string Route = "messages/record"; + private static readonly TimeSpan RetryInterval = FaultInjectionClusterFixture.RetryInterval; + private readonly FaultInjectionClusterFixture _cluster; + + private static readonly Gen GenStep = Gen.Frequency( + (4, Gen.Const(new Step(StepKind.Send))), + (3, Gen.OneOfConst(DeliveryFaultKind.Throw, DeliveryFaultKind.TimeOut, DeliveryFaultKind.LoseResponse, DeliveryFaultKind.TimeOutInFlight) + .Select(static fault => new Step(StepKind.Fault, fault))), + (1, Gen.Const(new Step(StepKind.CrashSender))), + (1, Gen.Const(new Step(StepKind.CrashReceiver))), + (2, Gen.Const(new Step(StepKind.Advance)))); + + public DeliverySimulationTests() : this(new FaultInjectionClusterFixture()) + { + } + + private DeliverySimulationTests(FaultInjectionClusterFixture cluster) : base(cluster) => _cluster = cluster; + + [Fact] + public Task GeneratedFaultSchedules_ApplyEachMessageOnceAndDeliverItOnceFaultsStop() => + GenStep.Array[1, 12].SampleAsync(RunScheduleAsync, iter: 6, threads: 1); + + private async Task RunScheduleAsync(Step[] schedule) + { + var sender = NewGrain(); + var receiver = NewGrain(); + var sent = new List<(MessageId Id, TestMessage Message)>(); + foreach (var step in schedule) + { + switch (step.Kind) + { + case StepKind.Send: + var message = NewMessage(sent.Count, "simulated"); + sent.Add((await sender.SendAsync(receiver.GetGrainId(), Route, message), message)); + break; + case StepKind.Fault: + _cluster.Deliveries.Enqueue(Create(step.Fault)); + break; + case StepKind.CrashSender: + if (Fixture.Cluster.TryGetGrainContext(sender.GetGrainId(), out _)) await _cluster.CrashAsync(sender); + break; + case StepKind.CrashReceiver: + if (Fixture.Cluster.TryGetGrainContext(receiver.GetGrainId(), out _)) await _cluster.CrashAsync(receiver); + break; + case StepKind.Advance: + _ = await sender.GetSnapshotAsync(); + await _cluster.AdvanceAndRunPumpAsync(sender, RetryInterval); + break; + } + + // Safety holds during faults: no message's effect is ever applied twice. + Assert.All((await receiver.GetSnapshotAsync()).Effects, static effect => Assert.Equal(1, effect.Count)); + } + + // Faults stop; each retry becomes due and succeeds, unless the message's retry timeout elapsed first. + _cluster.Deliveries.Clear(); + for (var round = 0; round < 8; round++) + { + var snapshot = await sender.GetSnapshotAsync(); + if (snapshot.OutboxCount == 0) break; + await _cluster.AdvanceAndRunPumpAsync(sender, RetryInterval); + var outbox = Fixture.GetGrainContext(sender).ActivationServices.GetRequiredService(); + await Fixture.WaitUntilAsync(sender, () => !outbox.IsDelivering); + } + + var final = await sender.GetSnapshotAsync(); + Assert.Equal(0, final.OutboxCount); + var deadLettered = final.OutboxDeadLetters.Select(static letter => letter.MessageId).ToHashSet(); + var received = await Fixture.WaitForInboxCountAsync(receiver, 0); + Assert.Empty(received.InboxDeadLetters); + foreach (var (id, message) in sent) + { + var count = received.Effects.Where(effect => effect.LogicalId == message.LogicalId).Sum(static effect => effect.Count); + if (deadLettered.Contains(id)) Assert.InRange(count, 0, 1); + else Assert.Equal(1, count); + } + } + + private static DeliveryFault Create(DeliveryFaultKind kind) => kind switch + { + DeliveryFaultKind.Throw => DeliveryFault.Throw(), + DeliveryFaultKind.TimeOut => DeliveryFault.TimeOut(), + DeliveryFaultKind.LoseResponse => DeliveryFault.LoseResponse(), + _ => DeliveryFault.TimeOutInFlight() + }; + + private enum StepKind { Send, Fault, CrashSender, CrashReceiver, Advance } + + private readonly record struct Step(StepKind Kind, DeliveryFaultKind Fault = default); +} diff --git a/test/Orleans.Messaging.Tests/Functional/EarlyActivationValidationTests.cs b/test/Orleans.Messaging.Tests/Functional/EarlyActivationValidationTests.cs new file mode 100644 index 00000000000..59eb2994701 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/EarlyActivationValidationTests.cs @@ -0,0 +1,82 @@ +using Microsoft.Extensions.DependencyInjection; +using Orleans.Journaling; +using Orleans.Messaging.Tests.Support; +using Orleans.Runtime; +using Xunit; + +namespace Orleans.Messaging.Tests.Functional; + +[Collection(MessagingClusterCollection.Name)] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class EarlyActivationValidationTests : MessagingBehaviorTestBase +{ + [Fact] + public async Task StatelessWorker_FailsBeforeJournalInitializationReplayAndWork() + { + var grainType = typeof(StatelessActivationValidationTestGrain); + var grain = Fixture.Client.GetGrain(Guid.NewGuid(), grainType.FullName!); + await AssertRejectedBeforeRecoveryAsync(grain, grainType, "one activation"); + } + + [Theory] + [InlineData(typeof(ReentrantActivationValidationTestGrain))] + [InlineData(typeof(MayInterleaveActivationValidationTestGrain))] + [InlineData(typeof(AlwaysInterleaveActivationValidationTestGrain))] + [InlineData(typeof(MetadataReentrantActivationValidationGrain))] + [InlineData(typeof(MetadataMayInterleaveActivationValidationGrain))] + public async Task ReentrantAndInterleavingModels_InitializeAndReplayNormally(Type grainType) + { + await AssertSupportedReplayAsync(grainType); + } + + private async Task AssertRejectedBeforeRecoveryAsync(IActivationValidationTestGrain grain, Type grainType, string diagnostic) + { + var failure = await Assert.ThrowsAnyAsync(() => grain.IncrementAsync()); + Assert.Contains(diagnostic, failure.ToString(), StringComparison.Ordinal); + Assert.Contains(grainType.Name, failure.ToString(), StringComparison.Ordinal); + var observation = Assert.Single(Fixture.ActivationProbe.Get(grain.GetGrainId())); + Assert.False(observation.InstanceAvailableDuringConstruction); + Assert.Equal(0, observation.Activated); + Assert.Equal(0, observation.Calls); + var journal = JournalId.FromGrainId(grain.GetGrainId()); + Assert.Equal(0, Fixture.Storage.GetInitializationCount(journal)); + Assert.Equal(0, Fixture.Storage.GetReadCount(journal)); + Assert.Equal(0, Fixture.Storage.GetSuccessfulWriteCount(journal)); + Assert.Equal(0, Fixture.JobManagerProbe.GetAttemptCount(MessagingPump.JobName, grain.GetGrainId())); + } + + [Fact] + public async Task SupportedModel_InitializesAndReplaysFreshActivationNormally() + { + await AssertSupportedReplayAsync(typeof(SupportedActivationValidationTestGrain)); + } + + [Fact] + public async Task ResolvedNonReentrantMetadata_InitializesAndReplaysNormally() + { + await AssertSupportedReplayAsync(typeof(MetadataNonReentrantActivationValidationGrain)); + } + + private async Task AssertSupportedReplayAsync(Type grainType) + { + var grain = Fixture.Client.GetGrain(Guid.NewGuid(), grainType.FullName!); + Assert.Equal(1, await grain.IncrementAsync()); + var first = Assert.Single(Fixture.ActivationProbe.Get(grain.GetGrainId())); + Assert.False(first.InstanceAvailableDuringConstruction); + Assert.NotNull(first.Context.GrainInstance); + Assert.Equal(1, first.Activated); + Assert.Equal(1, first.Calls); + Assert.Equal(1, Fixture.Storage.GetReadCount(JournalId.FromGrainId(grain.GetGrainId()))); + await grain.DeactivateAsync(); + await first.Context.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + Assert.Equal(2, await grain.IncrementAsync()); + var observations = Fixture.ActivationProbe.Get(grain.GetGrainId()); + Assert.Equal(2, observations.Length); + Assert.NotSame(first.Context, observations[1].Context); + Assert.Equal(1, observations[1].Activated); + Assert.Equal(1, observations[1].Calls); + Assert.Equal(2, Fixture.Storage.GetReadCount(JournalId.FromGrainId(grain.GetGrainId()))); + } +} diff --git a/test/Orleans.Messaging.Tests/Functional/FaultInjectionTests.cs b/test/Orleans.Messaging.Tests/Functional/FaultInjectionTests.cs new file mode 100644 index 00000000000..29c3f15ac16 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/FaultInjectionTests.cs @@ -0,0 +1,1964 @@ +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; +using System.Security.Cryptography; +using Orleans.Configuration; +using Orleans.DurableJobs; +using Orleans.Journaling; +using Orleans.Messaging.Tests.Support; +using Orleans.Runtime; +using Xunit; + +namespace Orleans.Messaging.Tests.Functional; + +/// +/// Injects faults at each boundary of the delivery chain and asserts that transient infrastructure faults only delay a +/// message, which is then processed and delivered exactly once in effect, while only explicit rejections and expired +/// retries are permanent. +/// +/// +/// Faults are injected through for the outbox's call to the receiver, +/// for journal writes, for wake-up +/// scheduling, and for handlers. Time moves only through +/// , and each retry runs through an explicit pump run. +/// +[Collection(MessagingClusterCollection.Name)] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class FaultInjectionTests : MessagingBehaviorTestBase +{ + private const string Route = "messages/record"; + private const string HeldRoute = "messages/held"; + private static readonly TimeSpan RetryInterval = FaultInjectionClusterFixture.RetryInterval; + private static readonly TimeSpan RetryTimeout = FaultInjectionClusterFixture.OutboxRetryTimeout; + private readonly FaultInjectionClusterFixture _cluster; + + public FaultInjectionTests() : this(new FaultInjectionClusterFixture()) + { + } + + private FaultInjectionTests(FaultInjectionClusterFixture cluster) : base(cluster) => _cluster = cluster; + + private DeliveryFaultInjector Deliveries => _cluster.Deliveries; + + [Fact] + public async Task OutboxWakeup_BareStagingWaitsForCommitAfterPriorDeliveryDrains() + { + var sender = NewGrain(); + var receiver = NewGrain(); + _ = await sender.GetSnapshotAsync(); + Fixture.JobManagerProbe.SkipNext(MessagingPump.JobName); + await sender.SendAsync(receiver.GetGrainId(), Route, NewMessage(20, "committed")); + await Fixture.OnGrainAsync(sender, () => Fixture.GetPump(sender).EnsureStarted()); + await WaitForDrainedOutboxAsync(sender); + var pump = Fixture.GetPump(sender); + Assert.NotNull(await Fixture.OnGrainAsync(sender, () => InboxTestAccess.WakeupJobId(pump))); + var attempts = Fixture.JobManagerProbe.GetAttemptCount(MessagingPump.JobName, sender.GetGrainId()); + await sender.StageWithoutCommitAsync(receiver.GetGrainId(), Route, NewMessage(21, "staged")); + + Assert.Equal(DurableJobRunStatus.Completed, (await Fixture.RunPumpAsync(sender)).Status); + Assert.Equal(DurableJobRunStatus.Completed, (await Fixture.RunPumpAsync(sender)).Status); + Assert.Equal(attempts, Fixture.JobManagerProbe.GetAttemptCount(MessagingPump.JobName, sender.GetGrainId())); + Assert.Equal(1, (await sender.GetSnapshotAsync()).OutboxCount); + Assert.Equal(1, Deliveries.ReceiverCalls); + + Fixture.JobManagerProbe.SkipNext(MessagingPump.JobName); + await sender.RetryWriteStateAsync(); + Assert.Equal(attempts + 1, Fixture.JobManagerProbe.GetAttemptCount(MessagingPump.JobName, sender.GetGrainId())); + await WaitForDrainedOutboxAsync(sender); + var completed = await Fixture.WaitForEffectCountAsync(receiver, 2); + Assert.Equal(new[] { "committed", "staged" }, completed.Effects.Select(effect => effect.Value)); + Assert.All(completed.Effects, effect => Assert.Equal(1, effect.Count)); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task OutboxWakeup_CapturedWriteRetainsCoverageUntilAcknowledgedAndRecovered(bool snapshot) + { + var sender = NewGrain(); + var receiver = NewGrain(); + var before = await sender.GetSnapshotAsync(); + var previous = Fixture.GetGrainContext(sender); + var journal = JournalId.FromGrainId(sender.GetGrainId()); + if (snapshot) Fixture.Storage.RequestSnapshot(journal); + using var acknowledgement = Fixture.Storage.BlockAcknowledgement(journal); + var message = NewMessage(22, "captured"); + var sending = sender.SendAsync(receiver.GetGrainId(), Route, message); + await acknowledgement.WaitUntilEnteredAsync(); + Assert.False(sending.IsCompleted); + Assert.Equal(0, Deliveries.ReceiverCalls); + + // Captured entries retain a wake-up while delivery awaits the write acknowledgement. + var wakeup = await Fixture.RunPumpAsync(sender); + Assert.Equal(DurableJobRunStatus.RescheduleRequested, wakeup.Status); + Assert.Equal(_cluster.JobClock.GetUtcNow() + TimeSpan.FromSeconds(1), wakeup.RescheduleTime); + Assert.Equal(0, Deliveries.ReceiverCalls); + + acknowledgement.Fail(); + await Assert.ThrowsAsync(() => sending); + await previous.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + var recovered = await sender.GetSnapshotAsync(); + Assert.NotEqual(before.ActivationId, recovered.ActivationId); + await WaitForDrainedOutboxAsync(sender); + await AssertProcessedOnceAsync(receiver, message); + Assert.Equal(1, Deliveries.ReceiverCalls); + Assert.Equal([DeliveryStatus.Accepted], Deliveries.Answers); + } + + [Fact] + public async Task OutboxWakeup_CompletedJobDuringPreparationIsReplacedBeforeCapture() + { + var sender = NewGrain(); + var receiver = NewGrain(); + _ = await sender.GetSnapshotAsync(); + var journal = JournalId.FromGrainId(sender.GetGrainId()); + using var write = Fixture.Storage.BlockWrite(journal); + Fixture.JobManagerProbe.SkipNext(MessagingPump.JobName); + using var scheduling = Fixture.JobManagerProbe.BlockNext(MessagingPump.JobName); + var message = NewMessage(23, "prepared"); + var sending = sender.SendAsync(receiver.GetGrainId(), Route, message); + await scheduling.WaitUntilEnteredAsync(); + + Assert.Equal(DurableJobRunStatus.Completed, (await Fixture.RunPumpAsync(sender)).Status); + Assert.False(sending.IsCompleted); + Assert.Equal(0, Fixture.Storage.GetSuccessfulWriteCount(journal)); + scheduling.Continue(); + await write.WaitUntilEnteredAsync(); + Assert.Equal(2, Fixture.JobManagerProbe.GetAttemptCount(MessagingPump.JobName, sender.GetGrainId())); + Assert.Single(Fixture.JobManagerProbe.GetScheduledJobs(MessagingPump.JobName, sender.GetGrainId())); + Assert.Equal(0, Deliveries.ReceiverCalls); + Assert.Equal(DurableJobRunStatus.RescheduleRequested, (await Fixture.RunPumpAsync(sender)).Status); + + write.Release(); + await sending; + await WaitForDrainedOutboxAsync(sender); + await AssertProcessedOnceAsync(receiver, message); + Assert.Equal(1, Deliveries.ReceiverCalls); + } + + [Theory] + [InlineData(DeliveryFaultKind.Throw, "Injected transient delivery failure.")] + [InlineData(DeliveryFaultKind.TimeOut, "TimeoutException")] + public async Task TransientDeliveryFault_DelaysTheMessageThenDeliversItOnce(DeliveryFaultKind kind, string error) + { + var sender = NewGrain(); + var receiver = NewGrain(); + Deliveries.Enqueue(kind == DeliveryFaultKind.Throw ? DeliveryFault.Throw() : DeliveryFault.TimeOut()); + var startedAt = _cluster.JobClock.GetUtcNow(); + var message = NewMessage(1, "transient"); + + var id = await sender.SendAsync(receiver.GetGrainId(), Route, message); + + // The fault only schedules a retry; the receiver was never called. + var attempt = await WaitForFailedAttemptAsync(sender, id, 1); + Assert.Contains(error, attempt.LastError); + Assert.Equal(startedAt + RetryInterval, attempt.NextAttemptAt); + Assert.Equal(startedAt, attempt.SentAt); + Assert.Equal(0, Deliveries.ReceiverCalls); + + await _cluster.AdvanceAndRunPumpAsync(sender, RetryInterval); + + await WaitForDrainedOutboxAsync(sender); + await AssertProcessedOnceAsync(receiver, message); + Assert.Equal([DeliveryStatus.Accepted], Deliveries.Answers); + Assert.Empty((await sender.GetSnapshotAsync()).OutboxDeadLetters); + } + + [Fact] + public async Task UncleanStop_ChargesEachDueMessageOneAttemptWithoutDeadLetteringIt() + { + var receiver = NewGrain(); + _ = await receiver.GetSnapshotAsync(); + var running = CreateEnvelope(receiver, NewMessage(180, "interrupted"), "messages/interrupted"); + var queued = CreateEnvelope(receiver, NewMessage(181, "queued")); + using (var barrier = Fixture.HandlerProbe.Arm(receiver.GetGrainId(), "messages/interrupted")) + { + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, running)).Status); + await WaitForBarrierAsync(receiver, barrier); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, queued)).Status); + await receiver.ConfigureHandlerAsync(false); + + // The clean-stop write fails, so the journal ends as if the process had died mid-attempt. + Fixture.Storage.FailWrite(JournalId.FromGrainId(receiver.GetGrainId())); + await _cluster.CrashAsync(receiver); + } + + // Recovery charges the interrupted attempt and the queued message which hadn't run, one attempt each. Neither + // had failed before, so even this fixture's zero InboxRetryTimeout dead-letters neither. + var recovered = await receiver.GetSnapshotAsync(); + var inbox = Fixture.GetInbox(receiver); + var states = await Fixture.OnGrainAsync(receiver, () => new[] + { + InboxTestAccess.GetMessageState(inbox, running.Envelope.MessageId), + InboxTestAccess.GetMessageState(inbox, queued.Envelope.MessageId) + }); + Assert.All(states, static state => Assert.Equal(1, state.AttemptCount)); + Assert.Equal(2, recovered.InboxCount); + Assert.Empty(recovered.InboxDeadLetters); + + await receiver.ConfigureHandlerAsync(true); + await _cluster.AdvanceAndRunPumpAsync(receiver, TimeSpan.Zero); + var completed = await Fixture.WaitForEffectCountAsync(receiver, 2); + Assert.All(completed.Effects, static effect => Assert.Equal(1, effect.Count)); + Assert.Empty(completed.InboxDeadLetters); + } + + [Fact] + public async Task FailedAttemptRetry_StaysCoveredByTheSendsWakeupThroughACrash() + { + var sender = NewGrain(); + var receiver = NewGrain(); + Deliveries.Enqueue(DeliveryFault.Throw()); + var message = NewMessage(9, "retry-wakeup"); + + var id = await sender.SendAsync(receiver.GetGrainId(), Route, message); + var attempt = await WaitForFailedAttemptAsync(sender, id, 1); + + // The retry write schedules no job of its own: the wake-up which the send ensured before it became durable keeps + // running while the outbox holds the entry, and its run reschedules to the retry time. + Assert.Single(Fixture.JobManagerProbe.GetScheduledJobs(MessagingPump.JobName, sender.GetGrainId())); + var wakeup = await Fixture.RunPumpAsync(sender); + Assert.Equal(DurableJobRunStatus.RescheduleRequested, wakeup.Status); + Assert.Equal(attempt.NextAttemptAt, wakeup.RescheduleTime); + + // A crash loses the activation; recovery re-arms the wake-up from the journaled retry time. + await _cluster.CrashAsync(sender); + _ = await sender.GetSnapshotAsync(); + await _cluster.AdvanceAndRunPumpAsync(sender, RetryInterval); + + await WaitForDrainedOutboxAsync(sender); + await AssertProcessedOnceAsync(receiver, message); + Assert.Equal([DeliveryStatus.Accepted], Deliveries.Answers); + } + + [Fact] + public async Task LostResponse_AfterTheReceiverCommitted_IsADuplicateAtTheReceiverAndDeliveredAtTheSender() + { + var sender = NewGrain(); + var receiver = NewGrain(); + var lost = DeliveryFault.LoseResponse(); + Deliveries.Enqueue(lost); + var message = NewMessage(2, "lost-response"); + + var id = await sender.SendAsync(receiver.GetGrainId(), Route, message); + + Assert.Equal(DeliveryStatus.Accepted, (await lost.ReceiverAnswer).Status); + Assert.Contains("Injected loss of the receiver's response.", (await WaitForFailedAttemptAsync(sender, id, 1)).LastError); + await Fixture.WaitForEffectCountAsync(receiver, 1); + + await _cluster.AdvanceAndRunPumpAsync(sender, RetryInterval); + + await WaitForDrainedOutboxAsync(sender); + Assert.Equal([DeliveryStatus.Accepted, DeliveryStatus.Duplicate], Deliveries.Answers); + await AssertProcessedOnceAsync(receiver, message); + Assert.Empty((await sender.GetSnapshotAsync()).OutboxDeadLetters); + } + + [Fact] + public async Task LostResponse_AfterTheSinkAccepted_RetiresTheSourceMessageAsDelivered() + { + var sender = NewSinkGrain(); + var receiver = NewSinkGrain(); + var lost = DeliveryFault.LoseResponse(); + Deliveries.Enqueue(lost); + var id = NewSourceMessageId("lost-response"); + var payload = RandomNumberGenerator.GetBytes(32); + + await sender.StageFromSourceAsync(receiver.GetGrainId(), id, payload, write: true); + + Assert.Equal(DeliveryStatus.Accepted, (await lost.ReceiverAnswer).Status); + await WaitForFailedAttemptAsync(sender, id, 1); + await _cluster.AdvanceAndRunPumpAsync(sender, RetryInterval); + + await Fixture.WaitUntilAsync(sender, () => Sink(sender).GetState().Retired.Count == 1); + Assert.Equal(nameof(OutboxMessageRetirementStatus.Delivered), (await OnSinkAsync(sender, static grain => grain.GetState())).Retired[id.ToString()]); + Assert.Equal([DeliveryStatus.Accepted, DeliveryStatus.Duplicate], Deliveries.Answers); + Assert.Equal(payload, (await OnSinkAsync(receiver, static grain => grain.GetState())).Received[id.ToString()]); + + // The receiver's deduplication answered the retry before offering it to the sink again. + var probe = _cluster.SourceSinkProbe.For(receiver); + Assert.Equal(1, probe.SinkOffers); + Assert.Equal(1, probe.SinkAcceptances); + } + + [Theory] + [InlineData(ReceiverRecovery.None)] + [InlineData(ReceiverRecovery.Journal)] + [InlineData(ReceiverRecovery.Snapshot)] + public async Task LostResponse_AfterTheSinkRejected_RetiresTheSourceMessageAsRejected(ReceiverRecovery recovery) + { + var sender = NewSinkGrain(); + var receiver = NewSinkGrain(); + var probe = _cluster.SourceSinkProbe.For(receiver); + probe.Mode = SinkMode.DeadLetter; + + // The rejection's write is the receiver's first, so a requested snapshot holds its processed record. + if (recovery == ReceiverRecovery.Snapshot) Fixture.Storage.RequestSnapshot(JournalId.FromGrainId(receiver.GetGrainId())); + var lost = DeliveryFault.LoseResponse(); + Deliveries.Enqueue(lost); + var id = NewSourceMessageId("lost-rejection"); + + await sender.StageFromSourceAsync(receiver.GetGrainId(), id, RandomNumberGenerator.GetBytes(32), write: true); + + var rejection = await lost.ReceiverAnswer; + Assert.Equal(DeliveryStatus.DeadLettered, rejection.Status); + await WaitForFailedAttemptAsync(sender, id, 1); + if (recovery != ReceiverRecovery.None) + { + var previous = Fixture.GetGrainContext(receiver); + await receiver.DeactivateAsync(); + await previous.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + } + + // The receiver answers the retry from its processed record, without offering the message to the sink again. + probe.Mode = SinkMode.Accept; + await _cluster.AdvanceAndRunPumpAsync(sender, RetryInterval); + + await Fixture.WaitUntilAsync(sender, () => Sink(sender).GetState().Retired.Count == 1); + var (retired, reasons, codes) = await OnSinkAsync(sender, static grain => (grain.GetState().Retired, grain.GetRetirementReasons(), grain.GetRetirementCodes())); + Assert.Equal(nameof(OutboxMessageRetirementStatus.Rejected), retired[id.ToString()]); + Assert.Equal(SourceSinkGrain.Rejection, reasons[id.ToString()]); + Assert.Equal(SourceSinkGrain.RejectionCode, codes[id.ToString()]); + Assert.Equal([DeliveryStatus.DeadLettered, DeliveryStatus.DeadLettered], Deliveries.Answers); + Assert.Equal(1, probe.SinkOffers); + Assert.Equal(0, probe.SinkAcceptances); + } + + public enum ReceiverRecovery + { + None, + Journal, + Snapshot + } + + [Fact] + public async Task TimedOutDelivery_WhoseCallIsStillCommitting_IsDeduplicatedAgainstTheRetry() + { + var sender = NewGrain(); + var receiver = NewGrain(); + _ = await receiver.GetSnapshotAsync(); + using var acceptance = Fixture.Storage.BlockWrite(JournalId.FromGrainId(receiver.GetGrainId())); + var timeout = DeliveryFault.TimeOutInFlight(); + Deliveries.Enqueue(timeout); + var message = NewMessage(3, "in-flight"); + + var id = await sender.SendAsync(receiver.GetGrainId(), Route, message); + + // The sender gave up on the call while the receiver's acceptance write is still in progress, then retries. + await acceptance.WaitUntilEnteredAsync(); + await WaitForFailedAttemptAsync(sender, id, 1); + await _cluster.AdvanceAndRunPumpAsync(sender, RetryInterval); + await Deliveries.WaitForReceiverCallsAsync(2); + acceptance.Release(); + + var answers = await Deliveries.WaitForAnswersAsync(2); + Assert.Equal([DeliveryStatus.Accepted, DeliveryStatus.Duplicate], answers.Order()); + Assert.Equal(DeliveryStatus.Accepted, (await timeout.ReceiverAnswer).Status); + await WaitForDrainedOutboxAsync(sender); + await AssertProcessedOnceAsync(receiver, message); + } + + [Fact] + public async Task PersistentDeliveryFault_DeadLettersAtTheSenderWithTheLastErrorAfterTheRetryTimeout() + { + var sender = NewSinkGrain(); + var receiver = NewSinkGrain(); + Deliveries.Repeat(static () => DeliveryFault.Throw(new IOException("Injected outage of the receiver's silo."))); + var startedAt = _cluster.JobClock.GetUtcNow(); + var id = NewSourceMessageId("persistent-fault"); + + await sender.StageFromSourceAsync(receiver.GetGrainId(), id, RandomNumberGenerator.GetBytes(32), write: true); + + await WaitForFailedAttemptAsync(sender, id, 1); + await _cluster.AdvanceAndRunPumpAsync(sender, RetryInterval); + var second = await WaitForFailedAttemptAsync(sender, id, 2); + Assert.Equal(startedAt, second.SentAt); + Assert.Equal(startedAt + RetryInterval + 2 * RetryInterval, second.NextAttemptAt); + + // Once the retry timeout passes, the next due attempt dead-letters the message without another try. + await _cluster.AdvanceAndRunPumpAsync(sender, RetryTimeout); + + await Fixture.WaitUntilAsync(sender, () => Sink(sender).GetState().Retired.Count == 1); + var (retired, reasons, deadLetter) = await Fixture.OnGrainAsync(sender, () => ( + Sink(sender).GetState().Retired, + Sink(sender).GetRetirementReasons(), + _cluster.GetOutboxState(sender).DeadLetter(id))); + Assert.Equal(nameof(OutboxMessageRetirementStatus.DeadLettered), retired[id.ToString()]); + Assert.Contains("Injected outage of the receiver's silo.", reasons[id.ToString()]); + Assert.Equal(reasons[id.ToString()], deadLetter.Reason); + Assert.Equal(2, deadLetter.AttemptCount); + Assert.Equal(2, Deliveries.Executions); + Assert.Equal(0, Deliveries.ReceiverCalls); + Assert.Equal(0, _cluster.SourceSinkProbe.For(receiver).SinkOffers); + } + + [Fact] + public async Task SinkBackpressure_ThroughTheOutbox_ExpiresAtTheOriginalAbsoluteDeadline() + { + var sender = NewSinkGrain(); + var receiver = NewSinkGrain(); + var probe = _cluster.SourceSinkProbe.For(receiver); + probe.Mode = SinkMode.Backpressure; + var id = NewSourceMessageId("backpressure"); + var payload = RandomNumberGenerator.GetBytes(32); + + var sentAt = _cluster.JobClock.GetUtcNow(); + await sender.StageFromSourceAsync(receiver.GetGrainId(), id, payload, write: true); + + var first = await WaitForFailedAttemptAsync(sender, id, 1); + Assert.Equal(sentAt, first.SentAt); + Assert.Equal("The receiver is backpressured.", first.LastError); + + // Backpressure never renews the logical send's bounded retry budget. + var step = RetryTimeout * 3 / 4; + await _cluster.AdvanceAndRunPumpAsync(sender, step); + var second = await WaitForFailedAttemptAsync(sender, id, 2); + Assert.Equal(sentAt, second.SentAt); + Assert.Empty(await Fixture.OnGrainAsync(sender, () => _cluster.GetOutboxState(sender).DeadLetterKeys.ToArray())); + + await _cluster.AdvanceAndRunPumpAsync(sender, RetryTimeout - step); + Assert.Equal(sentAt + RetryTimeout, _cluster.JobClock.GetUtcNow()); + + await Fixture.WaitUntilAsync(sender, () => Sink(sender).GetState().Retired.Count == 1); + Assert.Equal(nameof(OutboxMessageRetirementStatus.DeadLettered), (await OnSinkAsync(sender, static grain => grain.GetState())).Retired[id.ToString()]); + Assert.Equal([DeliveryStatus.Backpressured, DeliveryStatus.Backpressured], Deliveries.Answers); + Assert.Equal(2, probe.SinkOffers); + Assert.Equal(0, probe.SinkAcceptances); + Assert.Empty((await OnSinkAsync(receiver, static grain => grain.GetState())).Received); + var letter = await Fixture.OnGrainAsync(sender, () => _cluster.GetOutboxState(sender).DeadLetter(id)); + Assert.Contains("retry timeout elapsed", letter.Reason, StringComparison.Ordinal); + Assert.Empty(await OnSinkAsync(sender, static grain => grain.GetRetirementCodes())); + } + + [Fact] + public async Task DeliveryAttempts_KeepTheOriginalSentAtAcrossRetryAndRecovery() + { + var sender = NewSinkGrain(); + var receiver = NewSinkGrain(); + var probe = _cluster.SourceSinkProbe.For(receiver); + probe.Mode = SinkMode.Backpressure; + var id = NewSourceMessageId("redelivery-deadline"); + var sentAt = _cluster.JobClock.GetUtcNow(); + await sender.StageFromSourceAsync(receiver.GetGrainId(), id, RandomNumberGenerator.GetBytes(32), write: true); + await WaitForFailedAttemptAsync(sender, id, 1); + var step = RetryInterval; + await _cluster.AdvanceAndRunPumpAsync(sender, step); + await WaitForFailedAttemptAsync(sender, id, 2); + var beforeRecovery = await Fixture.OnGrainAsync(sender, () => _cluster.GetOutboxState(sender).Attempt(id)); + Assert.Equal(sentAt, beforeRecovery.SentAt); + var previous = Fixture.GetGrainContext(sender); + Fixture.Storage.RequestSnapshot(JournalId.FromGrainId(sender.GetGrainId())); + await sender.WriteAsync(); + await sender.DeactivateAsync(); + await previous.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + Assert.Equal(1, await sender.GetOutboxCountAsync()); + var recovered = await Fixture.OnGrainAsync(sender, () => _cluster.GetOutboxState(sender).Attempt(id)); + Assert.Equal(sentAt, recovered.SentAt); + probe.Mode = SinkMode.Accept; + await _cluster.AdvanceAndRunPumpAsync(sender, 2 * step); + await Fixture.WaitUntilAsync(sender, () => Sink(sender).GetState().Retired.Count == 1); + + // Transport timeouts and recovery cannot change the journaled logical send time. + Assert.Equal( + [sentAt, sentAt, sentAt], + Deliveries.SentTimes); + Assert.Equal([DeliveryStatus.Backpressured, DeliveryStatus.Backpressured, DeliveryStatus.Accepted], Deliveries.Answers); + Assert.Equal(1, probe.SinkAcceptances); + } + + [Fact] + public async Task SourceMessage_TransportsAndRetainsItsActualStagingSentAt() + { + var sender = NewSinkGrain(); + var receiver = NewSinkGrain(); + var sentAt = _cluster.JobClock.GetUtcNow(); + var id = NewSourceMessageId("beyond-window"); + + await sender.StageFromSourceAsync(receiver.GetGrainId(), id, RandomNumberGenerator.GetBytes(32), write: true); + + await Fixture.WaitUntilAsync(sender, () => Sink(sender).GetState().Retired.Count == 1); + var (retired, reasons, codes, deadLetters) = await Fixture.OnGrainAsync(sender, () => ( + Sink(sender).GetState().Retired, + Sink(sender).GetRetirementReasons(), + Sink(sender).GetRetirementCodes(), + _cluster.GetOutboxState(sender).DeadLetterKeys.ToArray())); + Assert.Equal(nameof(OutboxMessageRetirementStatus.Delivered), retired[id.ToString()]); + Assert.Empty(reasons); + Assert.Empty(codes); + Assert.Empty(deadLetters); + Assert.Equal([DeliveryStatus.Accepted], Deliveries.Answers); + Assert.Equal([sentAt], Deliveries.SentTimes); + var inbox = Fixture.GetGrainContext(receiver).ActivationServices.GetRequiredService(); + Assert.Equal(sentAt, await Fixture.OnGrainAsync(receiver, () => InboxTestAccess.GetProcessedSentAt(inbox, id))); + Assert.Equal(1, _cluster.SourceSinkProbe.For(receiver).SinkOffers); + + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task ReceiverAcceptanceWriteFailure_IsRetriedBySenderAndProcessedOnce(bool ambiguous) + { + var sender = NewGrain(); + var receiver = NewGrain(); + var before = await receiver.GetSnapshotAsync(); + var journal = JournalId.FromGrainId(receiver.GetGrainId()); + if (ambiguous) + { + Fixture.Storage.FailAfterWrite(journal); + } + else + { + Fixture.Storage.FailWrite(journal); + } + + var message = NewMessage(4, ambiguous ? "ambiguous-acceptance" : "failed-acceptance"); + + var id = await sender.SendAsync(receiver.GetGrainId(), Route, message); + + // The receiver failed the call and deactivated; the sender holds the message for a retry. + Assert.Contains("Injected", (await WaitForFailedAttemptAsync(sender, id, 1)).LastError); + Assert.Empty(Deliveries.Answers); + await _cluster.AdvanceAndRunPumpAsync(sender, RetryInterval); + + await WaitForDrainedOutboxAsync(sender); + // An acceptance which became durable despite the failure is recovered, so the retry is a duplicate. + Assert.Equal([ambiguous ? DeliveryStatus.Duplicate : DeliveryStatus.Accepted], Deliveries.Answers); + var processed = await AssertProcessedOnceAsync(receiver, message); + Assert.NotEqual(before.ActivationId, processed.ActivationId); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task SenderRetryStateWriteFailure_RecoversAndDeliversOnce(bool ambiguous) + { + var sender = NewGrain(); + var receiver = NewGrain(); + var hold = DeliveryFault.HoldBeforeCall(); + Deliveries.Enqueue(hold); + var startedAt = _cluster.JobClock.GetUtcNow(); + var message = NewMessage(5, ambiguous ? "ambiguous-retry-state" : "failed-retry-state"); + var id = await sender.SendAsync(receiver.GetGrainId(), Route, message); + await hold.WaitUntilEnteredAsync(); + ArmSenderWriteFailure(sender, ambiguous); + var previous = Fixture.GetGrainContext(sender); + + // The delivery fails, and so does the write which records the failed attempt. + hold.Fail(new IOException("Injected delivery failure.")); + await previous.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + _ = await sender.GetSnapshotAsync(); + + if (ambiguous) + { + // The recorded attempt became durable, so the recovered message waits for its backoff. + var recovered = await Fixture.OnGrainAsync(sender, () => _cluster.GetOutboxState(sender).Attempt(id)); + Assert.Equal(1, recovered.AttemptCount); + Assert.Equal(startedAt + RetryInterval, recovered.NextAttemptAt); + await _cluster.AdvanceAndRunPumpAsync(sender, RetryInterval); + } + + await WaitForDrainedOutboxAsync(sender); + Assert.Equal([DeliveryStatus.Accepted], Deliveries.Answers); + Assert.Equal(1, Deliveries.ReceiverCalls); + await AssertProcessedOnceAsync(receiver, message); + Assert.Empty((await sender.GetSnapshotAsync()).OutboxDeadLetters); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task SenderRemovalWriteFailure_AfterTheReceiverAccepted_DeliversOnceInEffect(bool ambiguous) + { + var sender = NewGrain(); + var receiver = NewGrain(); + var hold = DeliveryFault.HoldAfterCall(); + Deliveries.Enqueue(hold); + var message = NewMessage(6, ambiguous ? "ambiguous-removal" : "failed-removal"); + _ = await sender.SendAsync(receiver.GetGrainId(), Route, message); + await hold.WaitUntilEnteredAsync(); + Assert.Equal(DeliveryStatus.Accepted, (await hold.ReceiverAnswer).Status); + ArmSenderWriteFailure(sender, ambiguous); + var previous = Fixture.GetGrainContext(sender); + + hold.Release(); + await previous.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + _ = await sender.GetSnapshotAsync(); + + await WaitForDrainedOutboxAsync(sender); + // A removal which never became durable is redelivered, and the receiver reports the duplicate. + DeliveryStatus[] expected = ambiguous ? [DeliveryStatus.Accepted] : [DeliveryStatus.Accepted, DeliveryStatus.Duplicate]; + Assert.Equal(expected, Deliveries.Answers); + await AssertProcessedOnceAsync(receiver, message); + Assert.Empty((await sender.GetSnapshotAsync()).OutboxDeadLetters); + } + + [Fact] + public async Task SenderRemovalWriteFailure_RecoveredAfterTheRetryTimeout_DeadLettersWithoutRedelivery() + { + var sender = NewGrain(); + var receiver = NewGrain(); + var hold = DeliveryFault.HoldAfterCall(); + Deliveries.Enqueue(hold); + var message = NewMessage(7, "late-recovery"); + _ = await sender.SendAsync(receiver.GetGrainId(), Route, message); + await hold.WaitUntilEnteredAsync(); + Assert.Equal(DeliveryStatus.Accepted, (await hold.ReceiverAnswer).Status); + ArmSenderWriteFailure(sender, ambiguous: false); + var previous = Fixture.GetGrainContext(sender); + hold.Release(); + await previous.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + + // The recovered entry's retry timeout runs from its send, so an attempt now could reach the receiver after its + // deduplication window. The outbox dead-letters it instead of redelivering it. + _cluster.JobClock.Advance(RetryTimeout); + var snapshot = await Fixture.WaitForDeadLetterCountAsync(sender, 1); + + var deadLetter = Assert.Single(snapshot.OutboxDeadLetters); + Assert.Contains("retry timeout elapsed", deadLetter.Reason, StringComparison.Ordinal); + Assert.Equal(0, snapshot.OutboxCount); + Assert.Equal([DeliveryStatus.Accepted], Deliveries.Answers); + await AssertProcessedOnceAsync(receiver, message); + } + + [Fact] + public async Task HeldReceiver_DelaysOnlyItsOwnMessage() + { + var sender = NewGrain(); + var slow = NewGrain(); + var fast = NewGrain(); + var held = DeliveryFault.HoldBeforeCall(); + Deliveries.Enqueue(held); + var slowMessage = NewMessage(20, "slow"); + var fastMessage = NewMessage(21, "fast"); + + _ = await sender.SendAsync(slow.GetGrainId(), Route, slowMessage); + await held.WaitUntilEnteredAsync(); + _ = await sender.SendAsync(fast.GetGrainId(), Route, fastMessage); + + // Attempts complete independently, so the message to the other receiver is delivered while the first is held. + await AssertProcessedOnceAsync(fast, fastMessage); + Assert.Equal(1, (await sender.GetSnapshotAsync()).OutboxCount); + + held.Release(); + await WaitForDrainedOutboxAsync(sender); + await AssertProcessedOnceAsync(slow, slowMessage); + Assert.Equal(2, Deliveries.Executions); + } + + [Fact] + public async Task UnknownDeliveryStatus_IsRetriedAsAnUnknownOutcome() + { + var sender = NewGrain(); + var receiver = NewGrain(); + Deliveries.Enqueue(DeliveryFault.Answer(new DeliveryResult { Status = (DeliveryStatus)99 })); + var message = NewMessage(22, "unknown-status"); + + var id = await sender.SendAsync(receiver.GetGrainId(), Route, message); + + // A newer receiver's status which this sender can't interpret may hide an acceptance, so it retries. + Assert.Contains("Unexpected delivery status '99'", (await WaitForFailedAttemptAsync(sender, id, 1)).LastError); + await _cluster.AdvanceAndRunPumpAsync(sender, RetryInterval); + await WaitForDrainedOutboxAsync(sender); + await AssertProcessedOnceAsync(receiver, message); + Assert.Empty((await sender.GetSnapshotAsync()).OutboxDeadLetters); + } + + [Fact] + public async Task Withdrawal_DuringAnAttempt_IgnoresTheAttemptsOutcome() + { + var sender = NewSinkGrain(); + var receiver = NewSinkGrain(); + var hold = DeliveryFault.HoldAfterCall(); + Deliveries.Enqueue(hold); + var id = NewSourceMessageId("withdrawn-in-flight"); + + await sender.StageFromSourceAsync(receiver.GetGrainId(), id, RandomNumberGenerator.GetBytes(32), write: true); + await hold.WaitUntilEnteredAsync(); + await sender.WithdrawAsync(id); + await sender.WriteAsync(); + + // The attempt already reached the receiver, but the withdrawn entry neither retires nor delivers again. + hold.Release(); + var outbox = Fixture.GetGrainContext(sender).ActivationServices.GetRequiredService(); + await Fixture.WaitUntilAsync(sender, () => !outbox.IsDelivering); + Assert.Empty((await OnSinkAsync(sender, static grain => grain.GetState())).Retired); + Assert.Equal(0, await Fixture.OnGrainAsync(sender, () => outbox.Count)); + Assert.Empty(await Fixture.OnGrainAsync(sender, () => _cluster.GetOutboxState(sender).DeadLetterKeys.ToArray())); + Assert.Equal([DeliveryStatus.Accepted], Deliveries.Answers); + Assert.Equal(1, Deliveries.Executions); + } + [Fact] + public async Task SenderCrash_AfterTheReceiverCommitted_RedeliversAsADuplicate() + { + var sender = NewGrain(); + var receiver = NewGrain(); + var hold = DeliveryFault.HoldAfterCall(); + Deliveries.Enqueue(hold); + var message = NewMessage(7, "sender-crash"); + _ = await sender.SendAsync(receiver.GetGrainId(), Route, message); + await hold.WaitUntilEnteredAsync(); + Assert.Equal(DeliveryStatus.Accepted, (await hold.ReceiverAnswer).Status); + + var wakeup = await Fixture.RunPumpAsync(sender); + Assert.Equal(DurableJobRunStatus.RescheduleRequested, wakeup.Status); + Assert.Equal(_cluster.JobClock.GetUtcNow() + TimeSpan.FromSeconds(1), wakeup.RescheduleTime); + Assert.Equal(1, Deliveries.ReceiverCalls); + + // Deactivation cancels the in-flight delivery, whose outcome is never applied or counted as an attempt. + await _cluster.CrashAsync(sender); + _ = await sender.GetSnapshotAsync(); + + await WaitForDrainedOutboxAsync(sender); + Assert.Equal([DeliveryStatus.Accepted, DeliveryStatus.Duplicate], Deliveries.Answers); + await AssertProcessedOnceAsync(receiver, message); + Assert.Empty((await sender.GetSnapshotAsync()).OutboxDeadLetters); + } + + [Fact] + public async Task ReceiverCrash_MidBatch_RecoveryProcessesEachMessageOnce() + { + var receiver = NewGrain(); + _ = await receiver.GetSnapshotAsync(); + var deliverAt = _cluster.JobClock.GetUtcNow() + RetryTimeout; + var first = NewMessage(8, "first"); + var held = NewMessage(9, "held"); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, Schedule(CreateEnvelope(receiver, first, "messages/first"), deliverAt))).Status); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, Schedule(CreateEnvelope(receiver, held, HeldRoute), deliverAt))).Status); + var barrier = Fixture.HandlerProbe.Arm(receiver.GetGrainId(), HeldRoute); + + // Both messages become due in one batch; the first is handled and the second is in its handler at the crash. + _cluster.JobClock.Advance(RetryTimeout); + var run = Fixture.RunPumpAsync(receiver); + await WaitForBarrierAsync(receiver, barrier); + Assert.Equal(1, Assert.Single((await receiver.GetSnapshotAsync()).Effects).Count); + await _cluster.CrashAsync(receiver); + barrier.Dispose(); + _ = run.ContinueWith(static task => task.Exception, TaskScheduler.Default); + + _ = await receiver.GetSnapshotAsync(); + var recovered = await Fixture.SnapshotProbe.WaitAsync( + receiver.GetGrainId(), + static snapshot => snapshot.Effects.Count == 2 && snapshot.InboxCount == 0); + Assert.All(recovered.Effects, static effect => Assert.Equal(1, effect.Count)); + Assert.Equal([first.LogicalId, held.LogicalId], recovered.Effects.Select(static effect => effect.LogicalId)); + Assert.Empty(recovered.InboxDeadLetters); + } + + [Fact] + public async Task CanceledPumpRun_KeepsItsJobAndTheActivationKeepsProcessing() + { + var receiver = NewGrain(); + var before = await receiver.GetSnapshotAsync(); + var barrier = Fixture.HandlerProbe.Arm(receiver.GetGrainId(), HeldRoute); + var message = NewMessage(10, "canceled-run"); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, CreateEnvelope(receiver, message, HeldRoute))).Status); + await WaitForBarrierAsync(receiver, barrier); + var attempts = Fixture.JobManagerProbe.GetAttemptCount(MessagingPump.JobName, receiver.GetGrainId()); + + // Durable Jobs cancels a run only when its shard moves, and the job stays stored for the next owner, so the + // activation schedules nothing and keeps working. + using var cancellation = new CancellationTokenSource(); + var run = Fixture.RunCancelablePumpAsync(receiver, cancellation.Token); + cancellation.Cancel(); + await Assert.ThrowsAnyAsync(() => run); + Assert.Equal(attempts, Fixture.JobManagerProbe.GetAttemptCount(MessagingPump.JobName, receiver.GetGrainId())); + + barrier.Dispose(); + var processed = await AssertProcessedOnceAsync(receiver, message); + Assert.Equal(before.ActivationId, processed.ActivationId); + } + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task ScheduledDeliveryWakeupSchedulingFailure_IsRetriedBySenderAndProcessedAtDeliverAt(bool scheduledBeforeFailure) + { + var sender = NewGrain(); + var receiver = NewGrain(); + _ = await receiver.GetSnapshotAsync(); + var receiverContext = Fixture.GetGrainContext(receiver); + var hold = DeliveryFault.HoldBeforeCall(); + Deliveries.Enqueue(hold); + var deliverAt = _cluster.JobClock.GetUtcNow() + RetryTimeout + RetryTimeout; + var message = NewMessage(11, "scheduled"); + var outgoing = TestApplicationProtocol.Create(Sessions, sender.GetGrainId(), receiver.GetGrainId(), Route, message); + outgoing = outgoing with { Envelope = outgoing.Envelope with { DeliverAt = deliverAt } }; + await sender.StageOutputAsync(outgoing); + await sender.RetryWriteStateAsync(); + await hold.WaitUntilEnteredAsync(); + + // The receiver's acceptance write schedules its wake-up for the message, and the scheduling fails. + if (scheduledBeforeFailure) + { + Fixture.JobManagerProbe.FailAfterNext(MessagingPump.JobName); + } + else + { + Fixture.JobManagerProbe.FailNext(MessagingPump.JobName); + } + + hold.Release(); + Assert.Contains("Injected", (await WaitForFailedAttemptAsync(sender, outgoing.Envelope.MessageId, 1)).LastError); + await receiverContext.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + Assert.Empty(Deliveries.Answers); + + await _cluster.AdvanceAndRunPumpAsync(sender, RetryInterval); + await WaitForDrainedOutboxAsync(sender); + Assert.Equal([DeliveryStatus.Accepted], Deliveries.Answers); + var pending = await Fixture.WaitForInboxCountAsync(receiver, 1); + Assert.Empty(pending.Effects); + + await _cluster.AdvanceAndRunPumpAsync(receiver, RetryTimeout + RetryTimeout); + await AssertProcessedOnceAsync(receiver, message); + } + + [Fact] + public async Task OutboxDepth_IsReleasedByDeactivationAndRestoredByRecovery() + { + var sender = NewGrain(); + var receiver = NewGrain(); + Deliveries.Enqueue(DeliveryFault.Throw()); + var message = NewMessage(12, "depth-across-activations"); + var id = await sender.SendAsync(receiver.GetGrainId(), Route, message); + _ = await WaitForFailedAttemptAsync(sender, id, 1); + await Fixture.WaitForOutboxDepthAsync(1); + + await _cluster.CrashAsync(sender); + await Fixture.WaitForOutboxDepthAsync(0); + + // Recovery reports the pending message again, and retiring it releases that depth. + Assert.Equal(1, (await sender.GetSnapshotAsync()).OutboxCount); + await Fixture.WaitForOutboxDepthAsync(1); + await _cluster.AdvanceAndRunPumpAsync(sender, RetryInterval); + await WaitForDrainedOutboxAsync(sender); + await Fixture.WaitForOutboxDepthAsync(0); + await AssertProcessedOnceAsync(receiver, message); + } + + [Fact] + public async Task OutboxDepth_AfterAFailedWrite_CountsOnlyDurableMessages() + { + var sender = NewGrain(); + var receiver = NewGrain(); + Deliveries.Enqueue(DeliveryFault.Throw()); + var durable = NewMessage(13, "durable"); + var id = await sender.SendAsync(receiver.GetGrainId(), Route, durable); + _ = await WaitForFailedAttemptAsync(sender, id, 1); + await Fixture.WaitForOutboxDepthAsync(1); + var previous = Fixture.GetGrainContext(sender); + ArmSenderWriteFailure(sender, ambiguous: false); + + // The failed write fences the activation, which releases the depth of both messages. + await Assert.ThrowsAnyAsync(() => sender.SendAsync(receiver.GetGrainId(), Route, NewMessage(14, "lost"))); + await previous.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + await Fixture.WaitForOutboxDepthAsync(0); + + Assert.Equal(1, (await sender.GetSnapshotAsync()).OutboxCount); + await Fixture.WaitForOutboxDepthAsync(1); + await _cluster.AdvanceAndRunPumpAsync(sender, RetryInterval); + await WaitForDrainedOutboxAsync(sender); + await Fixture.WaitForOutboxDepthAsync(0); + await AssertProcessedOnceAsync(receiver, durable); + } + + [Fact] + public async Task EquivalentDuplicate_OfACommittedMessage_LeavesItsDeliveryUnchanged() + { + var sender = NewGrain(); + var receiver = NewGrain(); + Deliveries.Enqueue(DeliveryFault.Throw()); + var message = NewMessage(15, "duplicate-after-commit"); + var outgoing = TestApplicationProtocol.Create(Sessions, sender.GetGrainId(), receiver.GetGrainId(), Route, message); + var id = outgoing.Envelope.MessageId; + await sender.StageOutputAsync(outgoing); + await sender.RetryWriteStateAsync(); + var attempt = await WaitForFailedAttemptAsync(sender, id, 1); + var before = await sender.GetSnapshotAsync(); + + // The grain call delivers a copy of the envelope, which is equivalent to the committed one. + await sender.StageOutputAsync(outgoing); + await sender.RetryWriteStateAsync(); + + Assert.Single(Fixture.GetStagedOutput(sender)); + var unchanged = await Fixture.OnGrainAsync(sender, () => _cluster.GetOutboxState(sender).Attempt(id)); + Assert.Equal(attempt.AttemptCount, unchanged.AttemptCount); + Assert.Equal(attempt.NextAttemptAt, unchanged.NextAttemptAt); + Assert.Equal(attempt.SentAt, unchanged.SentAt); + Assert.Equal(before.ActivationId, (await sender.GetSnapshotAsync()).ActivationId); + + await _cluster.AdvanceAndRunPumpAsync(sender, RetryInterval); + await WaitForDrainedOutboxAsync(sender); + Assert.Equal([DeliveryStatus.Accepted], Deliveries.Answers); + Assert.Equal(1, Deliveries.ReceiverCalls); + await AssertProcessedOnceAsync(receiver, message); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task ConflictingDuplicate_IsRejectedWithoutChangingThePendingMessage(bool commitFirst) + { + var sender = NewGrain(); + var receiver = NewGrain(); + Deliveries.Enqueue(DeliveryFault.Throw()); + var message = NewMessage(16, "original"); + var outgoing = TestApplicationProtocol.Create(Sessions, sender.GetGrainId(), receiver.GetGrainId(), Route, message); + var id = outgoing.Envelope.MessageId; + var conflicting = TestApplicationProtocol.Create( + Sessions, sender.GetGrainId(), receiver.GetGrainId(), Route, message with { Value = "conflicting" }); + Assert.Equal(id, conflicting.Envelope.MessageId); + await sender.StageOutputAsync(outgoing); + if (commitFirst) + { + await sender.RetryWriteStateAsync(); + _ = await WaitForFailedAttemptAsync(sender, id, 1); + } + + var before = await sender.GetSnapshotAsync(); + + var error = await Assert.ThrowsAsync(() => sender.StageOutputAsync(conflicting)); + Assert.Contains(id.ToString(), error.Message); + Assert.Equal(outgoing.Envelope.Payload, Assert.Single(Fixture.GetStagedOutput(sender)).Envelope.Payload); + if (!commitFirst) + { + await sender.RetryWriteStateAsync(); + _ = await WaitForFailedAttemptAsync(sender, id, 1); + } + + Assert.Equal(before.ActivationId, (await sender.GetSnapshotAsync()).ActivationId); + await _cluster.AdvanceAndRunPumpAsync(sender, RetryInterval); + await WaitForDrainedOutboxAsync(sender); + Assert.Equal([DeliveryStatus.Accepted], Deliveries.Answers); + var processed = await AssertProcessedOnceAsync(receiver, message); + Assert.Equal(message.Value, Assert.Single(processed.Effects).Value); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task AdmissionFloor_ForwardThenBackwardClockRemainsFencedAcrossRecovery(bool snapshot) + { + var receiver = NewGrain(); + _ = await receiver.GetSnapshotAsync(); + var originalTime = _cluster.JobClock.GetUtcNow(); + _cluster.JobClock.Advance(TimeSpan.FromDays(2)); + var floor = _cluster.JobClock.GetUtcNow() - TimeSpan.FromDays(1); + var expired = CreateEnvelope(receiver, NewMessage(80, "floor-equality")) with { SentAt = floor }; + if (snapshot) Fixture.Storage.RequestSnapshot(JournalId.FromGrainId(receiver.GetGrainId())); + var result = await DeliverAsync(receiver, expired); + Assert.Equal(DeliveryStatus.Expired, result.Status); + Assert.Null(result.Code); + Assert.Empty((await receiver.GetSnapshotAsync()).Effects); + + _cluster.JobClock.Advance(originalTime - _cluster.JobClock.GetUtcNow()); + await ReactivateAsync(receiver); + var inbox = Fixture.GetInbox(receiver); + Assert.Equal(floor, await Fixture.OnGrainAsync(receiver, () => InboxTestAccess.AdmissionFloor(inbox))); + Assert.Equal(floor, await Fixture.OnGrainAsync(receiver, () => InboxTestAccess.DurableAdmissionFloor(inbox))); + foreach (var delta in new[] { -1L, 0L }) + { + var expiredMessage = CreateEnvelope(receiver, NewMessage(81, "expired-by-floor")) with { SentAt = floor.AddTicks(delta) }; + Assert.Equal(DeliveryStatus.Expired, (await DeliverAsync(receiver, expiredMessage)).Status); + } + + var context = Fixture.GetGrainContext(receiver); + var receiverClock = context.ActivationServices.GetRequiredKeyedService(DurableJobTimeProviderNames.DurableJobs); + var receiverTime = await Fixture.OnGrainAsync(receiver, receiverClock.GetUtcNow); + Assert.Equal(originalTime, receiverTime); + var window = context.ActivationServices.GetRequiredService>().Value.DeduplicationWindow; + var manager = context.ActivationServices.GetRequiredService(); + var journal = JournalId.FromGrainId(receiver.GetGrainId()); + var before = await Fixture.OnGrainAsync(receiver, () => ( + Writes: Fixture.Storage.GetSuccessfulWriteCount(journal), + StagedBytes: manager.PendingWriteByteCount, + Keys: InboxTestAccess.GetProcessedKeys(inbox), + Floor: InboxTestAccess.AdmissionFloor(inbox), + DurableFloor: InboxTestAccess.DurableAdmissionFloor(inbox))); + foreach (var sentAt in new[] { floor.AddTicks(1), floor + window / 2 }) + { + Assert.True(sentAt > receiverTime + window); + var tooFuture = CreateEnvelope(receiver, NewMessage(82, "too-future-after-rollback")) with { SentAt = sentAt }; + await Assert.ThrowsAsync(() => DeliverAsync(receiver, tooFuture)); + var after = await Fixture.OnGrainAsync(receiver, () => ( + Writes: Fixture.Storage.GetSuccessfulWriteCount(journal), + StagedBytes: manager.PendingWriteByteCount, + Keys: InboxTestAccess.GetProcessedKeys(inbox), + Floor: InboxTestAccess.AdmissionFloor(inbox), + DurableFloor: InboxTestAccess.DurableAdmissionFloor(inbox))); + Assert.Equal(before.Writes, after.Writes); + Assert.Equal(before.StagedBytes, after.StagedBytes); + Assert.True(before.Keys.SetEquals(after.Keys)); + Assert.Equal(before.Floor, after.Floor); + Assert.Equal(before.DurableFloor, after.DurableFloor); + Assert.Equal(receiverTime, await Fixture.OnGrainAsync(receiver, receiverClock.GetUtcNow)); + } + + // Keep the floor+1 boundary at the permitted future upper bound after the earlier clock rollback. + _cluster.JobClock.Advance(TimeSpan.FromTicks(1)); + var available = CreateEnvelope(receiver, NewMessage(82, "above-floor")) with { SentAt = floor.AddTicks(1) }; + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, available)).Status); + await Fixture.WaitForInboxCountAsync(receiver, 0); + var effect = Assert.Single((await receiver.GetSnapshotAsync()).Effects); + Assert.Equal("above-floor", effect.Value); + Assert.Equal(1, effect.Count); + Assert.Equal(floor.AddTicks(1), await Fixture.OnGrainAsync(receiver, () => + InboxTestAccess.GetProcessedSentAt(inbox, available.Envelope.MessageId))); + } + + [Fact] + public async Task AdmissionFloor_AtMinimumClock_ClampsWithoutUnderflow() + { + var receiver = NewSinkGrain(); + _cluster.JobClock.Advance(DateTimeOffset.MinValue - _cluster.JobClock.GetUtcNow()); + var boundary = new InboxMessage + { + Envelope = new() { MessageId = NewSourceMessageId("minimum"), Payload = [85], Subject = SourceSinkGrain.Subject }, + SentAt = DateTimeOffset.MinValue + }; + Assert.Equal(DeliveryStatus.Expired, (await receiver.AsReference() + .DeliverAsync(boundary, TestContext.Current.CancellationToken)).Status); + var firstAvailable = boundary with + { + Envelope = boundary.Envelope with { MessageId = NewSourceMessageId("first-tick"), Payload = [86] }, + SentAt = DateTimeOffset.MinValue.AddTicks(1) + }; + Assert.Equal(DeliveryStatus.Accepted, (await receiver.AsReference() + .DeliverAsync(firstAvailable, TestContext.Current.CancellationToken)).Status); + var received = Assert.Single((await OnSinkAsync(receiver, static grain => grain.GetState())).Received); + Assert.Equal(firstAvailable.Envelope.MessageId.ToString(), received.Key); + Assert.Equal([86], received.Value); + Assert.Equal(1, _cluster.SourceSinkProbe.For(receiver).SinkOffers); + } + + [Fact] + public async Task SnapshotFloor_OmittedEvidenceRemainsUntilTheSnapshotAcknowledgement() + { + var receiver = NewGrain(); + var sentAt = _cluster.JobClock.GetUtcNow(); + var message = CreateEnvelope(receiver, NewMessage(83, "snapshot-fence")) with { SentAt = sentAt }; + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, message)).Status); + await Fixture.WaitForInboxCountAsync(receiver, 0); + var inbox = Fixture.GetInbox(receiver); + var oldFloor = await Fixture.OnGrainAsync(receiver, () => InboxTestAccess.DurableAdmissionFloor(inbox)); + _cluster.JobClock.Advance(TimeSpan.FromDays(2)); + var journal = JournalId.FromGrainId(receiver.GetGrainId()); + Fixture.Storage.RequestSnapshot(journal); + using var acknowledgement = Fixture.Storage.BlockAcknowledgement(journal); + var writing = receiver.RetryWriteStateAsync(); + await acknowledgement.WaitUntilEnteredAsync(); + + Assert.False(writing.IsCompleted); + Assert.Contains(message.Envelope.MessageId, await Fixture.OnGrainAsync(receiver, () => InboxTestAccess.GetProcessedKeys(inbox))); + Assert.Equal(oldFloor, await Fixture.OnGrainAsync(receiver, () => InboxTestAccess.DurableAdmissionFloor(inbox))); + acknowledgement.Release(); + await writing; + Assert.DoesNotContain(message.Envelope.MessageId, await Fixture.OnGrainAsync(receiver, () => InboxTestAccess.GetProcessedKeys(inbox))); + Assert.Equal(_cluster.JobClock.GetUtcNow() - TimeSpan.FromDays(1), + await Fixture.OnGrainAsync(receiver, () => InboxTestAccess.DurableAdmissionFloor(inbox))); + Assert.Equal(DeliveryStatus.Expired, (await DeliverAsync(receiver, message)).Status); + await ReactivateAsync(receiver); + Assert.Equal(DeliveryStatus.Expired, (await DeliverAsync(receiver, message)).Status); + Assert.Equal(1, Assert.Single((await receiver.GetSnapshotAsync()).Effects).Count); + } + + [Fact] + public async Task ProcessedHistory_FloorOnlyAndOrdinaryAcksRetainItUntilAnUnchangedFloorSnapshotCommits() + { + var receiver = NewSinkGrain(); + var started = _cluster.JobClock.GetUtcNow(); + var finite = Enumerable.Range(0, 8).Select(index => new InboxMessage + { + Envelope = new() { MessageId = NewSourceMessageId($"finite-{index}"), Payload = [(byte)index], Subject = SourceSinkGrain.Subject }, + SentAt = started + }).ToArray(); + var extension = receiver.AsReference(); + foreach (var message in finite) + { + Assert.Equal(DeliveryStatus.Accepted, (await extension.DeliverAsync(message, TestContext.Current.CancellationToken)).Status); + } + _cluster.JobClock.Advance(TimeSpan.FromHours(12)); + var recent = Enumerable.Range(0, 8).Select(index => new InboxMessage + { + Envelope = new() { MessageId = NewSourceMessageId($"recent-{index}"), Payload = [(byte)(index + 8)], Subject = SourceSinkGrain.Subject }, + SentAt = _cluster.JobClock.GetUtcNow() + }).ToArray(); + var future = new InboxMessage + { + Envelope = new() { MessageId = NewSourceMessageId("retained-recent"), Payload = [16], Subject = SourceSinkGrain.Subject }, + SentAt = _cluster.JobClock.GetUtcNow() + }; + foreach (var message in recent.Append(future)) + { + Assert.Equal(DeliveryStatus.Accepted, (await extension.DeliverAsync(message, TestContext.Current.CancellationToken)).Status); + } + + var inbox = Fixture.GetGrainContext(receiver).ActivationServices.GetRequiredService(); + var retained = finite.Concat(recent).Append(future).Select(static message => message.Envelope.MessageId).ToHashSet(); + var journal = JournalId.FromGrainId(receiver.GetGrainId()); + var writes = Fixture.Storage.GetSuccessfulWriteCount(journal); + _cluster.JobClock.Advance(started + TimeSpan.FromDays(1) + TimeSpan.FromHours(1) - _cluster.JobClock.GetUtcNow()); + DateTimeOffset floor = default; + for (var index = 0; index < 32; index++) + { + _cluster.JobClock.Advance(TimeSpan.FromTicks(1)); + floor = _cluster.JobClock.GetUtcNow() - TimeSpan.FromDays(1); + var stale = new InboxMessage + { + Envelope = new() { MessageId = NewSourceMessageId($"floor-only-{index}"), Payload = [], Subject = SourceSinkGrain.Subject }, + SentAt = floor + }; + Assert.Equal(DeliveryStatus.Expired, (await extension.DeliverAsync(stale, TestContext.Current.CancellationToken)).Status); + Assert.Equal(writes + index + 1, Fixture.Storage.GetSuccessfulWriteCount(journal)); + Assert.Equal(floor, await Fixture.OnGrainAsync(receiver, () => InboxTestAccess.DurableAdmissionFloor(inbox))); + // These finite records are already eligible. Their continued presence proves scalar acks did not compact. + Assert.True(retained.SetEquals(await Fixture.OnGrainAsync(receiver, () => InboxTestAccess.GetProcessedKeys(inbox)))); + } + + var ordinary = new InboxMessage + { + Envelope = new() { MessageId = NewSourceMessageId("ordinary-append"), Payload = [17], Subject = SourceSinkGrain.Subject }, + SentAt = _cluster.JobClock.GetUtcNow() + }; + Assert.Equal(DeliveryStatus.Accepted, (await extension.DeliverAsync(ordinary, TestContext.Current.CancellationToken)).Status); + retained.Add(ordinary.Envelope.MessageId); + Assert.Equal(writes + 33, Fixture.Storage.GetSuccessfulWriteCount(journal)); + Assert.True(retained.SetEquals(await Fixture.OnGrainAsync(receiver, () => InboxTestAccess.GetProcessedKeys(inbox)))); + + // The floor is already durable. This snapshot must reclaim history even though its candidate cannot advance. + _cluster.JobClock.Advance(started - _cluster.JobClock.GetUtcNow()); + Fixture.Storage.RequestSnapshot(journal); + using var acknowledgement = Fixture.Storage.BlockAcknowledgement(journal); + var snapshot = receiver.WriteAsync(); + await acknowledgement.WaitUntilEnteredAsync(); + Assert.False(snapshot.IsCompleted); + Assert.Equal(writes + 34, Fixture.Storage.GetSuccessfulWriteCount(journal)); + Assert.Equal(floor, await Fixture.OnGrainAsync(receiver, () => InboxTestAccess.AdmissionFloor(inbox))); + Assert.Equal(floor, await Fixture.OnGrainAsync(receiver, () => InboxTestAccess.DurableAdmissionFloor(inbox))); + Assert.True(retained.SetEquals(await Fixture.OnGrainAsync(receiver, () => InboxTestAccess.GetProcessedKeys(inbox)))); + acknowledgement.Release(); + await snapshot; + + retained.ExceptWith(finite.Select(static message => message.Envelope.MessageId)); + Assert.True(retained.SetEquals(await Fixture.OnGrainAsync(receiver, () => InboxTestAccess.GetProcessedKeys(inbox)))); + Assert.Equal(floor, await Fixture.OnGrainAsync(receiver, () => InboxTestAccess.DurableAdmissionFloor(inbox))); + await _cluster.CrashAsync(receiver); + Assert.Equal(0, await receiver.GetOutboxCountAsync()); + inbox = Fixture.GetGrainContext(receiver).ActivationServices.GetRequiredService(); + Assert.Equal(floor, await Fixture.OnGrainAsync(receiver, () => InboxTestAccess.DurableAdmissionFloor(inbox))); + Assert.True(retained.SetEquals(await Fixture.OnGrainAsync(receiver, () => InboxTestAccess.GetProcessedKeys(inbox)))); + foreach (var message in finite) + { + Assert.Equal(DeliveryStatus.Expired, (await extension.DeliverAsync(message, TestContext.Current.CancellationToken)).Status); + } + // Do not present the later retained record outside the receiver's new future-time bound after rollback. + _cluster.JobClock.Advance(ordinary.SentAt - _cluster.JobClock.GetUtcNow()); + foreach (var message in recent.Append(future).Append(ordinary)) + { + Assert.Equal(DeliveryStatus.Duplicate, (await extension.DeliverAsync(message, TestContext.Current.CancellationToken)).Status); + } + Assert.Equal(18, _cluster.SourceSinkProbe.For(receiver).SinkOffers); + Assert.Equal(18, _cluster.SourceSinkProbe.For(receiver).SinkAcceptances); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task LostResponse_AfterCompactionAndClockRollback_CannotApplyTheMessageTwice(bool realSink) + { + var sender = NewSinkGrain(); + IAddressable receiver = realSink ? NewSinkGrain() : NewGrain(); + var id = NewSourceMessageId("late-lost-response"); + var payload = realSink + ? RandomNumberGenerator.GetBytes(32) + : TestApplicationProtocol.Encode(Sessions, new TestApplicationMessage(Route, new TestMessage(id, 84, "late-redelivery"))); + using var events = new DeliveryEventObserver(sender.GetGrainId(), receiver.GetGrainId(), id, faults: 0); + var retryScheduled = events.RetryScheduled.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + var retired = events.Retired.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + var lost = DeliveryFault.LoseResponse(); + var delayed = DeliveryFault.HoldBeforeCall(); + Deliveries.Enqueue(lost, delayed); + var started = _cluster.JobClock.GetUtcNow(); + // The source's payload is raw in the sink case and a real application envelope in the handler case. + if (realSink) + { + await sender.StageFromSourceAsync(receiver.GetGrainId(), id, payload, write: true); + } + else + { + _ = await sender.GetOutboxCountAsync(); + // Use an ordinary outbox envelope to preserve the application's route. + await Fixture.OnGrainAsync(sender, () => Fixture.GetGrainContext(sender).ActivationServices.GetRequiredService().Send(new OutboxMessage + { + ReceiverId = receiver.GetGrainId(), + Envelope = new() { MessageId = id, Payload = payload, Subject = Route } + })); + await sender.WriteAsync(); + } + + Assert.Equal(DeliveryStatus.Accepted, (await lost.ReceiverAnswer).Status); + Assert.Equal(id, (await retryScheduled).MessageId); + if (!realSink) await Fixture.WaitForInboxCountAsync((IMessagingTestGrain)receiver, 0); + await sender.WriteAsync(); + // Start the retry inside its original budget, then delay its arrival beyond receiver compaction. + // A pending retry could otherwise legitimately expire while the shared clock is two days ahead. + await _cluster.AdvanceAndRunPumpAsync(sender, RetryInterval); + await delayed.WaitUntilEnteredAsync(); + Assert.Equal(1, Deliveries.ReceiverCalls); + _cluster.JobClock.Advance(TimeSpan.FromDays(2)); + await Fixture.RunPumpAsync(sender); + var pending = await Fixture.OnGrainAsync(sender, () => + { + Assert.True(Fixture.GetGrainContext(sender).ActivationServices.GetRequiredService().IsDelivering); + return _cluster.GetOutboxState(sender).Attempt(id); + }); + Assert.Equal(1, pending.AttemptCount); + Assert.Equal(started, pending.SentAt); + Assert.False(retired.IsCompleted); + Fixture.Storage.RequestSnapshot(JournalId.FromGrainId(receiver.GetGrainId())); + await WriteJournalAsync(receiver); + var inbox = Fixture.GetGrainContext(receiver).ActivationServices.GetRequiredService(); + Assert.DoesNotContain(id, await Fixture.OnGrainAsync(receiver, () => InboxTestAccess.GetProcessedKeys(inbox))); + await _cluster.CrashAsync(receiver); + _cluster.JobClock.Advance(started - _cluster.JobClock.GetUtcNow()); + + var replay = new InboxMessage + { + Envelope = new() { MessageId = id, Payload = payload, Subject = realSink ? SourceSinkGrain.Subject : Route }, + SentAt = started + }; + var expired = await receiver.AsReference().DeliverAsync(replay, TestContext.Current.CancellationToken); + Assert.Equal(DeliveryStatus.Expired, expired.Status); + Assert.Null(expired.Code); + if (realSink) + { + Assert.Equal(payload, (await OnSinkAsync(receiver, static grain => grain.GetState())).Received[id.ToString()]); + Assert.Equal(1, _cluster.SourceSinkProbe.For(receiver).SinkOffers); + Assert.Equal(1, _cluster.SourceSinkProbe.For(receiver).SinkAcceptances); + } + else + { + var effect = Assert.Single((await ((IMessagingTestGrain)receiver).GetSnapshotAsync()).Effects); + Assert.Equal("late-redelivery", effect.Value); + Assert.Equal(1, effect.Count); + } + + // Observe this delayed attempt's answer and retirement, not an empty outbox caused by an earlier timeout. + // Its earlier acceptance is real, but the source must receive UNKNOWN, not a false rejection. + var answers = Deliveries.WaitForAnswersAsync(2); + delayed.Release(); + Assert.Equal(DeliveryStatus.Expired, (await delayed.ReceiverAnswer).Status); + Assert.Equal([DeliveryStatus.Accepted, DeliveryStatus.Expired], + await answers.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken)); + Assert.Equal(OutboxMessageRetirementStatus.DeadLettered, (await retired).Retirement.Status); + await sender.WriteAsync(); + Assert.Empty(await Fixture.OnGrainAsync(sender, () => _cluster.GetOutboxState(sender).AttemptKeys.ToArray())); + var terminal = await Fixture.OnGrainAsync(sender, () => _cluster.GetOutboxState(sender).DeadLetter(id)); + Assert.Equal(id, terminal.Message.Envelope.MessageId); + if (realSink) + { + Assert.Equal(nameof(OutboxMessageRetirementStatus.DeadLettered), + (await OnSinkAsync(sender, static grain => grain.GetState())).Retired[id.ToString()]); + Assert.Empty(await OnSinkAsync(sender, static grain => grain.GetRetirementCodes())); + Assert.Equal(1, _cluster.SourceSinkProbe.For(receiver).SinkAcceptances); + Assert.Equal(1, _cluster.SourceSinkProbe.For(receiver).SinkOffers); + } + else + { + Assert.Equal(1, Assert.Single((await ((IMessagingTestGrain)receiver).GetSnapshotAsync()).Effects).Count); + } + } + + [Fact] + public async Task SnapshotFloor_CompletionAfterCaptureRemainsUntilItsOwnWriteCommits() + { + var receiver = NewGrain(); + const string route = "messages/snapshot-interleaved-completion"; + using var handler = Fixture.HandlerProbe.Arm(receiver.GetGrainId(), route); + var message = CreateEnvelope(receiver, NewMessage(87, "interleaved"), route) with + { + SentAt = _cluster.JobClock.GetUtcNow() + }; + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, message)).Status); + await handler.WaitUntilEnteredAsync(); + _cluster.JobClock.Advance(TimeSpan.FromDays(2)); + var journal = JournalId.FromGrainId(receiver.GetGrainId()); + Fixture.Storage.RequestSnapshot(journal); + using var snapshotAcknowledgement = Fixture.Storage.BlockAcknowledgement(journal); + var snapshot = receiver.RetryWriteStateAsync(); + await snapshotAcknowledgement.WaitUntilEnteredAsync(); + using var completionWrite = Fixture.Storage.BlockWrite(journal); + handler.Release(); + var inbox = Fixture.GetInbox(receiver); + await Fixture.WaitUntilAsync(receiver, () => InboxTestAccess.GetProcessedKeys(inbox).Contains(message.Envelope.MessageId)); + snapshotAcknowledgement.Release(); + await snapshot; + await completionWrite.WaitUntilEnteredAsync(); + Assert.Contains(message.Envelope.MessageId, await Fixture.OnGrainAsync(receiver, () => InboxTestAccess.GetProcessedKeys(inbox))); + var duplicate = await Fixture.OnGrainAsync(receiver, + () => inbox.DeliverAsync(message, TestContext.Current.CancellationToken).AsTask()); + Assert.False(duplicate.IsCompleted); + completionWrite.Release(); + Assert.Equal(DeliveryStatus.Duplicate, (await duplicate).Status); + await ReactivateAsync(receiver); + Assert.Equal(DeliveryStatus.Expired, (await DeliverAsync(receiver, message)).Status); + Assert.Equal("interleaved", Assert.Single((await receiver.GetSnapshotAsync()).Effects).Value); + Assert.Equal(1, Assert.Single((await receiver.GetSnapshotAsync()).Effects).Count); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task AdmissionFloor_ExpiredAnswerWaitsForDurabilityAndRecoversAfterWriteFailure(bool ambiguous) + { + var receiver = NewSinkGrain(); + _ = await receiver.GetOutboxCountAsync(); + var previous = Fixture.GetGrainContext(receiver); + _cluster.JobClock.Advance(TimeSpan.FromDays(2)); + var floor = _cluster.JobClock.GetUtcNow() - TimeSpan.FromDays(1); + var stale = new InboxMessage + { + Envelope = new() { MessageId = NewSourceMessageId("floor-write"), Payload = [88], Subject = SourceSinkGrain.Subject }, + SentAt = floor + }; + var journal = JournalId.FromGrainId(receiver.GetGrainId()); + using var write = ambiguous ? Fixture.Storage.BlockAcknowledgement(journal) : Fixture.Storage.BlockWrite(journal); + var delivery = receiver.AsReference().DeliverAsync(stale, TestContext.Current.CancellationToken).AsTask(); + await write.WaitUntilEnteredAsync(); + Assert.False(delivery.IsCompleted); + Assert.Equal(0, _cluster.SourceSinkProbe.For(receiver).SinkOffers); + write.Fail(); + await Assert.ThrowsAsync(() => delivery); + await previous.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + Assert.Equal(0, await receiver.GetOutboxCountAsync()); + var inbox = Fixture.GetGrainContext(receiver).ActivationServices.GetRequiredService(); + Assert.Equal(ambiguous ? floor : DateTimeOffset.MinValue, + await Fixture.OnGrainAsync(receiver, () => InboxTestAccess.DurableAdmissionFloor(inbox))); + Assert.Equal(DeliveryStatus.Expired, (await receiver.AsReference() + .DeliverAsync(stale, TestContext.Current.CancellationToken)).Status); + Assert.Equal(floor, await Fixture.OnGrainAsync(receiver, () => InboxTestAccess.DurableAdmissionFloor(inbox))); + Assert.Empty((await OnSinkAsync(receiver, static grain => grain.GetState())).Received); + Assert.Equal(0, _cluster.SourceSinkProbe.For(receiver).SinkOffers); + } + + [Theory] + [InlineData(false, false, false)] + [InlineData(false, false, true)] + [InlineData(false, true, false)] + [InlineData(false, true, true)] + [InlineData(true, false, false)] + [InlineData(true, false, true)] + [InlineData(true, true, false)] + [InlineData(true, true, true)] + public async Task NewerSentAt_WriteFailureRecoversOnlyTheCommittedRetentionTime(bool completed, bool ambiguous, bool snapshot) + { + var receiver = NewGrain(); + const string route = "messages/retention-update-recovery"; + using var handler = Fixture.HandlerProbe.Arm(receiver.GetGrainId(), route); + var original = CreateEnvelope(receiver, NewMessage(90, "retention-recovery"), route); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, original)).Status); + await handler.WaitUntilEnteredAsync(); + if (completed) + { + handler.Release(); + await Fixture.WaitForInboxCountAsync(receiver, 0); + } + await receiver.RetryWriteStateAsync(); + _cluster.JobClock.Advance(TimeSpan.FromHours(2)); + var newer = original with { SentAt = _cluster.JobClock.GetUtcNow() }; + var context = Fixture.GetGrainContext(receiver); + var journal = JournalId.FromGrainId(receiver.GetGrainId()); + if (snapshot) Fixture.Storage.RequestSnapshot(journal); + using var barrier = ambiguous ? Fixture.Storage.BlockAcknowledgement(journal) : Fixture.Storage.BlockWrite(journal); + var updating = DeliverAsync(receiver, newer); + await barrier.WaitUntilEnteredAsync(); + Assert.False(updating.IsCompleted); + barrier.Fail(); + await Assert.ThrowsAsync(() => updating); + await context.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + handler.Dispose(); + _ = await receiver.GetSnapshotAsync(); + await Fixture.WaitForEffectCountAsync(receiver, 1); + await Fixture.WaitForInboxCountAsync(receiver, 0); + var inbox = Fixture.GetInbox(receiver); + Assert.Equal(ambiguous ? newer.SentAt : original.SentAt, await Fixture.OnGrainAsync(receiver, () => + InboxTestAccess.GetProcessedSentAt(inbox, original.Envelope.MessageId))); + var effect = Assert.Single((await receiver.GetSnapshotAsync()).Effects); + Assert.Equal("retention-recovery", effect.Value); + Assert.Equal(1, effect.Count); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(receiver, original)).Status); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task NewerSentAt_ExtendsRetentionThroughTheOldBoundaryAndPreservesTheKnownOutcome(bool rejected) + { + var receiver = NewSinkGrain(); + var probe = _cluster.SourceSinkProbe.For(receiver); + if (rejected) probe.Mode = SinkMode.DeadLetter; + var original = new InboxMessage + { + Envelope = new() { MessageId = NewSourceMessageId("extended-retention"), Payload = [91], Subject = SourceSinkGrain.Subject }, + SentAt = _cluster.JobClock.GetUtcNow() + }; + var extension = receiver.AsReference(); + Assert.Equal(rejected ? DeliveryStatus.DeadLettered : DeliveryStatus.Accepted, + (await extension.DeliverAsync(original, TestContext.Current.CancellationToken)).Status); + probe.Mode = SinkMode.Accept; + _cluster.JobClock.Advance(TimeSpan.FromHours(2)); + var newer = original with { SentAt = _cluster.JobClock.GetUtcNow() }; + var journal = JournalId.FromGrainId(receiver.GetGrainId()); + using var acknowledgement = Fixture.Storage.BlockAcknowledgement(journal); + var updating = extension.DeliverAsync(newer, TestContext.Current.CancellationToken).AsTask(); + await acknowledgement.WaitUntilEnteredAsync(); + Assert.False(updating.IsCompleted); + Assert.Equal(1, probe.SinkOffers); + acknowledgement.Release(); + AssertKnown(await updating); + await _cluster.CrashAsync(receiver); + Assert.Equal(0, await receiver.GetOutboxCountAsync()); + var inbox = Fixture.GetGrainContext(receiver).ActivationServices.GetRequiredService(); + Assert.Equal(newer.SentAt, await Fixture.OnGrainAsync(receiver, () => + InboxTestAccess.GetProcessedSentAt(inbox, original.Envelope.MessageId))); + + _cluster.JobClock.Advance(original.SentAt + TimeSpan.FromDays(1) - _cluster.JobClock.GetUtcNow()); + Fixture.Storage.RequestSnapshot(journal); + await receiver.WriteAsync(); + Assert.Contains(original.Envelope.MessageId, await Fixture.OnGrainAsync(receiver, () => InboxTestAccess.GetProcessedKeys(inbox))); + AssertKnown(await extension.DeliverAsync(original, TestContext.Current.CancellationToken)); + _cluster.JobClock.Advance(newer.SentAt + TimeSpan.FromDays(1) - _cluster.JobClock.GetUtcNow()); + Fixture.Storage.RequestSnapshot(journal); + await receiver.WriteAsync(); + Assert.DoesNotContain(original.Envelope.MessageId, await Fixture.OnGrainAsync(receiver, () => InboxTestAccess.GetProcessedKeys(inbox))); + Assert.Equal(DeliveryStatus.Expired, (await extension.DeliverAsync(newer, TestContext.Current.CancellationToken)).Status); + await _cluster.CrashAsync(receiver); + _cluster.JobClock.Advance(original.SentAt - _cluster.JobClock.GetUtcNow()); + Assert.Equal(DeliveryStatus.Expired, (await extension.DeliverAsync(newer, TestContext.Current.CancellationToken)).Status); + Assert.Equal(1, probe.SinkOffers); + Assert.Equal(rejected ? 0 : 1, probe.SinkAcceptances); + + void AssertKnown(DeliveryResult result) + { + Assert.Equal(rejected ? DeliveryStatus.DeadLettered : DeliveryStatus.Duplicate, result.Status); + Assert.Equal(rejected ? SourceSinkGrain.RejectionCode : null, result.Code); + if (rejected) Assert.Equal(SourceSinkGrain.Rejection, result.Message); + } + } + + [Fact] + public async Task PendingSentAt_ExtensionDuringAcceptanceCaptureRetainsWakeupAndCompletesWithLatestTime() + { + var receiver = NewGrain(); + _ = await receiver.GetSnapshotAsync(); + const string route = "messages/pending-sent-time-capture"; + using var handler = Fixture.HandlerProbe.Arm(receiver.GetGrainId(), route); + var journal = JournalId.FromGrainId(receiver.GetGrainId()); + using var acceptance = Fixture.Storage.BlockAcknowledgement(journal); + var original = CreateEnvelope(receiver, NewMessage(92, "latest-completion-time"), route); + var accepting = DeliverAsync(receiver, original); + await acceptance.WaitUntilEnteredAsync(); + _cluster.JobClock.Advance(TimeSpan.FromHours(2)); + var newer = original with { SentAt = _cluster.JobClock.GetUtcNow() }; + var inbox = Fixture.GetInbox(receiver); + using var updateWrite = Fixture.Storage.BlockWrite(journal); + var updating = await Fixture.OnGrainAsync(receiver, () => inbox.DeliverAsync(newer, TestContext.Current.CancellationToken).AsTask()); + Assert.False(updating.IsCompleted); + Assert.Equal(newer.SentAt, await Fixture.OnGrainAsync(receiver, () => + InboxTestAccess.GetPendingMessages(inbox)[original.Envelope.MessageId].Message.SentAt)); + acceptance.Release(); + Assert.Equal(DeliveryStatus.Accepted, (await accepting).Status); + await updateWrite.WaitUntilEnteredAsync(); + Assert.Equal(DurableJobRunStatus.RescheduleRequested, (await Fixture.RunPumpAsync(receiver)).Status); + Assert.Empty((await receiver.GetSnapshotAsync()).Effects); + Assert.NotEmpty(Fixture.JobManagerProbe.GetScheduledJobs(MessagingPump.JobName, receiver.GetGrainId())); + updateWrite.Release(); + Assert.Equal(DeliveryStatus.Duplicate, (await updating).Status); + await handler.WaitUntilEnteredAsync(); + handler.Release(); + await Fixture.WaitForInboxCountAsync(receiver, 0); + Assert.Equal(newer.SentAt, await Fixture.OnGrainAsync(receiver, () => + InboxTestAccess.GetProcessedSentAt(inbox, original.Envelope.MessageId))); + Assert.Equal(1, Assert.Single((await receiver.GetSnapshotAsync()).Effects).Count); + await ReactivateAsync(receiver); + inbox = Fixture.GetInbox(receiver); + Assert.Equal(newer.SentAt, await Fixture.OnGrainAsync(receiver, () => + InboxTestAccess.GetProcessedSentAt(inbox, original.Envelope.MessageId))); + } + + [Theory] + [InlineData("orleans-binary", false, false)] + [InlineData("orleans-binary", false, true)] + [InlineData("orleans-binary", true, false)] + [InlineData("orleans-binary", true, true)] + [InlineData("json", false, false)] + [InlineData("json", false, true)] + [InlineData("json", true, false)] + [InlineData("json", true, true)] + public async Task ProcessedSentAt_ExtensionAfterSnapshotCaptureSurvivesUntilItsOwnDurability( + string format, bool aboveFloor, bool rejected) + { + await using var alternate = format == "json" ? new FaultInjectionClusterFixture("json") : null; + if (alternate is not null) await alternate.InitializeAsync(); + var cluster = alternate ?? _cluster; + IAddressable receiver = rejected + ? cluster.Client.GetGrain(Guid.NewGuid()) + : cluster.Client.GetGrain(Guid.NewGuid()); + var sentAt = cluster.JobClock.GetUtcNow(); + var sessions = cluster.Client.ServiceProvider.GetRequiredService(); + var original = rejected + ? new InboxMessage + { + Envelope = new() { MessageId = NewSourceMessageId("captured-rejection"), Payload = [93], Subject = SourceSinkGrain.Subject }, + SentAt = sentAt + } + : TestApplicationProtocol.CreateInbox(sessions, GrainId.Create("replay-test-sender", "ordinary"), Route, + NewMessage(93, "captured-retention-update"), sentAt); + if (rejected) cluster.SourceSinkProbe.For(receiver).Mode = SinkMode.DeadLetter; + var extension = receiver.AsReference(); + var initial = await extension.DeliverAsync(original, TestContext.Current.CancellationToken); + Assert.Equal(rejected ? DeliveryStatus.DeadLettered : DeliveryStatus.Accepted, initial.Status); + if (!rejected) + { + await cluster.WaitForEffectCountAsync((IMessagingTestGrain)receiver, 1); + await cluster.WaitForInboxCountAsync((IMessagingTestGrain)receiver, 0); + await ((IMessagingTestGrain)receiver).RetryWriteStateAsync(); + } + else + { + Assert.Equal(SourceSinkGrain.Rejection, initial.Message); + Assert.Equal(SourceSinkGrain.RejectionCode, initial.Code); + cluster.SourceSinkProbe.For(receiver).Mode = SinkMode.Accept; + } + var oldContext = cluster.GetGrainContext(receiver); + var inbox = oldContext.ActivationServices.GetRequiredService(); + var journal = JournalId.FromGrainId(receiver.GetGrainId()); + var appends = cluster.Storage.GetSuccessfulAppendCount(journal); + var snapshots = cluster.Storage.GetSuccessfulSnapshotCount(journal); + cluster.JobClock.Advance(TimeSpan.FromDays(2)); + var floor = cluster.JobClock.GetUtcNow() - TimeSpan.FromDays(1); + Assert.True(original.SentAt <= floor); + cluster.Storage.RequestSnapshot(journal); + using var snapshotAck = cluster.Storage.BlockAcknowledgement(journal); + var snapshot = WriteAsync(); + await snapshotAck.WaitUntilEnteredAsync(); + Assert.Equal(snapshots + 1, cluster.Storage.GetSuccessfulSnapshotCount(journal)); + Assert.Contains(original.Envelope.MessageId, await cluster.OnGrainAsync(receiver, () => InboxTestAccess.GetProcessedKeys(inbox))); + var newerTime = floor.AddTicks(aboveFloor ? 1 : -1); + cluster.JobClock.Advance(newerTime - cluster.JobClock.GetUtcNow()); + var newer = original with { SentAt = cluster.JobClock.GetUtcNow() }; + using var extensionWrite = cluster.Storage.BlockWrite(journal); + var updating = await cluster.OnGrainAsync(receiver, () => inbox.DeliverAsync(newer, TestContext.Current.CancellationToken).AsTask()); + Assert.False(updating.IsCompleted); + snapshotAck.Release(); + await snapshot; + await extensionWrite.WaitUntilEnteredAsync(); + Assert.False(updating.IsCompleted); + Assert.Equal(appends, cluster.Storage.GetSuccessfulAppendCount(journal)); + Assert.Equal(floor, await cluster.OnGrainAsync(receiver, () => InboxTestAccess.DurableAdmissionFloor(inbox))); + Assert.Contains(original.Envelope.MessageId, await cluster.OnGrainAsync(receiver, () => InboxTestAccess.GetProcessedKeys(inbox))); + Assert.Equal(newer.SentAt, await cluster.OnGrainAsync(receiver, () => InboxTestAccess.GetProcessedSentAt(inbox, original.Envelope.MessageId))); + var olderReply = await cluster.OnGrainAsync(receiver, () => inbox.DeliverAsync(original, TestContext.Current.CancellationToken).AsTask()); + Assert.False(olderReply.IsCompleted); + extensionWrite.Release(); + AssertKnown(await updating); + AssertKnown(await olderReply); + Assert.Equal(appends + 1, cluster.Storage.GetSuccessfulAppendCount(journal)); + Assert.Equal(snapshots + 1, cluster.Storage.GetSuccessfulSnapshotCount(journal)); + + // Crash in the replay gap: the old snapshot omitted the identity and only this APPEND can restore it. + // Taking another snapshot here would mask a delta which cannot replay without its missing base record. + await cluster.CrashAsync(receiver); + Assert.Equal(snapshots + 1, cluster.Storage.GetSuccessfulSnapshotCount(journal)); + if (rejected) Assert.Equal(0, await ((ISourceSinkGrain)receiver).GetOutboxCountAsync()); + else _ = await ((IMessagingTestGrain)receiver).GetSnapshotAsync(); + Assert.NotSame(oldContext, cluster.GetGrainContext(receiver)); + inbox = cluster.GetGrainContext(receiver).ActivationServices.GetRequiredService(); + Assert.Equal(floor, await cluster.OnGrainAsync(receiver, () => InboxTestAccess.DurableAdmissionFloor(inbox))); + var recoveredResult = await extension.DeliverAsync(newer, TestContext.Current.CancellationToken); + if (aboveFloor) + { + AssertKnown(recoveredResult); + Assert.Equal(newer.SentAt, await cluster.OnGrainAsync(receiver, () => InboxTestAccess.GetProcessedSentAt(inbox, original.Envelope.MessageId))); + } + else + { + Assert.Equal(DeliveryStatus.Expired, recoveredResult.Status); + Assert.Null(recoveredResult.Code); + Assert.DoesNotContain(original.Envelope.MessageId, await cluster.OnGrainAsync(receiver, () => InboxTestAccess.GetProcessedKeys(inbox))); + } + Assert.Equal(snapshots + 1, cluster.Storage.GetSuccessfulSnapshotCount(journal)); + if (rejected) + { + var diagnostics = cluster.GetGrainContext(receiver).ActivationServices.GetRequiredService(); + var rejection = Assert.Single(diagnostics.InboxDeadLetters); + Assert.Equal(SourceSinkGrain.Rejection, rejection.Reason); + Assert.Equal(SourceSinkGrain.RejectionCode, rejection.Code); + Assert.Equal(1, cluster.SourceSinkProbe.For(receiver).SinkOffers); + Assert.Equal(0, cluster.SourceSinkProbe.For(receiver).SinkAcceptances); + } + else + { + var effect = Assert.Single((await ((IMessagingTestGrain)receiver).GetSnapshotAsync()).Effects); + Assert.Equal("captured-retention-update", effect.Value); + Assert.Equal(1, effect.Count); + } + + Task WriteAsync() => rejected + ? ((ISourceSinkGrain)receiver).WriteAsync() + : ((IMessagingTestGrain)receiver).RetryWriteStateAsync(); + + void AssertKnown(DeliveryResult result) + { + Assert.Equal(rejected ? DeliveryStatus.DeadLettered : DeliveryStatus.Duplicate, result.Status); + Assert.Equal(rejected ? SourceSinkGrain.RejectionCode : null, result.Code); + if (rejected) Assert.Equal(SourceSinkGrain.Rejection, result.Message); + } + } + + [Fact] + public async Task PendingSentAt_ExtensionDuringPreparationDoesNotTurnAnUnmutatedHandlerErrorIntoPumpFailure() + { + var receiver = NewGrain(); + var before = await receiver.GetSnapshotAsync(); + var context = Fixture.GetGrainContext(receiver); + var grain = Assert.IsType(context.GrainInstance); + const string route = "messages/retention-excluded-from-handler"; + using var preparation = Fixture.HandlerProbe.Arm(receiver.GetGrainId(), route + "/application-preparation"); + var original = CreateEnvelope(receiver, NewMessage(95, "no-handler-mutation") with { ThrowOnceDuringPreparation = true }, route); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, original)).Status); + await preparation.WaitUntilEnteredAsync(); + _cluster.JobClock.Advance(TimeSpan.FromHours(2)); + var newer = original with { SentAt = _cluster.JobClock.GetUtcNow() }; + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(receiver, newer)).Status); + var inbox = Fixture.GetInbox(receiver); + Assert.Equal(newer.SentAt, await Fixture.OnGrainAsync(receiver, () => + InboxTestAccess.GetPendingMessages(inbox)[original.Envelope.MessageId].Message.SentAt)); + Assert.Empty((await receiver.GetSnapshotAsync()).Effects); + var completed = Fixture.SnapshotProbe.WaitAsync(receiver.GetGrainId(), + snapshot => snapshot.InboxCount == 0 && snapshot.InboxDeadLetters.Count == 1); + preparation.Release(); + Assert.Same(completed, await Task.WhenAny(completed, grain.DeactivationFailure.Task)); + var result = await completed; + var letter = Assert.Single(result.InboxDeadLetters); + Assert.Contains($"Injected handler preparation failure for {original.Envelope.MessageId}", letter.Reason, StringComparison.Ordinal); + Assert.Equal(1, letter.AttemptCount); + Assert.Empty(result.Effects); + Assert.Equal(0, result.OutboxCount); + Assert.Equal(before.ActivationId, result.ActivationId); + Assert.Same(context, Fixture.GetGrainContext(receiver)); + Assert.False(context.Deactivated.IsCompleted); + Assert.False(grain.DeactivationFailure.Task.IsCompleted); + Assert.False(grain.ApplyAttempted.Task.IsCompleted); + Assert.False(Fixture.GetPump(receiver).ShutdownToken.IsCancellationRequested); + Assert.Equal(newer.SentAt, await Fixture.OnGrainAsync(receiver, () => InboxTestAccess.GetProcessedSentAt(inbox, original.Envelope.MessageId))); + var diagnostic = Assert.Single(await Fixture.OnGrainAsync(receiver, () => + context.ActivationServices.GetRequiredService().InboxDeadLetters.ToArray())); + Assert.Equal(newer.SentAt, diagnostic.Message.SentAt); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(receiver, newer)).Status); + Assert.Equal(before.ActivationId, (await receiver.GetSnapshotAsync()).ActivationId); + } + + [Theory] + [InlineData(false, false)] + [InlineData(false, true)] + [InlineData(true, false)] + [InlineData(true, true)] + public async Task RealOutbox_TimestampOnlyResendDuplicatesOnItsFirstAttemptAndRetiresDelivered( + bool completed, bool incomingLater) + { + var sender = NewSinkGrain(); + var receiver = NewGrain(); + var started = _cluster.JobClock.GetUtcNow(); + const string route = "messages/real-outbox-timestamp-only"; + using var handler = Fixture.HandlerProbe.Arm(receiver.GetGrainId(), route); + if (!incomingLater) _cluster.JobClock.Advance(TimeSpan.FromHours(2)); + var original = CreateEnvelope(receiver, NewMessage(94, "one-real-outbox-effect"), route); + var id = original.Envelope.MessageId; + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, original)).Status); + await handler.WaitUntilEnteredAsync(); + if (completed) + { + handler.Release(); + await Fixture.WaitForInboxCountAsync(receiver, 0); + } + await receiver.RetryWriteStateAsync(); + var journal = JournalId.FromGrainId(receiver.GetGrainId()); + var writes = Fixture.Storage.GetSuccessfulWriteCount(journal); + if (incomingLater) _cluster.JobClock.Advance(TimeSpan.FromHours(2)); + else _cluster.JobClock.Advance(started - _cluster.JobClock.GetUtcNow()); + var incomingSentAt = _cluster.JobClock.GetUtcNow(); + Assert.True(incomingLater ? incomingSentAt > original.SentAt : incomingSentAt < original.SentAt); + var expectedSentAt = incomingLater ? incomingSentAt : original.SentAt; + _cluster.SourceSinkProbe.For(sender).SubjectOverride = route; + using var events = new DeliveryEventObserver(sender.GetGrainId(), receiver.GetGrainId(), id, faults: 0); + var retired = events.Retired.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + var held = DeliveryFault.HoldAfterCall(); + var firstAnswer = held.ReceiverAnswer; + Deliveries.Enqueue(held); + using var acknowledgement = incomingLater ? Fixture.Storage.BlockAcknowledgement(journal) : null; + await sender.StageFromSourceAsync(receiver.GetGrainId(), id, original.Envelope.Payload, write: true); + if (acknowledgement is not null) + { + await acknowledgement.WaitUntilEnteredAsync(); + Assert.False(firstAnswer.IsCompleted); + Assert.False(retired.IsCompleted); + Assert.Empty(Deliveries.Answers); + Assert.Equal(writes + 1, Fixture.Storage.GetSuccessfulWriteCount(journal)); + acknowledgement.Release(); + } + await held.WaitUntilEnteredAsync(); + Assert.Equal(DeliveryStatus.Duplicate, (await firstAnswer).Status); + Assert.Equal(1, Deliveries.Executions); + Assert.Equal(1, Deliveries.ReceiverCalls); + Assert.Equal([DeliveryStatus.Duplicate], Deliveries.Answers); + Assert.Equal([incomingSentAt], Deliveries.SentTimes); + Assert.Empty(events.Retries); + Assert.Equal(0, (await Fixture.OnGrainAsync(sender, () => _cluster.GetOutboxState(sender).Attempt(id))).AttemptCount); + Assert.Equal(writes + (incomingLater ? 1 : 0), Fixture.Storage.GetSuccessfulWriteCount(journal)); + var inbox = Fixture.GetInbox(receiver); + Assert.Equal(expectedSentAt, await Fixture.OnGrainAsync(receiver, () => completed + ? InboxTestAccess.GetProcessedSentAt(inbox, id) + : InboxTestAccess.GetPendingMessages(inbox)[id].Message.SentAt)); + held.Release(); + Assert.Equal(OutboxMessageRetirementStatus.Delivered, (await retired).Retirement.Status); + await sender.WriteAsync(); + var state = await OnSinkAsync(sender, static grain => grain.GetState()); + Assert.Empty(state.Outgoing); + Assert.Equal(nameof(OutboxMessageRetirementStatus.Delivered), Assert.Single(state.Retired).Value); + Assert.Empty(await Fixture.OnGrainAsync(sender, () => _cluster.GetOutboxState(sender).AttemptKeys.ToArray())); + Assert.Empty(await Fixture.OnGrainAsync(sender, () => _cluster.GetOutboxState(sender).DeadLetterKeys.ToArray())); + Assert.Empty(events.Retries); + + handler.Release(); + await Fixture.WaitForInboxCountAsync(receiver, 0); + Assert.Equal(expectedSentAt, await Fixture.OnGrainAsync(receiver, () => InboxTestAccess.GetProcessedSentAt(inbox, id))); + // The earlier timestamp must not erase later history; a later one must extend history past the old boundary. + var oldBoundary = (incomingLater ? original.SentAt : incomingSentAt) + TimeSpan.FromDays(1); + _cluster.JobClock.Advance(oldBoundary - _cluster.JobClock.GetUtcNow()); + Fixture.Storage.RequestSnapshot(journal); + await receiver.RetryWriteStateAsync(); + await ReactivateAsync(receiver); + inbox = Fixture.GetInbox(receiver); + Assert.Equal(expectedSentAt, await Fixture.OnGrainAsync(receiver, () => InboxTestAccess.GetProcessedSentAt(inbox, id))); + var effect = Assert.Single((await receiver.GetSnapshotAsync()).Effects); + Assert.Equal("one-real-outbox-effect", effect.Value); + Assert.Equal(1, effect.Count); + Assert.Equal(1, Deliveries.Executions); + Assert.Equal(1, Deliveries.ReceiverCalls); + Assert.Empty(events.Retries); + } + + public enum RetirementFault { CallbackStaging, BeforeCommit, LostCommitAcknowledgement } + public enum TerminalOutcome { Delivered, Rejected, Expired } + + [Theory] + [InlineData(RetirementFault.CallbackStaging, TerminalOutcome.Delivered, false)] + [InlineData(RetirementFault.CallbackStaging, TerminalOutcome.Delivered, true)] + [InlineData(RetirementFault.CallbackStaging, TerminalOutcome.Rejected, false)] + [InlineData(RetirementFault.CallbackStaging, TerminalOutcome.Rejected, true)] + [InlineData(RetirementFault.CallbackStaging, TerminalOutcome.Expired, false)] + [InlineData(RetirementFault.CallbackStaging, TerminalOutcome.Expired, true)] + [InlineData(RetirementFault.BeforeCommit, TerminalOutcome.Delivered, false)] + [InlineData(RetirementFault.BeforeCommit, TerminalOutcome.Delivered, true)] + [InlineData(RetirementFault.BeforeCommit, TerminalOutcome.Rejected, false)] + [InlineData(RetirementFault.BeforeCommit, TerminalOutcome.Rejected, true)] + [InlineData(RetirementFault.BeforeCommit, TerminalOutcome.Expired, false)] + [InlineData(RetirementFault.BeforeCommit, TerminalOutcome.Expired, true)] + [InlineData(RetirementFault.LostCommitAcknowledgement, TerminalOutcome.Delivered, false)] + [InlineData(RetirementFault.LostCommitAcknowledgement, TerminalOutcome.Delivered, true)] + [InlineData(RetirementFault.LostCommitAcknowledgement, TerminalOutcome.Rejected, false)] + [InlineData(RetirementFault.LostCommitAcknowledgement, TerminalOutcome.Rejected, true)] + [InlineData(RetirementFault.LostCommitAcknowledgement, TerminalOutcome.Expired, false)] + [InlineData(RetirementFault.LostCommitAcknowledgement, TerminalOutcome.Expired, true)] + public async Task SourceRetirement_RecoveryCommitsOneOutcomeAndExecutesOneSelfInboxFollowUp( + RetirementFault fault, TerminalOutcome outcome, bool snapshot) + { + var sender = NewSinkGrain(); + var receiver = NewSinkGrain(); + _ = await sender.GetOutboxCountAsync(); + _ = await receiver.GetOutboxCountAsync(); + var started = _cluster.JobClock.GetUtcNow(); + var sourceProbe = _cluster.SourceSinkProbe.For(sender); + sourceProbe.FollowUpAt = started + TimeSpan.FromHours(2); + if (outcome == TerminalOutcome.Rejected) _cluster.SourceSinkProbe.For(receiver).Mode = SinkMode.DeadLetter; + if (outcome == TerminalOutcome.Expired) + { + // The receiver has durably advanced its admission floor past this send time, then its clock moves backward. + _cluster.JobClock.Advance(TimeSpan.FromDays(2)); + var stale = new InboxMessage + { + Envelope = new() { MessageId = NewSourceMessageId("advance-floor"), Payload = [], Subject = SourceSinkGrain.Subject }, + SentAt = _cluster.JobClock.GetUtcNow() - TimeSpan.FromDays(1) + }; + Assert.Equal(DeliveryStatus.Expired, + (await receiver.AsReference().DeliverAsync(stale, TestContext.Current.CancellationToken)).Status); + _cluster.JobClock.Advance(started - _cluster.JobClock.GetUtcNow()); + } + + var held = DeliveryFault.HoldAfterCall(); + Deliveries.Enqueue(held); + var id = NewSourceMessageId("atomic-retirement"); + var payload = RandomNumberGenerator.GetBytes(32); + await sender.StageFromSourceAsync(receiver.GetGrainId(), id, payload, write: true); + await held.WaitUntilEnteredAsync(); + var answer = await held.ReceiverAnswer; + Assert.Equal(outcome switch + { + TerminalOutcome.Delivered => DeliveryStatus.Accepted, + TerminalOutcome.Rejected => DeliveryStatus.DeadLettered, + _ => DeliveryStatus.Expired + }, answer.Status); + var journal = JournalId.FromGrainId(sender.GetGrainId()); + var oldContext = Fixture.GetGrainContext(sender); + if (snapshot) Fixture.Storage.RequestSnapshot(journal); + switch (fault) + { + case RetirementFault.CallbackStaging: sourceProbe.FailNextRetirements(1); break; + case RetirementFault.BeforeCommit: Fixture.Storage.FailWrite(journal); break; + case RetirementFault.LostCommitAcknowledgement: Fixture.Storage.FailAfterWrite(journal); break; + } + + held.Release(); + await oldContext.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + _ = await sender.GetOutboxCountAsync(); + await Fixture.WaitUntilAsync(sender, () => Sink(sender).GetTerminalCounts().ContainsKey(id.ToString()) + && !_cluster.GetOutboxState(sender).AttemptKeys.Any()); + await sender.WriteAsync(); + var expectedStatus = outcome switch + { + TerminalOutcome.Delivered => OutboxMessageRetirementStatus.Delivered, + TerminalOutcome.Rejected => OutboxMessageRetirementStatus.Rejected, + _ => OutboxMessageRetirementStatus.DeadLettered + }; + var state = await OnSinkAsync(sender, static grain => grain.GetState()); + Assert.Empty(state.Outgoing); + Assert.Equal(expectedStatus.ToString(), Assert.Single(state.Retired).Value); + Assert.Equal(new KeyValuePair(id.ToString(), 1), + Assert.Single(await OnSinkAsync(sender, static grain => grain.GetTerminalCounts()))); + Assert.Empty(await OnSinkAsync(sender, static grain => grain.GetFollowUpCounts())); + var pending = await Fixture.OnGrainAsync(sender, () => + InboxTestAccess.GetPendingMessages(Fixture.GetGrainContext(sender).ActivationServices.GetRequiredService()).Values.ToArray()); + var work = Assert.Single(pending); + Assert.Equal(SourceSinkGrain.FollowUpSubject, work.Message.Envelope.Subject); + Assert.Equal(id.ToString(), System.Text.Encoding.UTF8.GetString(work.Message.Envelope.Payload)); + Assert.Equal(sourceProbe.FollowUpAt, work.Message.Envelope.DeliverAt); + Assert.Equal(fault == RetirementFault.LostCommitAcknowledgement ? 1 : 2, sourceProbe.RetirementCallbacks); + var codes = await OnSinkAsync(sender, static grain => grain.GetRetirementCodes()); + if (outcome == TerminalOutcome.Rejected) Assert.Equal(SourceSinkGrain.RejectionCode, codes[id.ToString()]); + else Assert.Empty(codes); + if (outcome == TerminalOutcome.Expired) + { + Assert.Equal(0, _cluster.SourceSinkProbe.For(receiver).SinkOffers); + var letter = await Fixture.OnGrainAsync(sender, () => _cluster.GetOutboxState(sender).DeadLetter(id)); + Assert.Equal(id, letter.Message.Envelope.MessageId); + } + else + { + Assert.Equal(1, _cluster.SourceSinkProbe.For(receiver).SinkOffers); + Assert.Equal(outcome == TerminalOutcome.Delivered ? 1 : 0, _cluster.SourceSinkProbe.For(receiver).SinkAcceptances); + } + + // Crash again with durable pending work, then execute it through the recovered real inbox pump. + await _cluster.CrashAsync(sender); + Assert.Equal(0, await sender.GetOutboxCountAsync()); + await _cluster.AdvanceAndRunPumpAsync(sender, sourceProbe.FollowUpAt!.Value - _cluster.JobClock.GetUtcNow()); + await Fixture.WaitUntilAsync(sender, () => Sink(sender).GetFollowUpCounts().Count == 1); + await _cluster.CrashAsync(sender); + Assert.Equal(0, await sender.GetOutboxCountAsync()); + await Fixture.RunPumpAsync(sender); + Assert.Equal(new KeyValuePair(id.ToString(), 1), + Assert.Single(await OnSinkAsync(sender, static grain => grain.GetFollowUpCounts()))); + Assert.Equal(1, Assert.Single(await OnSinkAsync(sender, static grain => grain.GetTerminalCounts())).Value); + Assert.Equal(1, sourceProbe.Handled); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task SourceFailure_AtomicallyRetiresAndRecoversExecutableSelfInboxFollowUp(bool snapshot) + { + var sender = NewSinkGrain(); + var receiver = NewSinkGrain(); + var probe = _cluster.SourceSinkProbe.For(sender); + probe.FollowUpAt = _cluster.JobClock.GetUtcNow() + TimeSpan.FromHours(2); + var id = NewSourceMessageId("source-error-follow-up"); + await sender.StageFromSourceAsync(receiver.GetGrainId(), id, RandomNumberGenerator.GetBytes(32), write: false); + probe.Faults[id] = SourceFault.Throw; + if (snapshot) Fixture.Storage.RequestSnapshot(JournalId.FromGrainId(sender.GetGrainId())); + await sender.WriteAsync(); + await Fixture.WaitUntilAsync(sender, () => Sink(sender).GetTerminalCounts().Count == 1); + if (snapshot) Fixture.Storage.RequestSnapshot(JournalId.FromGrainId(sender.GetGrainId())); + await sender.WriteAsync(); + await _cluster.CrashAsync(sender); + Assert.Equal(0, await sender.GetOutboxCountAsync()); + var state = await OnSinkAsync(sender, static grain => grain.GetState()); + Assert.Empty(state.Outgoing); + Assert.Equal(nameof(OutboxMessageRetirementStatus.DeadLettered), state.Retired[id.ToString()]); + Assert.Contains(SourceSinkGrain.SourceFailure, + (await OnSinkAsync(sender, static grain => grain.GetRetirementReasons()))[id.ToString()], StringComparison.Ordinal); + Assert.Empty(await OnSinkAsync(sender, static grain => grain.GetRetirementCodes())); + Assert.Equal(1, Assert.Single(await OnSinkAsync(sender, static grain => grain.GetTerminalCounts())).Value); + Assert.Empty(await OnSinkAsync(sender, static grain => grain.GetFollowUpCounts())); + await _cluster.AdvanceAndRunPumpAsync(sender, probe.FollowUpAt.Value - _cluster.JobClock.GetUtcNow()); + await Fixture.WaitUntilAsync(sender, () => Sink(sender).GetFollowUpCounts().Count == 1); + Assert.Equal(new KeyValuePair(id.ToString(), 1), + Assert.Single(await OnSinkAsync(sender, static grain => grain.GetFollowUpCounts()))); + Assert.Equal(0, Deliveries.ReceiverCalls); + } + + private async Task WriteJournalAsync(IAddressable grain) + { + var write = await Fixture.OnGrainAsync(grain, () => + Fixture.GetGrainContext(grain).ActivationServices.GetRequiredService() + .WriteStateAsync(TestContext.Current.CancellationToken).AsTask()); + await write; + } + + private void ArmSenderWriteFailure(IMessagingTestGrain sender, bool ambiguous) + { + var journal = JournalId.FromGrainId(sender.GetGrainId()); + if (ambiguous) + { + Fixture.Storage.FailAfterWrite(journal); + } + else + { + Fixture.Storage.FailWrite(journal); + } + } + + private async Task WaitForFailedAttemptAsync(IAddressable sender, MessageId id, int attempts) + { + await _cluster.WaitForFailedAttemptsAsync(sender, id, attempts); + return await Fixture.OnGrainAsync(sender, () => _cluster.GetOutboxState(sender).Attempt(id)); + } + + private Task WaitForDrainedOutboxAsync(IMessagingTestGrain sender) => + Fixture.WaitUntilAsync(sender, () => Fixture.GetStagedOutput(sender).Length == 0); + + private async Task AssertProcessedOnceAsync(IMessagingTestGrain receiver, TestMessage message) + { + var processed = await Fixture.SnapshotProbe.WaitAsync( + receiver.GetGrainId(), + snapshot => snapshot.Effects.Any(effect => effect.LogicalId == message.LogicalId) && snapshot.InboxCount == 0); + var current = await receiver.GetSnapshotAsync(); + Assert.Equal(1, Assert.Single(current.Effects).Count); + Assert.Equal(message.LogicalId, current.Effects[0].LogicalId); + Assert.Equal(0, current.InboxCount); + Assert.Empty(current.InboxDeadLetters); + return processed; + } + + private ISourceSinkGrain NewSinkGrain() => Fixture.Client.GetGrain(Guid.NewGuid()); + + private SourceSinkGrain Sink(IAddressable grain) => Assert.IsType(Fixture.GetGrainContext(grain).GrainInstance); + + private Task OnSinkAsync(IAddressable grain, Func func) => + Fixture.OnGrainAsync(grain, () => func(Sink(grain))); + + private static MessageId NewSourceMessageId(string name) => MessageId.Create("fault-injection", name, Guid.NewGuid().ToString("N")); + + private static InboxMessage Schedule(InboxMessage message, DateTimeOffset deliverAt) => + message with { Envelope = message.Envelope with { DeliverAt = deliverAt } }; +} diff --git a/test/Orleans.Messaging.Tests/Functional/FutureSentAtTests.cs b/test/Orleans.Messaging.Tests/Functional/FutureSentAtTests.cs new file mode 100644 index 00000000000..671bc55530e --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/FutureSentAtTests.cs @@ -0,0 +1,505 @@ +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; +using Orleans.DurableJobs; +using Orleans.Journaling; +using Orleans.Messaging.Configuration; +using Orleans.Messaging.Diagnostics; +using Orleans.Messaging.Tests.Support; +using Orleans.Runtime; +using Orleans.TestingHost.Diagnostics; +using Xunit; + +namespace Orleans.Messaging.Tests.Functional; + +[Collection(MessagingClusterCollection.Name)] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class FutureSentAtTests : MessagingBehaviorTestBase +{ + private readonly IndependentJobClockFixture _cluster; + private static CancellationToken Token => TestContext.Current.CancellationToken; + + public FutureSentAtTests() : this(new IndependentJobClockFixture()) { } + + private FutureSentAtTests(IndependentJobClockFixture cluster) : base(cluster) => _cluster = cluster; + + [Theory] + [InlineData(0, true)] + [InlineData(1, false)] + public async Task UpperBoundary_AllowsEqualityAndRejectsTheNextTickWithoutStaging(long extraTicks, bool accepted) + { + var receiver = NewSink(); + var clock = new ManualJobClock(_cluster.JobClock.GetUtcNow()); + _cluster.Clocks.Set(receiver, clock); + Assert.Equal(0, await receiver.GetOutboxCountAsync()); + var before = await CaptureAsync(receiver); + var sentAt = clock.GetUtcNow() + Window(receiver) + TimeSpan.FromTicks(extraTicks); + var message = Message(sentAt); + if (accepted) + { + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, message)).Status); + Assert.Equal(before.Writes + 1, _cluster.Storage.GetSuccessfulWriteCount(Journal(receiver))); + var state = await OnSinkAsync(receiver, static grain => grain.GetState()); + Assert.Equal(message.Envelope.Payload, Assert.Single(state.Received).Value); + Assert.Equal(1, _cluster.SourceSinkProbe.For(receiver).SinkOffers); + Assert.Equal(1, _cluster.SourceSinkProbe.For(receiver).SinkAcceptances); + } + else + { + await AssertRejectedAsync(receiver, message); + await AssertUnchangedAsync(receiver, before); + Assert.Empty((await OnSinkAsync(receiver, static grain => grain.GetState())).Received); + Assert.Equal(0, _cluster.SourceSinkProbe.For(receiver).SinkOffers); + } + Assert.Equal(clock.GetUtcNow(), await ActualClockAsync(receiver)); + } + + [Theory] + [InlineData("pending")] + [InlineData("processed")] + [InlineData("rejected")] + public async Task KnownIdentity_RejectsNewerTimePastBoundBeforeRetentionExtension(string history) + { + var rejected = history == "rejected"; + IAddressable receiver = rejected ? NewSink() : NewGrain(); + var clock = new ManualJobClock(_cluster.JobClock.GetUtcNow()); + _cluster.Clocks.Set(receiver, clock); + const string route = "messages/known-future-rejection"; + using var handler = rejected ? null : _cluster.HandlerProbe.Arm(receiver.GetGrainId(), route); + var original = rejected ? Message(clock.GetUtcNow()) + : TestApplicationProtocol.CreateInbox(Sessions, GrainId.Create("future-test-sender", "known"), route, + NewMessage(96, "one-known-effect"), clock.GetUtcNow()); + if (rejected) _cluster.SourceSinkProbe.For(receiver).Mode = SinkMode.DeadLetter; + Assert.Equal(rejected ? DeliveryStatus.DeadLettered : DeliveryStatus.Accepted, + (await DeliverAsync(receiver, original)).Status); + if (handler is not null) + { + await handler.WaitUntilEnteredAsync(); + if (history == "processed") + { + handler.Release(); + await _cluster.WaitForInboxCountAsync((IMessagingTestGrain)receiver, 0); + } + await ((IMessagingTestGrain)receiver).RetryWriteStateAsync(); + } + else + { + _cluster.SourceSinkProbe.For(receiver).Mode = SinkMode.Accept; + } + var before = await CaptureAsync(receiver); + var invalid = original with { SentAt = clock.GetUtcNow() + Window(receiver) + TimeSpan.FromTicks(1) }; + await AssertRejectedAsync(receiver, invalid); + await AssertUnchangedAsync(receiver, before); + var inbox = Inbox(receiver); + Assert.Equal(original.SentAt, await _cluster.OnGrainAsync(receiver, () => history == "pending" + ? InboxTestAccess.GetPendingMessages(inbox)[original.Envelope.MessageId].Message.SentAt + : InboxTestAccess.GetProcessedSentAt(inbox, original.Envelope.MessageId))); + var known = await DeliverAsync(receiver, original); + Assert.Equal(rejected ? DeliveryStatus.DeadLettered : DeliveryStatus.Duplicate, known.Status); + if (rejected) + { + Assert.Equal(SourceSinkGrain.Rejection, known.Message); + Assert.Equal(SourceSinkGrain.RejectionCode, known.Code); + Assert.Equal(1, _cluster.SourceSinkProbe.For(receiver).SinkOffers); + Assert.Equal(0, _cluster.SourceSinkProbe.For(receiver).SinkAcceptances); + } + else + { + handler!.Release(); + await _cluster.WaitForInboxCountAsync((IMessagingTestGrain)receiver, 0); + var effect = Assert.Single((await ((IMessagingTestGrain)receiver).GetSnapshotAsync()).Effects); + Assert.Equal("one-known-effect", effect.Value); + Assert.Equal(1, effect.Count); + } + } + + [Theory] + [InlineData("pending", false)] + [InlineData("pending", true)] + [InlineData("processed", false)] + [InlineData("processed", true)] + [InlineData("rejected", false)] + [InlineData("rejected", true)] + public async Task KnownIdentity_EqualOrEarlierPastBoundKeepsRecordedAnswerWithoutWrites(string history, bool earlier) + { + var rejected = history == "rejected"; + IAddressable receiver = rejected ? NewSink() : NewGrain(); + var clock = new ManualJobClock(_cluster.JobClock.GetUtcNow()); + _cluster.Clocks.Set(receiver, clock); + const string route = "messages/known-answer-after-receiver-rollback"; + using var handler = rejected ? null : _cluster.HandlerProbe.Arm(receiver.GetGrainId(), route); + var original = rejected ? Message(clock.GetUtcNow()) + : TestApplicationProtocol.CreateInbox(Sessions, GrainId.Create("ordering-test-sender", "known"), route, + NewMessage(98, "known-despite-rollback"), clock.GetUtcNow()); + if (rejected) _cluster.SourceSinkProbe.For(receiver).Mode = SinkMode.DeadLetter; + Assert.Equal(rejected ? DeliveryStatus.DeadLettered : DeliveryStatus.Accepted, + (await DeliverAsync(receiver, original)).Status); + if (handler is not null) + { + await handler.WaitUntilEnteredAsync(); + if (history == "processed") + { + handler.Release(); + await _cluster.WaitForInboxCountAsync((IMessagingTestGrain)receiver, 0); + } + await ((IMessagingTestGrain)receiver).RetryWriteStateAsync(); + } + else + { + _cluster.SourceSinkProbe.For(receiver).Mode = SinkMode.Accept; + } + var window = Window(receiver); + var before = await CaptureAsync(receiver); + clock.Advance(-2 * window); + var receiverTime = await ActualClockAsync(receiver); + Assert.Equal(original.SentAt - 2 * window, receiverTime); + var incoming = original with { SentAt = earlier ? original.SentAt - window / 2 : original.SentAt }; + Assert.True(incoming.SentAt > receiverTime + window); + Assert.True(incoming.SentAt <= original.SentAt); + var known = await DeliverAsync(receiver, incoming); + Assert.Equal(rejected ? DeliveryStatus.DeadLettered : DeliveryStatus.Duplicate, known.Status); + Assert.Equal(rejected ? SourceSinkGrain.RejectionCode : null, known.Code); + if (rejected) Assert.Equal(SourceSinkGrain.Rejection, known.Message); + await AssertUnchangedAsync(receiver, before); + var inbox = Inbox(receiver); + Assert.Equal(original.SentAt, await _cluster.OnGrainAsync(receiver, () => history == "pending" + ? InboxTestAccess.GetPendingMessages(inbox)[original.Envelope.MessageId].Message.SentAt + : InboxTestAccess.GetProcessedSentAt(inbox, original.Envelope.MessageId))); + Assert.Equal(receiverTime, await ActualClockAsync(receiver)); + if (rejected) + { + Assert.Equal(1, _cluster.SourceSinkProbe.For(receiver).SinkOffers); + Assert.Equal(0, _cluster.SourceSinkProbe.For(receiver).SinkAcceptances); + Assert.Empty((await OnSinkAsync(receiver, static grain => grain.GetState())).Received); + } + else + { + handler!.Release(); + await _cluster.WaitForInboxCountAsync((IMessagingTestGrain)receiver, 0); + var effect = Assert.Single((await ((IMessagingTestGrain)receiver).GetSnapshotAsync()).Effects); + Assert.Equal("known-despite-rollback", effect.Value); + Assert.Equal(1, effect.Count); + } + } + + [Theory] + [InlineData(-1)] + [InlineData(0)] + public async Task UnknownPastBound_AtOrBelowDurableFloorExpiresBeforeFutureRejection(long floorTicks) + { + var receiver = NewSink(); + var started = _cluster.JobClock.GetUtcNow(); + var clock = new ManualJobClock(started); + _cluster.Clocks.Set(receiver, clock); + Assert.Equal(0, await receiver.GetOutboxCountAsync()); + var window = Window(receiver); + clock.Advance(3 * window); + var floor = clock.GetUtcNow() - window; + Assert.Equal(DeliveryStatus.Expired, (await DeliverAsync(receiver, Message(floor))).Status); + Assert.Equal(floor, await _cluster.OnGrainAsync(receiver, () => InboxTestAccess.DurableAdmissionFloor(Inbox(receiver)))); + clock.Advance(started - clock.GetUtcNow()); + var receiverTime = await ActualClockAsync(receiver); + Assert.Equal(started, receiverTime); + Assert.True(floor > receiverTime + window); + var before = await CaptureAsync(receiver); + var incoming = Message(floor.AddTicks(floorTicks)); + Assert.True(incoming.SentAt > receiverTime + window); + Assert.True(incoming.SentAt <= floor); + var expired = await DeliverAsync(receiver, incoming); + Assert.Equal(DeliveryStatus.Expired, expired.Status); + Assert.Null(expired.Code); + await AssertUnchangedAsync(receiver, before); + Assert.Empty((await OnSinkAsync(receiver, static grain => grain.GetState())).Received); + Assert.Equal(0, _cluster.SourceSinkProbe.For(receiver).SinkOffers); + await AssertRejectedAsync(receiver, Message(floor.AddTicks(1))); + await AssertUnchangedAsync(receiver, before); + Assert.Equal(receiverTime, await ActualClockAsync(receiver)); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task MaxValue_RejectsAtNormalTimeAndAcceptsClampedUpperBoundEquality(bool nearMaximum) + { + var receiver = NewSink(); + var now = nearMaximum + ? DateTimeOffset.MaxValue - TimeSpan.FromHours(12) + : _cluster.JobClock.GetUtcNow(); + var clock = new ManualJobClock(now); + _cluster.Clocks.Set(receiver, clock); + Assert.Equal(0, await receiver.GetOutboxCountAsync()); + var before = await CaptureAsync(receiver); + var message = Message(DateTimeOffset.MaxValue); + if (nearMaximum) + { + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, message)).Status); + Assert.Equal(DateTimeOffset.MaxValue, await _cluster.OnGrainAsync(receiver, () => + InboxTestAccess.GetProcessedSentAt(Inbox(receiver), message.Envelope.MessageId))); + Assert.Equal(message.Envelope.Payload, Assert.Single((await OnSinkAsync(receiver, static grain => grain.GetState())).Received).Value); + Assert.Equal(1, _cluster.SourceSinkProbe.For(receiver).SinkOffers); + } + else + { + await AssertRejectedAsync(receiver, message); + await AssertUnchangedAsync(receiver, before); + Assert.Equal(0, _cluster.SourceSinkProbe.For(receiver).SinkOffers); + } + Assert.Equal(now, await ActualClockAsync(receiver)); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task RealOutbox_IndependentClockSkewDeliversWithinWindowOrRetriesGuardUntilOriginalDeadline(bool tooFar) + { + var sender = NewSink(); + var receiver = NewSink(); + var receiverTime = _cluster.JobClock.GetUtcNow(); + var receiverClock = new ManualJobClock(receiverTime); + var senderClock = new ManualJobClock(receiverTime + (tooFar ? TimeSpan.FromDays(2) : TimeSpan.FromHours(12))); + _cluster.Clocks.Set(sender, senderClock); + _cluster.Clocks.Set(receiver, receiverClock); + Assert.Equal(0, await sender.GetOutboxCountAsync()); + Assert.Equal(0, await receiver.GetOutboxCountAsync()); + var before = await CaptureAsync(receiver); + var sentAt = senderClock.GetUtcNow(); + Assert.Equal(tooFar, sentAt > receiverTime + Window(receiver)); + await AssertClocksAsync(); + var id = TestApplicationProtocol.NewMessageId(); + byte[] payload = [97, 98, 99]; + using var events = new DeliveryEventObserver(sender.GetGrainId(), receiver.GetGrainId(), id, faults: 0); + using var diagnostics = new DiagnosticEventCollector(MessagingEvents.ListenerName); + var retired = events.Retired.WaitAsync(TimeSpan.FromSeconds(30), Token); + var firstRetry = tooFar ? WaitForRetryAsync(1) : null; + await sender.StageFromSourceAsync(receiver.GetGrainId(), id, payload, write: true); + await _cluster.RunPumpAsync(sender); + if (!tooFar) + { + Assert.Equal(OutboxMessageRetirementStatus.Delivered, (await retired).Retirement.Status); + await sender.WriteAsync(); + Assert.Equal([DeliveryStatus.Accepted], _cluster.Deliveries.Answers); + Assert.Equal([sentAt], _cluster.Deliveries.SentTimes); + Assert.Empty(_cluster.Deliveries.ReceiverFailures); + Assert.Empty(events.Retries); + Assert.Equal(1, _cluster.Deliveries.Executions); + Assert.Equal(1, _cluster.Deliveries.ReceiverCalls); + Assert.Equal(payload, (await OnSinkAsync(receiver, static grain => grain.GetState())).Received[id.ToString()]); + Assert.Equal(1, _cluster.SourceSinkProbe.For(receiver).SinkOffers); + Assert.Equal(1, _cluster.SourceSinkProbe.For(receiver).SinkAcceptances); + } + else + { + var deadline = sentAt + FaultInjectionClusterFixture.OutboxRetryTimeout; + OutboxMessageState? state = null; + for (var attempt = 1; attempt <= 3; attempt++) + { + var retry = attempt == 1 ? firstRetry! : WaitForRetryAsync(attempt); + if (attempt > 1) + { + senderClock.Advance(state!.NextAttemptAt!.Value - senderClock.GetUtcNow()); + await _cluster.RunPumpAsync(sender); + } + await retry; + await sender.WriteAsync(); + state = await _cluster.OnGrainAsync(sender, () => _cluster.GetOutboxState(sender).Attempt(id)); + Assert.Equal(attempt, state.AttemptCount); + Assert.Equal(sentAt, state.SentAt); + var next = sentAt + FaultInjectionClusterFixture.RetryInterval * ((1 << attempt) - 1); + Assert.Equal(next < deadline ? next : deadline, state.NextAttemptAt); + Assert.Contains(nameof(ArgumentOutOfRangeException), state.LastError, StringComparison.Ordinal); + Assert.Equal(attempt, _cluster.Deliveries.ReceiverCalls); + Assert.Equal(attempt, _cluster.Deliveries.ReceiverFailures.Count); + Assert.All(_cluster.Deliveries.ReceiverFailures, error => AssertFutureException(Assert.IsType(error), sentAt)); + Assert.All(_cluster.Deliveries.SentTimes, value => Assert.Equal(sentAt, value)); + Assert.Empty(_cluster.Deliveries.Answers); + Assert.False(retired.IsCompleted); + await AssertUnchangedAsync(receiver, before); + await AssertClocksAsync(); + } + senderClock.Advance(deadline - senderClock.GetUtcNow()); + Assert.Equal(deadline, senderClock.GetUtcNow()); + await _cluster.RunPumpAsync(sender); + var terminal = (await retired).Retirement; + Assert.Equal(OutboxMessageRetirementStatus.DeadLettered, terminal.Status); + Assert.Null(terminal.Code); + await sender.WriteAsync(); + Assert.Empty(await OnSinkAsync(sender, static grain => grain.GetRetirementCodes())); + var letter = await _cluster.OnGrainAsync(sender, () => _cluster.GetOutboxState(sender).DeadLetter(id)); + Assert.Contains("retry timeout elapsed", letter.Reason, StringComparison.Ordinal); + Assert.Equal(3, letter.AttemptCount); + Assert.Equal(3, _cluster.Deliveries.Executions); + Assert.Equal(3, _cluster.Deliveries.ReceiverCalls); + Assert.Equal(3, events.Retries.Count); + await AssertUnchangedAsync(receiver, before); + Assert.Empty((await OnSinkAsync(receiver, static grain => grain.GetState())).Received); + Assert.Equal(0, _cluster.SourceSinkProbe.For(receiver).SinkOffers); + Assert.Equal(0, _cluster.SourceSinkProbe.For(receiver).SinkAcceptances); + await AssertClocksAsync(); + } + var final = await OnSinkAsync(sender, static grain => grain.GetState()); + Assert.Empty(final.Outgoing); + Assert.Equal(tooFar ? nameof(OutboxMessageRetirementStatus.DeadLettered) : nameof(OutboxMessageRetirementStatus.Delivered), + Assert.Single(final.Retired).Value); + Assert.Equal(0, await sender.GetOutboxCountAsync()); + + Task WaitForRetryAsync(int attempt) => diagnostics.WaitForEventAsync( + nameof(MessagingEvents.OutboxRetryScheduled), + entry => entry.Payload is MessagingEvents.OutboxRetryScheduled retry + && retry.GrainContext.GrainId == sender.GetGrainId() && retry.MessageId == id && retry.AttemptCount == attempt, + TimeSpan.FromSeconds(30), Token); + + async Task AssertClocksAsync() + { + Assert.Equal(senderClock.GetUtcNow(), await ActualClockAsync(sender)); + Assert.Equal(receiverTime, await ActualClockAsync(receiver)); + Assert.Equal(receiverTime, receiverClock.GetUtcNow()); + Assert.Equal(receiverTime, _cluster.JobClock.GetUtcNow()); + } + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task RealOutbox_LostReplyThenReceiverRollbackReturnsDuplicateAndRetiresDelivered(bool realSink) + { + var sender = NewSink(); + IAddressable receiver = realSink ? NewSink() : NewGrain(); + var started = _cluster.JobClock.GetUtcNow(); + var senderClock = new ManualJobClock(started); + var receiverClock = new ManualJobClock(started); + _cluster.Clocks.Set(sender, senderClock); + _cluster.Clocks.Set(receiver, receiverClock); + Assert.Equal(0, await sender.GetOutboxCountAsync()); + if (realSink) Assert.Equal(0, await ((ISourceSinkGrain)receiver).GetOutboxCountAsync()); + else _ = await ((IMessagingTestGrain)receiver).GetSnapshotAsync(); + var id = TestApplicationProtocol.NewMessageId(); + const string route = "messages/lost-reply-receiver-rollback"; + var payload = realSink ? new byte[] { 100, 101 } + : TestApplicationProtocol.Encode(Sessions, new TestApplicationMessage(route, new TestMessage(id, 100, "once-after-receiver-rollback"))); + if (!realSink) _cluster.SourceSinkProbe.For(sender).SubjectOverride = route; + using var events = new DeliveryEventObserver(sender.GetGrainId(), receiver.GetGrainId(), id, faults: 0); + var retry = events.RetryScheduled.WaitAsync(TimeSpan.FromSeconds(30), Token); + var retired = events.Retired.WaitAsync(TimeSpan.FromSeconds(30), Token); + var lost = DeliveryFault.LoseResponse(); + var held = DeliveryFault.HoldAfterCall(); + var retryAnswer = held.ReceiverAnswer; + _cluster.Deliveries.Enqueue(lost, held); + await sender.StageFromSourceAsync(receiver.GetGrainId(), id, payload, write: true); + await _cluster.RunPumpAsync(sender); + Assert.Equal(DeliveryStatus.Accepted, (await lost.ReceiverAnswer).Status); + await retry; + await sender.WriteAsync(); + if (!realSink) + { + await _cluster.WaitForEffectCountAsync((IMessagingTestGrain)receiver, 1); + await _cluster.WaitForInboxCountAsync((IMessagingTestGrain)receiver, 0); + await ((IMessagingTestGrain)receiver).RetryWriteStateAsync(); + } + var before = await CaptureAsync(receiver); + var window = Window(receiver); + receiverClock.Advance(-2 * window - TimeSpan.FromHours(1)); + Assert.True(started > receiverClock.GetUtcNow() + window); + var state = await _cluster.OnGrainAsync(sender, () => _cluster.GetOutboxState(sender).Attempt(id)); + Assert.Equal(1, state.AttemptCount); + Assert.Equal(started, state.SentAt); + senderClock.Advance(state.NextAttemptAt!.Value - senderClock.GetUtcNow()); + await _cluster.RunPumpAsync(sender); + await held.WaitUntilEnteredAsync(); + Assert.Equal(DeliveryStatus.Duplicate, (await retryAnswer).Status); + Assert.Equal([DeliveryStatus.Accepted, DeliveryStatus.Duplicate], _cluster.Deliveries.Answers); + Assert.Equal([started, started], _cluster.Deliveries.SentTimes); + Assert.Empty(_cluster.Deliveries.ReceiverFailures); + Assert.Single(events.Retries); + Assert.False(retired.IsCompleted); + await AssertUnchangedAsync(receiver, before); + Assert.Equal(started, await _cluster.OnGrainAsync(receiver, () => InboxTestAccess.GetProcessedSentAt(Inbox(receiver), id))); + held.Release(); + Assert.Equal(OutboxMessageRetirementStatus.Delivered, (await retired).Retirement.Status); + await sender.WriteAsync(); + var final = await OnSinkAsync(sender, static grain => grain.GetState()); + Assert.Empty(final.Outgoing); + Assert.Equal(nameof(OutboxMessageRetirementStatus.Delivered), Assert.Single(final.Retired).Value); + Assert.Equal(0, await sender.GetOutboxCountAsync()); + Assert.Empty(await _cluster.OnGrainAsync(sender, () => _cluster.GetOutboxState(sender).DeadLetterKeys.ToArray())); + Assert.Equal(2, _cluster.Deliveries.Executions); + Assert.Equal(2, _cluster.Deliveries.ReceiverCalls); + Assert.Single(events.Retries); + Assert.Equal(started + FaultInjectionClusterFixture.RetryInterval, await ActualClockAsync(sender)); + Assert.Equal(started - 2 * window - TimeSpan.FromHours(1), await ActualClockAsync(receiver)); + Assert.Equal(started, _cluster.JobClock.GetUtcNow()); + if (realSink) + { + Assert.Equal(payload, (await OnSinkAsync(receiver, static grain => grain.GetState())).Received[id.ToString()]); + Assert.Equal(1, _cluster.SourceSinkProbe.For(receiver).SinkOffers); + Assert.Equal(1, _cluster.SourceSinkProbe.For(receiver).SinkAcceptances); + } + else + { + var effect = Assert.Single((await ((IMessagingTestGrain)receiver).GetSnapshotAsync()).Effects); + Assert.Equal("once-after-receiver-rollback", effect.Value); + Assert.Equal(1, effect.Count); + } + } + + private ISourceSinkGrain NewSink() => _cluster.Client.GetGrain(Guid.NewGuid()); + private Inbox Inbox(IAddressable receiver) => _cluster.GetGrainContext(receiver).ActivationServices.GetRequiredService(); + private static JournalId Journal(IAddressable receiver) => JournalId.FromGrainId(receiver.GetGrainId()); + private TimeSpan Window(IAddressable receiver) => _cluster.GetGrainContext(receiver).ActivationServices + .GetRequiredService>().Value.DeduplicationWindow; + private Task OnSinkAsync(IAddressable grain, Func action) => + _cluster.OnGrainAsync(grain, () => action(Assert.IsType(_cluster.GetGrainContext(grain).GrainInstance))); + private Task ActualClockAsync(IAddressable grain) => _cluster.OnGrainAsync(grain, () => + _cluster.GetGrainContext(grain).ActivationServices.GetRequiredKeyedService(DurableJobTimeProviderNames.DurableJobs).GetUtcNow()); + private static async Task DeliverAsync(IAddressable receiver, InboxMessage message) => + await receiver.AsReference().DeliverAsync(message, Token); + private static InboxMessage Message(DateTimeOffset sentAt) => new() + { + Envelope = new() { MessageId = TestApplicationProtocol.NewMessageId(), Payload = [1, 2, 3], Subject = SourceSinkGrain.Subject }, + SentAt = sentAt + }; + + private static void AssertFutureException(ArgumentOutOfRangeException error, DateTimeOffset sentAt) + { + Assert.Equal("message", error.ParamName); + Assert.Equal(sentAt, Assert.IsType(error.ActualValue)); + Assert.Contains("SentAt must not be later than the receiver's clock plus DeduplicationWindow.", error.Message, StringComparison.Ordinal); + } + + private async Task AssertRejectedAsync(IAddressable receiver, InboxMessage message) + { + // Inspect the actual guard exception before external-client exception serialization normalizes its metadata. + var direct = await Assert.ThrowsAsync(async () => + { + var delivery = await _cluster.OnGrainAsync(receiver, () => Inbox(receiver).DeliverAsync(message, Token).AsTask()); + await delivery; + }); + AssertFutureException(direct, message.SentAt); + var remote = await Assert.ThrowsAsync(() => DeliverAsync(receiver, message)); + Assert.Contains("SentAt must not be later than the receiver's clock plus DeduplicationWindow.", remote.Message, StringComparison.Ordinal); + } + + private Task CaptureAsync(IAddressable receiver) => _cluster.OnGrainAsync(receiver, () => + { + var inbox = Inbox(receiver); + return new ReceiverState(_cluster.Storage.GetSuccessfulWriteCount(Journal(receiver)), + _cluster.GetGrainContext(receiver).ActivationServices.GetRequiredService().PendingWriteByteCount, + inbox.Count, InboxTestAccess.GetProcessedKeys(inbox), InboxTestAccess.AdmissionFloor(inbox), + InboxTestAccess.DurableAdmissionFloor(inbox), _cluster.SourceSinkProbe.For(receiver).SinkOffers); + }); + + private async Task AssertUnchangedAsync(IAddressable receiver, ReceiverState expected) + { + var actual = await CaptureAsync(receiver); + Assert.Equal(expected.Writes, actual.Writes); + Assert.Equal(expected.StagedBytes, actual.StagedBytes); + Assert.Equal(expected.PendingCount, actual.PendingCount); + Assert.True(expected.Processed.SetEquals(actual.Processed)); + Assert.Equal(expected.Floor, actual.Floor); + Assert.Equal(expected.DurableFloor, actual.DurableFloor); + Assert.Equal(expected.SinkOffers, actual.SinkOffers); + } + + private sealed record ReceiverState(int Writes, long StagedBytes, int PendingCount, HashSet Processed, + DateTimeOffset Floor, DateTimeOffset DurableFloor, int SinkOffers); +} diff --git a/test/Orleans.Messaging.Tests/Functional/InboxAcceptanceBehaviorTests.cs b/test/Orleans.Messaging.Tests/Functional/InboxAcceptanceBehaviorTests.cs new file mode 100644 index 00000000000..7b0970dfa0d --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/InboxAcceptanceBehaviorTests.cs @@ -0,0 +1,291 @@ +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; +using Orleans.DurableJobs; +using Orleans.Journaling; +using Orleans.Messaging.Configuration; +using Orleans.Messaging.Tests.Support; +using Orleans.Runtime; +using Orleans.Serialization.Session; +using Xunit; + +namespace Orleans.Messaging.Tests.Functional; + +[Collection(MessagingClusterCollection.Name)] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class InboxAcceptanceBehaviorTests : MessagingBehaviorTestBase +{ + [Fact] + public async Task Deliver_AcceptedOnlyAfterInboxIsDurableAndWakeupIsScheduled() + { + var receiver = NewGrain(); + _ = await receiver.GetSnapshotAsync(); + var journalId = JournalId.FromGrainId(receiver.GetGrainId()); + var barrier = Fixture.Storage.BlockWrite(journalId); + var envelope = CreateEnvelope(receiver, NewMessage(1, "durability")); + + var delivery = DeliverAsync(receiver, envelope); + await barrier.WaitUntilEnteredAsync(); + + Assert.False(delivery.IsCompleted); + var staged = Fixture.GetSnapshot(receiver); + Assert.Equal(1, staged.InboxCount); + Assert.Empty(staged.Effects); + Assert.Single(Fixture.JobManagerProbe.GetScheduledJobs(MessagingPump.JobName, receiver.GetGrainId())); + + barrier.Release(); + var result = await delivery; + Assert.Equal(DeliveryStatus.Accepted, result.Status); + EndpointSnapshot completed; + try + { + completed = await Fixture.WaitForEffectCountAsync(receiver, 1); + } + catch (TimeoutException exception) + { + var snapshot = await receiver.GetSnapshotAsync(); + throw new TimeoutException( + $"Accepted message did not drain. Inbox={snapshot.InboxCount}, effects={snapshot.Effects.Count}, deadLetters={snapshot.InboxDeadLetters.Count}.", + exception); + } + Assert.Single(completed.Effects); + Assert.Equal(0, completed.InboxCount); + Assert.Equal(2, Fixture.Storage.GetSuccessfulWriteCount(journalId)); + } + + [Fact] + public async Task ConcurrentDuplicateDeliveries_ConvergeToOneEffectWithinRetention() + { + var receiver = NewGrain(); + using var barrier = Fixture.HandlerProbe.Arm(receiver.GetGrainId(), "messages/blocked-duplicate"); + var envelope = CreateEnvelope(receiver, NewMessage(11, "duplicate"), "messages/blocked-duplicate"); + + var first = await DeliverAsync(receiver, envelope); + await barrier.WaitUntilEnteredAsync(); + var second = DeliverAsync(receiver, envelope); + + // The pending acceptance is durable, so the duplicate is answered without waiting for the handler. + Assert.Equal(DeliveryStatus.Accepted, first.Status); + Assert.Equal(DeliveryStatus.Duplicate, (await second).Status); + Assert.Equal(1, Fixture.GetSnapshot(receiver).InboxCount); + + barrier.Release(); + var state = await Fixture.WaitForEffectCountAsync(receiver, 1); + Assert.Equal(1, Assert.Single(state.Effects).Count); + Assert.Equal(1, state.MaxConcurrentHandlers); + } + + [Fact] + public async Task Delivery_IsAdmittedWhileANonReentrantRequestIsRunning() + { + var receiver = NewGrain(); + _ = await receiver.GetSnapshotAsync(); + var journal = JournalId.FromGrainId(receiver.GetGrainId()); + var inbox = Fixture.GetInbox(receiver); + + // The request holds the grain while its journal write waits. + await receiver.StageEffectAsync(new MessageEffect(MessageId.Create("request", "effect"), 1, 0, "request")); + using var write = Fixture.Storage.BlockWrite(journal); + var request = receiver.RetryWriteStateAsync(); + await write.WaitUntilEnteredAsync(); + + var envelope = CreateEnvelope(receiver, NewMessage(14, "interleaved")); + var delivery = DeliverAsync(receiver, envelope); + await Fixture.WaitUntilAsync(receiver, () => inbox.Count == 1); + Assert.False(request.IsCompleted); + + write.Release(); + await request; + Assert.Equal(DeliveryStatus.Accepted, (await delivery).Status); + Assert.Equal("interleaved", Assert.Single((await Fixture.WaitForEffectCountAsync(receiver, 2)).Effects, static effect => effect.Value == "interleaved").Value); + } + [Fact] + public async Task DuplicateOfCompletedMessage_WaitsUntilCompletionIsDurable() + { + var receiver = NewGrain(); + _ = await receiver.GetSnapshotAsync(); + var journalId = JournalId.FromGrainId(receiver.GetGrainId()); + var envelope = CreateEnvelope(receiver, NewMessage(12, "completion-durability")); + + // The first write accepts the message and the second records its completion. + var barrier = Fixture.Storage.BlockWrite(journalId, matchingWrite: 2); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + await barrier.WaitUntilEnteredAsync(); + + // Whenever the duplicate is answered, the completion it depends on must already be durable. + var duplicate = DeliverAsync(receiver, envelope).ContinueWith( + task => (task.Result.Status, Writes: Fixture.Storage.GetSuccessfulWriteCount(journalId)), + TaskScheduler.Default); + + barrier.Release(); + var (status, writes) = await duplicate; + Assert.Equal(DeliveryStatus.Duplicate, status); + Assert.True(writes >= 2, $"The duplicate was answered after {writes} durable writes, before its completion was durable."); + Assert.Equal(1, Assert.Single((await receiver.GetSnapshotAsync()).Effects).Count); + } + + [Fact] + public async Task DuplicateAfterReactivationWithinRetention_RemainsEffectivelyOnce() + { + var receiver = NewGrain(); + var envelope = CreateEnvelope(receiver, NewMessage(13, "reactivation")); + + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + var before = await Fixture.WaitForEffectCountAsync(receiver, 1); + await receiver.RequestDeactivationAsync(); + var after = await receiver.GetSnapshotAsync(); + + Assert.NotEqual(before.ActivationId, after.ActivationId); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(receiver, envelope)).Status); + Assert.Equal(1, Assert.Single((await receiver.GetSnapshotAsync()).Effects).Count); + } + + [Fact] + public async Task KnownSentAt_OlderAndEqualResendsDoNotWriteOrShortenRetention() + { + var receiver = NewGrain(); + var message = CreateEnvelope(receiver, NewMessage(31, "same-command")); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, message)).Status); + await Fixture.WaitForInboxCountAsync(receiver, 0); + await receiver.RetryWriteStateAsync(); + var journal = JournalId.FromGrainId(receiver.GetGrainId()); + var writes = Fixture.Storage.GetSuccessfulWriteCount(journal); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(receiver, message)).Status); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(receiver, message with { SentAt = message.SentAt.AddTicks(-1) })).Status); + Assert.Equal(writes, Fixture.Storage.GetSuccessfulWriteCount(journal)); + var inbox = Fixture.GetInbox(receiver); + Assert.Equal(message.SentAt, await Fixture.OnGrainAsync(receiver, () => InboxTestAccess.GetProcessedSentAt(inbox, message.Envelope.MessageId))); + Assert.Equal(1, Assert.Single((await receiver.GetSnapshotAsync()).Effects).Count); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task NewerSentAt_PendingAndProcessedResendsWaitForRetentionDurability(bool completed) + { + var receiver = NewGrain(); + const string route = "messages/sent-time-retention"; + using var handler = Fixture.HandlerProbe.Arm(receiver.GetGrainId(), route); + var message = CreateEnvelope(receiver, NewMessage(32, "retained-command"), route); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, message)).Status); + await handler.WaitUntilEnteredAsync(); + if (completed) + { + handler.Release(); + await Fixture.WaitForInboxCountAsync(receiver, 0); + } + + await receiver.RetryWriteStateAsync(); + var beforeResends = Fixture.Storage.GetSuccessfulWriteCount(JournalId.FromGrainId(receiver.GetGrainId())); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(receiver, message)).Status); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(receiver, message with { SentAt = message.SentAt.AddTicks(-1) })).Status); + Assert.Equal(beforeResends, Fixture.Storage.GetSuccessfulWriteCount(JournalId.FromGrainId(receiver.GetGrainId()))); + Fixture.Clock.Advance(TimeSpan.FromMinutes(1)); + var newer = message with { SentAt = Fixture.JobNow }; + var inbox = Fixture.GetInbox(receiver); + var journal = JournalId.FromGrainId(receiver.GetGrainId()); + var writes = Fixture.Storage.GetSuccessfulWriteCount(journal); + using var acknowledgement = Fixture.Storage.BlockAcknowledgement(journal); + var updating = DeliverAsync(receiver, newer); + await acknowledgement.WaitUntilEnteredAsync(); + Assert.False(updating.IsCompleted); + Assert.Equal(writes + 1, Fixture.Storage.GetSuccessfulWriteCount(journal)); + Assert.Equal(newer.SentAt, await Fixture.OnGrainAsync(receiver, () => completed + ? InboxTestAccess.GetProcessedSentAt(inbox, message.Envelope.MessageId) + : InboxTestAccess.GetPendingMessages(inbox)[message.Envelope.MessageId].Message.SentAt)); + var olderReply = await Fixture.OnGrainAsync(receiver, () => inbox.DeliverAsync(message, TestContext.Current.CancellationToken).AsTask()); + Assert.False(olderReply.IsCompleted); + acknowledgement.Release(); + Assert.Equal(DeliveryStatus.Duplicate, (await updating).Status); + Assert.Equal(DeliveryStatus.Duplicate, (await olderReply).Status); + Assert.Equal(writes + 1, Fixture.Storage.GetSuccessfulWriteCount(journal)); + + handler.Release(); + await Fixture.WaitForInboxCountAsync(receiver, 0); + var state = await receiver.GetSnapshotAsync(); + Assert.Equal("retained-command", Assert.Single(state.Effects).Value); + Assert.Equal(1, Assert.Single(state.Effects).Count); + Assert.Equal(1, state.ProcessedMessageCount); + Assert.Equal(newer.SentAt, await Fixture.OnGrainAsync(receiver, () => InboxTestAccess.GetProcessedSentAt(inbox, message.Envelope.MessageId))); + } + + [Fact] + public async Task SameMessageIdFromDistinctSenders_DeduplicatesOneReceiverCommand() + { + var receiver = NewGrain(); + var original = CreateEnvelope(receiver, NewMessage(14, "sender-scoped")); + var other = TestApplicationProtocol.With(original, Sessions, + sender: GrainId.Create("other-sender", "same-message-id")); + + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, original)).Status); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(receiver, other)).Status); + var completed = await Fixture.WaitForEffectCountAsync(receiver, 1); + Assert.Equal(1, Assert.Single(completed.Effects).Count); + Assert.Equal(1, completed.ProcessedMessageCount); + + var previous = Fixture.GetGrainContext(receiver); + await receiver.RequestDeactivationAsync(); + await previous.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(receiver, original)).Status); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(receiver, other)).Status); + var recovered = await receiver.GetSnapshotAsync(); + Assert.NotEqual(completed.ActivationId, recovered.ActivationId); + Assert.Equal(1, Assert.Single(recovered.Effects).Count); + Assert.Equal(1, recovered.ProcessedMessageCount); + } + + [Fact] + public async Task ReorderedDistinctAndDuplicateMessages_ConvergeByApplicationSequence() + { + var receiver = NewGrain(); + var messages = new[] + { + NewMessage(3, "third"), + NewMessage(1, "first"), + NewMessage(2, "second"), + }; + + foreach (var message in messages) + { + var envelope = CreateEnvelope(receiver, message); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(receiver, envelope)).Status); + } + + var state = await Fixture.WaitForEffectCountAsync(receiver, 3); + Assert.Equal([1, 2, 3], state.Effects.Select(static effect => effect.Sequence)); + Assert.All(state.Effects, static effect => Assert.Equal(1, effect.Count)); + } + + [Fact] + public async Task ConcurrentDelivery_WaitsWhileHandlersRemainSequential() + { + var receiver = NewGrain(); + using var barrier = Fixture.HandlerProbe.Arm(receiver.GetGrainId(), "messages/sequential"); + var first = CreateEnvelope(receiver, NewMessage(21, "first"), "messages/sequential"); + var second = CreateEnvelope(receiver, NewMessage(22, "second"), "messages/sequential"); + + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, first)).Status); + await WaitForBarrierAsync(receiver, barrier); + var secondDelivery = DeliverAsync(receiver, second); + Assert.False(secondDelivery.IsCompleted); + + barrier.Release(); + Assert.Equal(DeliveryStatus.Accepted, (await secondDelivery).Status); + EndpointSnapshot state; + try + { + state = await Fixture.WaitForEffectCountAsync(receiver, 2); + } + catch (TimeoutException exception) + { + var snapshot = await receiver.GetSnapshotAsync(); + throw new TimeoutException( + $"Second message did not complete. Inbox={snapshot.InboxCount}, effects={snapshot.Effects.Count}, deadLetters={snapshot.InboxDeadLetters.Count}.", + exception); + } + Assert.Equal(1, state.MaxConcurrentHandlers); + Assert.Equal([21, 22], state.Effects.Select(static effect => effect.Sequence)); + } +} diff --git a/test/Orleans.Messaging.Tests/Functional/InboxAdmissionRecoveryTests.cs b/test/Orleans.Messaging.Tests/Functional/InboxAdmissionRecoveryTests.cs new file mode 100644 index 00000000000..843a6c5ab9f --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/InboxAdmissionRecoveryTests.cs @@ -0,0 +1,183 @@ +using Microsoft.Extensions.DependencyInjection; +using System.Reflection; +using System.Text.Json; +using Orleans.Journaling; +using Orleans.Messaging.Tests.Support; +using Orleans.Runtime; +using Orleans.Serialization; +using Xunit; + +namespace Orleans.Messaging.Tests.Functional; + +/// Runs persisted admission boundaries through both supported journal encodings. +[Collection(MessagingClusterCollection.Name)] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class InboxAdmissionRecoveryTests +{ + [Theory] + [InlineData("orleans-binary", false)] + [InlineData("orleans-binary", true)] + [InlineData("json", false)] + [InlineData("json", true)] + public async Task SentAtAndAdmissionFloor_JournalAndSnapshotRecoveryPreserveFencing(string format, bool snapshot) + { + await using var fixture = new FaultInjectionClusterFixture(format); + await fixture.InitializeAsync(); + var receiver = fixture.Client.GetGrain(Guid.NewGuid()); + var started = fixture.JobClock.GetUtcNow(); + var original = Message("old", started); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, original)).Status); + await fixture.CrashAsync(receiver); + Assert.Equal(0, await receiver.GetOutboxCountAsync()); + var inbox = fixture.GetGrainContext(receiver).ActivationServices.GetRequiredService(); + Assert.Equal(started, await fixture.OnGrainAsync(receiver, () => + InboxTestAccess.GetProcessedSentAt(inbox, original.Envelope.MessageId))); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(receiver, original)).Status); + + fixture.JobClock.Advance(TimeSpan.FromDays(2)); + var floor = fixture.JobClock.GetUtcNow() - TimeSpan.FromDays(1); + var retained = Message("recent", fixture.JobClock.GetUtcNow()); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, retained)).Status); + if (snapshot) fixture.Storage.RequestSnapshot(JournalId.FromGrainId(receiver.GetGrainId())); + // A journal command (or its snapshot) durably advances the same floor. + Assert.Equal(DeliveryStatus.Expired, + (await DeliverAsync(receiver, Message("expired-by-floor", floor))).Status); + await fixture.CrashAsync(receiver); + fixture.JobClock.Advance(started - fixture.JobClock.GetUtcNow()); + Assert.Equal(0, await receiver.GetOutboxCountAsync()); + inbox = fixture.GetGrainContext(receiver).ActivationServices.GetRequiredService(); + Assert.Equal(floor, await fixture.OnGrainAsync(receiver, () => InboxTestAccess.AdmissionFloor(inbox))); + Assert.DoesNotContain(original.Envelope.MessageId, + await fixture.OnGrainAsync(receiver, () => InboxTestAccess.GetProcessedKeys(inbox))); + Assert.Equal(DeliveryStatus.Expired, (await DeliverAsync(receiver, original)).Status); + // The recent record was staged two days later; restore a legitimate receiver clock before replaying it. + fixture.JobClock.Advance(retained.SentAt - fixture.JobClock.GetUtcNow()); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(receiver, retained)).Status); + Assert.Equal(retained.SentAt, await fixture.OnGrainAsync(receiver, () => + InboxTestAccess.GetProcessedSentAt(inbox, retained.Envelope.MessageId))); + Assert.Equal(2, fixture.SourceSinkProbe.For(receiver).SinkOffers); + Assert.Equal(2, fixture.SourceSinkProbe.For(receiver).SinkAcceptances); + } + + [Theory] + [InlineData("orleans-binary", "pending")] + [InlineData("orleans-binary", "processed")] + [InlineData("orleans-binary", "dead-letter")] + [InlineData("json", "pending")] + [InlineData("json", "processed")] + [InlineData("json", "dead-letter")] + public async Task Recovery_RejectsMissingAdmittedSentAtBeforeCompaction(string format, string kind) + { + await using var fixture = new FaultInjectionClusterFixture(format); + await fixture.InitializeAsync(); + var receiver = fixture.Client.GetGrain(Guid.NewGuid()); + _ = await receiver.GetOutboxCountAsync(); + var message = Message("missing-send-time", DateTimeOffset.MinValue); + var inbox = fixture.GetGrainContext(receiver).ActivationServices.GetRequiredService(); + var snapshot = new InboxSnapshot + { + AdmissionFloor = fixture.JobClock.GetUtcNow(), + Messages = kind == "pending" ? [new() { Message = message, State = new() }] : [], + Processed = kind == "processed" ? [new() { MessageId = message.Envelope.MessageId, SentAt = DateTimeOffset.MinValue }] : [], + DeadLetters = kind == "dead-letter" ? [new() + { + Message = message, DeadLetteredAt = fixture.JobClock.GetUtcNow(), Reason = "missing send time fixture", AttemptCount = 1 + }] : [] + }; + InboxSnapshot recovered; + if (format == "json") + { + recovered = JsonSerializer.Deserialize(JsonSerializer.Serialize(snapshot, InboxJsonContext.Default.InboxSnapshot), + InboxJsonContext.Default.InboxSnapshot)!; + } + else + { + var serializer = fixture.Client.ServiceProvider.GetRequiredService>(); + recovered = serializer.Deserialize(serializer.SerializeToArray(snapshot))!; + } + await fixture.OnGrainAsync(receiver, () => + { + ((IDurableValueCommandHandler)inbox).ApplySet(new() + { + Kind = InboxCommandKind.Snapshot, + Snapshot = recovered + }); + var error = Assert.Throws(() => ((IStateMachine)inbox).OnRecoveryCompleted()); + Assert.Equal("A recovered inbox entry requires its admitted send time.", error.Message); + }); + Assert.Equal(0, fixture.SourceSinkProbe.For(receiver).SinkOffers); + } + + [Theory] + [InlineData("orleans-binary")] + [InlineData("json")] + public async Task ReplayPendingExtension_RequiresItsPendingEntry(string format) + { + await using var fixture = new FaultInjectionClusterFixture(format); + await fixture.InitializeAsync(); + var receiver = fixture.Client.GetGrain(Guid.NewGuid()); + Assert.Equal(0, await receiver.GetOutboxCountAsync()); + var command = new InboxCommand + { + Kind = InboxCommandKind.ExtendSentAt, + MessageId = TestApplicationProtocol.NewMessageId(), + SentAt = fixture.JobClock.GetUtcNow() + }; + InboxCommand recovered; + if (format == "json") + { + recovered = JsonSerializer.Deserialize(JsonSerializer.Serialize(command, InboxJsonContext.Default.InboxCommand), + InboxJsonContext.Default.InboxCommand)!; + } + else + { + var serializer = fixture.Client.ServiceProvider.GetRequiredService>(); + recovered = serializer.Deserialize(serializer.SerializeToArray(command))!; + } + var inbox = fixture.GetGrainContext(receiver).ActivationServices.GetRequiredService(); + await fixture.OnGrainAsync(receiver, () => + { + var error = Assert.Throws(() => + ((IDurableValueCommandHandler)inbox).ApplySet(recovered)); + Assert.Equal("An ExtendSentAt command requires its pending entry.", error.Message); + Assert.Empty(InboxTestAccess.GetPendingMessages(inbox)); + Assert.Empty(InboxTestAccess.GetProcessedKeys(inbox)); + }); + Assert.Equal(0, fixture.SourceSinkProbe.For(receiver).SinkOffers); + } + + [Fact] + public void PersistedAdmissionFields_UseAdditiveStableFieldIds() + { + Assert.Equal(8, (int)InboxCommandKind.ExtendSentAt); + Assert.Equal(3U, typeof(InboxProcessedEntry).GetProperty(nameof(InboxProcessedEntry.SentAt))!.GetCustomAttribute()!.Id); + Assert.Equal(3U, typeof(OutboxMessageState).GetProperty(nameof(OutboxMessageState.SentAt))!.GetCustomAttribute()!.Id); + Assert.Null(typeof(OutboxMessageState).GetProperty("ExpirationTime")); + Assert.Equal(3U, typeof(InboxSnapshot).GetProperty(nameof(InboxSnapshot.AdmissionFloor))!.GetCustomAttribute()!.Id); + Assert.Equal(9U, typeof(InboxCommand).GetProperty(nameof(InboxCommand.SentAt))!.GetCustomAttribute()!.Id); + Assert.Equal(10U, typeof(InboxCommand).GetProperty(nameof(InboxCommand.AdmissionFloor))!.GetCustomAttribute()!.Id); + Assert.Null(typeof(InboxProcessedEntry).GetProperty(nameof(InboxProcessedEntry.Sequence))!.GetCustomAttribute()); + Assert.Null(typeof(InboxProcessedEntry).GetProperty("ProcessedAt")); + Assert.Null(typeof(InboxCommand).GetProperty("ProcessedAt")); + Assert.DoesNotContain(typeof(InboxProcessedEntry).GetMembers().SelectMany(static member => member.GetCustomAttributes()), + static attribute => attribute.Id == 1); + Assert.DoesNotContain(typeof(InboxCommand).GetMembers().SelectMany(static member => member.GetCustomAttributes()), + static attribute => attribute.Id == 4); + } + + private static InboxMessage Message(string name, DateTimeOffset sentAt) => new() + { + SentAt = sentAt, + Envelope = new() + { + MessageId = MessageId.Create("admission-recovery", name, Guid.NewGuid().ToString("N")), + Payload = [1, 2, 3], + Subject = SourceSinkGrain.Subject + } + }; + + private static async Task DeliverAsync(ISourceSinkGrain receiver, InboxMessage message) => + await receiver.AsReference().DeliverAsync(message, TestContext.Current.CancellationToken); +} diff --git a/test/Orleans.Messaging.Tests/Functional/InboxAdmissionTests.cs b/test/Orleans.Messaging.Tests/Functional/InboxAdmissionTests.cs new file mode 100644 index 00000000000..ed14cb9c708 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/InboxAdmissionTests.cs @@ -0,0 +1,295 @@ +using Microsoft.Extensions.DependencyInjection; +using Orleans.DurableJobs; +using Orleans.Journaling; +using Orleans.Messaging.Tests.Support; +using Orleans.Runtime; +using Orleans.Runtime.Diagnostics; +using Orleans.Serialization.Session; +using Orleans.TestingHost.Diagnostics; +using Xunit; + +namespace Orleans.Messaging.Tests.Functional; + +[Collection(MessagingClusterCollection.Name)] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class InboxAdmissionTests : MessagingBehaviorTestBase +{ + [Fact] + public async Task ExplicitPreparation_AllowsPriorStateWriteThenCapturesAppliedCohortAtomically() + { + using var attempt = await PrepareAttemptAsync("atomic-admission"); + var state = attempt.Grain.GetSnapshotForTest(); + Assert.Equal(1, state.InboxCount); + Assert.Equal(0, state.ProcessedMessageCount); + Assert.Empty(state.Effects); + Assert.Empty(attempt.Outbox); + await OnTurnAsync(attempt.Context, () => + attempt.Context.ActivationServices.GetRequiredKeyedService>("inbox").Value = "previous-state"); + await attempt.Manager.WriteStateAsync(TestContext.Current.CancellationToken); + var priorCapture = attempt.Grain.Captures[^1]; + Assert.Empty(priorCapture.Effects); + Assert.Equal(1, priorCapture.InboxCount); + Assert.Equal(0, priorCapture.ProcessedMessageCount); + Assert.Equal(0, priorCapture.OutboxCount); + Assert.False(attempt.Grain.ApplyAttempted.Task.IsCompleted); + var writes = Fixture.Storage.GetSuccessfulWriteCount(attempt.JournalId); + using var storage = Fixture.Storage.BlockWrite(attempt.JournalId); + attempt.Preparation.Release(); + await storage.WaitUntilEnteredAsync(); + var queued = attempt.Manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + var coalesced = attempt.Manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + Assert.False(queued.IsCompleted); + Assert.False(coalesced.IsCompleted); + Assert.Equal(writes, Fixture.Storage.GetSuccessfulWriteCount(attempt.JournalId)); + + var captured = attempt.Grain.GetSnapshotForTest(); + Assert.Equal(0, captured.InboxCount); + Assert.Equal(1, captured.ProcessedMessageCount); + Assert.Equal(1, Assert.Single(captured.Effects).Count); + Assert.Single(attempt.Grain.OutputCaptures[^1]); + Assert.False(queued.IsCompleted); + storage.Release(); + await Task.WhenAll(queued, coalesced); + await attempt.Receiver.RequestDeactivationAsync(); + var recovered = await attempt.Receiver.GetSnapshotAsync(); + Assert.NotEqual(state.ActivationId, recovered.ActivationId); + Assert.Equal(1, Assert.Single(recovered.Effects).Count); + Assert.Equal(1, recovered.ProcessedMessageCount); + Assert.Equal(1, recovered.OutboxCount); + Assert.Equal(0, recovered.InboxCount); + } + + [Fact] + public async Task HandlerStorageFailure_DeactivatesAndFailsQueuedWritesAfterPriorAcknowledgement() + { + using var attempt = await PrepareAttemptAsync("fault-ordering"); + IMessagingExtension extension = attempt.Context.ActivationServices.GetRequiredService(); + await OnTurnAsync(attempt.Context, () => + { + attempt.Context.ActivationServices.GetRequiredKeyedService>("inbox").Value = "prior-captured"; + }); + using var storage = Fixture.Storage.BlockWrite(attempt.JournalId); + var preceding = attempt.Manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + await storage.WaitUntilEnteredAsync(); + Fixture.Storage.FailWrite(attempt.JournalId); + var queued = attempt.Manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + var writes = Fixture.Storage.GetSuccessfulWriteCount(attempt.JournalId); + attempt.Preparation.Release(); + await attempt.Grain.ApplyAttempted.Task.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + await OnTurnAsync(attempt.Context, () => + { + Assert.Equal(1, attempt.Outbox.SendCalls); + Assert.Single(attempt.Grain.GetSnapshotForTest().Effects); + }); + Assert.False(queued.IsCompleted); + storage.Release(); + await preceding; + + var failure = await Assert.ThrowsAsync(() => queued); + Assert.Same(failure, await attempt.Grain.DeactivationFailure.Task); + await attempt.Context.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + // A failed pump stops the retired activation's inbox; later callers are rerouted to a fresh activation. + await Assert.ThrowsAsync(async () => + await extension.DeliverAsync(attempt.Envelope, TestContext.Current.CancellationToken)); + Assert.Equal(writes + 1, Fixture.Storage.GetSuccessfulWriteCount(attempt.JournalId)); + var failed = attempt.Grain.GetSnapshotForTest(); + Assert.Single(failed.Effects); + Assert.Equal(0, failed.InboxCount); + Assert.Equal(1, failed.ProcessedMessageCount); + _ = await attempt.Receiver.GetSnapshotAsync(); + var recovered = await Fixture.WaitForEffectCountAsync(attempt.Receiver, 1); + Assert.NotEqual(failed.ActivationId, recovered.ActivationId); + Assert.Equal(1, Assert.Single(recovered.Effects).Count); + Assert.Equal(1, recovered.ProcessedMessageCount); + } + + [Fact] + public async Task LateOutgoingIntent_AfterCapture_WaitsForNextCapture() + { + var receiver = NewGrain(); + _ = await receiver.GetSnapshotAsync(); + var context = Fixture.GetGrainContext(receiver); + var outbox = (JournaledTestOutbox)context.ActivationServices.GetRequiredService(); + var grain = Assert.IsType(context.GrainInstance); + var durable = context.ActivationServices.GetRequiredKeyedService>("test-handler-output"); + var sessions = Fixture.Client.ServiceProvider.GetRequiredService(); + var first = TestApplicationProtocol.Create(sessions, receiver.GetGrainId(), receiver.GetGrainId(), "messages/output", 1); + var late = TestApplicationProtocol.Create(sessions, receiver.GetGrainId(), receiver.GetGrainId(), "messages/output", 2); + await receiver.StageOutputAsync(first); + var storage = Fixture.Storage.BlockWrite(JournalId.FromGrainId(receiver.GetGrainId())); + var write = Fixture.WriteStateAsync(receiver).AsTask(); + await storage.WaitUntilEnteredAsync(); + await OnTurnTaskAsync(context, async () => + { + outbox.Send(late); + }); + Assert.Equal(new[] { first.Envelope.MessageId }, grain.OutputCaptures[^1]); + Assert.Equal(2, outbox.Count); + storage.Release(); + await write; + Assert.Equal(new[] { first.Envelope.MessageId }, grain.OutputCaptures[^1]); + await receiver.RetryWriteStateAsync(); + Assert.Equal(new[] { first.Envelope.MessageId, late.Envelope.MessageId }.OrderBy(static key => key.ToString(), StringComparer.Ordinal), + grain.OutputCaptures[^1].OrderBy(static key => key.ToString(), StringComparer.Ordinal)); + Assert.Equal(2, durable.Count); + await receiver.RequestDeactivationAsync(); + Assert.Equal(2, (await receiver.GetSnapshotAsync()).OutboxCount); + } + + [Fact] + public async Task AcceptanceAfterCapture_IsAcknowledgedByLaterOperation() + { + var receiver = NewGrain(); + var seed = CreateEnvelope(receiver, NewMessage(110, "seed"), "messages/cutoff"); + _ = await receiver.GetSnapshotAsync(); + var context = Fixture.GetGrainContext(receiver); + + // The pump runs alongside grain calls, so holding the seed's handler keeps it pending in the inbox. + using var handler = Fixture.HandlerProbe.Arm(receiver.GetGrainId(), "messages/cutoff"); + IMessagingExtension extension = context.ActivationServices.GetRequiredService(); + Task initial = null!; + await OnTurnAsync(context, () => initial = extension.DeliverAsync(seed, TestContext.Current.CancellationToken).AsTask()); + Assert.Equal(DeliveryStatus.Accepted, (await initial).Status); + await handler.WaitUntilEnteredAsync(); + var grain = Assert.IsType(context.GrainInstance); + var storage = Fixture.Storage.BlockWrite(JournalId.FromGrainId(receiver.GetGrainId())); + grain.ClearCaptures(); + await OnTurnAsync(context, () => context.ActivationServices.GetRequiredKeyedService>("inbox").Value = "capture-cutoff"); + var preceding = Fixture.WriteStateAsync(receiver).AsTask(); + await storage.WaitUntilEnteredAsync(); + var incoming = CreateEnvelope(receiver, NewMessage(111, "late-acceptance"), "messages/cutoff"); + Task delivery = null!; + await OnTurnAsync(context, () => delivery = extension.DeliverAsync(incoming).AsTask()); + Assert.False(delivery.IsCompleted); + Assert.Equal(2, grain.GetSnapshotForTest().InboxCount); + storage.Release(); + await preceding; + Assert.Equal(DeliveryStatus.Accepted, (await delivery).Status); + Assert.Equal(1, grain.Captures[0].InboxCount); + Assert.Equal(2, grain.Captures[1].InboxCount); + Assert.Equal(2, grain.GetSnapshotForTest().InboxCount); + handler.Release(); + Assert.Equal(2, (await Fixture.WaitForEffectCountAsync(receiver, 2)).ProcessedMessageCount); + } + + [Fact] + public async Task CallerWaitCancellation_DoesNotWithdrawCapturedAcceptance() + { + var receiver = NewGrain(); + _ = await receiver.GetSnapshotAsync(); + var journal = JournalId.FromGrainId(receiver.GetGrainId()); + var storage = Fixture.Storage.BlockWrite(journal); + var envelope = CreateEnvelope(receiver, NewMessage(112, "cancel-wait")); + using var cancellation = new CancellationTokenSource(); + var delivery = DeliverWithCancellationAsync(receiver, envelope, cancellation.Token); + await storage.WaitUntilEnteredAsync(); + cancellation.Cancel(); + await Assert.ThrowsAnyAsync(() => delivery); + storage.Release(); + var completed = await Fixture.WaitForEffectCountAsync(receiver, 1); + Assert.Equal(1, completed.ProcessedMessageCount); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(receiver, envelope)).Status); + } + + [Fact] + public async Task MessagingPrimaryStates_AreTheRegisteredScopedDataInstances() + { + var receiver = NewGrain(); + _ = await receiver.GetSnapshotAsync(); + var services = Fixture.GetGrainContext(receiver).ActivationServices; + var manager = services.GetRequiredService(); + Assert.True(manager.TryGetStateMachine(MessagingStateNames.Inbox, out var inbox)); + Assert.Same(services.GetRequiredService(), inbox); + Assert.True(manager.TryGetStateMachine("test-handler-output", out var output)); + Assert.Same(((JournaledTestOutbox)services.GetRequiredService()).StoredMessages, output); + Assert.Same(services.GetRequiredKeyedService>("test-handler-output"), output); + } + + [Fact] + public async Task AcceptanceWriteFailure_DeactivatesAndFreshScopeRetries() + { + var receiver = NewGrain(); + var before = await receiver.GetSnapshotAsync(); + var context = Fixture.GetGrainContext(receiver); + var grain = Assert.IsType(context.GrainInstance); + var journal = JournalId.FromGrainId(receiver.GetGrainId()); + var writes = Fixture.Storage.GetSuccessfulWriteCount(journal); + Fixture.Storage.FailWrite(journal); + var envelope = CreateEnvelope(receiver, NewMessage(114, "acceptance-write-failure")); + var failure = await Assert.ThrowsAsync(() => DeliverAsync(receiver, envelope)); + Assert.Contains("Injected journal write failure", failure.Message, StringComparison.Ordinal); + await context.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + var rejected = grain.GetSnapshotForTest(); + Assert.Equal(before.ActivationId, rejected.ActivationId); + Assert.Equal(1, rejected.InboxCount); + Assert.Empty(rejected.Effects); + Assert.Equal(writes, Fixture.Storage.GetSuccessfulWriteCount(journal)); + var deactivation = await grain.DeactivationFailure.Task.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + Assert.IsType(deactivation); + Assert.Equal(failure.Message, deactivation.Message); + Assert.NotEmpty(Fixture.JobManagerProbe.GetScheduledJobs(MessagingPump.JobName, receiver.GetGrainId())); + await context.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + var recovered = await receiver.GetSnapshotAsync(); + Assert.NotEqual(rejected.ActivationId, recovered.ActivationId); + Assert.Equal(0, recovered.InboxCount); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + Assert.Equal(1, Assert.Single((await Fixture.WaitForEffectCountAsync(receiver, 1)).Effects).Count); + } + + private async Task PrepareAttemptAsync(string routeSuffix) + { + var receiver = NewGrain(); + _ = await receiver.GetSnapshotAsync(); + var context = Fixture.GetGrainContext(receiver); + var route = "messages/" + routeSuffix; + var handler = Fixture.HandlerProbe.Arm(receiver.GetGrainId(), route); + var envelope = CreateEnvelope(receiver, NewMessage(109, routeSuffix) with { ForwardTo = GrainId.Create("output", routeSuffix) }, route); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + await handler.WaitUntilEnteredAsync(); + var outbox = (JournaledTestOutbox)context.ActivationServices.GetRequiredService(); + var preparation = Fixture.HandlerProbe.Arm(receiver.GetGrainId(), route + "/application-preparation"); + handler.Release(); + await preparation.WaitUntilEnteredAsync(); + return new(receiver, context, outbox, handler, preparation, envelope); + } + + private static Task OnTurnAsync(IGrainContext context, Action action) + { + var completion = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + context.Scheduler.QueueAction(() => + { + try { action(); completion.SetResult(); } + catch (Exception exception) { completion.SetException(exception); } + }); + return completion.Task; + } + + private static Task OnTurnTaskAsync(IGrainContext context, Func action) + { + var started = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + context.Scheduler.QueueAction(() => + { + try { started.SetResult(action()); } + catch (Exception exception) { started.SetException(exception); } + }); + return started.Task.Unwrap(); + } + + private sealed record Attempt(IMessagingTestGrain Receiver, IGrainContext Context, JournaledTestOutbox Outbox, + HandlerProbe.Barrier Handler, HandlerProbe.Barrier Preparation, InboxMessage Envelope) : IDisposable + { + public MessagingTestGrain Grain { get; } = Assert.IsType(Context.GrainInstance); + public IJournaledStateManager Manager => Context.ActivationServices.GetRequiredService(); + public JournalId JournalId => JournalId.FromGrainId(Receiver.GetGrainId()); + public void Dispose() { Preparation.Dispose(); Handler.Dispose(); } + } + + private sealed class CallbackContext(DurableJob job) : IJobRunContext + { + public DurableJob Job { get; } = job; + public string RunId { get; } = Guid.NewGuid().ToString("N"); + public int DequeueCount => 1; + } +} diff --git a/test/Orleans.Messaging.Tests/Functional/InboxCapacityBehaviorTests.cs b/test/Orleans.Messaging.Tests/Functional/InboxCapacityBehaviorTests.cs new file mode 100644 index 00000000000..489b82eec88 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/InboxCapacityBehaviorTests.cs @@ -0,0 +1,82 @@ +using Microsoft.Extensions.DependencyInjection; +using Orleans.Messaging.Tests.Support; +using Orleans.Runtime; +using Orleans.Serialization.Session; +using Xunit; + +namespace Orleans.Messaging.Tests.Functional; + +[CollectionDefinition(Name, DisableParallelization = true)] +public sealed class InboxCapacityCollection : ICollectionFixture +{ + public const string Name = "Messaging inbox capacity"; +} + +[Collection(InboxCapacityCollection.Name)] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class InboxCapacityBehaviorTests(InboxCapacityClusterFixture fixture) +{ + [Fact] + public async Task InboxAtCapacity_BackpressuresWithoutPersistenceAndRecoversWhenCapacityFrees() + { + var receiver = fixture.Client.GetGrain(Guid.NewGuid()); + var sessions = fixture.Client.ServiceProvider.GetRequiredService(); + var sender = GrainId.Create("capacity-test-sender", Guid.NewGuid().ToString("N")); + var poison = TestApplicationProtocol.CreateInbox(sessions, sender, "messages/capacity", new TestMessage(TestApplicationProtocol.NewMessageId(), 31, "poison", ThrowDuringPreparation: true), fixture.JobNow); + var rejected = TestApplicationProtocol.CreateInbox(sessions, sender, "messages/capacity", new TestMessage(TestApplicationProtocol.NewMessageId(), 32, "accepted-after-capacity"), fixture.JobNow); + const string processedInstrument = "orleans-messaging-inbox-messages-processed"; + var firstRetryAcknowledged = fixture.Metrics.WaitForCountAsync(processedInstrument, 1, "retry"); + + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, poison)).Status); + var full = await fixture.WaitForInboxCountAsync(receiver, 1); + Assert.Empty(full.Effects); + Assert.Equal(DeliveryStatus.Backpressured, (await DeliverAsync(receiver, rejected)).Status); + Assert.Equal(1, (await receiver.GetSnapshotAsync()).InboxCount); + + await firstRetryAcknowledged; + Assert.Equal(1, fixture.Metrics.GetCount(processedInstrument, "retry")); + var inbox = fixture.GetInbox(receiver); + var state = await fixture.OnGrainAsync(receiver, () => InboxTestAccess.GetMessageState(inbox, poison.Envelope.MessageId)); + Assert.Equal(1, state.AttemptCount); + var nextAttempt = Assert.IsType(state.NextAttemptAt); + Assert.Equal(fixture.Clock.GetUtcNow() + TimeSpan.FromHours(1), nextAttempt); + fixture.Clock.Advance(TimeSpan.FromHours(2)); + Assert.True(nextAttempt <= fixture.Clock.GetUtcNow()); + var previous = fixture.GetGrainContext(receiver); + await receiver.RequestDeactivationAsync(); + await previous.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + _ = await receiver.GetSnapshotAsync(); + Assert.NotSame(previous, fixture.GetGrainContext(receiver)); + var current = fixture.GetGrainContext(receiver); + var currentGrain = Assert.IsType(current.GrainInstance); + try + { + await fixture.SnapshotProbe.WaitAsync(receiver.GetGrainId(), + snapshot => snapshot.ActivationId == currentGrain.GetSnapshotForTest().ActivationId + && snapshot.InboxDeadLetters.Count == 1); + } + catch (TimeoutException exception) + { + var snapshot = await receiver.GetSnapshotAsync(); + throw new TimeoutException( + $"Recovered retry did not complete. Activation={snapshot.ActivationId}, inbox={snapshot.InboxCount}, " + + $"processed={snapshot.ProcessedMessageCount}, deadLetters={snapshot.InboxDeadLetters.Count}, " + + $"nextAttempt={nextAttempt:O}, now={fixture.Clock.GetUtcNow():O}, " + + $"failureCompleted={currentGrain.DeactivationFailure.Task.IsCompleted}.", exception); + } + + Assert.Equal(DeliveryStatus.Expired, (await DeliverAsync(receiver, rejected)).Status); + rejected = rejected with { SentAt = fixture.JobNow }; + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, rejected)).Status); + var recovered = await fixture.WaitForEffectCountAsync(receiver, 1); + Assert.Equal("accepted-after-capacity", Assert.Single(recovered.Effects).Value); + Assert.Equal(2, Assert.Single(recovered.InboxDeadLetters).AttemptCount); + } + + private static async Task DeliverAsync( + IMessagingTestGrain receiver, + InboxMessage envelope) => + await receiver.AsReference().DeliverAsync(envelope); +} diff --git a/test/Orleans.Messaging.Tests/Functional/InboxEnvelopeValidationTests.cs b/test/Orleans.Messaging.Tests/Functional/InboxEnvelopeValidationTests.cs new file mode 100644 index 00000000000..fd9410b2c5c --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/InboxEnvelopeValidationTests.cs @@ -0,0 +1,73 @@ +using Microsoft.Extensions.DependencyInjection; +using Orleans.Journaling; +using Orleans.Messaging.Tests.Support; +using Orleans.Runtime; +using Xunit; + +namespace Orleans.Messaging.Tests.Functional; + +[Collection(MessagingClusterCollection.Name)] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class InboxEnvelopeValidationTests : MessagingBehaviorTestBase +{ + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task DefaultMessage_RejectsBeforeAcceptanceOrDuplicate(bool existingKey) + { + var receiver = NewGrain(); + _ = await receiver.GetSnapshotAsync(); + // External deserialization can materialize invalid default fields before admission. + var malformed = new InboxMessage { Envelope = default, SentAt = Fixture.JobNow }; + var context = Fixture.GetGrainContext(receiver); + if (existingKey) + { + var processedInbox = Fixture.GetInbox(receiver); + await OnTurnAsync(context, () => InboxTestAccess.AddProcessed(processedInbox, malformed.Envelope.MessageId, malformed.SentAt)); + } + var journal = JournalId.FromGrainId(receiver.GetGrainId()); + var writes = Fixture.Storage.GetSuccessfulWriteCount(journal); + + var exception = await Assert.ThrowsAsync(() => DeliverAsync(receiver, malformed)); + + IMessagingExtension extension = Fixture.GetInbox(receiver); + await OnTurnAsync(context, () => + { + var direct = extension.DeliverAsync(malformed, TestContext.Current.CancellationToken); + Assert.True(direct.IsCompleted); + Assert.Throws(() => direct.GetAwaiter().GetResult()); + }); + Assert.Contains("message", exception.Message, StringComparison.OrdinalIgnoreCase); + var after = await receiver.GetSnapshotAsync(); + Assert.Equal(0, after.InboxCount); + Assert.Empty(after.Effects); + Assert.Equal(existingKey ? 1 : 0, after.ProcessedMessageCount); + Assert.Equal(writes, Fixture.Storage.GetSuccessfulWriteCount(journal)); + Assert.Equal(0, Fixture.JobManagerProbe.GetAttemptCount(MessagingPump.JobName, receiver.GetGrainId())); + } + + [Fact] + public async Task DirectEnvelopeWithExplicitValidIdentity_PreservesDeliveryAndDedupe() + { + var receiver = NewGrain(); + var template = CreateEnvelope(receiver, NewMessage(187, "explicit-identity")); + var envelope = TestApplicationProtocol.With(template, Sessions, + messageId: MessageId.Parse("tenant/test/command/explicit", null)); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + Assert.Equal(1, Assert.Single((await Fixture.WaitForEffectCountAsync(receiver, 1)).Effects).Count); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(receiver, envelope)).Status); + } + + private static Task OnTurnAsync(IGrainContext context, Action action) + { + var done = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + context.Scheduler.QueueAction(() => + { + try { action(); done.SetResult(); } + catch (Exception exception) { done.SetException(exception); } + }); + return done.Task; + } +} diff --git a/test/Orleans.Messaging.Tests/Functional/InboxHandlerCompletionTests.cs b/test/Orleans.Messaging.Tests/Functional/InboxHandlerCompletionTests.cs new file mode 100644 index 00000000000..ec744f5be10 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/InboxHandlerCompletionTests.cs @@ -0,0 +1,773 @@ +using Microsoft.Extensions.DependencyInjection; +using Orleans.Journaling; +using Orleans.Messaging.Configuration; +using Orleans.Messaging.Tests.Support; +using Orleans.Runtime; +using Orleans.Serialization.Session; +using Xunit; + +namespace Orleans.Messaging.Tests.Functional; + +[Collection(MessagingClusterCollection.Name)] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class InboxHandlerCompletionTests : MessagingBehaviorTestBase +{ + [Fact] + public async Task DeadLetter_StagesReasonPayloadAndDedupeBeforeReturnAndPersistsAfterAcknowledgement() + { + var rig = await CreateAsync(); + using var handler = rig.Handler; + var calls = 0; + handler.Body = (self, _) => + { + calls++; + self.Context.Fail("unsupported input"); + AssertCompletedState(rig, self.Context.Message); + var retained = Assert.Single(rig.Grain.GetSnapshotForTest().InboxDeadLetters); + Assert.Equal("unsupported input", retained.Reason); + Assert.Equal(1, retained.AttemptCount); + Assert.Equal(self.Context.Message.Envelope.MessageId, retained.MessageId); + Assert.Equal(new TestMessage(self.Context.Message.Envelope.MessageId, 501, "async-handler"), + Assert.IsType(TestApplicationProtocol.Read( + rig.Context.ActivationServices.GetRequiredService(), self.Context.Message).Body)); + self.Context.Fail("unsupported input"); + self.Context.Complete(); + Assert.Equal(retained, Assert.Single(rig.Grain.GetSnapshotForTest().InboxDeadLetters)); + return ValueTask.CompletedTask; + }; + var input = await DeliverAsync(rig); + using var storage = Fixture.Storage.BlockAcknowledgement(rig.Journal); + var finished = FinishedAsync(rig); + const string counter = "orleans-messaging-inbox-messages-processed"; + var processed = Fixture.Metrics.WaitForCountAsync(counter, 1, "dead_lettered"); + handler.Release.TrySetResult(); + await storage.WaitUntilEnteredAsync(); + Assert.False(finished.IsCompleted); + AssertCompletedState(rig, input); + Assert.Empty(rig.Effects); + Assert.Empty(rig.Outbox); + storage.Release(); + await WaitAsync(finished); + await processed; + Assert.Equal(1, calls); + Assert.Equal(1, Fixture.Metrics.GetCount(counter, "dead_lettered")); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(rig.Receiver, input)).Status); + Assert.Equal(1, calls); + await AssertHealthyAsync(rig); + await DeactivateAsync(rig); + var recovered = await rig.Receiver.GetSnapshotAsync(); + var letter = Assert.Single(recovered.InboxDeadLetters); + Assert.Equal("unsupported input", letter.Reason); + Assert.Equal(1, letter.AttemptCount); + Assert.Equal(input.Envelope.MessageId, letter.MessageId); + Assert.Empty(recovered.Effects); + Assert.Equal(0, recovered.InboxCount); + Assert.Equal(1, recovered.ProcessedMessageCount); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(rig.Receiver, input)).Status); + var recoveredContext = Fixture.GetGrainContext(rig.Receiver); + await OnTurnAsync(recoveredContext, async () => + { + var diagnostics = recoveredContext.ActivationServices.GetRequiredService(); + var retained = Assert.Single(diagnostics.InboxDeadLetters); + Assert.Equal(input.Envelope.MessageId, retained.Message.Envelope.MessageId); + Assert.Equal(input.Envelope.Subject, retained.Message.Envelope.Subject); + Assert.Equal(input.Envelope.Payload, retained.Message.Envelope.Payload); + var expectedHeaders = Assert.IsType>(input.Envelope.Headers); + var retainedHeaders = Assert.IsType>(retained.Message.Envelope.Headers); + Assert.Equal(expectedHeaders.Keys.Order(StringComparer.Ordinal), retainedHeaders.Keys.Order(StringComparer.Ordinal)); + foreach (var (name, bytes) in expectedHeaders) + { + Assert.Equal(bytes, retainedHeaders[name]); + } + Assert.True(diagnostics.RemoveInboxDeadLetter(input.Envelope.MessageId)); + await recoveredContext.ActivationServices.GetRequiredService().WriteStateAsync(); + Assert.Empty(diagnostics.InboxDeadLetters); + }); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(rig.Receiver, input)).Status); + } + + [Collection(MessagingClusterCollection.Name)] + [TestSuite("BVT")] + [TestProvider("None")] + [TestArea("Messaging")] + public sealed class InboxPermanentFailureTests() : MessagingBehaviorTestBase(new PermanentFailureClusterFixture()) + { + [Fact] + public async Task DeadLetter_AfterPreparationRetryTerminatesBeforeLimitAndCountsCurrentAttempt() + { + var receiver = NewGrain(); + await receiver.GetSnapshotAsync(); + var context = Fixture.GetGrainContext(receiver); + var grain = Assert.IsType(context.GrainInstance); + var handler = new PermanentHandler(); + var installed = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + context.Scheduler.QueueAction(() => + { + grain.HandlerOverride = handler; + installed.SetResult(); + }); + await installed.Task.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + var input = CreateEnvelope(receiver, NewMessage(901, "permanent input"), "permanent/input"); + const string counter = "orleans-messaging-inbox-messages-processed"; + var retried = Fixture.Metrics.WaitForCountAsync(counter, 1, "retry"); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, input)).Status); + await retried; + Assert.Equal(1, handler.Attempts); + Assert.Equal(1, (await receiver.GetSnapshotAsync()).InboxCount); + + var terminated = Fixture.Metrics.WaitForCountAsync(counter, 1, "dead_lettered"); + Fixture.Clock.Advance(TimeSpan.FromHours(1)); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(receiver, input)).Status); + await terminated; + var snapshot = await receiver.GetSnapshotAsync(); + Assert.Equal(2, handler.Attempts); + Assert.Equal(0, snapshot.InboxCount); + Assert.Equal(1, snapshot.ProcessedMessageCount); + Assert.Empty(snapshot.Effects); + var deadLetter = Assert.Single(snapshot.InboxDeadLetters); + Assert.Equal(input.Envelope.MessageId, deadLetter.MessageId); + Assert.Equal("permanent input", deadLetter.Reason); + Assert.Equal(2, deadLetter.AttemptCount); + Assert.Equal(1, Fixture.Metrics.GetCount(counter, "retry")); + Assert.Equal(1, Fixture.Metrics.GetCount(counter, "dead_lettered")); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(receiver, input)).Status); + Assert.Equal(2, handler.Attempts); + } + + private sealed class PermanentHandler : IInboxHandler + { + public int Attempts { get; private set; } + + public bool CanHandle(InboxMessage message) => true; + + + public ValueTask HandleAsync(IInboxHandlerContext context, CancellationToken cancellationToken) + { + if (++Attempts == 1) + { + throw new ArgumentException("preparation failed"); + } + context.Fail("permanent input"); + return ValueTask.CompletedTask; + } + } + + private sealed class PermanentFailureClusterFixture : MessagingClusterFixture + { + protected override void ConfigureOptions(DurableMessagingOptions options) + { + base.ConfigureOptions(options); + options.InboxRetryTimeout = TimeSpan.FromHours(4); + options.RetryInitialInterval = TimeSpan.FromHours(1); + options.DeduplicationWindow = TimeSpan.FromDays(7); + } + } + } + + [Theory] + [InlineData(null)] + [InlineData("")] + [InlineData(" ")] + public async Task DeadLetter_InvalidReasonLeavesTerminalStateUnchanged(string? reason) + { + var rig = await CreateAsync(); + using var handler = rig.Handler; + handler.Body = (self, _) => + { + var error = Assert.ThrowsAny(() => self.Context.Fail(reason!)); + Assert.Equal("reason", error.ParamName); + Assert.Single(rig.Inbox); + Assert.Empty(rig.Processed); + Assert.Empty(rig.Grain.GetSnapshotForTest().InboxDeadLetters); + self.Mutate(); + self.Context.Complete(); + return ValueTask.CompletedTask; + }; + var input = await DeliverAsync(rig); + var finished = FinishedAsync(rig); + handler.Release.TrySetResult(); + await WaitAsync(finished); + AssertSuccess(rig, input, outputCount: 0); + await AssertHealthyAsync(rig); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task DeadLetter_ConflictingTerminalOutcomeRejectsAttemptAndDeactivates(bool completeFirst) + { + var rig = await CreateAsync(); + using var handler = rig.Handler; + Exception? expected = null; + handler.Body = (self, _) => + { + if (completeFirst) self.Context.Complete(); + else self.Context.Fail("original reason"); + expected = Assert.Throws(() => self.Context.Fail("different reason")); + return ValueTask.CompletedTask; + }; + var input = await DeliverAsync(rig); + handler.Release.TrySetResult(); + var failure = await WaitAsync(rig.Grain.DeactivationFailure.Task); + Assert.Same(expected, failure); + Assert.Empty(rig.Effects); + Assert.Empty(rig.Outbox); + await AssertFailureReplayAsync(rig, input); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task DeadLetter_HandlerErrorAfterFailPreservesTerminalOutcomeUnlessStorageFails(bool failWrite) + { + var rig = await CreateAsync(); + using var handler = rig.Handler; + var handlerError = new IOException("after dead letter"); + handler.Body = (self, _) => + { + self.Context.Fail("permanent input failure"); + throw handlerError; + }; + var input = await DeliverAsync(rig); + using var storage = Fixture.Storage.BlockWrite(rig.Journal); + var finished = FinishedAsync(rig); + handler.Release.TrySetResult(); + await storage.WaitUntilEnteredAsync(); + AssertCompletedState(rig, input); + Assert.Equal("permanent input failure", Assert.Single(rig.Grain.GetSnapshotForTest().InboxDeadLetters).Reason); + var storageError = new IOException("actual persistence failure"); + if (failWrite) storage.Fail(storageError); + else storage.Release(); + await WaitAsync(finished); + if (failWrite) + { + Assert.Same(storageError, await WaitAsync(rig.Grain.DeactivationFailure.Task)); + await AssertFailureReplayAsync(rig, input); + Assert.NotEqual("permanent input failure", Assert.Single((await rig.Receiver.GetSnapshotAsync()).InboxDeadLetters).Reason); + } + else + { + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(rig.Receiver, input)).Status); + Assert.Empty(rig.Effects); + Assert.Empty(rig.Outbox); + await AssertHealthyAsync(rig); + } + } + + [Fact] + public async Task Complete_StagesBusinessOutputAndDedupeBeforeHandlerReturn() + { + var rig = await CreateAsync(); + using var handler = rig.Handler; + handler.Body = (self, token) => + { + self.Mutate(); + rig.Outbox.Send(self.Output); + self.Context.Complete(); + AssertCompletedState(rig, self.Context.Message); + Assert.Equal(1, Assert.Single(rig.Effects).Value.Count); + Assert.Single(rig.Outbox); + return ValueTask.CompletedTask; + }; + var input = await DeliverAsync(rig); + using var storage = Fixture.Storage.BlockAcknowledgement(rig.Journal); + var finished = FinishedAsync(rig); + handler.Release.TrySetResult(); + await storage.WaitUntilEnteredAsync(); + Assert.False(finished.IsCompleted); + storage.Release(); + await WaitAsync(finished); + AssertSuccess(rig, input, outputCount: 1); + await DeactivateAsync(rig); + var recovered = await rig.Receiver.GetSnapshotAsync(); + Assert.Equal(1, Assert.Single(recovered.Effects).Count); + Assert.Equal(1, recovered.ProcessedMessageCount); + Assert.Equal(1, recovered.OutboxCount); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(rig.Receiver, input)).Status); + } + + [Fact] + public async Task Complete_PrecedingWriterCapturesCompletionWithBusinessAcrossReturnContinuation() + { + var rig = await CreateAsync(); + using var handler = rig.Handler; + var input = await DeliverAsync(rig); + var writes = Writes(rig); + using var storage = Fixture.Storage.BlockWrite(rig.Journal); + var preceding = OnTurnAsync(rig.Context, async () => + { + rig.Context.ActivationServices.GetRequiredKeyedService>("inbox").Value = "prior"; + await rig.Manager.WriteStateAsync(CancellationToken.None); + }); + await storage.WaitUntilEnteredAsync(); + var priorCapture = rig.Grain.Captures[^1]; + Assert.Empty(priorCapture.Effects); + Assert.Equal(1, priorCapture.InboxCount); + Assert.Equal(0, priorCapture.ProcessedMessageCount); + Assert.Equal(0, priorCapture.OutboxCount); + var queued = OnTurnAsync(rig.Context, async () => await rig.Manager.WriteStateAsync(CancellationToken.None)); + var staged = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var returnContinuation = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + handler.Body = async (self, _) => + { + self.Mutate(); + rig.Outbox.Send(self.Output); + self.Context.Complete(); + AssertCompletedState(rig, input); + staged.TrySetResult(); + await returnContinuation.Task; + }; + var finished = FinishedAsync(rig); + handler.Release.TrySetResult(); + try + { + await WaitAsync(staged.Task); + Assert.False(finished.IsCompleted); + storage.Release(); + await WaitAsync(Task.WhenAll(preceding, queued)); + Assert.False(finished.IsCompleted); + Assert.Equal(writes + 2, Writes(rig)); + var capturedEffects = Assert.Single(rig.Grain.Captures, snapshot => snapshot.Effects.Count != 0); + Assert.Equal(0, capturedEffects.InboxCount); + Assert.Equal(1, capturedEffects.ProcessedMessageCount); + Assert.Equal(1, capturedEffects.OutboxCount); + Assert.Equal(new MessageEffect(input.Envelope.MessageId, 1, 501, "async-handler"), Assert.Single(capturedEffects.Effects)); + } + finally + { + returnContinuation.TrySetResult(); + } + await WaitAsync(finished); + AssertSuccess(rig, input, outputCount: 1); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task HandlerFailure_BeforeVsAfterCompleteUsesActualLogicalOutcome(bool completed) + { + var rig = await CreateAsync(); + using var handler = rig.Handler; + var error = new IOException("Handler outcome."); + handler.Body = (self, _) => + { + if (completed) + { + self.Mutate(); + rig.Outbox.Send(self.Output); + self.Context.Complete(); + } + throw error; + }; + var input = await DeliverAsync(rig); + var finished = FinishedAsync(rig); + handler.Release.TrySetResult(); + await WaitAsync(finished); + if (completed) + { + AssertSuccess(rig, input, outputCount: 1); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(rig.Receiver, input)).Status); + } + else + { + Assert.Empty(rig.Effects); + Assert.Empty(rig.Outbox); + Assert.Equal(input.Envelope.MessageId, Assert.Single(rig.Grain.GetSnapshotForTest().InboxDeadLetters).MessageId); + } + await AssertHealthyAsync(rig); + await DeactivateAsync(rig); + var replay = await rig.Receiver.GetSnapshotAsync(); + Assert.Equal(completed ? 1 : 0, replay.Effects.Count); + Assert.Equal(1, replay.ProcessedMessageCount); + Assert.Equal(completed ? 0 : 1, replay.InboxDeadLetters.Count); + } + + [Fact] + public async Task SuccessfulReturnWithoutCompleteOrStaging_FailsLikeAnException() + { + var rig = await CreateAsync(); + using var handler = rig.Handler; + handler.Body = (_, _) => ValueTask.CompletedTask; + var input = await DeliverAsync(rig); + var finished = FinishedAsync(rig); + handler.Release.TrySetResult(); + await WaitAsync(finished); + + // This fixture's InboxRetryTimeout is zero, so the first failure dead-letters the message. + var letter = Assert.Single(rig.Grain.GetSnapshotForTest().InboxDeadLetters); + Assert.Equal(input.Envelope.MessageId, letter.MessageId); + Assert.Contains("must call Complete", letter.Reason, StringComparison.Ordinal); + Assert.Empty(rig.Inbox); + Assert.Empty(rig.Effects); + await AssertHealthyAsync(rig); + } + + [Theory] + [InlineData(true)] + [InlineData(false)] + public async Task AttemptWithoutOutcomeAfterStaging_DiscardsItsChangesAndRunsAgainAfterRecovery(bool throws) + { + var rig = await CreateAsync(); + using var handler = rig.Handler; + handler.Body = (self, _) => + { + self.Mutate(); + if (throws) throw new InvalidOperationException("A downstream call failed after staging."); + return ValueTask.CompletedTask; + }; + var input = await DeliverAsync(rig); + var writes = Writes(rig); + handler.Release.TrySetResult(); + + var error = Assert.IsType(await WaitAsync(rig.Grain.DeactivationFailure.Task)); + Assert.Contains("after staging changes", error.Message, StringComparison.Ordinal); + await WaitAsync(rig.Context.Deactivated); + Assert.Equal(writes, Writes(rig)); + + // The retried wake-up recovers the grain from the journal without the first attempt's effect, and the message + // runs again: this fixture's replay handler dead-letters it. + var recovered = await Fixture.WaitForDeadLetterCountAsync(rig.Receiver, 1); + Assert.NotEqual(rig.Grain.GetSnapshotForTest().ActivationId, recovered.ActivationId); + Assert.Empty(recovered.Effects); + Assert.Equal(0, recovered.InboxCount); + Assert.Equal(input.Envelope.MessageId, Assert.Single(recovered.InboxDeadLetters).MessageId); + } + [Fact] + public async Task AttemptFailure_WhileADeliveryInterleaves_DoesNotTreatTheAcceptanceAsItsOwn() + { + var rig = await CreateAsync(); + using var handler = rig.Handler; + handler.Body = (_, _) => throw new InvalidOperationException("A downstream call failed."); + var input = await DeliverAsync(rig); + var finished = FinishedAsync(rig); + + // A delivery interleaves while the attempt waits; the acceptance it stages is the delivery's change. + var interleaved = CreateEnvelope(rig.Receiver, NewMessage(502, "interleaved")); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(rig.Receiver, interleaved)).Status); + + // The attempt fails without staging anything itself, so it records its failure as usual: this fixture's zero + // InboxRetryTimeout dead-letters the message at once, and the activation stays. + handler.Release.TrySetResult(); + await WaitAsync(finished); + + Assert.Contains(input.Envelope.MessageId, rig.Grain.GetSnapshotForTest().InboxDeadLetters.Select(static letter => letter.MessageId)); + await AssertHealthyAsync(rig); + } + + [Fact] + public async Task RepeatedComplete_SameActiveAttemptStagesExactlyOnce() + { + var rig = await CreateAsync(); + using var handler = rig.Handler; + handler.Body = (self, _) => + { + self.Mutate(); + self.Context.Complete(); + var completedId = Assert.Single(rig.Processed); + Assert.Equal(self.Context.Message.Envelope.MessageId, completedId); + var stagedBytes = rig.Manager.PendingWriteByteCount; + self.Context.Complete(); + Assert.Equal(completedId, Assert.Single(rig.Processed)); + Assert.Equal(stagedBytes, rig.Manager.PendingWriteByteCount); + return ValueTask.CompletedTask; + }; + var input = await DeliverAsync(rig); + var finished = FinishedAsync(rig); + handler.Release.TrySetResult(); + await WaitAsync(finished); + AssertSuccess(rig, input, outputCount: 0); + await AssertHealthyAsync(rig); + } + + [Theory] + [InlineData("complete")] + [InlineData("dead-letter")] + public async Task RetainedContext_AfterRetirementRejectsAndKeepsOwnerHealthy(string operation) + { + var rig = await CreateAsync(); + using var handler = rig.Handler; + var input = await DeliverAsync(rig); + var finished = FinishedAsync(rig); + handler.Release.TrySetResult(); + await WaitAsync(finished); + var error = await Assert.ThrowsAsync(() => + OnTurnAsync(rig.Context, () => InvokeContextOperation(handler, operation))); + Assert.Contains("inactive or different attempt", error.Message, StringComparison.Ordinal); + AssertSuccess(rig, input, outputCount: 0); + await AssertHealthyAsync(rig); + } + + [Theory] + [InlineData("complete")] + [InlineData("dead-letter")] + public async Task RetainedContext_DuringOtherAttemptThrowsWithoutAffectingCurrentAttempt(string operation) + { + var rig = await CreateAsync(); + using var first = rig.Handler; + var one = await DeliverAsync(rig); + var finished = FinishedAsync(rig); + first.Release.TrySetResult(); + await WaitAsync(finished); + using var second = new TestHandler(rig.Effects, rig.Context); + InvalidOperationException? rejection = null; + second.Body = (_, _) => + { + try { InvokeContextOperation(first, operation); } + catch (InvalidOperationException error) { rejection = error; } + throw new IOException("Replacement failure."); + }; + await OnTurnAsync(rig.Context, () => + rig.Grain.HandlerOverride = second); + var two = CreateEnvelope(rig.Receiver, NewMessage(502, "next"), "async/next"); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(rig.Receiver, two)).Status); + await WaitAsync(second.Entered.Task); + second.Release.TrySetResult(); + var snapshot = await Fixture.WaitForDeadLetterCountAsync(rig.Receiver, 1); + Assert.Contains("inactive or different attempt", Assert.IsType(rejection).Message, StringComparison.Ordinal); + var letter = Assert.Single(snapshot.InboxDeadLetters); + Assert.Equal(two.Envelope.MessageId, letter.MessageId); + Assert.Equal("Replacement failure.", letter.Reason); + Assert.Equal(one.Envelope.MessageId, Assert.Single(rig.Effects).Key); + await AssertHealthyAsync(rig); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task CompletionWrite_StorageOutcomeDecidesCommit(bool failWrite) + { + var rig = await CreateAsync(); + using var handler = rig.Handler; + handler.Body = (self, _) => + { + self.Mutate(); + rig.Outbox.Send(self.Output); + self.Context.Complete(); + return ValueTask.CompletedTask; + }; + var input = await DeliverAsync(rig); + using var storage = Fixture.Storage.BlockWrite(rig.Journal); + var finished = FinishedAsync(rig); + handler.Release.TrySetResult(); + await storage.WaitUntilEnteredAsync(); + AssertCompletedState(rig, input); + Assert.False(finished.IsCompleted); + var failure = new OperationCanceledException("Actual storage canceled.", new CancellationToken(canceled: true)); + if (failWrite) storage.Fail(failure); + else storage.Release(); + await WaitAsync(finished); + if (failWrite) + { + Assert.Same(failure, await WaitAsync(rig.Grain.DeactivationFailure.Task)); + await AssertFailureReplayAsync(rig, input); + } + else + { + AssertSuccess(rig, input, outputCount: 1); + await AssertHealthyAsync(rig); + } + } + + [Fact] + public async Task CompletionWrite_PreparationFailureReplaysCompletion() + { + var rig = await CreateAsync(); + using var handler = rig.Handler; + var input = await DeliverAsync(rig); + var error = new IOException("Journal preparation failed."); + await OnTurnAsync(rig.Context, () => rig.Grain.JournalPreparationFailure = error); + var finished = FinishedAsync(rig); + handler.Release.TrySetResult(); + await WaitAsync(finished); + var failure = Assert.IsType(await WaitAsync(rig.Grain.DeactivationFailure.Task)); + Assert.Same(error, failure.InnerException); + await AssertFailureReplayAsync(rig, input); + } + + [Fact] + public async Task PostCompleteHandlerFailure_WithStorageFailureKeepsStorageCauseAuthoritative() + { + var rig = await CreateAsync(); + using var handler = rig.Handler; + var handlerError = new InvalidOperationException("After Complete."); + handler.Body = (self, _) => { self.Mutate(); self.Context.Complete(); throw handlerError; }; + var input = await DeliverAsync(rig); + using var storage = Fixture.Storage.BlockWrite(rig.Journal); + var finished = FinishedAsync(rig); + handler.Release.TrySetResult(); + await storage.WaitUntilEnteredAsync(); + var storageError = new IOException("Storage failed."); + storage.Fail(storageError); + await WaitAsync(finished); + Assert.Same(storageError, await WaitAsync(rig.Grain.DeactivationFailure.Task)); + await AssertFailureReplayAsync(rig, input); + } + + private async Task CreateAsync() + { + var receiver = NewGrain(); + await receiver.GetSnapshotAsync(); + var context = Fixture.GetGrainContext(receiver); + var services = context.ActivationServices; + var effects = services.GetRequiredKeyedService>("test-effects"); + var handler = new TestHandler(effects, context); + var rig = new Rig(receiver, context, Assert.IsType(context.GrainInstance), + services.GetRequiredService(), (JournaledTestOutbox)services.GetRequiredService(), + effects, handler, services.GetRequiredService()); + await OnTurnAsync(context, () => rig.Grain.HandlerOverride = handler); + return rig; + } + + private async Task DeliverAsync(Rig rig) + { + var input = CreateEnvelope(rig.Receiver, NewMessage(501, "async-handler"), "async/handler"); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(rig.Receiver, input)).Status); + await WaitAsync(rig.Handler.Entered.Task); + return input; + } + + private static void InvokeContextOperation(TestHandler handler, string operation) + { + switch (operation) + { + case "complete": handler.Context.Complete(); break; + case "dead-letter": handler.Context.Fail("unusable input"); break; + default: throw new ArgumentException(nameof(operation)); + } + } + + private static void AssertCompletedState(Rig rig, InboxMessage input) + { + Assert.Empty(rig.Inbox); + Assert.Equal(input.Envelope.MessageId, Assert.Single(rig.Processed)); + } + + private static void AssertSuccess(Rig rig, InboxMessage input, int outputCount) + { + AssertCompletedState(rig, input); + Assert.Equal(new MessageEffect(input.Envelope.MessageId, 1, 501, "async-handler"), Assert.Single(rig.Effects).Value); + Assert.Equal(outputCount, rig.Outbox.Count); + Assert.Empty(rig.Grain.GetSnapshotForTest().InboxDeadLetters); + Assert.False(rig.Grain.DeactivationFailure.Task.IsCompleted); + } + + private int Writes(Rig rig) => Fixture.Storage.GetSuccessfulWriteCount(rig.Journal); + private static async Task AssertHealthyAsync(Rig rig) + { + await OnTurnAsync(rig.Context, async () => + { + rig.Context.ActivationServices.GetRequiredKeyedService>("inbox").Value = "healthy"; + await rig.Manager.WriteStateAsync(CancellationToken.None); + }); + Assert.False(rig.Grain.DeactivationFailure.Task.IsCompleted); + } + + private static async Task DeactivateAsync(Rig rig) + { + await rig.Receiver.RequestDeactivationAsync(); + await WaitAsync(rig.Context.Deactivated); + } + + private async Task AssertFailureReplayAsync(Rig rig, InboxMessage input) + { + await WaitAsync(rig.Context.Deactivated); + await rig.Receiver.GetSnapshotAsync(); + var recovered = await Fixture.WaitForDeadLetterCountAsync(rig.Receiver, 1); + Assert.Empty(recovered.Effects); + Assert.Equal(0, recovered.OutboxCount); + Assert.Equal(1, recovered.ProcessedMessageCount); + Assert.Equal(input.Envelope.MessageId, Assert.Single(recovered.InboxDeadLetters).MessageId); + } + + /// + /// Completes after the pump finishes its current pass (including the batch write) and one more pass. + /// + private Task FinishedAsync(Rig rig) + { + // Resolve the pump now: a failed write deactivates the grain and disposes its activation services. + var shutdown = rig.Context.ActivationServices.GetRequiredService().ShutdownToken; + return RunAsync(); + + async Task RunAsync() + { + try + { + await Fixture.RunPumpAsync(rig.Receiver); + } + catch (OperationCanceledException) when (shutdown.IsCancellationRequested) + { + // A failed write stops the pump; callers assert the failure explicitly. + } + } + } + + private static Task OnTurnAsync(IGrainContext context, Action action) => + OnTurnAsync(context, () => { action(); return Task.CompletedTask; }); + + private static Task OnTurnAsync(IGrainContext context, Func action) + { + var done = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + context.Scheduler.QueueAction(() => { _ = CompleteAsync(); }); + return done.Task; + + async Task CompleteAsync() + { + try + { + Assert.Same(context, ReceiverTestServices.CurrentGrainContext); + await action(); + done.SetResult(); + } + catch (Exception error) + { + done.SetException(error); + } + } + } + + private static Task WaitAsync(Task task) => task.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + private static Task WaitAsync(Task task) => task.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + private sealed record Rig(IMessagingTestGrain Receiver, IGrainContext Context, MessagingTestGrain Grain, + IJournaledStateManager Manager, JournaledTestOutbox Outbox, IDurableDictionary Effects, + TestHandler Handler, Inbox InboxState) + { + public IReadOnlyDictionary Inbox => InboxTestAccess.GetPendingMessages(InboxState); + public IReadOnlySet Processed => InboxTestAccess.GetProcessedKeys(InboxState); + + public JournalId Journal => JournalId.FromGrainId(Receiver.GetGrainId()); + } + + private sealed class TestHandler(IDurableDictionary effects, IGrainContext grainContext) : IInboxHandler, IDisposable + { + public TaskCompletionSource Entered { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + public TaskCompletionSource Release { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + public IInboxHandlerContext Context { get; private set; } = null!; + public OutboxMessage Output { get; private set; } + public Func Body { get; set; } = + static (self, _) => { self.Mutate(); self.Context.Complete(); return default; }; + public bool CanHandle(InboxMessage message) => true; + + public async ValueTask HandleAsync(IInboxHandlerContext context, CancellationToken cancellationToken) + { + Context = context; + + Output = TestApplicationProtocol.Create(grainContext.ActivationServices.GetRequiredService(), grainContext.GrainId, grainContext.GrainId, "output", 41); + Entered.TrySetResult(); + await Release.Task.WaitAsync(cancellationToken); + await Body(this, cancellationToken); + } + public void Mutate() + { + effects.TryGetValue(Context.Message.Envelope.MessageId, out var previous); + effects[Context.Message.Envelope.MessageId] = + new MessageEffect(Context.Message.Envelope.MessageId, (previous?.Count ?? 0) + 1, 501, "async-handler"); + } + public void Dispose() + { + Release.TrySetResult(); + + } + } +} diff --git a/test/Orleans.Messaging.Tests/Functional/InboxHandlerTransactionTests.cs b/test/Orleans.Messaging.Tests/Functional/InboxHandlerTransactionTests.cs new file mode 100644 index 00000000000..a87d8f9705d --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/InboxHandlerTransactionTests.cs @@ -0,0 +1,156 @@ +using Microsoft.Extensions.DependencyInjection; +using Orleans.DurableJobs; +using Orleans.Journaling; +using Orleans.Messaging.Tests.Support; +using Orleans.Runtime; +using Orleans.Serialization.Session; +using Xunit; + +namespace Orleans.Messaging.Tests.Functional; + +[Collection(MessagingClusterCollection.Name)] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class InboxHandlerTransactionTests : MessagingBehaviorTestBase +{ + public InboxHandlerTransactionTests() : base(receiverOnly: true) + { + } + + [Fact] + public async Task HandlerSuccess_CommitsEffectCompletionDedupeAndStagedOutputAtomically() + { + var receiver = NewGrain(); + var sink = NewGrain(); + var logicalId = TestApplicationProtocol.NewMessageId(); + var envelope = CreateEnvelope( + receiver, + new TestMessage(logicalId, 7, "atomic", sink.GetGrainId())); + + var result = await DeliverAsync(receiver, envelope); + var receiverState = await Fixture.WaitForEffectCountAsync(receiver, 1); + var output = Fixture.GetStagedOutput(receiver); + + Assert.Equal(DeliveryStatus.Accepted, result.Status); + var effect = Assert.Single(receiverState.Effects); + Assert.Equal(new MessageEffect(logicalId, 1, 7, "atomic"), effect); + Assert.Equal(0, receiverState.InboxCount); + Assert.Equal(1, receiverState.OutboxCount); + var outgoing = Assert.Single(output); + Assert.Equal(sink.GetGrainId(), outgoing.ReceiverId); + Assert.Equal("messages/forwarded", TestApplicationProtocol.Read(Fixture.Client.ServiceProvider.GetRequiredService(), outgoing).Route); + var body = TestApplicationProtocol.Read(Fixture.Client.ServiceProvider.GetRequiredService(), outgoing).Body; + Assert.Equal(new TestMessage(logicalId, 7, "atomic"), body); + Assert.Equal(1, receiverState.ProcessedMessageCount); + await receiver.RequestDeactivationAsync(); + var recovered = await receiver.GetSnapshotAsync(); + Assert.NotEqual(receiverState.ActivationId, recovered.ActivationId); + Assert.Equal(effect, Assert.Single(recovered.Effects)); + Assert.Equal(1, recovered.OutboxCount); + Assert.Equal(outgoing.Envelope.MessageId, Assert.Single(Fixture.GetStagedOutput(receiver)).Envelope.MessageId); + + var duplicate = await DeliverAsync(receiver, envelope); + Assert.Equal(DeliveryStatus.Duplicate, duplicate.Status); + Assert.Equal(1, Assert.Single((await receiver.GetSnapshotAsync()).Effects).Count); + Assert.Single(Fixture.GetStagedOutput(receiver)); + } + + [Fact] + public async Task HandlerPreparationFailure_DeadLettersWithoutStagingEffectsOrOutput() + { + var receiver = NewGrain(); + var sink = NewGrain(); + var envelope = CreateEnvelope( + receiver, + new TestMessage(TestApplicationProtocol.NewMessageId(), 9, "preparation-failure", sink.GetGrainId(), ThrowDuringPreparation: true)); + + var accepted = await DeliverAsync(receiver, envelope); + var state = await Fixture.WaitForDeadLetterCountAsync(receiver, 1); + + Assert.Equal(DeliveryStatus.Accepted, accepted.Status); + Assert.Empty(state.Effects); + Assert.Equal(0, state.InboxCount); + Assert.Equal(0, state.OutboxCount); + var deadLetter = Assert.Single(state.InboxDeadLetters); + Assert.Equal(envelope.Envelope.MessageId, deadLetter.MessageId); + Assert.Equal(1, deadLetter.AttemptCount); + Assert.Contains("Injected handler preparation failure", deadLetter.Reason, StringComparison.Ordinal); + Assert.Empty(Fixture.GetStagedOutput(receiver)); + await receiver.RequestDeactivationAsync(); + var recovered = await receiver.GetSnapshotAsync(); + Assert.NotEqual(state.ActivationId, recovered.ActivationId); + Assert.Empty(recovered.Effects); + Assert.Empty(Fixture.GetStagedOutput(receiver)); + Assert.Equal(envelope.Envelope.MessageId, Assert.Single(recovered.InboxDeadLetters).MessageId); + } + + [Fact] + public async Task HandlerPreparation_OrdinaryWriteLeavesEffectsLocalUntilApplyAndCompletion() + { + var receiver = NewGrain(); + _ = await receiver.GetSnapshotAsync(); + var oldContext = Fixture.GetGrainContext(receiver); + var oldGrain = Assert.IsType(oldContext.GrainInstance); + var envelope = CreateEnvelope(receiver, NewMessage(10, "premature-commit") with { CommitDuringHandling = true }); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + var completed = await Fixture.WaitForEffectCountAsync(receiver, 1); + Assert.Equal(1, Assert.Single(completed.Effects).Count); + Assert.Equal("premature-commit", Assert.Single(completed.Effects).Value); + Assert.Empty(completed.InboxDeadLetters); + Assert.Equal(0, completed.InboxCount); + Assert.Equal(1, completed.ProcessedMessageCount); + Assert.False(oldGrain.DeactivationFailure.Task.IsCompleted); + Assert.Same(oldContext, Fixture.GetGrainContext(receiver)); + } + + [Fact] + public async Task HandlerCompletionWriteFailure_FencesOldManagerAndFreshActivationRetries() + { + var receiver = NewGrain(); + _ = await receiver.GetSnapshotAsync(); + using var handler = Fixture.HandlerProbe.Arm(receiver.GetGrainId(), "messages/completion-failure"); + var envelope = CreateEnvelope(receiver, NewMessage(78, "completion-failure"), "messages/completion-failure"); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + await handler.WaitUntilEnteredAsync(); + var oldContext = Fixture.GetGrainContext(receiver); + var oldGrain = Assert.IsType(oldContext.GrainInstance); + var oldManager = oldContext.ActivationServices.GetRequiredService(); + var oldActivationId = oldGrain.GetSnapshotForTest().ActivationId; + Fixture.Storage.FailWrite(JournalId.FromGrainId(receiver.GetGrainId())); + handler.Release(); + await oldContext.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + await Assert.ThrowsAnyAsync(() => oldManager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask()); + _ = await receiver.GetSnapshotAsync(); + var completed = await Fixture.SnapshotProbe.WaitAsync(receiver.GetGrainId(), + snapshot => snapshot.ActivationId != oldActivationId + && snapshot.Effects.Sum(static effect => effect.Count) == 1); + Assert.NotEqual(oldActivationId, completed.ActivationId); + Assert.Equal(1, Assert.Single(completed.Effects).Count); + Assert.Empty(completed.InboxDeadLetters); + Assert.Equal(0, completed.InboxCount); + } + + [Fact] + public async Task PreparationFailureAccountingWriteFailure_RecoversInFreshActivation() + { + var receiver = NewGrain(); + var before = await receiver.GetSnapshotAsync(); + using var handler = Fixture.HandlerProbe.Arm(receiver.GetGrainId(), "messages/failure-accounting"); + var envelope = CreateEnvelope(receiver, NewMessage(79, "failure-accounting") with { ThrowDuringPreparation = true }, "messages/failure-accounting"); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + await handler.WaitUntilEnteredAsync(); + var oldContext = Fixture.GetGrainContext(receiver); + Fixture.Storage.FailWrite(JournalId.FromGrainId(receiver.GetGrainId())); + handler.Release(); + await oldContext.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + _ = await receiver.GetSnapshotAsync(); + var completed = await Fixture.WaitForDeadLetterCountAsync(receiver, 1); + Assert.NotEqual(before.ActivationId, completed.ActivationId); + // Recovery charges the attempt whose failure record didn't commit, so the rerun is the second attempt. + Assert.Equal(2, Assert.Single(completed.InboxDeadLetters).AttemptCount); + Assert.Empty(completed.Effects); + Assert.Empty(Fixture.GetStagedOutput(receiver)); + } + +} diff --git a/test/Orleans.Messaging.Tests/Functional/InboxHeaderBehaviorTests.cs b/test/Orleans.Messaging.Tests/Functional/InboxHeaderBehaviorTests.cs new file mode 100644 index 00000000000..bd965fec859 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/InboxHeaderBehaviorTests.cs @@ -0,0 +1,260 @@ +using System.Text; +using Microsoft.Extensions.DependencyInjection; +using Orleans.Journaling; +using Orleans.Messaging.Configuration; +using Orleans.Messaging.Tests.Support; +using Orleans.Runtime; +using Orleans.Serialization; +using Xunit; + +namespace Orleans.Messaging.Tests.Functional; + +[Collection(MessagingClusterCollection.Name)] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class InboxHeaderBehaviorTests : MessagingBehaviorTestBase +{ + public InboxHeaderBehaviorTests() : base(new HeaderFixture()) { } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task RawMessage_AbsentOptionalMetadataCommitsAndDeduplicates(bool empty) + { + var receiver = NewGrain(); + _ = await receiver.GetSnapshotAsync(); + var context = Fixture.GetGrainContext(receiver); + var grain = Assert.IsType(context.GrainInstance); + byte[] payload = empty ? [] : [0, 255, 128, 42]; + var message = Raw(MessageId.Create("raw", Guid.NewGuid().ToString("N")), payload); + Assert.Null(message.Envelope.Subject); + Assert.Null(message.Envelope.Headers); + using var handler = new RawHandler(context); + await OnTurnAsync(context, () => grain.HandlerOverride = handler); + using var storage = Fixture.Storage.BlockAcknowledgement(JournalId.FromGrainId(context.GrainId)); + var delivery = DeliverAsync(receiver, message); + await storage.WaitUntilEnteredAsync(); + Assert.False(delivery.IsCompleted); + Assert.False(handler.Entered.Task.IsCompleted); + storage.Release(); + Assert.Equal(DeliveryStatus.Accepted, (await delivery).Status); + await WaitAsync(handler.Entered.Task); + Assert.Equal(payload, handler.Message.Envelope.Payload.ToArray()); + handler.Release.TrySetResult(); + var completed = await Fixture.WaitForEffectCountAsync(receiver, 1); + Assert.Equal(message.Envelope.MessageId, Assert.Single(completed.Effects).LogicalId); + Assert.Equal(1, completed.ProcessedMessageCount); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(receiver, message)).Status); + await receiver.RequestDeactivationAsync(); + await WaitAsync(context.Deactivated); + var replayed = await receiver.GetSnapshotAsync(); + Assert.NotEqual(completed.ActivationId, replayed.ActivationId); + Assert.Equal(1, Assert.Single(replayed.Effects).Count); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(receiver, message)).Status); + } + + [Fact] + public async Task PendingDuplicate_ChangedCustomHeadersPreservesOriginal() + { + var receiver = NewGrain(); + _ = await receiver.GetSnapshotAsync(); + var context = Fixture.GetGrainContext(receiver); + var grain = Assert.IsType(context.GrainInstance); + var key = MessageId.Create("custom", Guid.NewGuid().ToString("N")); + var original = Raw(key, [1, 2, 3], "original"); + var changed = Raw(key, [1, 2, 3], "changed") with { SentAt = original.SentAt }; + using var handler = new RawHandler(context); + await OnTurnAsync(context, () => grain.HandlerOverride = handler); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, original)).Status); + await WaitAsync(handler.Entered.Task); + var pending = context.ActivationServices.GetRequiredService(); + Envelope first = default; + await OnTurnAsync(context, () => first = Assert.Single(pending.Messages()).Envelope); + var writes = Fixture.Storage.GetSuccessfulWriteCount(JournalId.FromGrainId(context.GrainId)); + var schedules = Fixture.JobManagerProbe.GetAttemptCount(MessagingPump.JobName, context.GrainId); + await OnTurnAsync(context, async () => + { + IMessagingExtension extension = context.ActivationServices.GetRequiredService(); + Assert.Equal(DeliveryStatus.Duplicate, (await extension.DeliverAsync(changed)).Status); + var retained = Assert.Single(pending.Messages()).Envelope; + Assert.Same(first.Payload, retained.Payload); + Assert.Same(first.Headers, retained.Headers); + Assert.True(Assert.IsType>(retained.Headers).TryGetValue("custom.trace", out var trace)); + Assert.Equal("original", Encoding.UTF8.GetString(trace)); + }); + Assert.Equal(writes, Fixture.Storage.GetSuccessfulWriteCount(JournalId.FromGrainId(context.GrainId))); + Assert.Equal(schedules, Fixture.JobManagerProbe.GetAttemptCount(MessagingPump.JobName, context.GrainId)); + handler.Release.TrySetResult(); + var completed = await Fixture.WaitForEffectCountAsync(receiver, 1); + Assert.Equal(1, Assert.Single(completed.Effects).Count); + Assert.False(grain.DeactivationFailure.Task.IsCompleted); + } + + [Fact] + public async Task UnknownHeaders_RecoverAndRemainAvailableAfterRetryAndDeadLetter() + { + var receiver = NewGrain(); + var key = MessageId.Create("headers", Guid.NewGuid().ToString("N")); + var body = NewMessage(810, "poison") with { LogicalId = key, ThrowDuringPreparation = true }; + var application = CreateEnvelope(receiver, body); + byte[] opaque = [0xff, 0, 0x80, 0x7f]; + var headers = new Dictionary(Assert.IsType>(application.Envelope.Headers), + StringComparer.Ordinal) + { + ["future.opaque"] = opaque, + ["future.number"] = new Serializer(Sessions).SerializeToArray(8675309) + }; + var message = application with { Envelope = application.Envelope with { Headers = headers } }; + var retry = Fixture.Metrics.WaitForCountAsync("orleans-messaging-inbox-messages-processed", 1, "retry"); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, message)).Status); + await retry; + var previous = Fixture.GetGrainContext(receiver); + await receiver.RequestDeactivationAsync(); + await WaitAsync(previous.Deactivated); + _ = await receiver.GetSnapshotAsync(); + var current = Fixture.GetGrainContext(receiver); + Assert.NotSame(previous, current); + var pending = await receiver.GetSnapshotAsync(); + Assert.Equal(1, pending.InboxCount); + Assert.Empty(pending.InboxDeadLetters); + await OnTurnAsync(current, () => + { + var stored = Assert.Single(current.ActivationServices.GetRequiredService().Messages()).Envelope; + var metadata = Assert.IsType>(stored.Headers); + Assert.True(metadata.TryGetValue("future.opaque", out var bytes)); + Assert.Equal(opaque, bytes); + Assert.True(metadata.TryGetValue("future.number", out var number)); + Assert.Equal(8675309, new Serializer(Sessions).Deserialize(number)); + }); + Fixture.Clock.Advance(TimeSpan.FromHours(1)); + await receiver.RequestDeactivationAsync(); + await WaitAsync(current.Deactivated); + _ = await receiver.GetSnapshotAsync(); + current = Fixture.GetGrainContext(receiver); + var dead = await Fixture.WaitForDeadLetterCountAsync(receiver, 1); + Assert.Equal(key, Assert.Single(dead.InboxDeadLetters).MessageId); + Assert.Equal(2, Assert.Single(dead.InboxDeadLetters).AttemptCount); + Assert.Empty(dead.Effects); + var diagnostics = current.ActivationServices.GetRequiredService(); + await OnTurnAsync(current, () => + { + var retained = Assert.Single(diagnostics.InboxDeadLetters).Message.Envelope; + var metadata = Assert.IsType>(retained.Headers); + Assert.True(metadata.TryGetValue("future.opaque", out var bytes)); + Assert.Equal(opaque, bytes); + Assert.True(metadata.TryGetValue("future.number", out var number)); + Assert.Equal(8675309, new Serializer(Sessions).Deserialize(number)); + Assert.Equal(application.Envelope.Payload.ToArray(), retained.Payload.ToArray()); + }); + await receiver.RequestDeactivationAsync(); + await WaitAsync(current.Deactivated); + _ = await receiver.GetSnapshotAsync(); + var recovered = Fixture.GetGrainContext(receiver); + await OnTurnAsync(recovered, () => + { + var retained = Assert.Single(recovered.ActivationServices.GetRequiredService() + .InboxDeadLetters).Message.Envelope; + var metadata = Assert.IsType>(retained.Headers); + Assert.True(metadata.TryGetValue("future.opaque", out var bytes)); + Assert.Equal(opaque, bytes); + Assert.True(metadata.TryGetValue("future.number", out var number)); + Assert.Equal(8675309, new Serializer(Sessions).Deserialize(number)); + }); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(receiver, Raw(key, []))).Status); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task InvalidMessageHeader_RejectsBeforeSchedulingOrMutation(bool completed) + { + var receiver = NewGrain(); + _ = await receiver.GetSnapshotAsync(); + var context = Fixture.GetGrainContext(receiver); + var key = MessageId.Create("malformed", Guid.NewGuid().ToString("N")); + var message = Raw(key, [1, 2]); + if (completed) + { + await OnTurnAsync(context, () => + InboxTestAccess.AddProcessed(context.ActivationServices.GetRequiredService(), key, message.SentAt)); + } + message = message with + { + Envelope = message.Envelope with + { + Headers = new(StringComparer.Ordinal) { ["invalid"] = null! } + } + }; + var writes = Fixture.Storage.GetSuccessfulWriteCount(JournalId.FromGrainId(context.GrainId)); + await OnTurnAsync(context, async () => + { + IMessagingExtension extension = context.ActivationServices.GetRequiredService(); + await Assert.ThrowsAsync(() => extension.DeliverAsync(message, TestContext.Current.CancellationToken).AsTask()); + }); + Assert.Equal(writes, Fixture.Storage.GetSuccessfulWriteCount(JournalId.FromGrainId(context.GrainId))); + Assert.Equal(0, Fixture.JobManagerProbe.GetAttemptCount(MessagingPump.JobName, context.GrainId)); + var snapshot = await receiver.GetSnapshotAsync(); + Assert.Equal(0, snapshot.InboxCount); + Assert.Equal(completed ? 1 : 0, snapshot.ProcessedMessageCount); + Assert.Empty(snapshot.Effects); + } + + private InboxMessage Raw(MessageId key, byte[] payload, string? trace = null) + { + Dictionary? headers = null; + if (trace is not null) (headers ??= new(StringComparer.Ordinal))["custom.trace"] = Encoding.UTF8.GetBytes(trace); + return new() { Envelope = new() { MessageId = key, Payload = payload, Headers = headers }, SentAt = Fixture.JobNow }; + } + + private static Task OnTurnAsync(IGrainContext context, Action action) => + OnTurnAsync(context, () => { action(); return Task.CompletedTask; }); + + private static Task OnTurnAsync(IGrainContext context, Func action) + { + var result = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + context.Scheduler.QueueAction(() => { _ = Run(); }); + return result.Task; + async Task Run() + { + try { await action(); result.SetResult(); } + catch (Exception exception) { result.SetException(exception); } + } + } + + private static Task WaitAsync(Task task) => + task.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + + private sealed class RawHandler(IGrainContext context) : IInboxHandler, IDisposable + { + public InboxMessage Message { get; private set; } + public TaskCompletionSource Entered { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + public TaskCompletionSource Release { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public bool CanHandle(InboxMessage message) => true; + + + public async ValueTask HandleAsync(IInboxHandlerContext handler, CancellationToken cancellationToken) + { + Message = handler.Message; + Entered.TrySetResult(); + await Release.Task.WaitAsync(cancellationToken); + var key = handler.Message.Envelope.MessageId; + var effects = context.ActivationServices.GetRequiredKeyedService>("test-effects"); + effects[key] = new(key, 1, 800, "raw"); + handler.Complete(); + } + + public void Dispose() => Release.TrySetResult(); + } + + private sealed class HeaderFixture : MessagingClusterFixture + { + protected override void ConfigureOptions(DurableMessagingOptions options) + { + base.ConfigureOptions(options); + options.InboxRetryTimeout = TimeSpan.FromHours(1); + options.RetryInitialInterval = TimeSpan.FromHours(1); + } + } +} diff --git a/test/Orleans.Messaging.Tests/Functional/InboxIdentityTests.cs b/test/Orleans.Messaging.Tests/Functional/InboxIdentityTests.cs new file mode 100644 index 00000000000..adfbda725a7 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/InboxIdentityTests.cs @@ -0,0 +1,464 @@ +using System.Reflection; +using Microsoft.Extensions.DependencyInjection; +using Orleans.Journaling; +using Orleans.Messaging.Tests.Support; +using Orleans.Runtime; +using Orleans.Serialization.Session; +using Xunit; + +namespace Orleans.Messaging.Tests.Functional; + +[Collection(MessagingClusterCollection.Name)] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class InboxIdentityTests : MessagingBehaviorTestBase +{ + [Fact] + public async Task PendingCommand_NewSenderCoalescesAndPreservesOriginalEnvelope() + { + var rig = await CreateAsync(); + using var handler = rig.Handler; + var key = MessageId.Create("tenant", "orders", "1042", "reserve"); + var original = Create(rig, key, "inventory.reserve.v1", "original"); + var repeated = TestApplicationProtocol.With(original, Sessions, sender: GrainId.Create("forwarder", "second")); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(rig.Receiver, original)).Status); + await WaitAsync(handler.Entered.Task); + var writes = Writes(rig); + var scheduled = Fixture.JobManagerProbe.GetAttemptCount(MessagingPump.JobName, rig.Context.GrainId); + byte[] storedPayload = null!; + await OnTurnAsync(rig.Context, () => storedPayload = Assert.Single(rig.Pending).Value.Message.Envelope.Payload); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverOnTurnAsync(rig, repeated)).Status); + await OnTurnAsync(rig.Context, () => + { + Assert.True(rig.Inbox.MessageState.TryGetValue(key, out var entry)); + var stored = entry.Message; + Assert.Equal(TestApplicationProtocol.Sender(Sessions, original.Envelope), TestApplicationProtocol.Sender(Sessions, stored.Envelope)); + Assert.Equal(TestApplicationProtocol.Subject(original.Envelope), TestApplicationProtocol.Subject(stored.Envelope)); + Assert.Same(storedPayload, stored.Envelope.Payload); + Assert.Equal(original.Envelope.Payload.ToArray(), stored.Envelope.Payload.ToArray()); + Assert.Single(rig.Pending); + Assert.Empty(rig.Processed); + }); + Assert.Equal(writes, Writes(rig)); + Assert.Equal(scheduled, Fixture.JobManagerProbe.GetAttemptCount(MessagingPump.JobName, rig.Context.GrainId)); + handler.Release.TrySetResult(); + await Fixture.WaitForEffectCountAsync(rig.Receiver, 1); + Assert.Equal(1, handler.Calls); + Assert.Equal(key, Assert.Single(rig.Grain.GetSnapshotForTest().Effects).LogicalId); + Assert.Equal(1, Assert.Single(rig.Grain.GetSnapshotForTest().Effects).Count); + } + + [Theory] + [InlineData("subject")] + [InlineData("subject-case")] + [InlineData("body")] + public async Task PendingCommand_ConflictingContentRejectsBeforeSchedulingOrMutation(string difference) + { + var rig = await CreateAsync(); + using var handler = rig.Handler; + var key = MessageId.Create("tenant", "orders", "1043", "reserve"); + var original = Create(rig, key, "inventory.reserve.v1", "original"); + var conflictingSubject = difference switch + { + "subject" => "inventory.release.v1", + "subject-case" => "Inventory.reserve.v1", + _ => Assert.IsType(TestApplicationProtocol.Subject(original.Envelope)) + }; + var replacement = Create(rig, key, conflictingSubject, "different-parameters", + GrainId.Create("forwarder", "other")); + var conflict = difference == "body" ? replacement : TestApplicationProtocol.With(original, Sessions, + sender: TestApplicationProtocol.Sender(Sessions, replacement.Envelope), subject: conflictingSubject); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(rig.Receiver, original)).Status); + await WaitAsync(handler.Entered.Task); + var writes = Writes(rig); + var scheduled = Fixture.JobManagerProbe.GetAttemptCount(MessagingPump.JobName, rig.Context.GrainId); + byte[] storedPayload = null!; + await OnTurnAsync(rig.Context, () => storedPayload = Assert.Single(rig.Pending).Value.Message.Envelope.Payload); + var failure = await Assert.ThrowsAsync(() => DeliverOnTurnAsync(rig, conflict)); + Assert.Contains(key.ToString(), failure.Message, StringComparison.Ordinal); + Assert.Contains("different command", failure.Message, StringComparison.Ordinal); + await OnTurnAsync(rig.Context, () => + { + Assert.Equal(TestApplicationProtocol.Sender(Sessions, original.Envelope), TestApplicationProtocol.Sender(Sessions, Assert.Single(rig.Pending).Value.Message.Envelope)); + Assert.Equal(TestApplicationProtocol.Subject(original.Envelope), TestApplicationProtocol.Subject(Assert.Single(rig.Pending).Value.Message.Envelope)); + Assert.Same(storedPayload, Assert.Single(rig.Pending).Value.Message.Envelope.Payload); + Assert.Equal(original.Envelope.Payload.ToArray(), Assert.Single(rig.Pending).Value.Message.Envelope.Payload.ToArray()); + Assert.Empty(rig.Processed); + Assert.Empty(rig.Grain.GetSnapshotForTest().Effects); + Assert.False(rig.Grain.DeactivationFailure.Task.IsCompleted); + }); + Assert.Equal(writes, Writes(rig)); + Assert.Equal(scheduled, Fixture.JobManagerProbe.GetAttemptCount(MessagingPump.JobName, rig.Context.GrainId)); + await OnTurnAsync(rig.Context, async () => + { + rig.Context.ActivationServices.GetRequiredKeyedService>("inbox").Value = "healthy"; + await rig.Manager.WriteStateAsync(CancellationToken.None); + }); + Assert.False(rig.Grain.DeactivationFailure.Task.IsCompleted); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverOnTurnAsync(rig, original)).Status); + handler.Release.TrySetResult(); + await Fixture.WaitForEffectCountAsync(rig.Receiver, 1); + Assert.Equal(1, handler.Calls); + Assert.Equal(1, Assert.Single(rig.Grain.GetSnapshotForTest().Effects).Count); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task CompletedCommand_ChangedSenderSubjectAndBodyAcknowledgesWithoutFingerprint(bool restart) + { + var rig = await CreateAsync(); + using var handler = rig.Handler; + var key = MessageId.Create("tenant", "orders", "1044", "reserve"); + var original = Create(rig, key, "inventory.reserve.v1", "original"); + var repeated = Create(rig, key, "completely.changed.subject.v2", "different-body", + GrainId.Create("forwarder", "new-owner")) with + { SentAt = original.SentAt }; + handler.Release.TrySetResult(); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(rig.Receiver, original)).Status); + await Fixture.WaitForEffectCountAsync(rig.Receiver, 1); + await Fixture.WaitForInboxCountAsync(rig.Receiver, 0); + var prior = rig.Context; + if (restart) + { + await rig.Receiver.RequestDeactivationAsync(); + await WaitAsync(prior.Deactivated); + await rig.Receiver.GetSnapshotAsync(); + Assert.NotSame(prior, Fixture.GetGrainContext(rig.Receiver)); + } + var writes = Writes(rig); + var scheduled = Fixture.JobManagerProbe.GetAttemptCount(MessagingPump.JobName, rig.Context.GrainId); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(rig.Receiver, repeated)).Status); + var completed = await rig.Receiver.GetSnapshotAsync(); + Assert.Equal(1, Assert.Single(completed.Effects).Count); + Assert.Equal(1, completed.ProcessedMessageCount); + Assert.Empty(completed.InboxDeadLetters); + Assert.Equal(writes, Writes(rig)); + Assert.Equal(scheduled, Fixture.JobManagerProbe.GetAttemptCount(MessagingPump.JobName, rig.Context.GrainId)); + } + + [Fact] + public async Task ExactParentChildAndSiblingCommandsHaveIndependentCompletions() + { + var rig = await CreateAsync(); + using var handler = rig.Handler; + handler.Release.TrySetResult(); + var parent = MessageId.Create("tenant", "orders", "1045"); + var keys = new[] { parent, new MessageId(parent.Key.CreateChildKey("reserve")), new MessageId(parent.Key.CreateChildKey("release")) }; + for (var index = 0; index < keys.Length; index++) + { + var command = Create(rig, keys[index], "inventory.command.v1", $"command-{index}"); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(rig.Receiver, command)).Status); + await Fixture.WaitForEffectCountAsync(rig.Receiver, index + 1); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(rig.Receiver, command)).Status); + } + var completed = await rig.Receiver.GetSnapshotAsync(); + Assert.Equal(3, completed.Effects.Count); + Assert.Equal(3, completed.ProcessedMessageCount); + Assert.All(completed.Effects, effect => Assert.Equal(1, effect.Count)); + Assert.Equal(keys.OrderBy(static key => key.ToString(), StringComparer.Ordinal), + completed.Effects.Select(static effect => effect.LogicalId).OrderBy(static key => key.ToString(), StringComparer.Ordinal)); + } + + [Fact] + public async Task SameCommandKey_InDifferentReceiversCompletesIndependently() + { + var first = await CreateAsync(); + var second = await CreateAsync(); + using var firstHandler = first.Handler; + using var secondHandler = second.Handler; + firstHandler.Release.TrySetResult(); + secondHandler.Release.TrySetResult(); + var key = MessageId.Create("tenant", "orders", "1049", "reserve"); + var one = Create(first, key, "inventory.reserve.v1", "first-receiver"); + var two = Create(second, key, "inventory.reserve.v1", "second-receiver"); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(first.Receiver, one)).Status); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(second.Receiver, two)).Status); + var firstOutcome = await Fixture.WaitForEffectCountAsync(first.Receiver, 1); + var secondOutcome = await Fixture.WaitForEffectCountAsync(second.Receiver, 1); + Assert.Equal(key, Assert.Single(firstOutcome.Effects).LogicalId); + Assert.Equal(key, Assert.Single(secondOutcome.Effects).LogicalId); + Assert.Equal(1, firstOutcome.ProcessedMessageCount); + Assert.Equal(1, secondOutcome.ProcessedMessageCount); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(first.Receiver, one)).Status); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(second.Receiver, two)).Status); + Assert.Equal(1, firstHandler.Calls); + Assert.Equal(1, secondHandler.Calls); + } + + [Fact] + public async Task PendingIdentityAfterReplayStillRejectsConflictAndDeduplicatesNewSender() + { + var rig = await CreateAsync(); + using var handler = rig.Handler; + var key = MessageId.Create("tenant", "orders", "1046", "reserve"); + var original = Create(rig, key, "inventory.reserve.v1", "original"); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(rig.Receiver, original)).Status); + await WaitAsync(handler.Entered.Task); + await OnTurnAsync(rig.Context, () => + { + rig.Context.Deactivate(new DeactivationReason( + DeactivationReasonCode.ApplicationRequested, "Verify pending command identity replay."), + TestContext.Current.CancellationToken); + }); + await WaitAsync(rig.Context.Deactivated); + using var recoveredHandler = new IdentityHandler(); + using var read = Fixture.Storage.BlockRead(JournalId.FromGrainId(rig.Context.GrainId)); + var activation = rig.Receiver.GetSnapshotAsync(); + await read.WaitUntilEnteredAsync(); + var recoveredContext = Fixture.GetGrainContext(rig.Receiver); + var recoveredGrain = Assert.IsType(recoveredContext.GrainInstance); + await OnTurnAsync(recoveredContext, () => recoveredGrain.HandlerOverride = recoveredHandler); + read.Release(); + await WaitAsync(recoveredHandler.Entered.Task); + var recovered = CreateRig(rig.Receiver, recoveredContext, recoveredHandler); + Assert.NotSame(rig.Context, recovered.Context); + var repeated = TestApplicationProtocol.With(original, Sessions, + sender: GrainId.Create("forwarder", "after-replay")); + var conflict = Create(recovered, key, "inventory.reserve.v1", "changed", + GrainId.Create("forwarder", "after-replay")); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverOnTurnAsync(recovered, repeated)).Status); + await Assert.ThrowsAsync(() => DeliverOnTurnAsync(recovered, conflict)); + Assert.Equal(TestApplicationProtocol.Sender(Sessions, original.Envelope), TestApplicationProtocol.Sender(Sessions, Assert.Single(recovered.Pending).Value.Message.Envelope)); + Assert.Empty(recovered.Processed); + Assert.False(recovered.Grain.DeactivationFailure.Task.IsCompleted); + recoveredHandler.Release.TrySetResult(); + await activation; + var completed = await Fixture.WaitForEffectCountAsync(rig.Receiver, 1); + Assert.Equal(1, Assert.Single(completed.Effects).Count); + Assert.Equal(1, completed.ProcessedMessageCount); + } + + [Fact] + public async Task DeadLetterInspectionAndRemovalUseReceiverLocalKeyAcrossReplay() + { + var receiver = NewGrain(); + var key = MessageId.Create("tenant", "orders", "1048", "invalid-command"); + var body = new TestMessage(key, 611, "poison", ThrowDuringPreparation: true); + var original = CreateEnvelope(receiver, body); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, original)).Status); + var deadLetter = Assert.Single((await Fixture.WaitForDeadLetterCountAsync(receiver, 1)).InboxDeadLetters); + Assert.Equal(key, deadLetter.MessageId); + var old = Fixture.GetGrainContext(receiver); + await receiver.RequestDeactivationAsync(); + await WaitAsync(old.Deactivated); + var recovered = await receiver.GetSnapshotAsync(); + Assert.Equal(key, Assert.Single(recovered.InboxDeadLetters).MessageId); + var duplicate = TestApplicationProtocol.With(original, Sessions, + sender: GrainId.Create("forwarder", "deadletter"), subject: "another.subject.v1"); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(receiver, duplicate)).Status); + Assert.True(await receiver.RemoveInboxDeadLetterAsync(key)); + Assert.False(await receiver.RemoveInboxDeadLetterAsync(key)); + var current = Fixture.GetGrainContext(receiver); + await receiver.RequestDeactivationAsync(); + await WaitAsync(current.Deactivated); + Assert.Empty((await receiver.GetSnapshotAsync()).InboxDeadLetters); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(receiver, duplicate)).Status); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task MissingOptionalMetadata_RawHandlerCompletesWithoutSubject(bool removeSender) + { + var rig = await CreateAsync(); + using var handler = rig.Handler; + var key = MessageId.Create("tenant", "invalid-subject", "command"); + var envelope = Create(rig, key, "valid.subject.v1", "missing-subject"); + var raw = TestApplicationProtocol.With(envelope, Sessions, removeSubject: true, removeSender: removeSender); + var bytes = envelope.Envelope.Payload.ToArray(); + handler.Release.TrySetResult(); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverOnTurnAsync(rig, raw)).Status); + var completed = await Fixture.WaitForEffectCountAsync(rig.Receiver, 1); + Assert.Equal(1, Assert.Single(completed.Effects).Count); + Assert.Equal(1, completed.ProcessedMessageCount); + Assert.Null(raw.Envelope.Subject); + if (removeSender) Assert.DoesNotContain("sender", Assert.IsType>(raw.Envelope.Headers)); + Assert.Equal(bytes, raw.Envelope.Payload.ToArray()); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverOnTurnAsync(rig, raw)).Status); + Assert.False(rig.Grain.DeactivationFailure.Task.IsCompleted); + } + + [Theory] + [InlineData("key-bytes")] + [InlineData("key-utf8")] + [InlineData("subject-bytes")] + [InlineData("subject-utf8")] + public async Task OversizedMetadata_RejectsBeforeSchedulingOrPersistence(string field) + { + var rig = await CreateAsync(); + using var handler = rig.Handler; + var key = field switch + { + "key-bytes" => MessageId.Create(new string('x', 1025)), + "key-utf8" => MessageId.Create(new string('\u00e9', 513)), + _ => MessageId.Create("tenant", "bounded", "command") + }; + var subject = field switch + { + "subject-bytes" => new string('x', 257), + "subject-utf8" => new string('\u00e9', 129), + _ => "inventory.reserve.v1" + }; + var writes = Writes(rig); + var message = Create(rig, key, subject, "oversized"); + await Assert.ThrowsAnyAsync(() => DeliverOnTurnAsync(rig, message)); + Assert.Equal(writes, Writes(rig)); + Assert.Equal(0, Fixture.JobManagerProbe.GetAttemptCount(MessagingPump.JobName, rig.Context.GrainId)); + Assert.Empty(rig.Pending); + Assert.Empty(rig.Processed); + Assert.Empty(rig.Grain.GetSnapshotForTest().Effects); + Assert.False(rig.Grain.DeactivationFailure.Task.IsCompleted); + Assert.False(handler.Entered.Task.IsCompleted); + } + + [Theory] + [InlineData("key-bytes")] + [InlineData("key-utf8")] + [InlineData("subject-bytes")] + [InlineData("subject-utf8")] + [InlineData("subject-space")] + public async Task MetadataAtExactDefaultLimit_IsAccepted(string field) + { + var rig = await CreateAsync(); + using var handler = rig.Handler; + handler.Release.TrySetResult(); + var key = field switch + { + "key-bytes" => MessageId.Create(new string('x', 1024)), + "key-utf8" => MessageId.Create(new string('\u00e9', 512)), + "key-depth" => MessageId.Create(Enumerable.Repeat("segment", 32).ToArray()), + _ => MessageId.Create("tenant", "bounded", "command") + }; + var subject = field switch + { + "subject-bytes" => new string('x', 256), + "subject-utf8" => new string('\u00e9', 128), + "subject-space" => " ", + _ => "inventory.reserve.v1" + }; + var envelope = Create(rig, key, subject, "maximum"); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(rig.Receiver, envelope)).Status); + var completed = await Fixture.WaitForEffectCountAsync(rig.Receiver, 1); + Assert.Equal(1, Assert.Single(completed.Effects).Count); + Assert.Equal(1, completed.ProcessedMessageCount); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(rig.Receiver, envelope)).Status); + } + + [Fact] + public async Task RetainedIdentityNeverReappliesChangedSenderSubjectAndBodyAtExactBoundary() + { + var rig = await CreateAsync(); + using var handler = rig.Handler; + handler.Release.TrySetResult(); + var key = MessageId.Create("tenant", "orders", "1047", "reserve"); + var original = Create(rig, key, "inventory.reserve.v1", "original"); + var repeated = Create(rig, key, "inventory.reserve.v2", "changed", + GrainId.Create("forwarder", "new")) with + { SentAt = original.SentAt }; + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(rig.Receiver, original)).Status); + await Fixture.WaitForEffectCountAsync(rig.Receiver, 1); + await Fixture.WaitForInboxCountAsync(rig.Receiver, 0); + Fixture.Clock.Advance(TimeSpan.FromMinutes(10) - TimeSpan.FromTicks(1)); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(rig.Receiver, repeated)).Status); + Fixture.Clock.Advance(TimeSpan.FromTicks(1)); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(rig.Receiver, repeated)).Status); + var completed = await rig.Receiver.GetSnapshotAsync(); + Assert.Equal(1, Assert.Single(completed.Effects).Count); + Assert.Equal(1, completed.ProcessedMessageCount); + Assert.Equal(1, handler.Calls); + } + + private async Task CreateAsync() + { + var receiver = NewGrain(); + await receiver.GetSnapshotAsync(); + var context = Fixture.GetGrainContext(receiver); + var handler = new IdentityHandler(); + var rig = CreateRig(receiver, context, handler); + await OnTurnAsync(context, () => rig.Grain.HandlerOverride = handler); + return rig; + } + + private static Rig CreateRig(IMessagingTestGrain receiver, IGrainContext context, IdentityHandler handler) => + new(receiver, context, Assert.IsType(context.GrainInstance), handler, + context.ActivationServices.GetRequiredService(), + context.ActivationServices.GetRequiredService()); + + private InboxMessage Create(Rig rig, MessageId key, string subject, string value, GrainId sender = default) => + TestApplicationProtocol.CreateInbox(Sessions, + sender.IsDefault ? GrainId.Create("producer", "first") : sender, + subject, new TestMessage(key, 610, value), Fixture.JobNow, key); + + private int Writes(Rig rig) => Fixture.Storage.GetSuccessfulWriteCount(JournalId.FromGrainId(rig.Context.GrainId)); + + private static Task DeliverOnTurnAsync(Rig rig, InboxMessage envelope) + { + var started = new TaskCompletionSource>(TaskCreationOptions.RunContinuationsAsynchronously); + rig.Context.Scheduler.QueueAction(() => + { + try + { + Assert.Same(rig.Context, ReceiverTestServices.CurrentGrainContext); + IMessagingExtension extension = rig.Context.ActivationServices.GetRequiredService(); + started.SetResult(extension.DeliverAsync(envelope, TestContext.Current.CancellationToken).AsTask()); + } + catch (Exception exception) { started.SetException(exception); } + }); + return started.Task.Unwrap(); + } + + private static Task OnTurnAsync(IGrainContext context, Action action) => + OnTurnAsync(context, () => { action(); return Task.CompletedTask; }); + + private static Task OnTurnAsync(IGrainContext context, Func action) + { + var done = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + context.Scheduler.QueueAction(() => { _ = CompleteAsync(); }); + return done.Task; + async Task CompleteAsync() + { + try + { + Assert.Same(context, ReceiverTestServices.CurrentGrainContext); + await action(); + done.SetResult(); + } + catch (Exception exception) { done.SetException(exception); } + } + } + + private static Task WaitAsync(Task task) => task.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + private sealed record Rig(IMessagingTestGrain Receiver, IGrainContext Context, + MessagingTestGrain Grain, IdentityHandler Handler, IJournaledStateManager Manager, Inbox Inbox) + { + public IReadOnlyDictionary Pending => InboxTestAccess.GetPendingMessages(Inbox); + public IReadOnlySet Processed => InboxTestAccess.GetProcessedKeys(Inbox); + } + + private sealed class IdentityHandler : IInboxHandler, IDisposable + { + public TaskCompletionSource Entered { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + public TaskCompletionSource Release { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + public int Calls { get; private set; } + public bool CanHandle(InboxMessage message) => true; + + public async ValueTask HandleAsync(IInboxHandlerContext context, CancellationToken cancellationToken) + { + var services = ReceiverTestServices.CurrentGrainContext!.ActivationServices; + var effects = services.GetRequiredKeyedService>("test-effects"); + var application = TestApplicationProtocol.Read(services.GetRequiredService(), context.Message); + var body = Assert.IsType(application.Body); + Calls++; + Entered.TrySetResult(); + await Release.Task.WaitAsync(cancellationToken); + cancellationToken.ThrowIfCancellationRequested(); + effects.TryGetValue(context.Message.Envelope.MessageId, out var prior); + effects[context.Message.Envelope.MessageId] = new MessageEffect(context.Message.Envelope.MessageId, + (prior?.Count ?? 0) + 1, body.Sequence, body.Value); + context.Complete(); + } + public void Dispose() => Release.TrySetResult(); + } +} diff --git a/test/Orleans.Messaging.Tests/Functional/InboxJournalRecoveryTests.cs b/test/Orleans.Messaging.Tests/Functional/InboxJournalRecoveryTests.cs new file mode 100644 index 00000000000..3fad020ba53 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/InboxJournalRecoveryTests.cs @@ -0,0 +1,131 @@ +using Microsoft.Extensions.DependencyInjection; +using Orleans.DurableJobs; +using Orleans.Journaling; +using Orleans.Messaging.Tests.Support; +using Xunit; + +namespace Orleans.Messaging.Tests.Functional; + +[Collection(MessagingClusterCollection.Name)] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class InboxJournalRecoveryTests : MessagingBehaviorTestBase +{ + [Fact] + public async Task FailedInboxAcceptance_FencesOldObjectsAndFreshReplayOmitsEnvelope() + { + var receiver = NewGrain(); + var before = await receiver.GetSnapshotAsync(); + var oldContext = Fixture.GetGrainContext(receiver); + var oldManager = oldContext.ActivationServices.GetRequiredService(); + Fixture.Storage.FailWrite(JournalId.FromGrainId(receiver.GetGrainId())); + var envelope = CreateEnvelope(receiver, NewMessage(2, "failed-acceptance")); + + await Assert.ThrowsAsync(() => DeliverAsync(receiver, envelope)); + await oldContext.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + + var recovered = await receiver.GetSnapshotAsync(); + Assert.NotEqual(before.ActivationId, recovered.ActivationId); + Assert.NotSame(oldManager, Fixture.GetGrainContext(receiver).ActivationServices.GetRequiredService()); + Assert.Equal(0, recovered.InboxCount); + Assert.Empty(recovered.Effects); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + Assert.Equal(1, Assert.Single((await Fixture.WaitForEffectCountAsync(receiver, 1)).Effects).Count); + } + + [Fact] + public async Task AmbiguousInboxAcceptanceCommit_PreservesAndProcessesRecoveredEnvelope() + { + var receiver = NewGrain(); + var before = await receiver.GetSnapshotAsync(); + var oldContext = Fixture.GetGrainContext(receiver); + Fixture.Storage.FailAfterWrite(JournalId.FromGrainId(receiver.GetGrainId())); + var envelope = CreateEnvelope(receiver, NewMessage(76, "ambiguous-acceptance")); + + await Assert.ThrowsAsync(() => DeliverAsync(receiver, envelope)); + await oldContext.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + + _ = await receiver.GetSnapshotAsync(); + var completed = await Fixture.WaitForEffectCountAsync(receiver, 1); + Assert.NotEqual(before.ActivationId, completed.ActivationId); + Assert.Equal("ambiguous-acceptance", Assert.Single(completed.Effects).Value); + Assert.Equal(0, completed.InboxCount); + } + + [Fact] + public async Task CompletionWriteFailure_RecoversInFreshActivation() + { + var receiver = NewGrain(); + var before = await receiver.GetSnapshotAsync(); + const string route = "messages/inbox-completion-retry"; + using var handler = Fixture.HandlerProbe.Arm(receiver.GetGrainId(), route); + var envelope = CreateEnvelope(receiver, NewMessage(57, "inbox-completion-retry"), route); + + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + await handler.WaitUntilEnteredAsync(); + Fixture.Storage.FailWrite(JournalId.FromGrainId(receiver.GetGrainId())); + handler.Release(); + + var cleaned = await Fixture.SnapshotProbe.WaitAsync( + receiver.GetGrainId(), + snapshot => snapshot.ActivationId != before.ActivationId && snapshot.InboxCount == 0 && snapshot.Effects.Count > 0); + + Assert.Equal(1, Assert.Single(cleaned.Effects).Count); + Assert.Empty(cleaned.InboxDeadLetters); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task CompletionWriteFailure_CancelsTheOwningJobPassBeforeRecovery(bool ambiguous) + { + var receiver = NewGrain(); + var before = await receiver.GetSnapshotAsync(); + var previous = Fixture.GetGrainContext(receiver); + const string route = "messages/failed-job-pass"; + using var handler = Fixture.HandlerProbe.Arm(receiver.GetGrainId(), route); + var envelope = CreateEnvelope(receiver, NewMessage(77, "failed-job-pass"), route); + Fixture.JobManagerProbe.SkipNext(MessagingPump.JobName); + await Fixture.OnGrainAsync(receiver, () => InboxTestAccess.StageAcceptance(Fixture.GetInbox(receiver), envelope)); + await receiver.RetryWriteStateAsync(); + + // This job owns the pass which enters the handler. + var running = Fixture.RunPumpAsync(receiver); + await handler.WaitUntilEnteredAsync(); + var journal = JournalId.FromGrainId(receiver.GetGrainId()); + if (ambiguous) Fixture.Storage.FailAfterWrite(journal); + else Fixture.Storage.FailWrite(journal); + handler.Release(); + + await Assert.ThrowsAnyAsync(() => running); + await previous.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + _ = await receiver.GetSnapshotAsync(); + var recovered = await Fixture.WaitForEffectCountAsync(receiver, 1); + Assert.NotEqual(before.ActivationId, recovered.ActivationId); + Assert.Equal("failed-job-pass", Assert.Single(recovered.Effects).Value); + Assert.Equal(1, Assert.Single(recovered.Effects).Count); + Assert.Equal(0, recovered.InboxCount); + Assert.Empty(recovered.InboxDeadLetters); + } + + [Fact] + public async Task WakeupSchedulingFailure_RetriedDeliveryProcessesOnce() + { + var receiver = NewGrain(); + _ = await receiver.GetSnapshotAsync(); + var oldContext = Fixture.GetGrainContext(receiver); + var envelope = CreateEnvelope(receiver, NewMessage(61, "schedule-failure")); + Fixture.JobManagerProbe.FailNext(MessagingPump.JobName); + + // The wakeup is scheduled while the acceptance write is prepared, so its failure prevents that commit. + var failure = await Assert.ThrowsAsync(() => DeliverAsync(receiver, envelope)); + Assert.IsType(failure.InnerException); + await oldContext.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + + var status = (await DeliverAsync(receiver, envelope)).Status; + Assert.True(status is DeliveryStatus.Accepted or DeliveryStatus.Duplicate, status.ToString()); + var completed = await Fixture.WaitForEffectCountAsync(receiver, 1); + Assert.Equal(1, Assert.Single(completed.Effects).Count); + } +} diff --git a/test/Orleans.Messaging.Tests/Functional/InboxLifecycleBehaviorTests.cs b/test/Orleans.Messaging.Tests/Functional/InboxLifecycleBehaviorTests.cs new file mode 100644 index 00000000000..55870a5c602 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/InboxLifecycleBehaviorTests.cs @@ -0,0 +1,61 @@ +using Orleans.Messaging.Tests.Support; +using Orleans.Runtime; +using Xunit; + +namespace Orleans.Messaging.Tests.Functional; + +[Collection(MessagingClusterCollection.Name)] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class InboxLifecycleBehaviorTests : MessagingBehaviorTestBase +{ + [Fact] + public async Task Deactivation_StopsPumpAndWaitsForActiveHandler() + { + var receiver = NewGrain(); + var before = await receiver.GetSnapshotAsync(); + var context = Fixture.GetGrainContext(receiver); + var shutdown = Fixture.GetPump(receiver).ShutdownToken; + using var handler = Fixture.HandlerProbe.Arm(receiver.GetGrainId(), "messages/teardown-active", observeCancellation: false); + var envelope = CreateEnvelope(receiver, NewMessage(95, "active-handler"), "messages/teardown-active"); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + await handler.WaitUntilEnteredAsync(); + var stopping = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + using var registration = shutdown.Register(stopping.SetResult); + + context.Deactivate(new(DeactivationReasonCode.ApplicationRequested, "Verify active handler teardown."), TestContext.Current.CancellationToken); + + await stopping.Task.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + Assert.False(context.Deactivated.IsCompleted); + handler.Release(); + await context.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + + var recovered = await Fixture.WaitForEffectCountAsync(receiver, 1); + Assert.NotEqual(before.ActivationId, recovered.ActivationId); + Assert.Equal(1, Assert.Single(recovered.Effects).Count); + Assert.Equal(0, recovered.InboxCount); + } + + [Fact] + public async Task DeactivationBeforeProcessing_RecoveryDrainsAcceptedMessageOnce() + { + var receiver = NewGrain(); + var before = await receiver.GetSnapshotAsync(); + var context = Fixture.GetGrainContext(receiver); + var envelope = CreateEnvelope(receiver, NewMessage(96, "queued"), "messages/queued"); + + // Deactivation cancels any handler the original activation starts, leaving the message to recovery. + using (Fixture.HandlerProbe.Arm(receiver.GetGrainId(), "messages/queued")) + { + Assert.Equal(DeliveryStatus.Accepted, (await receiver.AcceptAndDeactivateAsync(envelope)).Status); + await context.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + } + + _ = await receiver.GetSnapshotAsync(); + var recovered = await Fixture.WaitForEffectCountAsync(receiver, 1); + Assert.NotEqual(before.ActivationId, recovered.ActivationId); + Assert.Equal(1, Assert.Single(recovered.Effects).Count); + Assert.Equal(0, recovered.InboxCount); + } +} diff --git a/test/Orleans.Messaging.Tests/Functional/InboxQuiescenceTests.cs b/test/Orleans.Messaging.Tests/Functional/InboxQuiescenceTests.cs new file mode 100644 index 00000000000..8c357572a72 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/InboxQuiescenceTests.cs @@ -0,0 +1,92 @@ +using Microsoft.Extensions.DependencyInjection; +using Orleans.Journaling; +using Orleans.Messaging.Tests.Support; +using Orleans.Runtime; +using Xunit; + +namespace Orleans.Messaging.Tests.Functional; + +[Collection(MessagingClusterCollection.Name)] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class InboxQuiescenceTests : MessagingBehaviorTestBase +{ + [Fact] + public async Task OwnerDelete_AfterAcknowledgedWork_StopsPumpAndUsesFreshActivation() + { + var receiver = NewGrain(); + using var handler = Fixture.HandlerProbe.Arm(receiver.GetGrainId(), "messages/delete-ack"); + var first = CreateEnvelope(receiver, NewMessage(160, "first"), "messages/delete-ack"); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, first)).Status); + await handler.WaitUntilEnteredAsync(); + var context = Fixture.GetGrainContext(receiver); + var grain = Assert.IsType(context.GrainInstance); + var pump = Fixture.GetPump(receiver); + var outbox = GetOutbox(context); + var second = CreateEnvelope(receiver, NewMessage(161, "second"), "messages/delete-ack"); + var secondDelivery = DeliverAsync(receiver, second); + + handler.Release(); + Assert.Equal(DeliveryStatus.Accepted, (await secondDelivery).Status); + var completed = await Fixture.WaitForEffectCountAsync(receiver, 2); + Assert.Equal(2, completed.ProcessedMessageCount); + Assert.False(grain.DeactivationFailure.Task.IsCompleted); + using var storage = Fixture.Storage.BlockDelete(JournalId.FromGrainId(receiver.GetGrainId())); + var deletion = receiver.DeleteStateAndDeactivateAsync(); + await storage.WaitUntilEnteredAsync(); + Assert.True(outbox.Stopping.IsCompleted); + Assert.True(pump.ShutdownToken.IsCancellationRequested); + Assert.False(deletion.IsCompleted); + storage.Release(); + await deletion; + await context.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + var deleted = await receiver.GetSnapshotAsync(); + Assert.NotEqual(completed.ActivationId, deleted.ActivationId); + Assert.Empty(deleted.Effects); + Assert.Equal(0, deleted.ProcessedMessageCount); + } + + [Fact] + public async Task OwnerDelete_DiscardsStagedOutputAndLaterDeliveryUsesFreshActivation() + { + var receiver = NewGrain(); + await receiver.StageEffectAsync(new MessageEffect(TestApplicationProtocol.NewMessageId(), 1, 190, "delete-this")); + await receiver.RetryWriteStateAsync(); + var before = await receiver.GetSnapshotAsync(); + var context = Fixture.GetGrainContext(receiver); + var grain = Assert.IsType(context.GrainInstance); + var outbox = GetOutbox(context); + var staged = TestApplicationProtocol.Create(Sessions, receiver.GetGrainId(), receiver.GetGrainId(), + "output/staged", NewMessage(193, "staged-output")); + { + await receiver.StageOutputAsync(staged); + Assert.Equal(staged.Envelope.MessageId, Assert.Single(outbox.Messages).Envelope.MessageId); + var journal = JournalId.FromGrainId(receiver.GetGrainId()); + var writes = Fixture.Storage.GetSuccessfulWriteCount(journal); + await receiver.DeleteStateAndDeactivateAsync(); + await context.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + Assert.Equal(writes, Fixture.Storage.GetSuccessfulWriteCount(journal)); + Assert.Empty(outbox.Messages); + Assert.Empty(grain.GetSnapshotForTest().Effects); + Assert.True(outbox.Stopping.IsCompleted); + Assert.Throws(() => outbox.Send(staged)); + } + Assert.Empty(outbox.Messages); + var fresh = await receiver.GetSnapshotAsync(); + Assert.NotEqual(before.ActivationId, fresh.ActivationId); + Assert.Empty(fresh.Effects); + Assert.Equal(0, fresh.OutboxCount); + var envelope = CreateEnvelope(receiver, NewMessage(191, "new-state")); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + var after = await Fixture.WaitForEffectCountAsync(receiver, 1); + _ = await receiver.GetSnapshotAsync(); + Assert.Equal(fresh.ActivationId, after.ActivationId); + Assert.Equal("new-state", Assert.Single(after.Effects).Value); + Assert.Equal(1, after.ProcessedMessageCount); + Assert.False(grain.DeactivationFailure.Task.IsCompleted); + } + + private static JournaledTestOutbox GetOutbox(IGrainContext context) => + (JournaledTestOutbox)context.ActivationServices.GetRequiredService(); +} diff --git a/test/Orleans.Messaging.Tests/Functional/InboxRetryTests.cs b/test/Orleans.Messaging.Tests/Functional/InboxRetryTests.cs new file mode 100644 index 00000000000..21221e9b979 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/InboxRetryTests.cs @@ -0,0 +1,243 @@ +using Microsoft.Extensions.DependencyInjection; +using Orleans.DurableJobs; +using Orleans.Messaging.Configuration; +using Orleans.Messaging.Tests.Support; +using Xunit; + +namespace Orleans.Messaging.Tests.Functional; + +[Collection(MessagingClusterCollection.Name)] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class InboxRetryTests() : MessagingBehaviorTestBase(new RetryFixture()) +{ + private static readonly TimeSpan RetryDelay = TimeSpan.FromMinutes(1); + + // Attempts run at 0, 1, 3, 7, 15, 31 and 63 retry delays; the eighth and last runs when the timeout elapses. + private static readonly TimeSpan RetryTimeout = RetryDelay * 100; + + // Retry deadlines follow the Durable Jobs clock, which these tests move independently of the grain clock. + private ManualJobClock JobClock => ((RetryFixture)Fixture).JobClock; + + [Fact] + public async Task HandlerFailure_RetriesOnTheDurableJobsClockAndNotTheGrainClock() + { + var receiver = NewGrain(); + _ = await receiver.GetSnapshotAsync(); + var inbox = Fixture.GetInbox(receiver); + var envelope = CreateEnvelope(receiver, NewMessage(197, "job-clock") with { ThrowOnceDuringPreparation = true }); + var messageId = envelope.Envelope.MessageId; + var start = JobClock.GetUtcNow(); + + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + await Fixture.WaitUntilAsync(receiver, () => inbox.Count == 1 && InboxTestAccess.GetMessageState(inbox, messageId).AttemptCount == 1); + var state = await Fixture.OnGrainAsync(receiver, () => InboxTestAccess.GetMessageState(inbox, messageId)); + Assert.Equal(start + RetryDelay, state.NextAttemptAt); + Assert.Equal(start + RetryTimeout, state.ExpirationTime); + + // Moving the grain clock past the retry timeout neither retries nor dead-letters the message. + Fixture.Clock.Advance(RetryTimeout * 2); + Assert.Equal(DurableJobRunStatus.RescheduleRequested, (await Fixture.RunPumpAsync(receiver)).Status); + var pending = await receiver.GetSnapshotAsync(); + Assert.Equal(1, pending.InboxCount); + Assert.Empty(pending.Effects); + Assert.Empty(pending.InboxDeadLetters); + Assert.Equal(1, (await Fixture.OnGrainAsync(receiver, () => InboxTestAccess.GetMessageState(inbox, messageId))).AttemptCount); + + JobClock.Advance(RetryDelay); + Assert.Equal(DurableJobRunStatus.Completed, (await Fixture.RunPumpAsync(receiver)).Status); + Assert.Equal(1, Assert.Single((await Fixture.WaitForEffectCountAsync(receiver, 1)).Effects).Count); + } + + [Fact] + public async Task HandlerFailure_RetriesAfterBackoffAndEarlyWakeupsAreHarmless() + { + var receiver = NewGrain(); + _ = await receiver.GetSnapshotAsync(); + var inbox = Fixture.GetInbox(receiver); + var envelope = CreateEnvelope(receiver, NewMessage(192, "retry") with { ThrowOnceDuringPreparation = true }); + var messageId = envelope.Envelope.MessageId; + var now = JobClock.GetUtcNow(); + + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + await Fixture.WaitUntilAsync(receiver, () => inbox.Count == 1 && InboxTestAccess.GetMessageState(inbox, messageId).AttemptCount == 1); + var state = await Fixture.OnGrainAsync(receiver, () => InboxTestAccess.GetMessageState(inbox, messageId)); + Assert.Equal(now + RetryDelay, state.NextAttemptAt); + Assert.Contains("Injected handler preparation failure", state.LastError); + + // Wake-ups before the retry is due must not run the handler. + var early = await Fixture.RunPumpAsync(receiver); + Assert.Equal(DurableJobRunStatus.RescheduleRequested, early.Status); + _ = await Fixture.RunPumpAsync(receiver); + var pending = await receiver.GetSnapshotAsync(); + Assert.Equal(1, pending.InboxCount); + Assert.Empty(pending.Effects); + + JobClock.Advance(RetryDelay); + Assert.Equal(DurableJobRunStatus.Completed, (await Fixture.RunPumpAsync(receiver)).Status); + var completed = await Fixture.WaitForEffectCountAsync(receiver, 1); + Assert.Equal(1, Assert.Single(completed.Effects).Count); + Assert.Equal(0, completed.InboxCount); + Assert.Empty(completed.InboxDeadLetters); + + // Spurious wake-ups with no work complete immediately and change nothing. + Assert.Equal(DurableJobRunStatus.Completed, (await Fixture.RunPumpAsync(receiver)).Status); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(receiver, envelope)).Status); + Assert.Equal(1, Assert.Single((await receiver.GetSnapshotAsync()).Effects).Count); + } + + [Fact] + public async Task TransientHandlerFailures_RetryUntilTheHandlerSucceedsWithinTheRetryTimeout() + { + var receiver = NewGrain(); + _ = await receiver.GetSnapshotAsync(); + var inbox = Fixture.GetInbox(receiver); + var envelope = CreateEnvelope(receiver, NewMessage(196, "transient") with { ThrowForAttempts = 6 }); + var messageId = envelope.Envelope.MessageId; + var start = JobClock.GetUtcNow(); + + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + await Fixture.WaitUntilAsync(receiver, () => inbox.Count == 1 && InboxTestAccess.GetMessageState(inbox, messageId).AttemptCount == 1); + await RunFailedAttemptsAsync(receiver, inbox, messageId, start, failedAttempts: 6); + + Assert.Equal(DurableJobRunStatus.Completed, (await Fixture.RunPumpAsync(receiver)).Status); + var completed = await Fixture.WaitForEffectCountAsync(receiver, 1); + Assert.Equal(1, Assert.Single(completed.Effects).Count); + Assert.Equal(0, completed.InboxCount); + Assert.Empty(completed.InboxDeadLetters); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(receiver, envelope)).Status); + } + + // Verifies each failed attempt's retry state, running the pump at each retry, and leaves the clock at the attempt + // which follows the last failed one. + private async Task RunFailedAttemptsAsync(IMessagingTestGrain receiver, Inbox inbox, MessageId messageId, DateTimeOffset start, int failedAttempts) + { + var expirationTime = start + RetryTimeout; + for (var attempt = 1; ; attempt++) + { + var state = await Fixture.OnGrainAsync(receiver, () => InboxTestAccess.GetMessageState(inbox, messageId)); + Assert.Equal(attempt, state.AttemptCount); + Assert.Equal(expirationTime, state.ExpirationTime); + var backoff = start + RetryDelay * ((1 << attempt) - 1); + Assert.Equal(backoff < expirationTime ? backoff : expirationTime, state.NextAttemptAt); + JobClock.Advance(state.NextAttemptAt!.Value - JobClock.GetUtcNow()); + if (attempt == failedAttempts) + { + return; + } + + _ = await Fixture.RunPumpAsync(receiver); + await Fixture.WaitUntilAsync(receiver, () => InboxTestAccess.GetMessageState(inbox, messageId).AttemptCount == attempt + 1); + } + } + [Fact] + public async Task OrdinaryHandlerFailures_RetryUntilTheRetryTimeoutThenDeadLetter() + { + var receiver = NewGrain(); + _ = await receiver.GetSnapshotAsync(); + var inbox = Fixture.GetInbox(receiver); + var envelope = CreateEnvelope(receiver, NewMessage(193, "always-fails") with { ThrowDuringPreparation = true }); + var messageId = envelope.Envelope.MessageId; + var start = JobClock.GetUtcNow(); + + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + await Fixture.WaitUntilAsync(receiver, () => inbox.Count == 1 && InboxTestAccess.GetMessageState(inbox, messageId).AttemptCount == 1); + + // Backoff doubles per attempt, and the last attempt runs when InboxRetryTimeout elapses after the first failure. + await RunFailedAttemptsAsync(receiver, inbox, messageId, start, failedAttempts: 7); + Assert.Equal(start + RetryTimeout, JobClock.GetUtcNow()); + Assert.Equal(DurableJobRunStatus.Completed, (await Fixture.RunPumpAsync(receiver)).Status); + var deadLettered = await Fixture.WaitForDeadLetterCountAsync(receiver, 1); + var deadLetter = Assert.Single(deadLettered.InboxDeadLetters); + Assert.Equal(messageId, deadLetter.MessageId); + Assert.Equal(8, deadLetter.AttemptCount); + Assert.Equal(0, deadLettered.InboxCount); + Assert.Empty(deadLettered.Effects); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(receiver, envelope)).Status); + + await receiver.RequestDeactivationAsync(); + var recovered = await receiver.GetSnapshotAsync(); + Assert.Equal(deadLetter, Assert.Single(recovered.InboxDeadLetters)); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(receiver, envelope)).Status); + } + + [Fact] + public async Task AcceptedMessage_WaitsForHandlerRegistrationWithoutDeadLettering() + { + var receiver = NewGrain(); + _ = await receiver.GetSnapshotAsync(); + var context = Fixture.GetGrainContext(receiver); + var envelope = CreateEnvelope(receiver, NewMessage(194, "awaits-handler"), "messages/awaits-handler"); + using (var barrier = Fixture.HandlerProbe.Arm(receiver.GetGrainId(), "messages/awaits-handler")) + { + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + await WaitForBarrierAsync(receiver, barrier); + await receiver.ConfigureHandlerAsync(false); + + // Deactivation cancels the in-flight handler without consuming an attempt. + await receiver.RequestDeactivationAsync(); + await context.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + } + + var withoutHandler = await receiver.GetSnapshotAsync(); + Assert.Equal(1, withoutHandler.InboxCount); + Assert.Empty(withoutHandler.Effects); + Assert.Empty(withoutHandler.InboxDeadLetters); + + // Without a handler, the pending message holds no wake-up. + Assert.Equal(DurableJobRunStatus.Completed, (await Fixture.RunPumpAsync(receiver)).Status); + Assert.Equal(1, (await receiver.GetSnapshotAsync()).InboxCount); + + // Registering a handler on the same activation resumes the pending message. + await receiver.ConfigureHandlerAsync(true); + var completed = await Fixture.WaitForEffectCountAsync(receiver, 1); + Assert.Equal(withoutHandler.ActivationId, completed.ActivationId); + Assert.Equal("awaits-handler", Assert.Single(completed.Effects).Value); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(receiver, envelope)).Status); + } + + [Fact] + public async Task WakeupWhoseJobCompletesBeforeSchedulingReturns_DoesNotSuppressLaterWakeups() + { + var receiver = NewGrain(); + _ = await receiver.GetSnapshotAsync(); + var pump = Fixture.GetPump(receiver); + var target = receiver.GetGrainId(); + + // The scheduled job runs and completes, with no work remaining, before the scheduling call returns. + Fixture.JobManagerProbe.SkipNext(MessagingPump.JobName); + using var schedule = Fixture.JobManagerProbe.BlockNext(MessagingPump.JobName); + var scheduling = await Fixture.OnGrainAsync(receiver, () => pump.EnsureWakeupAsync(DateTimeOffset.MinValue, TestContext.Current.CancellationToken).AsTask()); + await schedule.WaitUntilEnteredAsync(); + Assert.Equal(DurableJobRunStatus.Completed, (await Fixture.RunPumpAsync(receiver)).Status); + schedule.Continue(); + await scheduling.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + + // Later work schedules its own wake-up. + var attempts = Fixture.JobManagerProbe.GetAttemptCount(MessagingPump.JobName, target); + var envelope = CreateEnvelope(receiver, NewMessage(195, "later-work")); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + Assert.Equal(attempts + 1, Fixture.JobManagerProbe.GetAttemptCount(MessagingPump.JobName, target)); + Assert.Equal(1, Assert.Single((await Fixture.WaitForEffectCountAsync(receiver, 1)).Effects).Count); + } + + private sealed class RetryFixture : MessagingClusterFixture + { + // Days apart from the grain clock, so a test which reads the wrong clock sees the wrong deadline. + public ManualJobClock JobClock { get; } = new(DateTimeOffset.UtcNow + TimeSpan.FromDays(3)); + + protected override void ConfigureOptions(DurableMessagingOptions options) + { + base.ConfigureOptions(options); + options.InboxRetryTimeout = RetryTimeout; + options.RetryInitialInterval = RetryDelay; + } + + protected override void ConfigureServices(IServiceCollection services) + { + base.ConfigureServices(services); + services.AddKeyedSingleton(DurableJobTimeProviderNames.DurableJobs, JobClock); + } + } +} diff --git a/test/Orleans.Messaging.Tests/Functional/InboxSentAtBoundTests.cs b/test/Orleans.Messaging.Tests/Functional/InboxSentAtBoundTests.cs new file mode 100644 index 00000000000..fdb70fb8b1a --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/InboxSentAtBoundTests.cs @@ -0,0 +1,128 @@ +using Microsoft.Extensions.DependencyInjection; +using Orleans.DurableJobs; +using Orleans.Journaling; +using Orleans.Messaging.Configuration; +using Orleans.Messaging.Tests.Support; +using Xunit; + +namespace Orleans.Messaging.Tests.Functional; + +/// +/// Pins the order in which the inbox evaluates a delivery's send time: the future bound applies only to a time the +/// inbox would store, a recorded identity is answered from its record, and an untracked identity at or below the +/// durable admission floor expires. +/// +[Collection(MessagingClusterCollection.Name)] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class InboxSentAtBoundTests() : MessagingBehaviorTestBase(new SentAtBoundFixture()) +{ + private static readonly TimeSpan Window = TimeSpan.FromMinutes(10); + + // The bound is exact only on a clock which stands still between reading it and delivering. + private ManualJobClock JobClock => ((SentAtBoundFixture)Fixture).JobClock; + + [Fact] + public async Task SentAtBeyondTheBound_ThrowsWithoutStagingAndTheBoundItselfIsAdmitted() + { + var receiver = NewGrain(); + _ = await receiver.GetSnapshotAsync(); + var journal = JournalId.FromGrainId(receiver.GetGrainId()); + var inbox = Fixture.GetInbox(receiver); + var bound = Fixture.JobNow + Window; + var message = CreateEnvelope(receiver, NewMessage(1, "future-bound")) with { SentAt = bound }; + var writes = Fixture.Storage.GetSuccessfulWriteCount(journal); + + await Assert.ThrowsAsync(() => DeliverAsync(receiver, message with { SentAt = bound.AddTicks(1) })); + Assert.Equal(writes, Fixture.Storage.GetSuccessfulWriteCount(journal)); + Assert.Equal(0, await Fixture.OnGrainAsync(receiver, () => InboxTestAccess.GetPendingMessages(inbox).Count)); + Assert.Equal(0, await Fixture.OnGrainAsync(receiver, () => InboxTestAccess.GetProcessedCount(inbox))); + + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, message)).Status); + await Fixture.WaitForInboxCountAsync(receiver, 0); + await receiver.RetryWriteStateAsync(); + writes = Fixture.Storage.GetSuccessfulWriteCount(journal); + + // A recorded identity may not extend its retention beyond the bound either. + await Assert.ThrowsAsync(() => DeliverAsync(receiver, message with { SentAt = bound.AddTicks(1) })); + Assert.Equal(writes, Fixture.Storage.GetSuccessfulWriteCount(journal)); + Assert.Equal(bound, await Fixture.OnGrainAsync(receiver, () => InboxTestAccess.GetProcessedSentAt(inbox, message.Envelope.MessageId))); + Assert.Equal(1, Assert.Single((await receiver.GetSnapshotAsync()).Effects).Count); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task RecordedIdentity_IsAnsweredFromItsRecordAfterClockRollbackPutsItBeyondTheBound(bool completed) + { + var receiver = NewGrain(); + const string route = "messages/sent-time-rollback"; + using var handler = Fixture.HandlerProbe.Arm(receiver.GetGrainId(), route); + var message = CreateEnvelope(receiver, NewMessage(2, "rolled-back"), route); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, message)).Status); + await WaitForBarrierAsync(receiver, handler); + if (completed) + { + handler.Release(); + await Fixture.WaitForInboxCountAsync(receiver, 0); + } + + await receiver.RetryWriteStateAsync(); + var journal = JournalId.FromGrainId(receiver.GetGrainId()); + var inbox = Fixture.GetInbox(receiver); + var writes = Fixture.Storage.GetSuccessfulWriteCount(journal); + JobClock.Advance(-(Window + TimeSpan.FromTicks(1))); + Assert.True(message.SentAt > Fixture.JobNow + Window); + + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(receiver, message)).Status); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(receiver, message with { SentAt = message.SentAt.AddTicks(-1) })).Status); + Assert.Equal(writes, Fixture.Storage.GetSuccessfulWriteCount(journal)); + + await Assert.ThrowsAsync(() => DeliverAsync(receiver, message with { SentAt = message.SentAt.AddTicks(1) })); + Assert.Equal(writes, Fixture.Storage.GetSuccessfulWriteCount(journal)); + Assert.Equal(message.SentAt, await Fixture.OnGrainAsync(receiver, () => completed + ? InboxTestAccess.GetProcessedSentAt(inbox, message.Envelope.MessageId) + : InboxTestAccess.GetPendingMessages(inbox)[message.Envelope.MessageId].Message.SentAt)); + + handler.Release(); + await Fixture.WaitForInboxCountAsync(receiver, 0); + Assert.Equal(1, Assert.Single((await receiver.GetSnapshotAsync()).Effects).Count); + } + + [Fact] + public async Task UntrackedIdentityAtTheDurableFloor_ExpiresAfterClockRollbackPutsItBeyondTheBound() + { + var receiver = NewGrain(); + var floor = Fixture.JobNow - Window; + var advancing = CreateEnvelope(receiver, NewMessage(3, "advances-floor")) with { SentAt = floor }; + Assert.Equal(DeliveryStatus.Expired, (await DeliverAsync(receiver, advancing)).Status); + var inbox = Fixture.GetInbox(receiver); + Assert.Equal(floor, await Fixture.OnGrainAsync(receiver, () => InboxTestAccess.DurableAdmissionFloor(inbox))); + + JobClock.Advance(-(Window + Window + TimeSpan.FromTicks(1))); + var untracked = CreateEnvelope(receiver, NewMessage(4, "at-floor")) with { SentAt = floor }; + Assert.True(untracked.SentAt > Fixture.JobNow + Window); + + Assert.Equal(DeliveryStatus.Expired, (await DeliverAsync(receiver, untracked)).Status); + await Assert.ThrowsAsync(() => DeliverAsync(receiver, untracked with { SentAt = floor.AddTicks(1) })); + Assert.Empty((await receiver.GetSnapshotAsync()).Effects); + } + + private sealed class SentAtBoundFixture : MessagingClusterFixture + { + public ManualJobClock JobClock { get; } = new(DateTimeOffset.UtcNow); + + protected override void ConfigureOptions(DurableMessagingOptions options) + { + base.ConfigureOptions(options); + options.DeduplicationWindow = Window; + } + + protected override void ConfigureServices(IServiceCollection services) + { + base.ConfigureServices(services); + services.AddKeyedSingleton(DurableJobTimeProviderNames.DurableJobs, JobClock); + } + } +} diff --git a/test/Orleans.Messaging.Tests/Functional/InboxStateEncodingTests.cs b/test/Orleans.Messaging.Tests/Functional/InboxStateEncodingTests.cs new file mode 100644 index 00000000000..7e06d07ee33 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/InboxStateEncodingTests.cs @@ -0,0 +1,126 @@ +using Microsoft.Extensions.DependencyInjection; +using Orleans.Journaling; +using Orleans.Messaging.Tests.Support; +using Xunit; + +namespace Orleans.Messaging.Tests.Functional; + +[Collection(MessagingClusterCollection.Name)] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class InboxStateEncodingTests : MessagingBehaviorTestBase +{ + public InboxStateEncodingTests() : base(new FaultingInboxCodecFixture()) { } + + [Fact] + public async Task CompleteCommandCodecFailure_PreservesOriginalCauseAndReplaysAcknowledgedState() + { + var receiver = NewGrain(); + _ = await receiver.GetSnapshotAsync(); + var context = Fixture.GetGrainContext(receiver); + var grain = Assert.IsType(context.GrainInstance); + var outbox = (JournaledTestOutbox)context.ActivationServices.GetRequiredService(); + using var handler = Fixture.HandlerProbe.Arm(receiver.GetGrainId(), "messages/complete-codec-fault"); + var envelope = CreateEnvelope(receiver, NewMessage(204, "complete-codec-fault"), "messages/complete-codec-fault"); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + await handler.WaitUntilEnteredAsync(); + var journal = JournalId.FromGrainId(receiver.GetGrainId()); + var writes = Fixture.Storage.GetSuccessfulWriteCount(journal); + var failure = new IOException("Completion remove command failed."); + ((FaultingInboxCodecFixture)Fixture).NextFailure = failure; + handler.Release(); + Assert.Same(failure, await grain.DeactivationFailure.Task.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken)); + await context.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + Assert.Equal(writes, Fixture.Storage.GetSuccessfulWriteCount(journal)); + Assert.Empty(outbox.Messages); + _ = await receiver.GetSnapshotAsync(); + var fresh = Assert.IsType(Fixture.GetGrainContext(receiver).GrainInstance); + var replayed = Assert.IsType(fresh.ReplayedSnapshot); + Assert.Equal(1, replayed.InboxCount); + Assert.Equal(0, replayed.ProcessedMessageCount); + Assert.Empty(replayed.Effects); + Assert.Equal(0, replayed.OutboxCount); + var completed = await Fixture.WaitForEffectCountAsync(receiver, 1); + Assert.Equal(1, Assert.Single(completed.Effects).Count); + Assert.Equal(1, completed.ProcessedMessageCount); + Assert.Empty(completed.InboxDeadLetters); + } + + [Fact] + public async Task HandlerErrorOutcomeCodecFailure_DeactivatesAndReplaysAcceptedMessage() + { + var receiver = NewGrain(); + _ = await receiver.GetSnapshotAsync(); + var context = Fixture.GetGrainContext(receiver); + var grain = Assert.IsType(context.GrainInstance); + using var handler = Fixture.HandlerProbe.Arm(receiver.GetGrainId(), "messages/error-codec-fault"); + var message = NewMessage(205, "error-codec-fault") with { ThrowOnceDuringPreparation = true }; + var envelope = CreateEnvelope(receiver, message, "messages/error-codec-fault"); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + await handler.WaitUntilEnteredAsync(); + var journal = JournalId.FromGrainId(receiver.GetGrainId()); + var writes = Fixture.Storage.GetSuccessfulWriteCount(journal); + var failure = new IOException("Handler error outcome command failed."); + ((FaultingInboxCodecFixture)Fixture).NextFailure = failure; + handler.Release(); + Assert.Same(failure, await grain.DeactivationFailure.Task.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken)); + await context.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + Assert.Equal(writes, Fixture.Storage.GetSuccessfulWriteCount(journal)); + _ = await receiver.GetSnapshotAsync(); + var completed = await Fixture.WaitForEffectCountAsync(receiver, 1); + Assert.Equal(1, Assert.Single(completed.Effects).Count); + Assert.Equal(1, completed.ProcessedMessageCount); + Assert.Empty(completed.InboxDeadLetters); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task InboxCodecFailure_PreventsStorageAndRetriedDeliveryProcessesOnce(bool snapshot) + { + var receiver = NewGrain(); + var before = await receiver.GetSnapshotAsync(); + var context = Fixture.GetGrainContext(receiver); + var grain = Assert.IsType(context.GrainInstance); + var outbox = (JournaledTestOutbox)context.ActivationServices.GetRequiredService(); + var journal = JournalId.FromGrainId(receiver.GetGrainId()); + var writes = Fixture.Storage.GetSuccessfulWriteCount(journal); + var failure = new IOException("Injected inbox command codec failure."); + var faulting = (FaultingInboxCodecFixture)Fixture; + faulting.NextFailure = failure; + faulting.FailOnSnapshot = snapshot; + if (snapshot) Fixture.Storage.RequestSnapshot(journal); + var envelope = CreateEnvelope(receiver, NewMessage(203, "codec")); + await Assert.ThrowsAsync(() => DeliverAsync(receiver, envelope)); + Assert.Equal(snapshot, faulting.FailedSnapshot); + Assert.Empty(outbox.Messages); + Assert.Equal(writes, Fixture.Storage.GetSuccessfulWriteCount(journal)); + if (snapshot) + { + // The snapshot fails while the write is encoded, after the acceptance was applied, so the activation is discarded. + Assert.Same(failure, await grain.DeactivationFailure.Task.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken)); + await context.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + Assert.Equal(1, grain.GetSnapshotForTest().InboxCount); + Assert.Empty(grain.GetSnapshotForTest().Effects); + var recovered = await receiver.GetSnapshotAsync(); + Assert.NotEqual(before.ActivationId, recovered.ActivationId); + Assert.Equal(0, recovered.InboxCount); + Assert.Empty(recovered.Effects); + Assert.Equal(0, recovered.ProcessedMessageCount); + } + else + { + // Commands are encoded before they are applied, so a rejected acceptance leaves the activation's state unchanged. + Assert.False(grain.DeactivationFailure.Task.IsCompleted); + var current = await receiver.GetSnapshotAsync(); + Assert.Equal(before.ActivationId, current.ActivationId); + Assert.Equal(0, current.InboxCount); + Assert.Empty(current.Effects); + Assert.Equal(0, current.ProcessedMessageCount); + } + + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + Assert.Equal(1, Assert.Single((await Fixture.WaitForEffectCountAsync(receiver, 1)).Effects).Count); + } +} diff --git a/test/Orleans.Messaging.Tests/Functional/InboxStateManagerBoundaryTests.cs b/test/Orleans.Messaging.Tests/Functional/InboxStateManagerBoundaryTests.cs new file mode 100644 index 00000000000..13b7358565c --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/InboxStateManagerBoundaryTests.cs @@ -0,0 +1,228 @@ +using System.Collections.Immutable; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using NSubstitute; +using Orleans.Hosting; +using Orleans.Journaling; +using Orleans.Journaling.Json; +using Orleans.Messaging.Tests.Support; +using Orleans.Metadata; +using Orleans.Runtime; +using Orleans.Serialization; +using Xunit; + +namespace Orleans.Messaging.Tests.Functional; + +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class InboxStateManagerBoundaryTests +{ + private static CancellationToken Cancellation => TestContext.Current.CancellationToken; + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task ActivationStandardState_UsesScopedCodecAndSingleOwnerEnrollment(bool keyedFirst) + { + const string selectedFormat = "orleans-binary"; + var builder = CreateBuilder(selectedFormat); + var codecType = GetDictionaryCodecType(builder.Services, selectedFormat); + builder.Services.AddKeyedScoped>(selectedFormat, (sp, _) => + new TrackingCodec((IDurableDictionaryCommandCodec)ActivatorUtilities.CreateInstance(sp, codecType))); + builder.Services.AddScoped(); + var grainType = GrainType.Create("codec-scope"); + var manifest = new GrainManifest( + ImmutableDictionary.Empty.Add( + grainType, new GrainProperties( + ImmutableDictionary.Empty.WithComparers(StringComparer.Ordinal, StringComparer.Ordinal))), + ImmutableDictionary.Empty); + var manifestProvider = Substitute.For(); + manifestProvider.LocalGrainManifest.Returns(manifest); + manifestProvider.Current.Returns(new ClusterManifest( + new MajorMinorVersion(1, 0), + ImmutableDictionary.Empty, + [manifest])); + builder.Services.AddSingleton(manifestProvider); + builder.Services.AddSingleton(); + builder.Services.AddScoped(sp => + { + var context = Substitute.For(); + context.ActivationServices.Returns(sp); + context.GrainId.Returns(GrainId.Create(grainType, IdSpan.Create(Guid.NewGuid().ToString("N")))); + context.ObservableLifecycle.Returns(Substitute.For()); + return context; + }); + builder.Services.AddScoped(sp => new OwnedComponent( + sp.GetRequiredKeyedService>("state"), sp.GetRequiredService())); + await using var services = builder.Services.BuildServiceProvider(validateScopes: true); + IDurableDictionary state; + OwnedComponent component; + Dependency dependency; + IJournaledStateManager journalOwner; + await using (var scope = services.CreateAsyncScope()) + { + var sp = scope.ServiceProvider; + Assert.Same(manifest.Grains[grainType], + sp.GetRequiredService().GetGrainProperties(sp.GetRequiredService().GrainId.Type)); + var app = sp.GetRequiredService(); + journalOwner = sp.GetRequiredService(); + Assert.Same(app, journalOwner); + var codec = Assert.IsType(sp.GetRequiredKeyedService>(selectedFormat)); + Assert.Throws(() => services.GetRequiredKeyedService>(selectedFormat)); + state = keyedFirst + ? sp.GetRequiredKeyedService>("state") + : app.GetOrAddState>("state"); + component = sp.GetRequiredService(); + dependency = sp.GetRequiredService(); + Assert.Same(state, component.State); + Assert.Same(dependency, component.Dependency); + Assert.Same(state, sp.GetRequiredKeyedService>("state")); + Assert.Same(state, app.GetOrAddState>("state")); + Assert.True(app.TryGetState>("state", out var found)); + Assert.Same(state, found); + Assert.True(journalOwner.TryGetStateMachine("state", out var machine)); + Assert.Same(state, machine); + sp.GetRequiredService().ObservableLifecycle.Received(1) + .Subscribe(Arg.Any(), GrainLifecycleStage.SetupState, Arg.Any()); + await journalOwner.InitializeAsync(Cancellation); + Assert.Equal(0, codec.Applies); + state.Add("committed", 17); + Assert.Equal(1, codec.SetsWritten); + await app.WriteStateAsync(Cancellation); + Assert.Equal(1, codec.SetsWritten); + Assert.Equal(17, state["committed"]); + Assert.Throws(() => app.GetOrAddState>("late")); + Assert.Throws(() => sp.GetRequiredKeyedService>("late")); + Assert.Same(state, app.GetOrAddState>("state")); + await journalOwner.DeleteStateAsync(Cancellation); + Assert.Empty(state); + Assert.False(dependency.Disposed); + Assert.Equal(0, component.Disposals); + await using var other = services.CreateAsyncScope(); + var otherOwner = other.ServiceProvider.GetRequiredService(); + Assert.NotSame(journalOwner, otherOwner); + Assert.NotSame(codec, other.ServiceProvider.GetRequiredKeyedService>(selectedFormat)); + } + Assert.True(dependency.Disposed); + Assert.Equal(1, component.Disposals); + await Assert.ThrowsAsync(() => journalOwner.WriteStateAsync(Cancellation).AsTask()); + } + + [Fact] + public async Task NamedStandaloneOwner_UsesSelectedFormatAndCallerOwnedComponentsAcrossReplay() + { + const string selectedFormat = "json"; + const string storageName = "caller-owned-store"; + var builder = CreateBuilder(selectedFormat); + builder.AddVolatileJournalStorage(storageName); + builder.UseJsonJournalFormat(StandardStateJsonContext.Default); + var codecType = GetDictionaryCodecType(builder.Services, selectedFormat); + builder.Services.AddKeyedSingleton>(selectedFormat, (sp, _) => + new TrackingCodec((IDurableDictionaryCommandCodec)ActivatorUtilities.CreateInstance(sp, codecType))); + var dependenciesCreated = 0; + builder.Services.AddScoped(_ => { dependenciesCreated++; return new Dependency(); }); + var id = new JournalId("inbox-state-lifetime/" + Guid.NewGuid().ToString("N")); + builder.Services.AddScoped(sp => + sp.GetRequiredKeyedService(storageName).CreateStandalone(id)); + builder.Services.AddScoped(sp => new OwnedComponent( + sp.GetRequiredKeyedService>("state"), sp.GetRequiredService())); + await using var services = builder.Services.BuildServiceProvider(validateScopes: true); + Assert.Equal(0, dependenciesCreated); + var codec = Assert.IsType(services.GetRequiredKeyedService>(selectedFormat)); + Dependency dependency; + OwnedComponent component; + await using (var scope = services.CreateAsyncScope()) + { + var first = scope.ServiceProvider.GetRequiredService(); + Assert.False(first is IDurableStateManager); + dependency = scope.ServiceProvider.GetRequiredService(); + Assert.False(first.TryGetStateMachine("state", out _)); + component = scope.ServiceProvider.GetRequiredService(); + var state = component.State; + Assert.Same(scope.ServiceProvider.GetRequiredKeyedService>("state"), state); + Assert.True(first.TryGetStateMachine("state", out var registered)); + Assert.Same(state, registered); + await first.InitializeAsync(Cancellation); + Assert.Equal(0, codec.Applies); + state.Add("persisted", 23); + await first.WriteStateAsync(Cancellation); + Assert.Equal(1, codec.SetsWritten); + await first.DisposeAsync(); + Assert.Equal(0, component.Disposals); + Assert.False(dependency.Disposed); + Assert.Equal(1, dependenciesCreated); + await using var recoveryScope = services.CreateAsyncScope(); + var recovered = recoveryScope.ServiceProvider.GetRequiredService(); + Assert.NotSame(first, recovered); + Assert.False(recovered is IDurableStateManager); + var recoveredCodec = recoveryScope.ServiceProvider.GetRequiredKeyedService>(selectedFormat); + Assert.Same(codec, recoveredCodec); + var fresh = recoveryScope.ServiceProvider.GetRequiredKeyedService>("state"); + await recovered.InitializeAsync(Cancellation); + Assert.NotSame(state, fresh); + Assert.Equal(new KeyValuePair("persisted", 23), Assert.Single(fresh)); + Assert.Equal(1, codec.Applies); + await recovered.DeleteStateAsync(Cancellation); + Assert.Empty(fresh); + await recovered.DisposeAsync(); + Assert.Equal(0, component.Disposals); + Assert.False(dependency.Disposed); + Assert.Equal(1, dependenciesCreated); + } + + Assert.True(dependency.Disposed); + Assert.Equal(1, component.Disposals); + } + + private static Type GetDictionaryCodecType(IServiceCollection services, string selectedFormat) => services.Last(entry => + entry.ServiceType == typeof(IDurableDictionaryCommandCodec<,>) && Equals(entry.ServiceKey, selectedFormat)) + .KeyedImplementationType!.MakeGenericType(typeof(string), typeof(int)); + + internal static TestSiloBuilder CreateBuilder(string selectedFormat) + { + var builder = new TestSiloBuilder(); + builder.Services.AddSerializer(); + builder.Services.AddLogging(); + builder.Services.AddSingleton(TimeProvider.System); + builder.Services.AddKeyedSingleton(KeyedService.AnyKey, static (sp, _) => sp.GetRequiredService()); + builder.AddVolatileJournalStorage(); + builder.Services.Configure(options => options.JournalFormatKey = selectedFormat); + return builder; + } + + internal sealed class TestSiloBuilder : ISiloBuilder + { + public IServiceCollection Services { get; } = new ServiceCollection(); + public IConfiguration Configuration { get; } = new ConfigurationBuilder().Build(); + } + + private sealed class Dependency : IDisposable + { + public bool Disposed { get; private set; } + public void Dispose() => Disposed = true; + } + + private sealed class OwnedComponent(IDurableDictionary state, Dependency dependency) : IDisposable + { + public IDurableDictionary State { get; } = state; + public Dependency Dependency { get; } = dependency; + public int Disposals { get; private set; } + public void Dispose() => Disposals++; + } + + private sealed class TrackingCodec(IDurableDictionaryCommandCodec inner) : IDurableDictionaryCommandCodec + { + public int SetsWritten { get; private set; } + public int Applies { get; private set; } + public void WriteSet(string key, int value, JournalStreamWriter writer) { SetsWritten++; inner.WriteSet(key, value, writer); } + public void WriteRemove(string key, JournalStreamWriter writer) => inner.WriteRemove(key, writer); + public void WriteClear(JournalStreamWriter writer) => inner.WriteClear(writer); + public void WriteSnapshot(IReadOnlyCollection> items, JournalStreamWriter writer) => inner.WriteSnapshot(items, writer); + public void Apply(JournalBufferReader input, IDurableDictionaryCommandHandler consumer) + { + Applies++; + inner.Apply(input, consumer); + } + } +} diff --git a/test/Orleans.Messaging.Tests/Functional/InboxStateProtocolTests.cs b/test/Orleans.Messaging.Tests/Functional/InboxStateProtocolTests.cs new file mode 100644 index 00000000000..cb2c412ba1d --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/InboxStateProtocolTests.cs @@ -0,0 +1,98 @@ +using Microsoft.Extensions.DependencyInjection; +using Orleans.Journaling; +using Orleans.Messaging.Tests.Support; +using Orleans.Runtime; +using Xunit; + +namespace Orleans.Messaging.Tests.Functional; + +[Collection(MessagingClusterCollection.Name)] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class InboxStateProtocolTests : MessagingBehaviorTestBase +{ + [Fact] + public async Task RawWriteDuringLocalPreparation_CapturesNoHandlerEffects() + { + var receiver = NewGrain(); + const string route = "messages/local-preparation"; + using var handler = Fixture.HandlerProbe.Arm(receiver.GetGrainId(), route); + var envelope = CreateEnvelope(receiver, NewMessage(201, "local"), route); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + await handler.WaitUntilEnteredAsync(); + var context = Fixture.GetGrainContext(receiver); + var grain = Assert.IsType(context.GrainInstance); + var manager = context.ActivationServices.GetRequiredService(); + await OnTurnAsync(context, () => context.ActivationServices.GetRequiredKeyedService>("inbox").Value = "raw-write"); + await manager.WriteStateAsync(TestContext.Current.CancellationToken); + Assert.False(grain.ApplyAttempted.Task.IsCompleted); + var beforeApply = grain.GetSnapshotForTest(); + Assert.Empty(beforeApply.Effects); + Assert.Equal(1, beforeApply.InboxCount); + Assert.Equal(0, beforeApply.ProcessedMessageCount); + var captured = grain.Captures[^1]; + Assert.Empty(captured.Effects); + Assert.Equal(1, captured.InboxCount); + Assert.Equal(0, captured.ProcessedMessageCount); + handler.Release(); + var completed = await Fixture.WaitForEffectCountAsync(receiver, 1); + Assert.Equal(0, completed.InboxCount); + Assert.Equal(1, completed.ProcessedMessageCount); + Assert.Equal(1, Assert.Single(completed.Effects).Count); + await receiver.RequestDeactivationAsync(); + var replayed = await receiver.GetSnapshotAsync(); + Assert.Equal(completed.Effects, replayed.Effects); + Assert.Equal(1, replayed.ProcessedMessageCount); + } + + [Fact] + public async Task HandlerFailureBeforeComplete_AccountsPreparationOutcomeWithoutMutationInference() + { + var receiver = NewGrain(); + const string route = "messages/no-earlier-write"; + using var handler = Fixture.HandlerProbe.Arm(receiver.GetGrainId(), route); + var envelope = CreateEnvelope(receiver, NewMessage(203, "no-earlier-write"), route); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + await handler.WaitUntilEnteredAsync(); + var context = Fixture.GetGrainContext(receiver); + var grain = Assert.IsType(context.GrainInstance); + var manager = context.ActivationServices.GetRequiredService(); + var journal = JournalId.FromGrainId(receiver.GetGrainId()); + var writes = Fixture.Storage.GetSuccessfulWriteCount(journal); + var failure = new IOException("Local preparation failed before Complete."); + await OnTurnAsync(context, () => + { + Assert.Equal(0, manager.PendingWriteByteCount); + grain.NextApplyFailure = failure; + }); + + handler.Release(); + var completed = await Fixture.WaitForDeadLetterCountAsync(receiver, 1); + Assert.Contains(failure.Message, Assert.Single(completed.InboxDeadLetters).Reason, StringComparison.Ordinal); + Assert.Equal(writes + 1, Fixture.Storage.GetSuccessfulWriteCount(journal)); + Assert.Empty(completed.Effects); + Assert.Equal(0, completed.InboxCount); + Assert.Equal(1, completed.ProcessedMessageCount); + Assert.False(grain.DeactivationFailure.Task.IsCompleted); + await receiver.RequestDeactivationAsync(); + await context.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + var recovered = await receiver.GetSnapshotAsync(); + Assert.NotEqual(grain.GetSnapshotForTest().ActivationId, recovered.ActivationId); + Assert.Empty(recovered.Effects); + Assert.Equal(completed.InboxDeadLetters, recovered.InboxDeadLetters); + Assert.Equal(1, recovered.ProcessedMessageCount); + Assert.Equal(0, recovered.InboxCount); + } + + private static Task OnTurnAsync(IGrainContext context, Action action) + { + var completion = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + context.Scheduler.QueueAction(() => + { + try { action(); completion.SetResult(); } + catch (Exception exception) { completion.SetException(exception); } + }); + return completion.Task; + } +} diff --git a/test/Orleans.Messaging.Tests/Functional/InboxWorkSourceTests.cs b/test/Orleans.Messaging.Tests/Functional/InboxWorkSourceTests.cs new file mode 100644 index 00000000000..eca2362bd90 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/InboxWorkSourceTests.cs @@ -0,0 +1,205 @@ +using System.Collections.Concurrent; +using Microsoft.Extensions.DependencyInjection; +using Orleans.DurableJobs; +using Orleans.Journaling; +using Orleans.Messaging.Tests.Support; +using Orleans.Runtime; +using Xunit; + +namespace Orleans.Messaging.Tests.Functional; + +/// +/// The inbox's durable wake-up reactivates a grain while a registered holds work. +/// +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class InboxWorkSourceTests() : MessagingBehaviorTestBase(receiverOnly: false) +{ + private static CancellationToken Token => TestContext.Current.CancellationToken; + + [Fact] + public async Task WriteWhichMakesWorkPending_SchedulesTheWakeupBeforeTheWriteIsStored() + { + var grain = NewWorkGrain(); + await grain.GetPendingAsync(); + using var write = Fixture.Storage.BlockWrite(Journal(grain)); + + var adding = grain.AddWorkAsync(TimeSpan.Zero); + await write.WaitUntilEnteredAsync(); + + Assert.Equal(1, Fixture.JobManagerProbe.GetSuccessCount(MessagingPump.JobName, grain.GetGrainId())); + write.Release(); + await adding; + await grain.CompleteWorkAsync(); + } + + [Fact] + public async Task NegativeSourceDelay_SchedulesAPromptWakeupBeforeTheWriteIsStored() + { + var grain = NewWorkGrain(); + await grain.GetPendingAsync(); + using var write = Fixture.Storage.BlockWrite(Journal(grain)); + + var adding = grain.AddWorkAsync(TimeSpan.MinValue); + await write.WaitUntilEnteredAsync(); + + Assert.Equal(1, Fixture.JobManagerProbe.GetSuccessCount(MessagingPump.JobName, grain.GetGrainId())); + write.Release(); + await adding; + Assert.Equal(1, await grain.GetPendingAsync()); + await grain.CompleteWorkAsync(); + } + + [Fact] + public async Task CrashAfterTheWriteWhichMakesWorkPending_ReactivatesTheGrain() + { + var grain = NewWorkGrain(); + await grain.GetPendingAsync(); + var crashed = Fixture.GetGrainContext(grain); + + // The write becomes durable but its acknowledgement fails, so the activation fences and stops. + Fixture.Storage.FailAfterWrite(Journal(grain)); + await Assert.ThrowsAsync(() => grain.AddWorkAsync(TimeSpan.Zero)); + await crashed.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), Token); + + await WaitForActivationsAsync(grain, 2); + Assert.Equal(1, await grain.GetPendingAsync()); + await grain.CompleteWorkAsync(); + } + + [Fact] + public async Task DeactivationWithPendingWork_ReactivatesTheGrain() + { + var grain = NewWorkGrain(); + await grain.AddWorkAsync(TimeSpan.Zero); + var previous = Fixture.GetGrainContext(grain); + + await grain.DeactivateAsync(); + await previous.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), Token); + + await WaitForActivationsAsync(grain, 2); + Assert.Equal(1, await grain.GetPendingAsync()); + await grain.CompleteWorkAsync(); + } + + [Fact] + public async Task Wakeup_ReschedulesAfterTheSourceDelay_AndCompletesOnceTheWorkClears() + { + var delay = TimeSpan.FromHours(1); + var grain = NewWorkGrain(); + await grain.AddWorkAsync(delay); + + var before = DateTimeOffset.UtcNow; + var pending = await Fixture.RunPumpAsync(grain); + var after = DateTimeOffset.UtcNow; + Assert.Equal(DurableJobRunStatus.RescheduleRequested, pending.Status); + Assert.InRange(pending.RescheduleTime!.Value, before + delay, after + delay); + + await grain.CompleteWorkAsync(); + Assert.Equal(DurableJobRunStatus.Completed, (await Fixture.RunPumpAsync(grain)).Status); + } + + [Fact] + public async Task WritesWithoutPendingWork_DoNotScheduleAWakeup() + { + var grain = NewWorkGrain(); + + await grain.AddWorkAsync(TimeSpan.Zero); + await grain.CompleteWorkAsync(); + var scheduled = Fixture.JobManagerProbe.GetAttemptCount(MessagingPump.JobName, grain.GetGrainId()); + Assert.Equal(DurableJobRunStatus.Completed, (await Fixture.RunPumpAsync(grain)).Status); + + await grain.TouchAsync(); + Assert.Equal(scheduled, Fixture.JobManagerProbe.GetAttemptCount(MessagingPump.JobName, grain.GetGrainId())); + } + + private IWorkSourceTestGrain NewWorkGrain() => Fixture.Client.GetGrain(Guid.NewGuid()); + + private static JournalId Journal(IAddressable grain) => JournalId.FromGrainId(grain.GetGrainId()); + + private static async Task WaitForActivationsAsync(IAddressable grain, int expected) + { + var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(30); + while (WorkSourceTestGrain.GetActivationCount(grain.GetGrainId()) < expected) + { + if (DateTime.UtcNow > deadline) + { + throw new TimeoutException($"Grain '{grain.GetGrainId()}' was not reactivated."); + } + + await Task.Delay(10, Token); + } + } +} + +public interface IWorkSourceTestGrain : IGrainWithGuidKey +{ + Task AddWorkAsync(TimeSpan wakeupDelay); + Task CompleteWorkAsync(); + Task TouchAsync(); + Task GetPendingAsync(); + Task DeactivateAsync(); +} + +/// Holds pending work in journaled state and reports it as an inbox work source. +public sealed class WorkSourceTestGrain : Grain, IWorkSourceTestGrain, IMessagingGrain, IInboxWorkSource +{ + private static readonly ConcurrentDictionary Activations = new(); + private readonly IJournaledStateManager _manager; + private readonly IDurableValue _pending; + private readonly IDurableValue _wakeupDelayTicks; + private readonly IDurableValue _touches; + + public WorkSourceTestGrain( + IJournaledStateManager manager, + IInbox inbox, + [FromKeyedServices("work-pending")] IDurableValue pending, + [FromKeyedServices("work-wakeup-delay")] IDurableValue wakeupDelayTicks, + [FromKeyedServices("work-touches")] IDurableValue touches) + { + _manager = manager; + _pending = pending; + _wakeupDelayTicks = wakeupDelayTicks; + _touches = touches; + inbox.Register(this); + } + + internal static int GetActivationCount(GrainId grainId) => Activations.TryGetValue(grainId, out var count) ? count : 0; + + public override Task OnActivateAsync(CancellationToken cancellationToken) + { + Activations.AddOrUpdate(this.GetGrainId(), 1, static (_, count) => count + 1); + return Task.CompletedTask; + } + + public async Task AddWorkAsync(TimeSpan wakeupDelay) + { + _pending.Value++; + _wakeupDelayTicks.Value = wakeupDelay.Ticks; + await _manager.WriteStateAsync(CancellationToken.None); + } + + public async Task CompleteWorkAsync() + { + _pending.Value = 0; + await _manager.WriteStateAsync(CancellationToken.None); + } + + public async Task TouchAsync() + { + _touches.Value++; + await _manager.WriteStateAsync(CancellationToken.None); + } + + public Task GetPendingAsync() => Task.FromResult(_pending.Value); + + public Task DeactivateAsync() + { + DeactivateOnIdle(); + return Task.CompletedTask; + } + + TimeSpan? IInboxWorkSource.GetWakeupDelay() => + _pending.Value > 0 ? TimeSpan.FromTicks(_wakeupDelayTicks.Value) : null; +} diff --git a/test/Orleans.Messaging.Tests/Functional/InterruptedAttemptTests.cs b/test/Orleans.Messaging.Tests/Functional/InterruptedAttemptTests.cs new file mode 100644 index 00000000000..c057d1c387b --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/InterruptedAttemptTests.cs @@ -0,0 +1,92 @@ +using Orleans.Messaging.Tests.Support; +using Orleans.Runtime; +using Xunit; + +namespace Orleans.Messaging.Tests.Functional; + +/// +/// An activation which stops without recording an attempt's outcome, because a handler failed after staging changes or +/// the silo crashed, has its due messages charged one attempt at recovery, so backoff and InboxRetryTimeout bound every +/// failure path. This fixture's InboxRetryTimeout is zero. +/// +[Collection(MessagingClusterCollection.Name)] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class InterruptedAttemptTests : MessagingBehaviorTestBase +{ + private const string InterruptedReason = "The activation stopped without recording the outcome of an attempt."; + + [Fact] + public async Task HandlerWhichAlwaysFailsAfterStaging_IsDeadLetteredOnceItsRetryWindowElapses() + { + var receiver = NewGrain(); + var envelope = CreateEnvelope(receiver, NewMessage(170, "stage-then-throw") with { ThrowAfterStaging = true }); + + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + + // The first recovery charges the interrupted attempt and starts the retry window; the rerun fails the same way, + // and the next recovery dead-letters the message because the window has elapsed. No staged effect commits. + var final = await Fixture.WaitForDeadLetterCountAsync(receiver, 1); + var letter = Assert.Single(final.InboxDeadLetters); + Assert.Equal(envelope.Envelope.MessageId, letter.MessageId); + Assert.Equal(InterruptedReason, letter.Reason); + Assert.Equal(2, letter.AttemptCount); + Assert.Empty(final.Effects); + Assert.Equal(0, final.InboxCount); + } + + [Fact] + public async Task FailureOnAGrainWithUnreportableState_IsDeadLetteredOnceItsRetryWindowElapses() + { + const string processed = "orleans-messaging-inbox-messages-processed"; + var receiver = Fixture.Client.GetGrain(Guid.NewGuid()); + _ = await receiver.GetSnapshotAsync(); + var deadLettered = Fixture.Metrics.WaitForCountAsync(processed, Fixture.Metrics.GetCount(processed, "dead_lettered") + 1, "dead_lettered"); + var envelope = TestApplicationProtocol.CreateInbox( + Sessions, GrainId.Create("external-test-sender", "persistent"), "messages/record", NewMessage(171, "unreportable"), Fixture.JobNow); + + Assert.Equal(DeliveryStatus.Accepted, (await receiver.AsReference() + .DeliverAsync(envelope, TestContext.Current.CancellationToken)).Status); + + // The grain's persistent state can't report changes, so each failure discards in-memory state through recovery. + // The charges still bound the retries: the second recovery dead-letters the message before activation completes, + // so the snapshot reads the committed outcome. + await deadLettered; + var snapshot = await receiver.GetSnapshotAsync(); + + Assert.Equal([InterruptedReason], snapshot.DeadLetterReasons); + Assert.Equal([2], snapshot.DeadLetterAttempts); + Assert.Equal(2, snapshot.Handled); + Assert.Equal(0, snapshot.InboxCount); + } + + [Fact] + public async Task GracefulDeactivationWithPendingMessages_ChargesNoAttempt() + { + var receiver = NewGrain(); + _ = await receiver.GetSnapshotAsync(); + var context = Fixture.GetGrainContext(receiver); + var envelope = CreateEnvelope(receiver, NewMessage(172, "graceful"), "messages/graceful"); + using (var barrier = Fixture.HandlerProbe.Arm(receiver.GetGrainId(), "messages/graceful")) + { + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + await WaitForBarrierAsync(receiver, barrier); + await receiver.ConfigureHandlerAsync(false); + + // Deactivation cancels the in-flight attempt and journals a clean stop. + await receiver.RequestDeactivationAsync(); + await context.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + } + + _ = await receiver.GetSnapshotAsync(); + var inbox = Fixture.GetInbox(receiver); + var state = await Fixture.OnGrainAsync(receiver, () => InboxTestAccess.GetMessageState(inbox, envelope.Envelope.MessageId)); + Assert.Equal(0, state.AttemptCount); + Assert.Null(state.ExpirationTime); + + await receiver.ConfigureHandlerAsync(true); + var completed = await Fixture.WaitForEffectCountAsync(receiver, 1); + Assert.Empty(completed.InboxDeadLetters); + } +} diff --git a/test/Orleans.Messaging.Tests/Functional/JournaledTestOutboxBehaviorTests.cs b/test/Orleans.Messaging.Tests/Functional/JournaledTestOutboxBehaviorTests.cs new file mode 100644 index 00000000000..221286e3766 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/JournaledTestOutboxBehaviorTests.cs @@ -0,0 +1,122 @@ +using Microsoft.Extensions.DependencyInjection; +using Orleans.Journaling; +using Orleans.Messaging.Tests.Support; +using Orleans.Runtime; +using Orleans.Serialization; +using Orleans.Serialization.Session; +using Xunit; + +namespace Orleans.Messaging.Tests.Functional; + +[Collection(MessagingClusterCollection.Name)] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class JournaledTestOutboxBehaviorTests : MessagingBehaviorTestBase +{ + [Fact] + public async Task EquivalentDuplicates_BeforeAndAfterCommit_PreserveOneOutput() + { + var owner = NewGrain(); + var original = CreateOutput(owner); + var copied = original; + var serializer = Fixture.Client.ServiceProvider.GetRequiredService>(); + var copy = serializer.Deserialize(serializer.SerializeToArray(original)); + Assert.Same(original.Envelope.Payload, copied.Envelope.Payload); + Assert.Same(original.Envelope.Headers, copied.Envelope.Headers); + Assert.NotSame(original.Envelope.Payload, copy.Envelope.Payload); + Assert.NotSame(original.Envelope.Headers, copy.Envelope.Headers); + + await owner.StageOutputAsync(original); + var retained = Assert.Single(Fixture.GetStagedOutput(owner)); + await owner.StageOutputAsync(copied); + Assert.Same(retained.Envelope.Headers, Assert.Single(Fixture.GetStagedOutput(owner)).Envelope.Headers); + Assert.Same(retained.Envelope.Payload, Assert.Single(Fixture.GetStagedOutput(owner)).Envelope.Payload); + await owner.RetryWriteStateAsync(); + await owner.StageOutputAsync(copy); + Assert.Same(retained.Envelope.Headers, Assert.Single(Fixture.GetStagedOutput(owner)).Envelope.Headers); + Assert.Same(retained.Envelope.Payload, Assert.Single(Fixture.GetStagedOutput(owner)).Envelope.Payload); + var before = await owner.GetSnapshotAsync(); + await owner.RequestDeactivationAsync(); + var recovered = await owner.GetSnapshotAsync(); + Assert.NotEqual(before.ActivationId, recovered.ActivationId); + await owner.StageOutputAsync(copy); + + var output = Assert.Single(Fixture.GetStagedOutput(owner)); + Assert.Equal(original.Envelope.MessageId, output.Envelope.MessageId); + Assert.Equal(0, TestApplicationProtocol.Read(Fixture.Client.ServiceProvider.GetRequiredService(), output).Body); + Assert.Equal(1, recovered.OutboxCount); + } + + [Theory] + [InlineData("receiver")] + [InlineData("body-bytes")] + public async Task ConflictingDuplicate_RejectsChangedEnvelopeAndPreservesCommittedOutput(string difference) + { + var owner = NewGrain(); + var original = CreateOutput(owner); + await owner.StageOutputAsync(original); + await owner.RetryWriteStateAsync(); + var retained = Assert.Single(Fixture.GetStagedOutput(owner)); + var conflicting = difference switch + { + "receiver" => original with { ReceiverId = GrainId.Create("other-receiver", "1") }, + _ => original with + { + Envelope = TestApplicationProtocol.With(original.ToInboxMessage(Fixture.JobNow), Sessions, + payload: CreateOutput(owner, changed: true).Envelope.Payload).Envelope + } + }; + + var exception = await Assert.ThrowsAsync(() => owner.StageOutputAsync(conflicting)); + + Assert.Contains(original.Envelope.MessageId.ToString(), exception.Message, StringComparison.Ordinal); + Assert.Same(retained.Envelope.Headers, Assert.Single(Fixture.GetStagedOutput(owner)).Envelope.Headers); + Assert.Same(retained.Envelope.Payload, Assert.Single(Fixture.GetStagedOutput(owner)).Envelope.Payload); + await owner.RetryWriteStateAsync(); + await owner.RequestDeactivationAsync(); + var recovered = await owner.GetSnapshotAsync(); + Assert.Equal(1, recovered.OutboxCount); + var output = Assert.Single(Fixture.GetStagedOutput(owner)); + Assert.Equal(original.Envelope.MessageId, output.Envelope.MessageId); + Assert.Equal(TestApplicationProtocol.Sender(Sessions, original.Envelope), TestApplicationProtocol.Sender(Sessions, output.Envelope)); + Assert.Equal(original.ReceiverId, output.ReceiverId); + Assert.Equal(0, TestApplicationProtocol.Read(Fixture.Client.ServiceProvider.GetRequiredService(), output).Body); + } + + [Fact] + public async Task HandlerEquivalentDuplicateOutput_CommitsOneEffectAndOneOutput() + { + var receiver = NewGrain(); + var target = GrainId.Create("output-target", "handler"); + var envelope = CreateEnvelope( + receiver, + NewMessage(97, "duplicate-output") with { ForwardTo = target }, + "messages/duplicate-output"); + + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + var completed = await Fixture.SnapshotProbe.WaitAsync(receiver.GetGrainId(), + static snapshot => snapshot.InboxCount == 0 && (snapshot.Effects.Count > 0 || snapshot.InboxDeadLetters.Count > 0)); + + Assert.Empty(completed.InboxDeadLetters); + Assert.Equal(1, Assert.Single(completed.Effects).Count); + Assert.Equal(1, completed.ProcessedMessageCount); + Assert.Equal(1, completed.OutboxCount); + var output = Assert.Single(Fixture.GetStagedOutput(receiver)); + Assert.Equal(target, output.ReceiverId); + await receiver.RequestDeactivationAsync(); + var recovered = await receiver.GetSnapshotAsync(); + Assert.NotEqual(completed.ActivationId, recovered.ActivationId); + Assert.Equal(1, Assert.Single(recovered.Effects).Count); + Assert.Equal(output.Envelope.MessageId, Assert.Single(Fixture.GetStagedOutput(receiver)).Envelope.MessageId); + Assert.Empty(recovered.InboxDeadLetters); + } + + + + + + private OutboxMessage CreateOutput(IMessagingTestGrain owner, bool changed = false) => + TestApplicationProtocol.Create(Fixture.Client.ServiceProvider.GetRequiredService(), + owner.GetGrainId(), owner.GetGrainId(), "output", changed ? 1 : 0); +} diff --git a/test/Orleans.Messaging.Tests/Functional/JsonJournalMessagingTests.cs b/test/Orleans.Messaging.Tests/Functional/JsonJournalMessagingTests.cs new file mode 100644 index 00000000000..3cb8e079ed8 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/JsonJournalMessagingTests.cs @@ -0,0 +1,264 @@ +using System.Collections.Concurrent; +using Microsoft.Extensions.DependencyInjection; +using Orleans.Hosting; +using Orleans.DurableJobs; +using Orleans.Journaling; +using Orleans.Messaging.Tests.Support; +using Orleans.Runtime; +using Orleans.Serialization; +using Orleans.TestingHost; +using Xunit; + +namespace Orleans.Messaging.Tests.Functional; + +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class JsonJournalMessagingTests(JsonJournalMessagingTests.Fixture fixture) : IClassFixture +{ + private static CancellationToken Token => TestContext.Current.CancellationToken; + private InProcessTestCluster Cluster => fixture.Cluster; + private JsonMessagingProbe Probe => fixture.Probe; + private DateTimeOffset JobNow => Cluster.Silos[0].ServiceProvider + .GetRequiredKeyedService(DurableJobTimeProviderNames.DurableJobs).GetUtcNow(); + + [Fact] + public async Task JsonJournal_RecoversOutboxAndInboxStateAcrossActivations() + { + var sender = Cluster.Client.GetGrain(Guid.NewGuid()); + var receiver = Cluster.Client.GetGrain(Guid.NewGuid()); + var messageId = MessageId.Create("json", Guid.NewGuid().ToString("N")); + + // The sender deactivates after committing the send, so delivery runs from the outbox recovered from the JSON journal. + await sender.SendAndDeactivateAsync(receiver.GetGrainId(), messageId, 42); + Assert.Equal(new[] { 42 }, await Probe.WaitAsync(receiver.GetGrainId(), 1, Token)); + Assert.True(Cluster.TryGetGrainContext(receiver.GetGrainId(), out var receiverContext)); + Assert.Contains("Json", receiverContext.GetComponent()!.InboxCodec.GetType().Name, StringComparison.Ordinal); + Assert.Same(receiverContext.ActivationServices.GetRequiredService(), receiverContext.ActivationServices.GetRequiredService()); + while (await sender.GetOutboxCountAsync() != 0) + { + await Task.Delay(10, Token); + } + + var sentAt = await ReadSentAtAsync(receiverContext, messageId); + await receiver.DeactivateAsync(); + await receiverContext.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), Token); + var type = new MessageType(JsonMessagingGrain.Subject, Cluster.Client.ServiceProvider.GetRequiredService>()); + var redelivery = type.Create(messageId, receiver.GetGrainId(), 42).ToInboxMessage(sentAt); + + Assert.Equal(DeliveryStatus.Duplicate, (await receiver.AsReference().DeliverAsync(redelivery, Token)).Status); + Assert.Equal(new[] { 42 }, Probe.Get(receiver.GetGrainId())); + } + + [Fact] + public async Task CustomInboxHandlers_FirstHandlerWhichCanHandleReceivesMessage() + { + var receiver = Cluster.Client.GetGrain(Guid.NewGuid()); + var message = new InboxMessage + { + Envelope = new Envelope { MessageId = MessageId.Create("custom", Guid.NewGuid().ToString("N")), Payload = [7], Subject = "custom" }, + SentAt = JobNow + }; + + var result = await receiver.AsReference().DeliverAsync(message, Token); + + Assert.Equal(DeliveryStatus.Accepted, result.Status); + Assert.Equal(new[] { CustomHandlerGrain.FirstMatch }, await Probe.WaitAsync(receiver.GetGrainId(), 1, Token)); + } + + [Fact] + public async Task JsonJournal_RecoversScheduledInboxMessageWithItsDeliverAt() + { + var receiver = Cluster.Client.GetGrain(Guid.NewGuid()); + var messageId = MessageId.Create("json", "scheduled", Guid.NewGuid().ToString("N")); + var type = new MessageType(JsonMessagingGrain.Subject, Cluster.Client.ServiceProvider.GetRequiredService>()); + var deliverAt = new DateTimeOffset(DateTime.UtcNow.AddDays(1).Ticks, TimeSpan.Zero).ToOffset(TimeSpan.FromHours(5)); + var scheduled = type.Create(messageId, receiver.GetGrainId(), 7, deliverAt).ToInboxMessage(JobNow); + var extension = receiver.AsReference(); + + Assert.Equal(DeliveryStatus.Accepted, (await extension.DeliverAsync(scheduled, Token)).Status); + Assert.True(Cluster.TryGetGrainContext(receiver.GetGrainId(), out var receiverContext)); + await receiver.DeactivateAsync(); + await receiverContext.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), Token); + + // Redelivery replays the JSON journal; pending-command equivalence requires the recovered DeliverAt. + Assert.Equal(DeliveryStatus.Duplicate, (await extension.DeliverAsync(scheduled, Token)).Status); + var conflicting = type.Create(messageId, receiver.GetGrainId(), 7, deliverAt.AddSeconds(1)).ToInboxMessage(scheduled.SentAt); + await Assert.ThrowsAsync(() => extension.DeliverAsync(conflicting, Token).AsTask()); + Assert.True(Cluster.TryGetGrainContext(receiver.GetGrainId(), out var recoveredContext)); + Assert.NotSame(receiverContext, recoveredContext); + var inbox = recoveredContext.ActivationServices.GetRequiredService(); + var recovered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + recoveredContext.Scheduler.QueueAction(() => recovered.SetResult(inbox.MessageState[messageId].Message.Envelope.DeliverAt)); + Assert.Equal(deliverAt, await recovered.Task.WaitAsync(TimeSpan.FromSeconds(30), Token)); + Assert.Empty(Probe.Get(receiver.GetGrainId())); + } + + [Fact] + public async Task JsonJournal_RecoversSelfSentMessagesFromTheInbox() + { + var grain = Cluster.Client.GetGrain(Guid.NewGuid()); + var scheduledId = MessageId.Create("json", "self-scheduled", Guid.NewGuid().ToString("N")); + var immediateId = MessageId.Create("json", "self", Guid.NewGuid().ToString("N")); + var deliverAt = new DateTimeOffset(DateTime.UtcNow.AddDays(1).Ticks, TimeSpan.Zero); + + // Each send commits the acceptance in the grain's own inbox; the outbox never holds the message. + await grain.SendAsync(grain.GetGrainId(), scheduledId, 7, deliverAt); + await grain.SendAsync(grain.GetGrainId(), immediateId, 8); + Assert.Equal(0, await grain.GetOutboxCountAsync()); + Assert.Equal(new[] { 8 }, await Probe.WaitAsync(grain.GetGrainId(), 1, Token)); + Assert.True(Cluster.TryGetGrainContext(grain.GetGrainId(), out var context)); + var sentAt = await ReadSentAtAsync(context, immediateId); + await grain.DeactivateAsync(); + await context.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), Token); + + // The next activation recovers the pending scheduled message and the completion record from the JSON journal. + Assert.Equal(0, await grain.GetOutboxCountAsync()); + Assert.True(Cluster.TryGetGrainContext(grain.GetGrainId(), out var recoveredContext)); + Assert.NotSame(context, recoveredContext); + var inbox = recoveredContext.ActivationServices.GetRequiredService(); + var recovered = new TaskCompletionSource<(int Count, DateTimeOffset? DeliverAt)>(TaskCreationOptions.RunContinuationsAsynchronously); + recoveredContext.Scheduler.QueueAction(() => recovered.SetResult((inbox.Count, inbox.MessageState[scheduledId].Message.Envelope.DeliverAt))); + Assert.Equal((1, deliverAt), await recovered.Task.WaitAsync(TimeSpan.FromSeconds(30), Token)); + var type = new MessageType(JsonMessagingGrain.Subject, Cluster.Client.ServiceProvider.GetRequiredService>()); + var redelivery = type.Create(immediateId, grain.GetGrainId(), 8).ToInboxMessage(sentAt); + Assert.Equal(DeliveryStatus.Duplicate, (await grain.AsReference().DeliverAsync(redelivery, Token)).Status); + Assert.Equal(new[] { 8 }, Probe.Get(grain.GetGrainId())); + } + + private static Task ReadSentAtAsync(IGrainContext context, MessageId messageId) + { + var result = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + context.Scheduler.QueueAction(() => + { + try + { + var inbox = context.ActivationServices.GetRequiredService(); + result.SetResult(inbox.MessageState.TryGetValue(messageId, out var pending) + ? pending.Message.SentAt + : InboxTestAccess.GetProcessedSentAt(inbox, messageId)); + } + catch (Exception exception) { result.SetException(exception); } + }); + return result.Task.WaitAsync(TimeSpan.FromSeconds(30), Token); + } + + public sealed class Fixture : IAsyncLifetime + { + public JsonMessagingProbe Probe { get; } = new(); + public InProcessTestCluster Cluster { get; private set; } = null!; + + public async ValueTask InitializeAsync() + { + var builder = new InProcessTestClusterBuilder(1); + builder.ConfigureSilo((_, silo) => + { + silo.AddVolatileJournalStorage(); + silo.Services.Configure(options => options.JournalFormatKey = "json"); + silo.UseInMemoryDurableJobs(); + silo.AddMessaging(); + silo.Services.AddSingleton(Probe); + }); + Cluster = builder.Build(); + await Cluster.DeployAsync(TestContext.Current.CancellationToken); + } + + public async ValueTask DisposeAsync() => await Cluster.DisposeAsync(); + } +} + +public interface ICustomHandlerGrain : IGrainWithGuidKey; + +public sealed class CustomHandlerGrain : Grain, ICustomHandlerGrain, IMessagingGrain +{ + internal const int Declined = 1; + internal const int FirstMatch = 2; + internal const int SecondMatch = 3; + + public CustomHandlerGrain(IInbox inbox, IGrainContext context, JsonMessagingProbe probe) => + inbox.Register(new Handler(Declined, false, context, probe)) + .Register(new Handler(FirstMatch, true, context, probe)) + .Register(new Handler(SecondMatch, true, context, probe)); + + private sealed class Handler(int id, bool canHandle, IGrainContext grain, JsonMessagingProbe probe) : IInboxHandler + { + public bool CanHandle(InboxMessage message) => canHandle && message.Envelope.Subject == "custom"; + + public ValueTask HandleAsync(IInboxHandlerContext context, CancellationToken cancellationToken) + { + probe.Record(grain.GrainId, id); + context.Complete(); + return ValueTask.CompletedTask; + } + } +} +public interface IJsonMessagingGrain : IGrainWithGuidKey +{ + Task SendAsync(GrainId destination, MessageId messageId, int value, DateTimeOffset? deliverAt = null); + Task SendAndDeactivateAsync(GrainId destination, MessageId messageId, int value); + Task GetOutboxCountAsync(); + Task DeactivateAsync(); +} + +public sealed class JsonMessagingProbe +{ + private readonly ConcurrentDictionary> _received = new(); + + public void Record(GrainId grainId, int value) => _received.GetOrAdd(grainId, static _ => new()).Enqueue(value); + + public int[] Get(GrainId grainId) => _received.TryGetValue(grainId, out var values) ? values.ToArray() : []; + + public async Task WaitAsync(GrainId grainId, int count, CancellationToken cancellationToken) + { + var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(30); + while (Get(grainId).Length < count) + { + if (DateTime.UtcNow > deadline) throw new TimeoutException($"No message was handled by '{grainId}'."); + await Task.Delay(10, cancellationToken); + } + + return Get(grainId); + } +} + +public sealed class JsonMessagingGrain : Grain, IJsonMessagingGrain, IMessagingGrain +{ + internal const string Subject = "json/value"; + private readonly IJournaledStateManager _manager; + private readonly IOutbox _outbox; + private readonly MessageType _type; + private readonly JsonMessagingProbe _probe; + + public JsonMessagingGrain(IJournaledStateManager manager, IInbox inbox, IOutbox outbox, Serializer serializer, JsonMessagingProbe probe) + { + _manager = manager; + _outbox = outbox; + _probe = probe; + _type = new MessageType(Subject, serializer); + inbox.Register(_type, this, static (value, grain, context) => + { + grain._probe.Record(grain.GetGrainId(), value); + context.Complete(); + }); + } + + public async Task SendAsync(GrainId destination, MessageId messageId, int value, DateTimeOffset? deliverAt = null) + { + _outbox.Send(_type, messageId, destination, value, deliverAt); + await _manager.WriteStateAsync(CancellationToken.None); + } + + public async Task SendAndDeactivateAsync(GrainId destination, MessageId messageId, int value) + { + await SendAsync(destination, messageId, value); + DeactivateOnIdle(); + } + + public Task GetOutboxCountAsync() => Task.FromResult(_outbox.Count()); + + public Task DeactivateAsync() + { + DeactivateOnIdle(); + return Task.CompletedTask; + } +} diff --git a/test/Orleans.Messaging.Tests/Functional/MessageSourceAndSinkTests.cs b/test/Orleans.Messaging.Tests/Functional/MessageSourceAndSinkTests.cs new file mode 100644 index 00000000000..75f11c2d3e5 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/MessageSourceAndSinkTests.cs @@ -0,0 +1,1321 @@ +using System.Collections.Concurrent; +using System.Security.Cryptography; +using System.Text; +using System.Text.Json.Serialization; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Options; +using Microsoft.Extensions.Time.Testing; +using Orleans.Hosting; +using Orleans.DurableJobs; +using Orleans.Journaling; +using Orleans.Journaling.Json; +using Orleans.Messaging.Configuration; +using Orleans.Messaging.Diagnostics; +using Orleans.Messaging.Tests.Support; +using Orleans.Runtime; +using Orleans.TestingHost; +using Xunit; + +namespace Orleans.Messaging.Tests.Functional; + +/// +/// Outbox sources and inbox sinks keep application-owned payloads as the only journaled copy. +/// +public abstract class MessageSourceAndSinkTests(MessageSourceAndSinkFixture fixture) +{ + private static CancellationToken Token => TestContext.Current.CancellationToken; + private InProcessTestCluster Cluster => fixture.Cluster; + private RecordingJournalStorageProvider Storage => fixture.Storage; + private SourceSinkProbe Probe => fixture.Probe; + + [Fact] + public async Task SourceMessage_JournalsEachPayloadOnceAndRetiresInTheRemovalWrite() + { + var sender = NewGrain(); + var receiver = NewGrain(); + var messageId = NewMessageId("delivered"); + var payload = NewPayload(); + + await sender.StageFromSourceAsync(receiver.GetGrainId(), messageId, payload, write: true); + + await WaitUntilAsync(sender, context => Outbox(context).Count == 0 && Writes(sender).Length == 2); + var senderState = await OnGrainAsync(sender, context => Grain(context).GetState()); + Assert.Empty(senderState.Outgoing); + Assert.Equal(nameof(OutboxMessageRetirementStatus.Delivered), Assert.Single(senderState.Retired).Value); + Assert.Empty(await OnGrainAsync(sender, context => Grain(context).GetRetirementReasons())); + // The outbox materializes the message once to validate the send and once for the delivery attempt. + Assert.Equal(2, Probe.For(sender).Materializations); + + // The payload is journaled once by each grain: in the sender's source state and in the receiver's sink state. + Assert.Equal(1, Storage.CountPayloadOccurrences(Journal(sender), payload)); + Assert.Equal(1, Storage.CountPayloadOccurrences(Journal(receiver), payload)); + + // The retirement commits in the write which removes the outbox entry. + var writes = Writes(sender); + var retirement = Encoding.UTF8.GetBytes(nameof(OutboxMessageRetirementStatus.Delivered)); + Assert.True(writes[0].AsSpan().IndexOf(retirement) < 0); + Assert.True(writes[1].AsSpan().IndexOf(retirement) >= 0); + + // The sink accepted the message without an inbox entry; the inbox journals only its processed record. + var receiverState = await OnGrainAsync(receiver, context => ( + Grain(context).GetState(), + Inbox(context).Count, + InboxTestAccess.GetProcessedKeys(Inbox(context)).Contains(messageId))); + Assert.Equal(payload, receiverState.Item1.Received[messageId.ToString()]); + Assert.Equal(0, receiverState.Count); + Assert.True(receiverState.Item3); + Assert.Equal(1, Probe.For(receiver).SinkAcceptances); + Assert.Equal(0, Probe.For(receiver).Handled); + Assert.Single(Writes(receiver)); + + // A recovered sender agrees: the entry is removed and its retirement is recorded. + Assert.True(Cluster.TryGetGrainContext(sender.GetGrainId(), out var senderContext)); + await sender.DeactivateAsync(); + await senderContext.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), Token); + Assert.Equal(0, await sender.GetOutboxCountAsync()); + var recovered = await OnGrainAsync(sender, context => Grain(context).GetState()); + Assert.Empty(recovered.Outgoing); + Assert.Single(recovered.Retired); + Assert.Equal(2, Writes(sender).Length); + } + + [Fact] + public async Task SourceMessage_RecoveredFromTheJournal_IsDeliveredFromItsSource() + { + var sender = NewGrain(); + var receiver = NewGrain(); + var messageId = NewMessageId("recovered"); + var payload = NewPayload(); + + await sender.StageFromSourceAndDeactivateAsync(receiver.GetGrainId(), messageId, payload); + + await WaitUntilAsync(receiver, context => Grain(context).GetState().Received.ContainsKey(messageId.ToString())); + await WaitUntilAsync(sender, context => Outbox(context).Count == 0 && Grain(context).GetState().Retired.Count == 1); + Assert.Equal(1, Probe.For(receiver).SinkAcceptances); + Assert.Equal(1, Storage.CountPayloadOccurrences(Journal(sender), payload)); + Assert.Equal(1, Storage.CountPayloadOccurrences(Journal(receiver), payload)); + } + + [Fact] + public async Task SourceMessage_DeadLetteredByTheOutbox_MaterializesItsMessageBeforeRetiring() + { + var sender = NewGrain(); + var receiver = Cluster.Client.GetGrain(Guid.NewGuid()); + await receiver.PingAsync(); + var messageId = NewMessageId("outbox-dead-letter"); + var payload = NewPayload(); + + await sender.StageFromSourceAsync(receiver.GetGrainId(), messageId, payload, write: true); + + // The receiver has no handler, so delivery attempts fail until OutboxRetryTimeout elapses. + await WaitUntilAsync(sender, context => Outbox(context).Count == 0 && Grain(context).GetState().Retired.Count == 1); + var (state, deadLetters) = await OnGrainAsync(sender, context => ( + Grain(context).GetState(), + context.ActivationServices.GetRequiredService().OutboxDeadLetters)); + Assert.Empty(state.Outgoing); + Assert.Equal(nameof(OutboxMessageRetirementStatus.DeadLettered), state.Retired[messageId.ToString()]); + var letter = Assert.Single(deadLetters); + Assert.Equal(letter.Reason, (await OnGrainAsync(sender, context => Grain(context).GetRetirementReasons()))[messageId.ToString()]); + Assert.Equal(messageId, letter.Message.Envelope.MessageId); + Assert.Equal(receiver.GetGrainId(), letter.Message.ReceiverId); + Assert.Equal(payload, letter.Message.Envelope.Payload); + Assert.Equal(SourceSinkGrain.Subject, letter.Message.Envelope.Subject); + Assert.True(letter.AttemptCount > 1); + // Each attempt materializes the message, as does a final wake which finds the retry timeout expired. + Assert.InRange(Probe.For(sender).Materializations, letter.AttemptCount + 1, letter.AttemptCount + 2); + + // The dead letter keeps its own copy for diagnosis. + await WaitUntilAsync(sender, _ => Storage.CountPayloadOccurrences(Journal(sender), payload) == 2); + } + + [Fact] + public async Task SourceMessage_DeadLetteredByTheOutbox_KeepsItsPayloadAndHeadersWhenTheSourceReleasesThem() + { + var sender = NewGrain(); + var receiver = Cluster.Client.GetGrain(Guid.NewGuid()); + await receiver.PingAsync(); + var messageId = NewMessageId("released-dead-letter"); + var payload = NewPayload(); + + // The source shares its payload array as a header value and clears that array in OnRetired. + Probe.For(sender).ReleaseBuffersOnRetired = true; + await sender.StageFromSourceAsync(receiver.GetGrainId(), messageId, payload, write: true); + + await WaitUntilAsync(sender, context => Outbox(context).Count == 0 && Grain(context).GetState().Retired.Count == 1); + AssertCarriesPayload(Assert.Single(await OnGrainAsync(sender, context => + context.ActivationServices.GetRequiredService().OutboxDeadLetters))); + + // The source state journals the payload once, and the dead letter journals it as its payload and its header. + await WaitUntilAsync(sender, _ => Storage.CountPayloadOccurrences(Journal(sender), payload) == 3); + + // A snapshot written after the retirement journals the dead letter's own copy, which a new activation recovers. + Storage.RequestSnapshot(Journal(sender)); + await sender.WriteAsync(); + Assert.Equal(5, Storage.CountPayloadOccurrences(Journal(sender), payload)); + Assert.True(Cluster.TryGetGrainContext(sender.GetGrainId(), out var activation)); + await sender.DeactivateAsync(); + await activation.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), Token); + Assert.Equal(0, await sender.GetOutboxCountAsync()); + AssertCarriesPayload(Assert.Single(await OnGrainAsync(sender, context => + context.ActivationServices.GetRequiredService().OutboxDeadLetters))); + + void AssertCarriesPayload(DeadLetter letter) + { + Assert.Equal(messageId, letter.Message.Envelope.MessageId); + Assert.Equal(payload, letter.Message.Envelope.Payload); + Assert.Equal(payload, Assert.Single(letter.Message.Envelope.Headers!, header => header.Key == SourceSinkGrain.PayloadHeader).Value); + } + } + + [Fact] + public async Task SourceMessage_DeadLetteredByTheReceiver_RetiresAsRejectedWithTheReceiversReasonAndCode() + { + const string code = SourceSinkGrain.RejectionCode; + var sender = NewGrain(); + var receiver = NewGrain(); + Probe.For(receiver).Mode = SinkMode.DeadLetter; + var messageId = NewMessageId("receiver-dead-letter"); + var payload = NewPayload(); + + await sender.StageFromSourceAsync(receiver.GetGrainId(), messageId, payload, write: true); + + await WaitUntilAsync(sender, context => Outbox(context).Count == 0 && Grain(context).GetState().Retired.Count == 1); + var (state, outboxDeadLetters) = await OnGrainAsync(sender, context => ( + Grain(context).GetState(), + context.ActivationServices.GetRequiredService().OutboxDeadLetters)); + Assert.Equal(nameof(OutboxMessageRetirementStatus.Rejected), state.Retired[messageId.ToString()]); + Assert.Equal(SourceSinkGrain.Rejection, (await OnGrainAsync(sender, context => Grain(context).GetRetirementReasons()))[messageId.ToString()]); + Assert.Equal(code, (await OnGrainAsync(sender, context => Grain(context).GetRetirementCodes())).GetValueOrDefault(messageId.ToString())); + Assert.Empty(state.Outgoing); + Assert.Empty(outboxDeadLetters); + var inboxLetter = Assert.Single(await OnGrainAsync(receiver, context => + context.ActivationServices.GetRequiredService().InboxDeadLetters)); + Assert.Equal(payload, inboxLetter.Message.Envelope.Payload); + Assert.Equal(SourceSinkGrain.Rejection, inboxLetter.Reason); + Assert.Equal(code, inboxLetter.Code); + } + + [Fact] + public async Task SourceMessage_WhoseSinkThrowsTransiently_IsRetriedUntilTheSinkAcceptsIt() + { + var sender = NewGrain(); + var receiver = NewGrain(); + Probe.For(receiver).FailNextOffers(2); + var messageId = NewMessageId("sink-transient-failure"); + var payload = NewPayload(); + + await sender.StageFromSourceAsync(receiver.GetGrainId(), messageId, payload, write: true); + + await WaitUntilAsync(sender, context => Outbox(context).Count == 0 && Grain(context).GetState().Retired.Count == 1); + var (state, outboxDeadLetters) = await OnGrainAsync(sender, context => ( + Grain(context).GetState(), + context.ActivationServices.GetRequiredService().OutboxDeadLetters)); + Assert.Equal(nameof(OutboxMessageRetirementStatus.Delivered), state.Retired[messageId.ToString()]); + Assert.Empty(outboxDeadLetters); + var (received, inboxDeadLetters) = await OnGrainAsync(receiver, context => ( + Grain(context).GetState().Received, + context.ActivationServices.GetRequiredService().InboxDeadLetters)); + Assert.Equal(payload, received[messageId.ToString()]); + Assert.Empty(inboxDeadLetters); + Assert.Equal(3, Probe.For(receiver).SinkOffers); + Assert.Equal(1, Probe.For(receiver).SinkAcceptances); + } + + [Fact] + public async Task SourceMessage_WhoseSinkKeepsThrowing_DeadLettersAtTheSenderWithTheLastErrorAfterTheRetryTimeout() + { + var sender = NewGrain(); + var receiver = NewGrain(); + Probe.For(receiver).Mode = SinkMode.Throw; + var messageId = NewMessageId("sink-persistent-failure"); + + await sender.StageFromSourceAsync(receiver.GetGrainId(), messageId, NewPayload(), write: true); + + await WaitUntilAsync(sender, context => Outbox(context).Count == 0 && Grain(context).GetState().Retired.Count == 1); + var (state, outboxDeadLetters, reasons, codes) = await OnGrainAsync(sender, context => ( + Grain(context).GetState(), + context.ActivationServices.GetRequiredService().OutboxDeadLetters, + Grain(context).GetRetirementReasons(), + Grain(context).GetRetirementCodes())); + Assert.Equal(nameof(OutboxMessageRetirementStatus.DeadLettered), state.Retired[messageId.ToString()]); + var letter = Assert.Single(outboxDeadLetters); + Assert.Contains(SourceSinkGrain.SinkFailure, letter.Reason, StringComparison.Ordinal); + Assert.Equal(letter.Reason, reasons[messageId.ToString()]); + Assert.Null(letter.Code); + Assert.Empty(codes); + Assert.True(letter.AttemptCount > 1); + + var (received, inboxDeadLetters, inboxCount) = await OnGrainAsync(receiver, context => ( + Grain(context).GetState().Received, + context.ActivationServices.GetRequiredService().InboxDeadLetters, + Inbox(context).Count)); + Assert.Empty(received); + Assert.Empty(inboxDeadLetters); + Assert.Equal(0, inboxCount); + Assert.Equal(letter.AttemptCount, Probe.For(receiver).SinkOffers); + } + + [Fact] + public async Task DiagnosticEvents_InjectADeliveryFaultAndReportTheRetryAndTheRetirement() + { + var sender = NewGrain(); + var receiver = NewGrain(); + var messageId = NewMessageId("diagnostic-events"); + using var events = new DeliveryEventObserver(sender.GetGrainId(), receiver.GetGrainId(), messageId, faults: 1); + + await sender.StageFromSourceAsync(receiver.GetGrainId(), messageId, NewPayload(), write: true); + + // The observer's exception failed the first attempt before the inbox offered the message to its sink, so the + // outbox retried the delivery, and the retry delivered it. + var retired = await events.Retired.WaitAsync(TimeSpan.FromSeconds(30), Token); + Assert.Equal(sender.GetGrainId(), retired.GrainContext.GrainId); + Assert.Equal(OutboxMessageRetirementStatus.Delivered, retired.Retirement.Status); + var retry = Assert.Single(events.Retries); + Assert.Equal((sender.GetGrainId(), receiver.GetGrainId(), 1), (retry.GrainContext.GrainId, retry.ReceiverId, retry.AttemptCount)); + Assert.Contains(DeliveryEventObserver.Fault, retry.Error, StringComparison.Ordinal); + Assert.Equal(2, events.Receivings); + Assert.Equal((1, 1), (Probe.For(receiver).SinkOffers, Probe.For(receiver).SinkAcceptances)); + } + + [Fact] + public async Task SourceMessage_CoalescesWithItsPendingReferenceAndRejectsConflicts() + { + var sender = NewGrain(); + var receiver = NewGrain().GetGrainId(); + var messageId = NewMessageId("coalesced"); + var sentId = NewMessageId("sent"); + + await sender.StageFromSourceAsync(receiver, messageId, NewPayload(), write: false); + await sender.SendFromSourceAsync(messageId); + Assert.Equal(1, await sender.GetOutboxCountAsync()); + + await Assert.ThrowsAsync(() => sender.SendAsync(receiver, messageId, NewPayload(), write: false)); + await sender.SendAsync(receiver, sentId, NewPayload(), write: false); + await Assert.ThrowsAsync(() => sender.SendFromSourceAsync(sentId)); + await Assert.ThrowsAsync(() => sender.SendFromSourceAsync(NewMessageId("unprovided"))); + await Assert.ThrowsAsync(() => sender.StageFromSourceAsync(sender.GetGrainId(), NewMessageId("self"), NewPayload(), write: false)); + Assert.Equal(2, await sender.GetOutboxCountAsync()); + + await sender.WriteAsync(); + await WaitUntilAsync(sender, context => Outbox(context).Count == 0); + + // Materialized to validate the first send, to reject the self-addressed send, and for delivery; an unprovided + // message is never materialized. + Assert.Equal(3, Probe.For(sender).Materializations); + } + + [Fact] + public async Task SourceMessage_WithdrawnBySource_IsRemovedWithoutDeliveryOrRetirement() + { + var sender = NewGrain(); + var receiver = NewGrain(); + var messageId = NewMessageId("withdrawn"); + + await sender.StageFromSourceAsync(receiver.GetGrainId(), messageId, NewPayload(), write: false); + await sender.WithdrawAsync(messageId); + Assert.Equal(0, await sender.GetOutboxCountAsync()); + Assert.False(await OnGrainAsync(sender, context => Outbox(context).Withdraw(messageId))); + await sender.WriteAsync(); + + // The withdrawal and the source's release commit in the write which stages them. + await WaitUntilAsync(sender, context => Outbox(context).Count == 0 && Writes(sender).Length == 1); + var state = await OnGrainAsync(sender, context => Grain(context).GetState()); + Assert.Empty(state.Outgoing); + Assert.Empty(state.Retired); + Assert.Equal(1, Probe.For(sender).Materializations); + Assert.Equal(0, Probe.For(receiver).SinkOffers); + + Assert.True(Cluster.TryGetGrainContext(sender.GetGrainId(), out var senderContext)); + await sender.DeactivateAsync(); + await senderContext.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), Token); + Assert.Equal(0, await sender.GetOutboxCountAsync()); + } + + [Theory] + [InlineData(SourceFault.Throw)] + [InlineData(SourceFault.InvalidMessage)] + [InlineData(SourceFault.MismatchedId)] + public async Task SourceFailureAfterSend_DeadLettersOnlyThatEntryAndKeepsTheGrainWorking(SourceFault fault) + { + var sender = NewGrain(); + var receiver = NewGrain(); + var first = NewMessageId("sibling"); + var failing = NewMessageId("failing"); + var second = NewMessageId("sibling"); + await sender.StageFromSourceAsync(receiver.GetGrainId(), first, NewPayload(), write: false); + await sender.StageFromSourceAsync(receiver.GetGrainId(), failing, NewPayload(), write: false); + await sender.StageFromSourceAsync(receiver.GetGrainId(), second, NewPayload(), write: false); + Assert.True(Cluster.TryGetGrainContext(sender.GetGrainId(), out var activation)); + using var metrics = new MessageMetricListener(activation.ActivationServices.GetRequiredService().Meter); + + // The source provided every message when it was sent, and fails to provide one for its delivery attempt. + Probe.For(sender).Faults[failing] = fault; + await sender.WriteAsync(); + + await WaitUntilAsync(sender, context => Outbox(context).Count == 0 && Grain(context).GetState().Retired.Count == 3); + var (state, deadLetters) = await OnGrainAsync(sender, context => ( + Grain(context).GetState(), + context.ActivationServices.GetRequiredService().OutboxDeadLetters)); + Assert.Empty(state.Outgoing); + Assert.Equal(nameof(OutboxMessageRetirementStatus.DeadLettered), state.Retired[failing.ToString()]); + Assert.Equal(nameof(OutboxMessageRetirementStatus.Delivered), state.Retired[first.ToString()]); + Assert.Equal(nameof(OutboxMessageRetirementStatus.Delivered), state.Retired[second.ToString()]); + var received = (await OnGrainAsync(receiver, context => Grain(context).GetState())).Received; + Assert.Equal(new[] { first.ToString(), second.ToString() }.Order(), received.Keys.Order()); + AssertSourceFailureDeadLetter(Assert.Single(deadLetters), failing, fault); + Assert.Equal(fixture.Clock.GetUtcNow(), Assert.Single(deadLetters).DeadLetteredAt); + Assert.Equal( + Assert.Single(deadLetters).Reason, + Assert.Single(await OnGrainAsync(sender, context => Grain(context).GetRetirementReasons())).Value); + + // The dead letter and its retirement commit in one write, before the writes which retire the delivered siblings + // as their attempts complete. + var writes = Writes(sender); + Assert.InRange(writes.Length, 3, 4); + var deadLettered = Encoding.UTF8.GetBytes(nameof(OutboxMessageRetirementStatus.DeadLettered)); + var delivered = Encoding.UTF8.GetBytes(nameof(OutboxMessageRetirementStatus.Delivered)); + Assert.True(writes[1].AsSpan().IndexOf(deadLettered) >= 0); + Assert.True(writes[1].AsSpan().IndexOf(delivered) < 0); + Assert.All(writes[2..], write => Assert.True(write.AsSpan().IndexOf(deadLettered) < 0)); + Assert.All(writes[2..], write => Assert.True(write.AsSpan().IndexOf(delivered) >= 0)); + + var grainType = sender.GetGrainId().Type.ToString(); + var outcomes = metrics.Read("outbox-messages-delivered"); + Assert.Equal(3, outcomes.Length); + Assert.Single(outcomes, row => row.Tags.Contains(new("status", "source_error"))).AssertCounter(grainType, "source_error"); + Assert.Equal(2, outcomes.Count(row => row.Tags.Contains(new("status", "accepted")))); + + // The activation keeps processing messages; the source was called once per send and once per delivery attempt. + var later = NewMessageId("later"); + await sender.StageFromSourceAsync(receiver.GetGrainId(), later, NewPayload(), write: true); + await WaitUntilAsync(sender, context => Outbox(context).Count == 0 && Grain(context).GetState().Retired.Count == 4); + Assert.Equal(nameof(OutboxMessageRetirementStatus.Delivered), (await OnGrainAsync(sender, context => Grain(context).GetState())).Retired[later.ToString()]); + Assert.True(Cluster.TryGetGrainContext(sender.GetGrainId(), out var current)); + Assert.Same(activation, current); + Assert.Equal(1, Probe.For(sender).Activations); + Assert.Equal(8, Probe.For(sender).Materializations); + + // The persistent fault never fails a later activation: the dead letter recovers and the source isn't called again. + await sender.DeactivateAsync(); + await activation.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), Token); + Assert.Equal(0, await sender.GetOutboxCountAsync()); + var recovered = await OnGrainAsync(sender, context => + context.ActivationServices.GetRequiredService().OutboxDeadLetters); + AssertSourceFailureDeadLetter(Assert.Single(recovered), failing, fault); + Assert.Equal(fixture.Clock.GetUtcNow(), Assert.Single(recovered).DeadLetteredAt); + Assert.Equal(2, Probe.For(sender).Activations); + Assert.Equal(8, Probe.For(sender).Materializations); + } + + [Fact] + public async Task SourceFailureOnRecoveredEntry_DeadLettersItAndRetiresItWithTheFailingSource() + { + var grain = NewGrain(); + var receiver = NewGrain().GetGrainId(); + _ = await grain.GetOutboxCountAsync(); + var messageId = NewMessageId("recovered-failure"); + Probe.For(grain).Faults[messageId] = SourceFault.Throw; + + // Replays a reference entry whose source can no longer provide it, then runs a delivery pass. + await OnGrainAsync(grain, context => + { + Grain(context).StoreOutgoing(receiver, messageId, NewPayload()); + var outbox = Outbox(context); + ApplyReference(outbox, messageId); + ((IStateMachine)outbox).OnRecoveryCompleted(); + var pump = context.ActivationServices.GetRequiredService(); + pump.EnsureStarted(); + pump.Signal(); + return true; + }); + + await WaitUntilAsync(grain, context => Outbox(context).Count == 0); + var (state, deadLetters) = await OnGrainAsync(grain, context => ( + Grain(context).GetState(), + context.ActivationServices.GetRequiredService().OutboxDeadLetters)); + Assert.Empty(state.Outgoing); + Assert.Equal(nameof(OutboxMessageRetirementStatus.DeadLettered), state.Retired[messageId.ToString()]); + AssertSourceFailureDeadLetter(Assert.Single(deadLetters), messageId, SourceFault.Throw); + + // The activation keeps processing messages. + var later = NewMessageId("after-recovered-failure"); + await grain.StageFromSourceAsync(receiver, later, NewPayload(), write: true); + await WaitUntilAsync(grain, context => Outbox(context).Count == 0 && Grain(context).GetState().Retired.Count == 2); + Assert.Equal(nameof(OutboxMessageRetirementStatus.Delivered), (await OnGrainAsync(grain, context => Grain(context).GetState())).Retired[later.ToString()]); + Assert.Equal(1, Probe.For(grain).Activations); + } + + [Theory] + [InlineData(SourceFault.Throw)] + [InlineData(SourceFault.InvalidMessage)] + [InlineData(SourceFault.MismatchedId)] + public async Task SourceFailureDuringSend_ThrowsToTheCallerAndStagesNothing(SourceFault fault) + { + var sender = NewGrain(); + var receiver = NewGrain(); + var messageId = NewMessageId("send-failure"); + Probe.For(sender).Faults[messageId] = fault; + + var stage = sender.StageFromSourceAsync(receiver.GetGrainId(), messageId, NewPayload(), write: false); + var error = fault == SourceFault.InvalidMessage + ? await Assert.ThrowsAsync(() => stage) + : (Exception)await Assert.ThrowsAsync(() => stage); + Assert.Contains(ExpectedSourceError(fault), error.Message, StringComparison.Ordinal); + Assert.Equal(0, await sender.GetOutboxCountAsync()); + + var later = NewMessageId("after-send-failure"); + await sender.StageFromSourceAsync(receiver.GetGrainId(), later, NewPayload(), write: true); + await WaitUntilAsync(sender, context => Outbox(context).Count == 0 && Grain(context).GetState().Retired.Count == 1); + Assert.Empty(await OnGrainAsync(sender, context => + context.ActivationServices.GetRequiredService().OutboxDeadLetters)); + Assert.Equal(1, Probe.For(sender).Activations); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task RecoveredDeadLetterWithoutReceiver_IsValidOnlyWithTheSourceFailureShape(bool withPayload) + { + var grain = NewGrain(); + _ = await grain.GetOutboxCountAsync(); + var messageId = NewMessageId("recovered-dead-letter"); + + // Replays a dead letter without a receiver, then validates the recovered state as recovery does. + var recovery = OnGrainAsync(grain, context => + { + var outbox = Outbox(context); + ((IDurableValueCommandHandler)outbox).ApplySet(new OutboxCommand + { + Kind = OutboxCommandKind.Delivery, + MessageId = messageId, + DeadLetter = new() + { + Message = new() { Envelope = new() { MessageId = messageId, Payload = withPayload ? NewPayload() : [] }, ReceiverId = default }, + DeadLetteredAt = DateTimeOffset.UtcNow, + Reason = "The outbox message source failed to provide the message.", + AttemptCount = 1 + } + }); + ((IStateMachine)outbox).OnRecoveryCompleted(); + return outbox.DeadLetterState.Count; + }); + + if (withPayload) + { + var error = await Assert.ThrowsAsync(() => recovery); + Assert.Contains("receiver must not be default", error.Message, StringComparison.Ordinal); + } + else + { + Assert.Equal(1, await recovery); + } + } + + [Fact] + public async Task RecoveredReferenceEntry_IsDeadLetteredWhenNoSourceProvidesItAndFailsWithoutSources() + { + var grain = NewGrain(); + var withoutSources = Cluster.Client.GetGrain(Guid.NewGuid()); + _ = await grain.GetOutboxCountAsync(); + await withoutSources.PingAsync(); + var unprovided = NewMessageId("unprovided"); + + // Replays a reference entry, validates the recovered state as recovery does, then runs a delivery pass. + await OnGrainAsync(grain, context => + { + var outbox = Outbox(context); + ApplyReference(outbox, unprovided); + ((IStateMachine)outbox).OnRecoveryCompleted(); + var pump = context.ActivationServices.GetRequiredService(); + pump.EnsureStarted(); + pump.Signal(); + return true; + }); + // Withdrawal is explicit, so an entry which no source provides is lost data: it is dead-lettered with no owner to + // notify. + await WaitUntilAsync(grain, context => Outbox(context).Count == 0); + Assert.Empty((await OnGrainAsync(grain, context => Grain(context).GetState())).Retired); + var letter = Assert.Single(await OnGrainAsync(grain, context => + context.ActivationServices.GetRequiredService().OutboxDeadLetters)); + Assert.Equal(unprovided, letter.Message.Envelope.MessageId); + Assert.Equal("No registered outbox message source provides the message.", letter.Reason); + + var error = await Assert.ThrowsAsync(() => OnGrainAsync(withoutSources, context => + { + var outbox = Outbox(context); + ApplyReference(outbox, unprovided); + ((IStateMachine)outbox).OnRecoveryCompleted(); + return true; + })); + Assert.Contains("no outbox message source is registered", error.Message, StringComparison.Ordinal); + } + + private void ApplyReference(Outbox outbox, MessageId messageId) => + ((IDurableValueCommandHandler)outbox).ApplySet(new OutboxCommand + { + Kind = OutboxCommandKind.Enqueue, + Reference = new() { MessageId = messageId }, + State = new() { SentAt = fixture.JobNow } + }); + + private static void AssertSourceFailureDeadLetter(DeadLetter letter, MessageId messageId, SourceFault fault) + { + // No valid message exists, so the dead letter carries only the message ID. + Assert.Equal(messageId, letter.Message.Envelope.MessageId); + Assert.True(letter.Message.ReceiverId.IsDefault); + Assert.Empty(letter.Message.Envelope.Payload); + Assert.Null(letter.Message.Envelope.Subject); + Assert.Null(letter.Message.Envelope.Headers); + Assert.Null(letter.Message.Envelope.DeliverAt); + Assert.StartsWith("The outbox message source failed to provide the message: ", letter.Reason, StringComparison.Ordinal); + Assert.Contains(ExpectedSourceError(fault), letter.Reason, StringComparison.Ordinal); + Assert.Equal(1, letter.AttemptCount); + } + + private static string ExpectedSourceError(SourceFault fault) => fault switch + { + SourceFault.Throw => SourceSinkGrain.SourceFailure, + SourceFault.InvalidMessage => "The payload must not be null.", + _ => "The outbox message source returned message ID" + }; + + [Fact] + public async Task SinkAcceptance_TrackedSentAtDeduplicatesWithoutReofferingTheSink() + { + var receiver = NewGrain(); + var message = NewInboxMessage("deduplicated", NewPayload()); + var extension = receiver.AsReference(); + + Assert.Equal(DeliveryStatus.Accepted, (await extension.DeliverAsync(message, Token)).Status); + Assert.Equal(DeliveryStatus.Duplicate, (await extension.DeliverAsync(message, Token)).Status); + Assert.Equal(1, Probe.For(receiver).SinkOffers); + Assert.Equal(1, Probe.For(receiver).SinkAcceptances); + Assert.Equal(0, await OnGrainAsync(receiver, context => Inbox(context).Count)); + + fixture.Clock.Advance(MessageSourceAndSinkFixture.DeduplicationWindow); + Assert.Equal(DeliveryStatus.Duplicate, (await extension.DeliverAsync(message, Token)).Status); + Assert.Equal(1, Probe.For(receiver).SinkOffers); + Assert.Equal(1, Probe.For(receiver).SinkAcceptances); + Assert.Equal(0, Probe.For(receiver).Handled); + Assert.Equal(0, await OnGrainAsync(receiver, context => Inbox(context).Count)); + } + + [Fact] + public async Task SinkRejection_DeadLettersTheMessageAndAnswersItsDuplicatesWithTheSameRejection() + { + const string reason = SourceSinkGrain.Rejection; + const string code = SourceSinkGrain.RejectionCode; + var receiver = NewGrain(); + Probe.For(receiver).Mode = SinkMode.DeadLetter; + var payload = NewPayload(); + var message = NewInboxMessage("rejected", payload); + var extension = receiver.AsReference(); + await receiver.WriteAsync(); + using var metrics = new MessageMetricListener(await OnGrainAsync(receiver, context => context.ActivationServices.GetRequiredService().Meter)); + + var result = await extension.DeliverAsync(message, Token); + + Assert.Equal(DeliveryStatus.DeadLettered, result.Status); + Assert.Equal(reason, result.Message); + Assert.Equal(code, result.Code); + var grainType = receiver.GetGrainId().Type.ToString(); + Assert.Single(metrics.Read("inbox-messages-received")).AssertCounter(grainType, "dead_lettered"); + Assert.Single(metrics.Read("inbox-messages-processed")).AssertCounter(grainType, "dead_lettered"); + var (state, count, letters) = await OnGrainAsync(receiver, context => ( + Grain(context).GetState(), + Inbox(context).Count, + context.ActivationServices.GetRequiredService().InboxDeadLetters)); + Assert.Empty(state.Received); + Assert.Equal(0, count); + var letter = Assert.Single(letters); + Assert.Equal(payload, letter.Message.Envelope.Payload); + Assert.Equal(reason, letter.Reason); + Assert.Equal(code, letter.Code); + Assert.Equal(1, letter.AttemptCount); + Assert.Equal(1, Storage.CountPayloadOccurrences(Journal(receiver), payload)); + + // A duplicate receives the rejection from the processed record, which outlives the dead letter, without + // reaching the sink. + Probe.For(receiver).Mode = SinkMode.Accept; + AssertRejected(await extension.DeliverAsync(message, Token)); + Assert.True(await OnGrainAsync(receiver, context => + context.ActivationServices.GetRequiredService().RemoveInboxDeadLetter(message.Envelope.MessageId))); + AssertRejected(await extension.DeliverAsync(message, Token)); + Assert.Equal(1, Probe.For(receiver).SinkOffers); + Assert.Equal(0, Probe.For(receiver).SinkAcceptances); + Assert.Equal(0, Probe.For(receiver).Handled); + + // A tracked rejection remains the answer until an acknowledged floor compacts its send time. + fixture.Clock.Advance(MessageSourceAndSinkFixture.DeduplicationWindow); + AssertRejected(await extension.DeliverAsync(message, Token)); + Assert.Equal(1, Probe.For(receiver).SinkOffers); + Assert.Equal(0, Probe.For(receiver).SinkAcceptances); + + // Removing the diagnostic dead letter does not erase the durable receiver decision. + AssertRejected(await extension.DeliverAsync(message, Token)); + await receiver.DeactivateAsync(); + AssertRejected(await extension.DeliverAsync(message, Token)); + Assert.Equal(1, Probe.For(receiver).SinkOffers); + + static void AssertRejected(DeliveryResult duplicate) + { + Assert.Equal(DeliveryStatus.DeadLettered, duplicate.Status); + Assert.Equal(reason, duplicate.Message); + Assert.Equal(code, duplicate.Code); + } + } + + [Theory] + [InlineData(SinkMode.Throw, SourceSinkGrain.SinkFailure)] + [InlineData(SinkMode.UnsupportedStatus, "The inbox message sink reported the unsupported delivery status 'HandlerNotFound'.")] + public async Task SinkFailure_FailsTheDeliveryWithoutStagingAndTheRetryIsAccepted(SinkMode mode, string error) + { + var receiver = NewGrain(); + Probe.For(receiver).Mode = mode; + var payload = NewPayload(); + var message = NewInboxMessage("sink-failure", payload); + var extension = receiver.AsReference(); + + var exception = await Assert.ThrowsAsync(async () => await extension.DeliverAsync(message, Token)); + + Assert.Equal(error, exception.Message); + var (state, count, letters) = await OnGrainAsync(receiver, context => ( + Grain(context).GetState(), + Inbox(context).Count, + context.ActivationServices.GetRequiredService().InboxDeadLetters)); + Assert.Empty(state.Received); + Assert.Equal(0, count); + Assert.Empty(letters); + Assert.Equal(0, Storage.CountPayloadOccurrences(Journal(receiver), payload)); + + Probe.For(receiver).Mode = SinkMode.Accept; + Assert.Equal(DeliveryStatus.Accepted, (await extension.DeliverAsync(message, Token)).Status); + Assert.Equal(2, Probe.For(receiver).SinkOffers); + Assert.Equal(1, Probe.For(receiver).SinkAcceptances); + Assert.Equal(1, Storage.CountPayloadOccurrences(Journal(receiver), payload)); + } + + [Fact] + public async Task SinkBackpressure_StagesNothingAndTheRetryIsAccepted() + { + var receiver = NewGrain(); + Probe.For(receiver).Mode = SinkMode.Backpressure; + var payload = NewPayload(); + var message = NewInboxMessage("backpressured", payload); + var extension = receiver.AsReference(); + + Assert.Equal(DeliveryStatus.Backpressured, (await extension.DeliverAsync(message, Token)).Status); + var (count, letters) = await OnGrainAsync(receiver, context => ( + Inbox(context).Count, + context.ActivationServices.GetRequiredService().InboxDeadLetters)); + Assert.Equal(0, count); + Assert.Empty(letters); + Assert.Equal(0, Storage.CountPayloadOccurrences(Journal(receiver), payload)); + + Probe.For(receiver).Mode = SinkMode.Accept; + Assert.Equal(DeliveryStatus.Accepted, (await extension.DeliverAsync(message, Token)).Status); + Assert.Equal(2, Probe.For(receiver).SinkOffers); + Assert.Equal(1, Probe.For(receiver).SinkAcceptances); + Assert.Equal(0, Probe.For(receiver).Handled); + } + + [Fact] + public async Task ScheduledMessage_IsOfferedToTheSinkBeforeItIsDue() + { + var receiver = NewGrain(); + var extension = receiver.AsReference(); + var payload = NewPayload(); + var scheduled = NewInboxMessage("scheduled", payload, DateTimeOffset.UtcNow.AddDays(1)); + + Assert.Equal(DeliveryStatus.Accepted, (await extension.DeliverAsync(scheduled, Token)).Status); + + var (state, count) = await OnGrainAsync(receiver, context => (Grain(context).GetState(), Inbox(context).Count)); + Assert.Equal(payload, state.Received[scheduled.Envelope.MessageId.ToString()]); + Assert.Equal(0, count); + Assert.Equal(1, Probe.For(receiver).SinkAcceptances); + Assert.Equal(0, Probe.For(receiver).Handled); + } + + [Fact] + public async Task DeclinedMessages_UseInboxAcceptanceAndHandlers() + { + var receiver = NewGrain(); + var extension = receiver.AsReference(); + Probe.For(receiver).Mode = SinkMode.Decline; + + var scheduled = NewInboxMessage("scheduled", NewPayload(), DateTimeOffset.UtcNow.AddDays(1)); + Assert.Equal(DeliveryStatus.Accepted, (await extension.DeliverAsync(scheduled, Token)).Status); + Assert.Equal(1, await OnGrainAsync(receiver, context => Inbox(context).Count)); + + var declinedPayload = NewPayload(); + Assert.Equal(DeliveryStatus.Accepted, (await extension.DeliverAsync(NewInboxMessage("declined", declinedPayload), Token)).Status); + await WaitUntilAsync(receiver, _ => Probe.For(receiver).Handled == 1); + Assert.Equal(2, Probe.For(receiver).SinkOffers); + Assert.Equal(0, Probe.For(receiver).SinkAcceptances); + Assert.Empty((await OnGrainAsync(receiver, context => Grain(context).GetState())).Received); + Assert.Equal(1, await OnGrainAsync(receiver, context => Inbox(context).Count)); + Assert.Equal(1, Storage.CountPayloadOccurrences(Journal(receiver), declinedPayload)); + } + + [Fact] + public async Task SelfSentMessage_IsAcceptedByTheSinkInTheSendingWrite() + { + var grain = NewGrain(); + _ = await grain.GetOutboxCountAsync(); + var writes = Writes(grain).Length; + var messageId = NewMessageId("self"); + var payload = NewPayload(); + + await grain.SendAsync(grain.GetGrainId(), messageId, payload, write: true); + + var (state, inboxCount, outboxCount) = await OnGrainAsync(grain, context => (Grain(context).GetState(), Inbox(context).Count, Outbox(context).Count)); + Assert.Equal(payload, state.Received[messageId.ToString()]); + Assert.Equal(0, inboxCount); + Assert.Equal(0, outboxCount); + Assert.Equal(1, Probe.For(grain).SinkAcceptances); + Assert.Equal(writes + 1, Writes(grain).Length); + Assert.Equal(1, Storage.CountPayloadOccurrences(Journal(grain), payload)); + } + + [Theory] + [InlineData(SinkMode.Backpressure)] + [InlineData(SinkMode.Throw)] + [InlineData(SinkMode.UnsupportedStatus)] + public async Task SelfSentMessage_FailsTheSendWithoutStagingWhenTheSinkBackpressuresOrFails(SinkMode mode) + { + var grain = NewGrain(); + _ = await grain.GetOutboxCountAsync(); + Probe.For(grain).Mode = mode; + var writes = Writes(grain).Length; + + await Assert.ThrowsAsync( + () => grain.SendAsync(grain.GetGrainId(), NewMessageId("self-backpressured"), NewPayload(), write: true)); + + var (state, inboxCount, outboxCount) = await OnGrainAsync(grain, context => (Grain(context).GetState(), Inbox(context).Count, Outbox(context).Count)); + Assert.Empty(state.Received); + Assert.Equal(0, inboxCount); + Assert.Equal(0, outboxCount); + Assert.Equal(writes, Writes(grain).Length); + } + + private ISourceSinkGrain NewGrain() => Cluster.Client.GetGrain(Guid.NewGuid()); + + private static MessageId NewMessageId(string name) => MessageId.Create("source-sink", name, Guid.NewGuid().ToString("N")); + + private static byte[] NewPayload() => RandomNumberGenerator.GetBytes(48); + + private InboxMessage NewInboxMessage(string name, byte[] payload, DateTimeOffset? deliverAt = null) => new() + { + Envelope = new Envelope { MessageId = NewMessageId(name), Payload = payload, Subject = SourceSinkGrain.Subject, DeliverAt = deliverAt }, + SentAt = fixture.JobNow + }; + + private static JournalId Journal(IAddressable grain) => JournalId.FromGrainId(grain.GetGrainId()); + + private byte[][] Writes(IAddressable grain) => Storage.GetWrites(Journal(grain)); + + private static SourceSinkGrain Grain(IGrainContext context) => Assert.IsType(context.GrainInstance); + + private static Inbox Inbox(IGrainContext context) => context.ActivationServices.GetRequiredService(); + + private static Outbox Outbox(IGrainContext context) => context.ActivationServices.GetRequiredService(); + + private Task OnGrainAsync(IAddressable grain, Func func) + { + if (!Cluster.TryGetGrainContext(grain.GetGrainId(), out var context)) + { + throw new InvalidOperationException($"Grain '{grain.GetGrainId()}' is not active."); + } + + var done = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + context.Scheduler.QueueAction(() => + { + try + { + done.SetResult(func(context)); + } + catch (Exception exception) + { + done.SetException(exception); + } + }); + return done.Task.WaitAsync(TimeSpan.FromSeconds(30), Token); + } + + private async Task WaitUntilAsync(IAddressable grain, Func predicate) + { + var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(30); + while (!Cluster.TryGetGrainContext(grain.GetGrainId(), out _) || !await OnGrainAsync(grain, predicate)) + { + if (DateTime.UtcNow > deadline) + { + throw new TimeoutException($"Condition was not met for grain '{grain.GetGrainId()}'."); + } + + await Task.Delay(10, Token); + } + } +} + +/// +/// Observes one message's messaging events, and fails its first deliveries by throwing from +/// . +/// +internal sealed class DeliveryEventObserver : IObserver, IDisposable +{ + public const string Fault = "The test injected a delivery fault."; + + private readonly GrainId _sender; + private readonly GrainId _receiver; + private readonly MessageId _messageId; + private readonly IDisposable _subscription; + private readonly ConcurrentQueue _retries = new(); + private readonly TaskCompletionSource _retryScheduled = new(TaskCreationOptions.RunContinuationsAsynchronously); + private readonly TaskCompletionSource _retired = new(TaskCreationOptions.RunContinuationsAsynchronously); + private int _faults; + private int _receivings; + + public DeliveryEventObserver(GrainId sender, GrainId receiver, MessageId messageId, int faults) + { + (_sender, _receiver, _messageId, _faults) = (sender, receiver, messageId, faults); + _subscription = MessagingEvents.AllEvents.Subscribe(this); + } + + public Task Retired => _retired.Task; + public Task RetryScheduled => _retryScheduled.Task; + public IReadOnlyCollection Retries => _retries; + public int Receivings => Volatile.Read(ref _receivings); + + public void OnNext(MessagingEvents.MessagingEvent value) + { + switch (value) + { + case MessagingEvents.InboxMessageReceiving receiving + when receiving.GrainContext.GrainId == _receiver && receiving.Message.Envelope.MessageId == _messageId: + Interlocked.Increment(ref _receivings); + if (Interlocked.Decrement(ref _faults) >= 0) + { + throw new InvalidOperationException(Fault); + } + + break; + case MessagingEvents.OutboxRetryScheduled retry when retry.GrainContext.GrainId == _sender && retry.MessageId == _messageId: + _retries.Enqueue(retry); + _retryScheduled.TrySetResult(retry); + break; + case MessagingEvents.OutboxMessageRetired retired when retired.GrainContext.GrainId == _sender && retired.MessageId == _messageId: + _retired.TrySetResult(retired); + break; + } + } + + public void OnCompleted() { } + + public void OnError(Exception error) { } + + public void Dispose() => _subscription.Dispose(); +} + +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class JsonMessageSourceAndSinkTests(JsonMessageSourceAndSinkTests.Fixture fixture) + : MessageSourceAndSinkTests(fixture), IClassFixture +{ + public sealed class Fixture() : MessageSourceAndSinkFixture("json"); +} + +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class BinaryMessageSourceAndSinkTests(BinaryMessageSourceAndSinkTests.Fixture fixture) + : MessageSourceAndSinkTests(fixture), IClassFixture +{ + public sealed class Fixture() : MessageSourceAndSinkFixture("orleans-binary"); +} + +public abstract class MessageSourceAndSinkFixture(string journalFormatKey) : IAsyncLifetime +{ + public static readonly TimeSpan DeduplicationWindow = TimeSpan.FromMinutes(10); + + public FakeTimeProvider Clock { get; } = new(DateTimeOffset.UtcNow); + public RecordingJournalStorageProvider Storage { get; } = new(); + public SourceSinkProbe Probe { get; } = new(); + public InProcessTestCluster Cluster { get; private set; } = null!; + public DateTimeOffset JobNow => Cluster.Silos[0].ServiceProvider + .GetRequiredKeyedService(DurableJobTimeProviderNames.DurableJobs).GetUtcNow(); + + public async ValueTask InitializeAsync() + { + var builder = new InProcessTestClusterBuilder(1); + builder.ConfigureSilo((_, silo) => + { + silo.AddVolatileJournalStorage(); + silo.Services.Configure(options => options.JournalFormatKey = journalFormatKey); + silo.Services.Configure(options => options.AddTypeInfoResolver(SourceSinkJsonContext.Default)); + silo.Services.AddSingleton(Clock); + silo.Services.AddKeyedSingleton(DurableJobTimeProviderNames.DurableJobs, + new MessagingClusterFixture.FollowingJobClock(Clock, Clock.GetUtcNow())); + silo.Services.UseTimeProviderForBackgroundAreas(TimeProvider.System); + silo.UseInMemoryDurableJobs(); + silo.AddMessaging(options => + { + options.DeduplicationWindow = DeduplicationWindow; + options.OutboxRetryTimeout = TimeSpan.FromSeconds(3); + options.RetryInitialInterval = TimeSpan.FromMilliseconds(10); + }); + silo.Services.AddSingleton(Probe); + silo.Services.RemoveAll(); + silo.Services.RemoveAll(); + silo.Services.AddSingleton(services => + { + Storage.Configure(services.GetRequiredService>()); + return Storage; + }); + silo.Services.AddSingleton( + services => (IJournalStorageCatalog)services.GetRequiredService()); + }); + Cluster = builder.Build(); + await Cluster.DeployAsync(TestContext.Current.CancellationToken); + } + + public async ValueTask DisposeAsync() => await Cluster.DisposeAsync(); +} + +[JsonSerializable(typeof(string))] +[JsonSerializable(typeof(byte[]))] +[JsonSerializable(typeof(int))] +[JsonSerializable(typeof(MessageEffect))] +internal sealed partial class SourceSinkJsonContext : JsonSerializerContext; + +public enum SinkMode +{ + Accept, + Decline, + Throw, + DeadLetter, + Backpressure, + UnsupportedStatus +} + +/// How the source fails to provide a message which it stores. +public enum SourceFault +{ + Throw, + InvalidMessage, + MismatchedId +} + +public sealed class SourceSinkProbe +{ + private readonly ConcurrentDictionary _counters = new(); + + public Counters For(IAddressable grain) => For(grain.GetGrainId()); + + public Counters For(GrainId grainId) => _counters.GetOrAdd(grainId, static _ => new()); + + public sealed class Counters + { + private int _activations; + private int _materializations; + private int _sinkOffers; + private int _sinkAcceptances; + private int _handled; + private int _failuresRemaining; + private int _retirementFailuresRemaining; + private int _retirementCallbacks; + + public volatile SinkMode Mode; + public volatile string? SubjectOverride; + + /// Makes the source share its payload array as a header value, and clear that array when the message retires. + public volatile bool ReleaseBuffersOnRetired; + /// Opt-in application policy: journal a terminal outcome and a real self-inbox follow-up. + public DateTimeOffset? FollowUpAt { get; set; } + public int RetirementCallbacks => Volatile.Read(ref _retirementCallbacks); + public ConcurrentDictionary Faults { get; } = new(); + public int Activations => Volatile.Read(ref _activations); + public int Materializations => Volatile.Read(ref _materializations); + public int SinkOffers => Volatile.Read(ref _sinkOffers); + public int SinkAcceptances => Volatile.Read(ref _sinkAcceptances); + public int Handled => Volatile.Read(ref _handled); + + public void OnActivated() => Interlocked.Increment(ref _activations); + public void OnMaterialized() => Interlocked.Increment(ref _materializations); + public void OnSinkOffered() => Interlocked.Increment(ref _sinkOffers); + public void OnSinkAccepted() => Interlocked.Increment(ref _sinkAcceptances); + public void OnHandled() => Interlocked.Increment(ref _handled); + + /// Makes the sink throw for the next offers before applying . + public void FailNextOffers(int count) => Volatile.Write(ref _failuresRemaining, count); + + public bool TryConsumeFailure() => Interlocked.Decrement(ref _failuresRemaining) >= 0; + public void FailNextRetirements(int count) => Volatile.Write(ref _retirementFailuresRemaining, count); + public void OnRetiring() => Interlocked.Increment(ref _retirementCallbacks); + public bool TryConsumeRetirementFailure() => Interlocked.Decrement(ref _retirementFailuresRemaining) >= 0; + } +} + +public interface ISourceSinkGrain : IGrainWithGuidKey +{ + Task StageFromSourceAsync(GrainId receiver, MessageId messageId, byte[] payload, bool write); + Task StageFromSourceAndDeactivateAsync(GrainId receiver, MessageId messageId, byte[] payload); + Task SendFromSourceAsync(MessageId messageId); + Task WithdrawAsync(MessageId messageId); + Task SendAsync(GrainId receiver, MessageId messageId, byte[] payload, bool write); + Task WriteAsync(); + Task GetOutboxCountAsync(); + Task DeactivateAsync(); +} + +/// Stores outgoing messages as an outbox source and received payloads as an inbox sink. +public sealed class SourceSinkGrain : Grain, ISourceSinkGrain, IMessagingGrain, IOutboxMessageSource, IInboxMessageSink +{ + internal const string Subject = "source-sink/payload"; + internal const string PayloadHeader = "source-sink/payload-copy"; + internal const string Rejection = "The sink rejected the message."; + internal const string RejectionCode = "source-sink/rejected"; + internal const string SinkFailure = "The sink failed before deciding the message's outcome."; + internal const string SourceFailure = "The source can't decode the stored message."; + internal const string FollowUpSubject = "source-sink/terminal-follow-up"; + internal const string RetirementFailure = "Injected failure after staging terminal outcome and follow-up."; + private readonly GrainId _grainId; + private readonly IJournaledStateManager _manager; + private readonly IOutbox _outbox; + private readonly IDurableDictionary _outgoing; + private readonly IDurableDictionary _receivers; + private readonly IDurableDictionary _received; + private readonly IDurableDictionary _retired; + private readonly IDurableDictionary _reasons; + private readonly IDurableDictionary _codes; + private readonly IDurableDictionary _terminalCounts; + private readonly IDurableDictionary _followUpCounts; + private readonly SourceSinkProbe.Counters _probe; + + public SourceSinkGrain( + IGrainContext context, + IJournaledStateManager manager, + IInbox inbox, + IOutbox outbox, + [FromKeyedServices("outgoing")] IDurableDictionary outgoing, + [FromKeyedServices("receivers")] IDurableDictionary receivers, + [FromKeyedServices("received")] IDurableDictionary received, + [FromKeyedServices("retired")] IDurableDictionary retired, + [FromKeyedServices("reasons")] IDurableDictionary reasons, + [FromKeyedServices("codes")] IDurableDictionary codes, + [FromKeyedServices("terminal-counts")] IDurableDictionary terminalCounts, + [FromKeyedServices("follow-up-counts")] IDurableDictionary followUpCounts, + SourceSinkProbe probe) + { + _grainId = context.GrainId; + _manager = manager; + _outbox = outbox; + _outgoing = outgoing; + _receivers = receivers; + _received = received; + _retired = retired; + _reasons = reasons; + _codes = codes; + _terminalCounts = terminalCounts; + _followUpCounts = followUpCounts; + _probe = probe.For(context.GrainId); + _probe.OnActivated(); + outbox.Register(this); + inbox.Register(this).Register(new Handler(_probe, _followUpCounts)); + } + + internal (Dictionary Outgoing, Dictionary Received, Dictionary Retired) GetState() => + (new(_outgoing), new(_received), new(_retired)); + + /// Gets the reasons of this source's dead-lettered messages, by message ID. + internal Dictionary GetRetirementReasons() => new(_reasons); + + /// Gets the receivers' rejection codes for this source's rejected messages, by message ID. + internal Dictionary GetRetirementCodes() => new(_codes); + internal Dictionary GetTerminalCounts() => new(_terminalCounts); + internal Dictionary GetFollowUpCounts() => new(_followUpCounts); + + /// Stores an outgoing message in the source's state without sending it. + internal void StoreOutgoing(GrainId receiver, MessageId messageId, byte[] payload) + { + _outgoing[messageId.ToString()] = payload; + _receivers[messageId.ToString()] = receiver.ToString(); + } + + public async Task StageFromSourceAsync(GrainId receiver, MessageId messageId, byte[] payload, bool write) + { + _outgoing[messageId.ToString()] = payload; + _receivers[messageId.ToString()] = receiver.ToString(); + _outbox.Send(messageId); + if (write) await _manager.WriteStateAsync(CancellationToken.None); + } + + public async Task StageFromSourceAndDeactivateAsync(GrainId receiver, MessageId messageId, byte[] payload) + { + await StageFromSourceAsync(receiver, messageId, payload, write: true); + DeactivateOnIdle(); + } + + public Task SendFromSourceAsync(MessageId messageId) + { + _outbox.Send(messageId); + return Task.CompletedTask; + } + + public Task WithdrawAsync(MessageId messageId) + { + Assert.True(_outgoing.Remove(messageId.ToString())); + Assert.True(_receivers.Remove(messageId.ToString())); + Assert.True(_outbox.Withdraw(messageId)); + return Task.CompletedTask; + } + + public async Task SendAsync(GrainId receiver, MessageId messageId, byte[] payload, bool write) + { + _outbox.Send(new OutboxMessage + { + Envelope = new Envelope { MessageId = messageId, Payload = payload, Subject = Subject }, + ReceiverId = receiver + }); + if (write) await _manager.WriteStateAsync(CancellationToken.None); + } + + public async Task WriteAsync() => await _manager.WriteStateAsync(CancellationToken.None); + + public Task GetOutboxCountAsync() => Task.FromResult(_outbox.Count()); + + public Task DeactivateAsync() + { + DeactivateOnIdle(); + return Task.CompletedTask; + } + + bool IOutboxMessageSource.TryGetMessage(MessageId messageId, out OutboxMessage message) + { + var key = messageId.ToString(); + if (!_outgoing.TryGetValue(key, out var payload)) + { + message = default; + return false; + } + + _probe.OnMaterialized(); + message = new() + { + Envelope = new() + { + MessageId = messageId, + Payload = payload, + Subject = _probe.SubjectOverride ?? Subject, + Headers = _probe.ReleaseBuffersOnRetired ? new() { [PayloadHeader] = payload } : null + }, + ReceiverId = GrainId.Parse(_receivers[key]) + }; + if (_probe.Faults.TryGetValue(messageId, out var fault)) + { + message = fault switch + { + SourceFault.Throw => throw new InvalidOperationException(SourceFailure), + SourceFault.InvalidMessage => message with { Envelope = message.Envelope with { Payload = null! } }, + _ => message with { Envelope = message.Envelope with { MessageId = new MessageId(messageId.Key.CreateChildKey("other")) } } + }; + } + + return true; + } + + void IOutboxMessageSource.OnRetired(MessageId messageId, OutboxMessageRetirement retirement) + { + _probe.OnRetiring(); + var key = messageId.ToString(); + if (_probe.ReleaseBuffersOnRetired) Array.Clear(_outgoing[key]); + Assert.True(_outgoing.Remove(key)); + Assert.True(_receivers.Remove(key)); + _retired[key] = retirement.Status.ToString(); + if (retirement.Reason is { } reason) _reasons[key] = reason; + if (retirement.Code is { } code) _codes[key] = code; + if (_probe.FollowUpAt is { } followUpAt) + { + _terminalCounts.TryGetValue(key, out var previous); + _terminalCounts[key] = previous + 1; + _outbox.Send(new OutboxMessage + { + ReceiverId = _grainId, + Envelope = new() + { + MessageId = new MessageId(messageId.Key.CreateChildKey("terminal-follow-up")), + Payload = Encoding.UTF8.GetBytes(key), + Subject = FollowUpSubject, + DeliverAt = followUpAt + } + }); + if (_probe.TryConsumeRetirementFailure()) throw new InvalidOperationException(RetirementFailure); + } + } + + bool IInboxMessageSink.TryAccept(InboxMessage message, out DeliveryResult result) + { + result = default; + if (message.Envelope.Subject != Subject) return false; + _probe.OnSinkOffered(); + var key = message.Envelope.MessageId.ToString(); + if (_probe.TryConsumeFailure()) throw new InvalidOperationException(SinkFailure); + switch (_probe.Mode) + { + case SinkMode.Decline: + return false; + case SinkMode.Throw: + throw new InvalidOperationException(SinkFailure); + case SinkMode.DeadLetter: + result = DeliveryResult.DeadLettered(Rejection, RejectionCode); + return true; + case SinkMode.Backpressure: + result = DeliveryResult.Backpressured(); + return true; + case SinkMode.UnsupportedStatus: + result = DeliveryResult.HandlerNotFound(); + return true; + } + + if (_received.ContainsKey(key)) + { + result = DeliveryResult.Duplicate(); + return true; + } + + _received[key] = message.Envelope.Payload; + _probe.OnSinkAccepted(); + result = DeliveryResult.Accepted(); + return true; + } + + private sealed class Handler(SourceSinkProbe.Counters probe, IDurableDictionary followUpCounts) : IInboxHandler + { + public bool CanHandle(InboxMessage message) => message.Envelope.Subject is Subject or FollowUpSubject; + + public ValueTask HandleAsync(IInboxHandlerContext context, CancellationToken cancellationToken) + { + probe.OnHandled(); + if (context.Message.Envelope.Subject == FollowUpSubject) + { + var key = Encoding.UTF8.GetString(context.Message.Envelope.Payload); + followUpCounts.TryGetValue(key, out var previous); + followUpCounts[key] = previous + 1; + } + context.Complete(); + return ValueTask.CompletedTask; + } + } +} + +public interface INoHandlerGrain : IGrainWithGuidKey +{ + Task PingAsync(); +} + +/// A messaging grain without handlers or sinks, whose inbox reports every delivery as handler-not-found. +public sealed class NoHandlerGrain : Grain, INoHandlerGrain, IMessagingGrain +{ + public Task PingAsync() => Task.CompletedTask; +} diff --git a/test/Orleans.Messaging.Tests/Functional/MessagingActivationBehaviorTests.cs b/test/Orleans.Messaging.Tests/Functional/MessagingActivationBehaviorTests.cs new file mode 100644 index 00000000000..9796d093abe --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/MessagingActivationBehaviorTests.cs @@ -0,0 +1,48 @@ +using Microsoft.Extensions.DependencyInjection; +using Orleans.DurableJobs; +using Orleans.Journaling; +using Orleans.Messaging.Tests.Support; +using Orleans.Runtime; +using Orleans.Serialization.Session; +using Xunit; + +namespace Orleans.Messaging.Tests.Functional; + +[Collection(MessagingClusterCollection.Name)] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class MessagingActivationBehaviorTests : MessagingBehaviorTestBase +{ + public MessagingActivationBehaviorTests() : base(receiverOnly: false) + { + } + + [Fact] + public async Task ApplicationJournaledStateNamesDoNotCollideWithMessagingState() + { + var grain = NewGrain(); + + var snapshot = await grain.GetSnapshotAsync(); + + Assert.Equal(0, snapshot.InboxCount); + Assert.Equal(0, snapshot.OutboxCount); + } + [Fact] + public async Task PublicHosting_BindsRealOutboxAndRegisteredMessagingStates() + { + var grain = NewGrain(); + _ = await grain.GetSnapshotAsync(); + var services = Fixture.GetGrainContext(grain).ActivationServices; + var outbox = services.GetRequiredService(); + var inbox = services.GetRequiredService(); + Assert.Same(outbox, services.GetRequiredService()); + Assert.Same(inbox, services.GetRequiredService()); + var manager = services.GetRequiredService(); + Assert.True(manager.TryGetStateMachine(MessagingStateNames.Outbox, out var outboxState)); + Assert.True(manager.TryGetStateMachine(MessagingStateNames.Inbox, out var inboxState)); + Assert.Same(outbox, outboxState); + Assert.Same(inbox, inboxState); + } + +} diff --git a/test/Orleans.Messaging.Tests/Functional/MessagingClusterCollection.cs b/test/Orleans.Messaging.Tests/Functional/MessagingClusterCollection.cs new file mode 100644 index 00000000000..39e60402481 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/MessagingClusterCollection.cs @@ -0,0 +1,9 @@ +using Xunit; + +namespace Orleans.Messaging.Tests.Functional; + +[CollectionDefinition(Name, DisableParallelization = true)] +public sealed class MessagingClusterCollection +{ + public const string Name = "Messaging cluster"; +} diff --git a/test/Orleans.Messaging.Tests/Functional/MessagingGrainTypeConfiguratorTests.cs b/test/Orleans.Messaging.Tests/Functional/MessagingGrainTypeConfiguratorTests.cs new file mode 100644 index 00000000000..3f58d11c556 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/MessagingGrainTypeConfiguratorTests.cs @@ -0,0 +1,490 @@ +using System.Globalization; +using System.Reflection; +using Microsoft.Extensions.DependencyInjection; +using NSubstitute; +using Orleans.Concurrency; +using Orleans.Hosting; +using Orleans.Journaling; +using Orleans.Messaging.Tests.Support; +using Orleans.Metadata; +using Orleans.Runtime; +using Orleans.Runtime.Placement; +using Orleans.Serialization.Session; +using Xunit; + +namespace Orleans.Messaging.Tests.Functional; + +[Collection(MessagingClusterCollection.Name)] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public class MessagingGrainTypeConfiguratorTests : MessagingBehaviorTestBase +{ + public MessagingGrainTypeConfiguratorTests() : this(new BootstrapClusterFixture()) + { + } + + protected MessagingGrainTypeConfiguratorTests(BootstrapClusterFixture fixture) : base(fixture) + { + } + + private BootstrapDeliveryProbe Delivery => ((BootstrapClusterFixture)Fixture).Delivery; + private BootstrapProbe Probe => ((BootstrapClusterFixture)Fixture).Probe; + private static CancellationToken Cancellation => TestContext.Current.CancellationToken; + + [Theory] + [InlineData(typeof(PlainBootstrapGrain))] + [InlineData(typeof(ApplicationBootstrapGrain))] + [InlineData(typeof(InterfaceBootstrapGrain))] + [InlineData(typeof(GenericBootstrapGrain))] + [InlineData(typeof(DurableBootstrapGrain))] + [InlineData(typeof(MarkedDurableBootstrapGrain))] + public async Task SelectedComposition_InstallsMessagingExtensionAndReplaysFreshScopedState(Type grainClass) + { + var grain = CreateGrain(grainClass); + Assert.Equal(0, await grain.GetValueAsync()); + var first = Assert.Single(Probe.Get(grain.GetGrainId())); + var state = first.Context.ActivationServices.GetRequiredService(); + AssertComposition(first, state, grainClass, expectedActivationValue: 0); + var setup = GetSetup(first.Context); + Assert.Single(setup.GetInvocationList()); + await grain.SetValueAsync(41); + var journal = JournalId.FromGrainId(grain.GetGrainId()); + + using var handler = Fixture.HandlerProbe.Arm(grain.GetGrainId(), BootstrapState.Route); + var envelope = CreateEnvelope(grain); + Assert.Equal(DeliveryStatus.Accepted, (await grain.AsReference().DeliverAsync(envelope, Cancellation)).Status); + await handler.WaitUntilEnteredAsync(); + Assert.Equal(41, first.Value!.Value); + Assert.Equal(1, first.Inbox!.Count); + Assert.Equal(0, first.Outbox!.Count()); + Assert.Equal(0, GetProcessedCount(first)); + Assert.Equal(0, state.HandlerCalls); + handler.Release(); + await Fixture.WaitUntilAsync(grain, () => state.HandlerCalls == 1 && first.Inbox!.Count == 0); + Assert.Equal(42, await grain.GetValueAsync()); + Assert.Equal(1, GetProcessedCount(first)); + var output = Assert.Single(first.Outbox!.Messages()); + Assert.NotEmpty(Fixture.JobManagerProbe.GetScheduledJobs(MessagingPump.JobName, grain.GetGrainId())); + Assert.Equal(DeliveryStatus.Duplicate, (await grain.AsReference().DeliverAsync(envelope, Cancellation)).Status); + Assert.Equal(1, state.HandlerCalls); + + var other = CreateGrain(grainClass); + Assert.Equal(0, await other.GetValueAsync()); + var isolated = Assert.Single(Probe.Get(other.GetGrainId())); + Assert.NotSame(first.Manager, isolated.Manager); + Assert.NotSame(first.Inbox, isolated.Inbox); + Assert.NotSame(first.Outbox, isolated.Outbox); + Assert.Empty(isolated.Outbox!.Messages()); + await other.DeactivateAsync(); + await isolated.Context.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), Cancellation); + + await grain.DeactivateAsync(); + await first.Context.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), Cancellation); + Assert.Equal(1, first.Disposals); + Assert.Equal(1, first.GrainDisposals); + Assert.Equal(1, state.Disposals); + Assert.Equal(42, await grain.GetValueAsync()); + var observations = Probe.Get(grain.GetGrainId()); + Assert.Equal(2, observations.Length); + var recovered = observations[1]; + var recoveredState = recovered.Context.ActivationServices.GetRequiredService(); + AssertComposition(recovered, recoveredState, grainClass, expectedActivationValue: 42); + Assert.NotSame(first.Context, recovered.Context); + Assert.NotSame(first.Inbox, recovered.Inbox); + Assert.NotSame(first.Outbox, recovered.Outbox); + Assert.Same(setup, GetSetup(recovered.Context)); + Assert.Equal(output.Envelope.MessageId, Assert.Single(recovered.Outbox!.Messages()).Envelope.MessageId); + Assert.Equal(1, GetProcessedCount(recovered)); + Assert.Equal(DeliveryStatus.Duplicate, (await grain.AsReference().DeliverAsync(envelope, Cancellation)).Status); + Assert.Equal(0, recoveredState.HandlerCalls); + Assert.Equal(2, Fixture.Storage.GetReadCount(journal)); + + var delivered = Delivery.WaitForOutputAsync(grain.GetGrainId()); + var drained = Delivery.WaitForDrainAsync(grain.GetGrainId()); + var sentAt = await Fixture.OnGrainAsync(grain, () => + new OutboxStateTestAccess(recovered.Outbox!).Attempt(output.Envelope.MessageId).SentAt); + Delivery.Release(); + var receipt = await delivered; + await drained; + Assert.Equal(output.Envelope.MessageId, receipt.MessageId); + Assert.Equal(42, receipt.Value); + Assert.Empty(recovered.Outbox!.Messages()); + var sink = Fixture.Client.GetGrain("capture"); + Assert.Equal(1, await sink.GetMessageCountAsync()); + Assert.Equal(DeliveryStatus.Duplicate, (await sink.AsReference() + .DeliverAsync(output.ToInboxMessage(sentAt!.Value), Cancellation)).Status); + Assert.Equal(1, await sink.GetMessageCountAsync()); + } + + [Fact] + public async Task OrdinarySend_SchedulesWakeupBeforeCaptureAndDispatchesAfterAck() + { + var grain = CreateGrain(typeof(PlainBootstrapGrain)); + await grain.SetValueAsync(11); + var observation = Assert.Single(Probe.Get(grain.GetGrainId())); + var journal = JournalId.FromGrainId(grain.GetGrainId()); + var writes = Fixture.Storage.GetSuccessfulWriteCount(journal); + using var scheduling = Fixture.JobManagerProbe.BlockNext(MessagingPump.JobName); + var operation = grain.SendValueAsync(42); + await scheduling.WaitUntilEnteredAsync(); + Assert.Equal(42, observation.Value!.Value); + Assert.Equal(1, observation.Outbox!.Count()); + Assert.Equal(writes, Fixture.Storage.GetSuccessfulWriteCount(journal)); + var storage = Fixture.Storage.BlockWrite(journal); + var delivered = Delivery.WaitForOutputAsync(grain.GetGrainId()); + Delivery.Release(); + scheduling.Continue(); + await storage.WaitUntilEnteredAsync(); + Assert.Single(observation.Outbox!.Messages()); + Assert.False(delivered.IsCompleted); + Assert.Equal(1, Fixture.JobManagerProbe.GetSuccessCount(MessagingPump.JobName, grain.GetGrainId())); + storage.Release(); + await operation; + var receipt = await delivered; + Assert.Equal(42, receipt.Value); + Assert.Equal(42, await grain.GetValueAsync()); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task OrdinarySend_SchedulingFailurePreservesPendingBusinessForExplicitRetry(bool ambiguous) + { + var grain = CreateGrain(typeof(PlainBootstrapGrain)); + await grain.SetValueAsync(11); + var observation = Assert.Single(Probe.Get(grain.GetGrainId())); + var journal = JournalId.FromGrainId(grain.GetGrainId()); + var writes = Fixture.Storage.GetSuccessfulWriteCount(journal); + if (ambiguous) Fixture.JobManagerProbe.FailAfterNext(MessagingPump.JobName); + else Fixture.JobManagerProbe.FailNext(MessagingPump.JobName); + var prerequisite = await Assert.ThrowsAsync(() => grain.SendValueAsync(42)); + Assert.IsType(prerequisite.InnerException); + Assert.Equal(42, await grain.GetValueAsync()); + Assert.Equal(1, observation.Outbox!.Count()); + Assert.Equal(writes, Fixture.Storage.GetSuccessfulWriteCount(journal)); + Assert.Equal(ambiguous ? 1 : 0, Fixture.JobManagerProbe.GetScheduledJobs(MessagingPump.JobName, grain.GetGrainId()).Count); + await grain.PersistAsync(); + Assert.Equal(writes + 1, Fixture.Storage.GetSuccessfulWriteCount(journal)); + Assert.Equal(42, TestApplicationProtocol.Read(Sessions, Assert.Single(observation.Outbox!.Messages())).Body); + Assert.NotEmpty(Fixture.JobManagerProbe.GetScheduledJobs(MessagingPump.JobName, grain.GetGrainId())); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task OrdinarySend_FreshReplayResolvesActualWriteOutcome(bool committed) + { + var grain = CreateGrain(typeof(PlainBootstrapGrain)); + await grain.SetValueAsync(11); + var observation = Assert.Single(Probe.Get(grain.GetGrainId())); + var journal = JournalId.FromGrainId(grain.GetGrainId()); + var writes = Fixture.Storage.GetSuccessfulWriteCount(journal); + if (committed) Fixture.Storage.FailAfterWrite(journal); + else Fixture.Storage.FailWrite(journal); + await Assert.ThrowsAsync(() => grain.SendValueAsync(42)); + Assert.Equal(writes + (committed ? 1 : 0), Fixture.Storage.GetSuccessfulWriteCount(journal)); + Assert.Single(Fixture.JobManagerProbe.GetScheduledJobs(MessagingPump.JobName, grain.GetGrainId())); + await grain.DeactivateAsync(); + await observation.Context.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), Cancellation); + Assert.Equal(committed ? 42 : 11, await grain.GetValueAsync()); + var recovered = Probe.Get(grain.GetGrainId())[^1]; + Assert.NotSame(observation.Context, recovered.Context); + Assert.Equal(committed ? 1 : 0, recovered.Outbox!.Count()); + if (committed) + { + Assert.Equal(42, TestApplicationProtocol.Read(Sessions, Assert.Single(recovered.Outbox!.Messages())).Body); + } + else + { + Assert.Empty(recovered.Outbox!.Messages()); + } + } + + [Fact] + public async Task SelectedWithoutConstructorDependencies_MaterializesMessagingStateBeforeRead() + { + var grain = Control(); + var journal = JournalId.FromGrainId(grain.GetGrainId()); + var read = Fixture.Storage.BlockRead(journal); + var activation = grain.PingAsync(); + try + { + await read.WaitUntilEnteredAsync(); + var observation = Assert.Single(Probe.Get(grain.GetGrainId())); + Assert.Same(observation.ConstructedGrain, observation.Context.GrainInstance); + var manager = observation.Context.ActivationServices.GetRequiredService(); + Assert.Equal(2, BootstrapState.ReadMessagingStates(manager).Count()); + Assert.Equal(0, observation.Activations); + Assert.Single(GetSetup(observation.Context).GetInvocationList()); + } + finally + { + read.Release(); + } + + await activation; + Assert.Equal(1, Fixture.Storage.GetReadCount(journal)); + } + + [Theory] + [InlineData(typeof(UnselectedBootstrapGrain), false)] + [InlineData(typeof(JournalOnlyBootstrapGrain), true)] + public async Task UnselectedComposition_LeavesMessagingUnresolved(Type grainClass, bool journaled) + { + var grain = Fixture.Client.GetGrain(Guid.NewGuid(), grainClass.FullName!); + await grain.PingAsync(); + var observation = Assert.Single(Probe.Get(grain.GetGrainId())); + Assert.Equal(1, observation.Activations); + Assert.Null(observation.Inbox); + Assert.Null(observation.Outbox); + Assert.Null(GetSetupOrDefault(observation.Context)); + Assert.Null(observation.Context.GetComponent()); + var journal = JournalId.FromGrainId(grain.GetGrainId()); + Assert.Equal(journaled ? 1 : 0, Fixture.Storage.GetCreationCount(journal)); + Assert.Equal(journaled ? 1 : 0, Fixture.Storage.GetReadCount(journal)); + Assert.Equal(0, Fixture.Storage.GetInitializationCount(journal)); + if (journaled) + { + Assert.NotNull(observation.Manager); + Assert.True(observation.Manager.TryGetStateMachine("bootstrap-journal-only", out var state)); + Assert.Same(observation.Value, state); + Assert.Empty(BootstrapState.ReadMessagingStates(observation.Manager)); + } + else + { + Assert.Null(observation.Manager); + } + + AssertNoScheduledJobs(grain.GetGrainId()); + } + + [Theory] + [InlineData(typeof(ReentrantBootstrapGrain))] + [InlineData(typeof(MayInterleaveBootstrapGrain))] + [InlineData(typeof(AlwaysInterleaveBootstrapGrain))] + [InlineData(typeof(MetadataReentrantBootstrapGrain))] + [InlineData(typeof(MetadataMayInterleaveBootstrapGrain))] + public async Task ReentrantAndInterleavingModels_AreAcceptedAndBindTheInbox(Type grainClass) + { + var grain = Fixture.Client.GetGrain(Guid.NewGuid(), grainClass.FullName!); + await grain.PingAsync(); + var observation = Assert.Single(Probe.Get(grain.GetGrainId())); + Assert.Equal(1, observation.Activations); + var inbox = observation.Context.ActivationServices.GetRequiredService(); + Assert.Same(inbox, observation.Context.GetComponent()!.GetExtension()); + + // No handler is registered by control grains, so the installed extension reports that explicitly. + var result = await grain.AsReference().DeliverAsync(CreateEnvelope(grain), Cancellation); + Assert.Equal(DeliveryStatus.HandlerNotFound, result.Status); + Assert.False(observation.Context.Deactivated.IsCompleted); + } + + [Fact] + public async Task StatelessWorker_FailsBeforeStorageAndCleansScope() + { + var grain = Control(); + var exception = await Assert.ThrowsAnyAsync(() => grain.PingAsync()); + Assert.Contains("one activation", exception.ToString(), StringComparison.Ordinal); + Assert.Contains(nameof(StatelessBootstrapGrain), exception.ToString(), StringComparison.Ordinal); + var observation = Assert.Single(Probe.Get(grain.GetGrainId())); + await observation.Context.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), Cancellation); + Assert.False(observation.InstanceAvailableInConstructor); + Assert.NotNull(observation.ConstructedGrain); + Assert.Equal(0, observation.Activations); + Assert.Equal(1, observation.GrainDisposals); + Assert.Equal(1, observation.Disposals); + Assert.Equal(0, Fixture.Storage.GetCreationCount(JournalId.FromGrainId(grain.GetGrainId()))); + AssertNoStorageWork(grain.GetGrainId()); + AssertNoScheduledJobs(grain.GetGrainId()); + } + + [Theory] + [InlineData(typeof(MetadataStatelessPlacementBootstrapGrain), "StatelessWorkerPlacement")] + [InlineData(typeof(AliasedStatelessPlacementBootstrapGrain), BootstrapClusterFixture.StatelessPlacementAlias)] + public async Task ResolvedStatelessPlacement_FailsBeforeStorageAndCleansScope(Type grainClass, string placementKey) + { + Assert.False(grainClass.IsDefined(typeof(StatelessWorkerAttribute), inherit: true)); + var grain = Fixture.Client.GetGrain(Guid.NewGuid(), grainClass.FullName!); + var services = Fixture.Cluster.Silos[0].ServiceProvider; + var properties = services.GetRequiredService().GetGrainProperties(grain.GetGrainId().Type); + Assert.Equal(placementKey, properties.Properties[WellKnownGrainTypeProperties.PlacementStrategy]); + var placement = services.GetRequiredService().GetPlacementStrategy(grain.GetGrainId().Type); + Assert.Equal(new StatelessWorkerAttribute().PlacementStrategy.GetType(), placement.GetType()); + if (placementKey == BootstrapClusterFixture.StatelessPlacementAlias) + { + Assert.Same(services.GetRequiredKeyedService(placementKey), placement); + } + + var exception = await Assert.ThrowsAnyAsync(() => grain.PingAsync()); + Assert.Contains("one activation", exception.ToString(), StringComparison.Ordinal); + Assert.Contains(grainClass.Name, exception.ToString(), StringComparison.Ordinal); + var observation = Assert.Single(Probe.Get(grain.GetGrainId())); + await observation.Context.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), Cancellation); + Assert.IsType(grainClass, observation.ConstructedGrain); + Assert.Equal(0, observation.Activations); + Assert.Equal(1, observation.GrainDisposals); + Assert.Equal(1, observation.Disposals); + Assert.Equal(0, Fixture.Storage.GetCreationCount(JournalId.FromGrainId(grain.GetGrainId()))); + AssertNoStorageWork(grain.GetGrainId()); + AssertNoScheduledJobs(grain.GetGrainId()); + } + + [Theory] + [InlineData(typeof(MetadataOrdinaryPlacementBootstrapGrain), nameof(RandomPlacement))] + [InlineData(typeof(AliasedOrdinaryPlacementBootstrapGrain), BootstrapClusterFixture.OrdinaryPlacementAlias)] + public async Task ResolvedOrdinaryPlacement_WithoutMessagingConstructorDependencies_InitializesNormally(Type grainClass, string placementKey) + { + var grain = Fixture.Client.GetGrain(Guid.NewGuid(), grainClass.FullName!); + var services = Fixture.Cluster.Silos[0].ServiceProvider; + var properties = services.GetRequiredService().GetGrainProperties(grain.GetGrainId().Type); + Assert.Equal(placementKey, properties.Properties[WellKnownGrainTypeProperties.PlacementStrategy]); + Assert.IsType(services.GetRequiredService().GetPlacementStrategy(grain.GetGrainId().Type)); + + var journal = JournalId.FromGrainId(grain.GetGrainId()); + var read = Fixture.Storage.BlockRead(journal); + var activation = grain.PingAsync(); + BootstrapObservation observation; + try + { + await read.WaitUntilEnteredAsync(); + observation = Assert.Single(Probe.Get(grain.GetGrainId())); + Assert.IsType(grainClass, observation.ConstructedGrain); + Assert.Equal(0, observation.Activations); + var activationServices = observation.Context.ActivationServices; + var manager = activationServices.GetRequiredService(); + Assert.True(manager.TryGetStateMachine(MessagingStateNames.Inbox, out var inbox)); + Assert.Same(activationServices.GetRequiredService(), inbox); + Assert.True(manager.TryGetStateMachine(MessagingStateNames.Outbox, out var outbox)); + Assert.Same(activationServices.GetRequiredService(), outbox); + Assert.Single(GetSetup(observation.Context).GetInvocationList()); + } + finally + { + read.Release(); + } + + await activation; + Assert.Equal(1, observation.Activations); + Assert.Equal(1, Fixture.Storage.GetCreationCount(journal)); + Assert.Equal(1, Fixture.Storage.GetReadCount(journal)); + Assert.Equal(0, Fixture.Storage.GetSuccessfulWriteCount(journal)); + AssertNoScheduledJobs(grain.GetGrainId()); + } + + [Fact] + public async Task SetupFailure_DisposesScopeAndPreservesOriginalError() + { + var grain = Control(); + var exception = await Assert.ThrowsAnyAsync(() => grain.PingAsync()); + var observation = Assert.Single(Probe.Get(grain.GetGrainId())); + Assert.IsType(observation.ExpectedFailure); + Assert.Contains(observation.ExpectedFailure.Message, exception.ToString(), StringComparison.Ordinal); + await observation.Context.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), Cancellation); + Assert.Equal(0, observation.Activations); + Assert.Equal(1, observation.GrainDisposals); + Assert.Equal(1, observation.Disposals); + Assert.NotNull(observation.Inbox); + Assert.NotNull(observation.Outbox); + AssertNoStorageWork(grain.GetGrainId()); + AssertNoScheduledJobs(grain.GetGrainId()); + } + + [Fact] + public async Task UnselectedEndpointDependency_FailsExplicitly() + { + var grain = Control(); + var exception = await Assert.ThrowsAnyAsync(() => grain.PingAsync()); + Assert.Contains("Messaging is not configured for grain type", exception.ToString(), StringComparison.Ordinal); + var observation = Assert.Single(Probe.Get(grain.GetGrainId())); + await observation.Context.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), Cancellation); + Assert.Equal(1, observation.Disposals); + Assert.Null(observation.Inbox); + Assert.Null(GetSetupOrDefault(observation.Context)); + AssertNoScheduledJobs(grain.GetGrainId()); + } + + [Fact] + public void RepeatedRegistration_InstallsOneTypeConfigurator() + { + var services = new ServiceCollection(); + var silo = Substitute.For(); + silo.Services.Returns(services); + silo.AddJournaling(); + services.AddMessaging(); + services.AddMessaging(); + var descriptor = Assert.Single(services, static descriptor => descriptor.ServiceType == typeof(IConfigureGrainTypeComponents) + && descriptor.ImplementationType == typeof(MessagingGrainTypeConfigurator)); + Assert.Equal(ServiceLifetime.Singleton, descriptor.Lifetime); + Assert.Single(services, static descriptor => descriptor.ServiceType == typeof(MessagingPump)); + Assert.Single(services, static descriptor => descriptor.ServiceType == typeof(Inbox)); + Assert.Single(services, static descriptor => descriptor.ServiceType == typeof(Outbox)); + Assert.Single(services, static descriptor => descriptor.ServiceType == typeof(IOutbox)); + Assert.DoesNotContain(services, static descriptor => descriptor.IsKeyedService && descriptor.ServiceType == typeof(IOutbox)); + var extension = Assert.Single(services, static descriptor => descriptor.ServiceType == typeof(IGrainExtension)); + Assert.Equal(typeof(Orleans.DurableJobs.DurableJob).Assembly, Assert.IsAssignableFrom(extension.ServiceKey).Assembly); + Assert.Empty(typeof(IMessagingGrain).GetInterfaces()); + Assert.Empty(typeof(IMessagingGrain).GetMethods()); + Assert.False(typeof(IAddressable).IsAssignableFrom(typeof(IMessagingGrain))); + } + + private static void AssertComposition(BootstrapObservation observation, BootstrapState state, Type grainClass, int expectedActivationValue) + { + Assert.Equal(grainClass, observation.ConstructedGrain!.GetType()); + Assert.False(observation.InstanceAvailableInConstructor); + Assert.Same(observation.ConstructedGrain, state.GrainAtActivation); + Assert.Equal(2, state.MessagingStateCountAtActivation); + Assert.Equal(1, observation.Activations); + Assert.Equal(expectedActivationValue, state.ActivationValue); + var services = observation.Context.ActivationServices; + Assert.Same(observation.Manager, services.GetRequiredService()); + var applicationManager = services.GetRequiredService(); + Assert.Same(observation.Manager, applicationManager); + Assert.Same(observation.Value, applicationManager.GetOrAddState>("bootstrap-value")); + var inbox = services.GetRequiredService(); + var outbox = services.GetRequiredService(); + Assert.Same(inbox, observation.Inbox); + Assert.Same(outbox, observation.Outbox); + Assert.True(observation.Manager!.TryGetStateMachine(MessagingStateNames.Inbox, out var inboxState)); + Assert.Same(inbox, inboxState); + Assert.True(observation.Manager.TryGetStateMachine(MessagingStateNames.Outbox, out var outboxState)); + Assert.Same(outbox, outboxState); + Assert.Same(inbox, observation.Context.GetComponent()!.GetExtension()); + Assert.Same(state, services.GetRequiredService()); + } + + private IBootstrapTestGrain CreateGrain(Type grainClass) => grainClass == typeof(GenericBootstrapGrain) + ? Fixture.Client.GetGrain>(Guid.NewGuid()) + : Fixture.Client.GetGrain(Guid.NewGuid(), grainClass.FullName!); + + private IBootstrapControlGrain Control() => Fixture.Client.GetGrain(Guid.NewGuid(), typeof(T).FullName!); + + private int _commandSequence; + + private InboxMessage CreateEnvelope(IAddressable grain) => + OutboxTestMessages.CreateApplication(Fixture.Client.ServiceProvider, + MessageId.Create("test", grain.GetGrainId().ToString(), "command", (++_commandSequence).ToString(CultureInfo.InvariantCulture)), + GrainId.Create("bootstrap-sender", "external"), grain.GetGrainId(), BootstrapState.Route, 1).ToInboxMessage(Fixture.JobNow); + + private static int GetProcessedCount(BootstrapObservation observation) => + InboxTestAccess.GetProcessedCount(Assert.IsType(observation.Inbox)); + + private static Delegate GetSetup(IGrainContext context) => Assert.IsAssignableFrom(GetSetupOrDefault(context)); + + private static object? GetSetupOrDefault(IGrainContext context) + { + var shared = context.GetType().GetField("_shared", BindingFlags.Instance | BindingFlags.NonPublic)!.GetValue(context)!; + return shared.GetType().GetField("_activationSetup", BindingFlags.Instance | BindingFlags.NonPublic)!.GetValue(shared); + } + + private void AssertNoStorageWork(GrainId grainId) + { + var journal = JournalId.FromGrainId(grainId); + Assert.Equal(0, Fixture.Storage.GetInitializationCount(journal)); + Assert.Equal(0, Fixture.Storage.GetReadCount(journal)); + Assert.Equal(0, Fixture.Storage.GetSuccessfulWriteCount(journal)); + } + + private void AssertNoScheduledJobs(GrainId grainId) => + Assert.Equal(0, Fixture.JobManagerProbe.GetAttemptCount(MessagingPump.JobName, grainId)); +} diff --git a/test/Orleans.Messaging.Tests/Functional/MessagingKeepAliveTests.cs b/test/Orleans.Messaging.Tests/Functional/MessagingKeepAliveTests.cs new file mode 100644 index 00000000000..8dc3674466c --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/MessagingKeepAliveTests.cs @@ -0,0 +1,33 @@ +using Orleans.Messaging.Tests.Support; +using Xunit; + +namespace Orleans.Messaging.Tests.Functional; + +[Collection(MessagingClusterCollection.Name)] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class MessagingKeepAliveTests : MessagingBehaviorTestBase +{ + private const string HeldRoute = "messages/held"; + + [Fact] + public async Task RunningHandler_KeepsActivationAliveUntilInboxDrains() + { + var receiver = NewGrain(); + using var handler = Fixture.HandlerProbe.Arm(receiver.GetGrainId(), HeldRoute); + var envelope = CreateEnvelope(receiver, NewMessage(1, "held"), HeldRoute); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + await handler.WaitUntilEnteredAsync(); + var context = Fixture.GetGrainContext(receiver); + + // The handler runs outside of any grain request, so only the pump's hold prevents idle collection. + var management = Fixture.Client.GetGrain(0); + await management.ForceActivationCollection(TimeSpan.Zero, TestContext.Current.CancellationToken); + Assert.False(context.Deactivated.IsCompleted); + + handler.Release(); + Assert.Equal("held", Assert.Single((await Fixture.WaitForEffectCountAsync(receiver, 1)).Effects).Value); + await Fixture.CollectAsync(context); + } +} diff --git a/test/Orleans.Messaging.Tests/Functional/MessagingMetricCardinalityTests.cs b/test/Orleans.Messaging.Tests/Functional/MessagingMetricCardinalityTests.cs new file mode 100644 index 00000000000..9620ef70fb5 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/MessagingMetricCardinalityTests.cs @@ -0,0 +1,272 @@ +using System.Diagnostics.Metrics; +using Microsoft.Extensions.DependencyInjection; +using Orleans.Messaging.Tests.Support; +using Orleans.Runtime; +using Xunit; + +namespace Orleans.Messaging.Tests.Functional; + +[Collection(MessagingClusterCollection.Name)] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class MessagingMetricCardinalityTests : MessagingBehaviorTestBase +{ + [Fact] + public async Task PendingAndProcessedDuplicates_RecordOneBoundedReceiptEach() + { + var receiver = NewGrain(); + await receiver.GetSnapshotAsync(); + var context = Fixture.GetGrainContext(receiver); + var services = context.ActivationServices; + IMessagingExtension extension = services.GetRequiredService(); + // The pump runs alongside grain calls, so holding the handler keeps the accepted message pending. + using var handler = Fixture.HandlerProbe.Arm(receiver.GetGrainId(), "messages/metric-pending"); + var envelope = CreateEnvelope(receiver, NewMessage(451, "duplicate-receipts"), "messages/metric-pending"); + Assert.Equal(DeliveryStatus.Accepted, (await StartDelivery()).Status); + await handler.WaitUntilEnteredAsync(); + Assert.NotEmpty(Fixture.JobManagerProbe.GetScheduledJobs(MessagingPump.JobName, receiver.GetGrainId())); + var pending = Fixture.GetSnapshot(receiver); + Assert.Equal(1, pending.InboxCount); + Assert.Equal(0, pending.ProcessedMessageCount); + Assert.Empty(pending.Effects); + using var probe = new MessageMetricListener(services.GetRequiredService().Meter); + var grainType = receiver.GetGrainId().Type.ToString(); + + Assert.Equal(DeliveryStatus.Duplicate, (await StartDelivery()).Status); + Assert.Single(probe.Read("inbox-messages-received")).AssertCounter(grainType, "duplicate"); + handler.Release(); + var processed = await Fixture.WaitForEffectCountAsync(receiver, 1); + Assert.Equal(0, processed.InboxCount); + Assert.Equal(1, processed.ProcessedMessageCount); + Assert.Equal(1, Assert.Single(processed.Effects).Count); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(receiver, envelope)).Status); + var receipts = probe.Read("inbox-messages-received"); + Assert.Equal(2, receipts.Length); + Assert.All(receipts, receipt => receipt.AssertCounter(grainType, "duplicate")); + Assert.Single(receipts.Select(receipt => receipt.Series).Distinct()); + + Task StartDelivery() + { + var started = new TaskCompletionSource>(TaskCreationOptions.RunContinuationsAsynchronously); + context.Scheduler.QueueAction(() => + { + try { started.SetResult(extension.DeliverAsync(envelope, TestContext.Current.CancellationToken).AsTask()); } + catch (Exception exception) { started.SetException(exception); } + }); + return started.Task.Unwrap(); + } + } + + [Theory] + [InlineData(1)] + [InlineData(64)] + public async Task DistinctApplicationRoutes_KeepFixedMetricSeriesAndMissingHandlerOutcomes(int routeCount) + { + var receiver = NewGrain(); + var missingHandler = NewGrain(); + await missingHandler.ConfigureHandlerAsync(false); + await ReactivateAsync(missingHandler); + await receiver.GetSnapshotAsync(); + var context = Fixture.GetGrainContext(receiver); + using var probe = new MessageMetricListener(context.ActivationServices.GetRequiredService().Meter); + var grainType = receiver.GetGrainId().Type.ToString(); + var routes = new HashSet(); + for (var index = 0; index < routeCount; index++) + { + var unknown = $"unknown/{Guid.NewGuid():N}"; + routes.Add(unknown); + var rejected = CreateEnvelope(missingHandler, NewMessage(index, "unknown"), unknown); + Assert.Equal(DeliveryStatus.HandlerNotFound, (await DeliverAsync(missingHandler, rejected)).Status); + Assert.Equal(unknown, TestApplicationProtocol.Read(Fixture.Client.ServiceProvider.GetRequiredService(), rejected).Route); + var valid = $"messages/cardinality/{Guid.NewGuid():N}"; + routes.Add(valid); + var accepted = CreateEnvelope(receiver, NewMessage(index, "accepted"), valid); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, accepted)).Status); + Assert.Equal(valid, TestApplicationProtocol.Read(Fixture.Client.ServiceProvider.GetRequiredService(), accepted).Route); + await Fixture.WaitForEffectCountAsync(receiver, index + 1); + } + + await probe.WaitForCountAsync("inbox-processing-duration", routeCount); + var receivedRows = probe.Read("inbox-messages-received"); + Assert.Equal(routeCount * 2, routes.Count); + Assert.Equal(routeCount * 2, receivedRows.Length); + Assert.Equal(2, receivedRows.Select(row => row.Series).Distinct().Count()); + var acceptedRows = receivedRows.Where(row => row.Tags.Any(tag => tag.Key == "status" && Equals(tag.Value, "accepted"))).ToArray(); + var rejectedRows = receivedRows.Where(row => row.Tags.Any(tag => tag.Key == "status" && Equals(tag.Value, "handler_not_found"))).ToArray(); + Assert.Equal(routeCount, acceptedRows.Length); + Assert.Equal(routeCount, rejectedRows.Length); + Assert.All(acceptedRows, row => row.AssertCounter(grainType, "accepted")); + Assert.All(rejectedRows, row => row.AssertCounter(grainType, "handler_not_found")); + var processed = probe.Read("inbox-messages-processed"); + Assert.Equal(routeCount, processed.Length); + Assert.Single(processed.Select(row => row.Series).Distinct()); + Assert.All(processed, row => row.AssertCounter(grainType, "success")); + var durations = probe.Read("inbox-processing-duration"); + Assert.Equal(routeCount, durations.Length); + Assert.Single(durations.Select(row => row.Series).Distinct()); + Assert.All(durations, row => + { + Assert.Equal("ms", row.Unit); + Assert.True(row.Value >= 0); + Assert.Equal(new[] { new KeyValuePair("grain_type", grainType) }, row.Tags); + }); + var completed = await receiver.GetSnapshotAsync(); + Assert.Equal(routeCount, completed.Effects.Count); + Assert.All(completed.Effects, effect => Assert.Equal(1, effect.Count)); + Assert.Equal(routeCount, completed.ProcessedMessageCount); + Assert.Equal(0, completed.InboxCount); + Assert.Equal(0, completed.OutboxCount); + Assert.Empty(completed.InboxDeadLetters); + } +} + +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class MessagingInstrumentTagTests +{ + [Theory] + [InlineData("inbox-messages-received", "accepted")] + [InlineData("inbox-messages-processed", "success")] + [InlineData("outbox-messages-sent", null)] + [InlineData("outbox-messages-delivered", "duplicate")] + public void Counters_EmitExactBoundedTags(string instrumentName, string? status) + { + using var factory = new TestMeterFactory(); + var instruments = new Orleans.Messaging.MessagingInstruments(new OrleansInstruments(factory)); + using var probe = new MessageMetricListener(factory.Meter); + const string grainType = "instrument-test"; + Action emit = instrumentName switch + { + "inbox-messages-received" => () => instruments.OnInboxMessageReceived(grainType, status!), + "inbox-messages-processed" => () => instruments.OnInboxMessageProcessed(grainType, status!), + "outbox-messages-sent" => () => instruments.OnOutboxMessageSent(grainType), + _ => () => instruments.OnOutboxMessageDelivered(grainType, status!), + }; + emit(); + var measurement = Assert.Single(probe.Read(instrumentName)); + Assert.Equal(MessageMetricListener.Prefix + instrumentName, measurement.Name); + measurement.AssertCounter(grainType, status); + } + + [Theory] + [InlineData("inbox-processing-duration")] + [InlineData("outbox-delivery-duration")] + public void Durations_RecordMillisecondsAndClampNegativeValues(string instrumentName) + { + using var factory = new TestMeterFactory(); + var instruments = new Orleans.Messaging.MessagingInstruments(new OrleansInstruments(factory)); + using var probe = new MessageMetricListener(factory.Meter); + const string grainType = "instrument-test"; + Action record = instrumentName == "inbox-processing-duration" + ? duration => instruments.OnInboxProcessingDuration(duration, grainType) + : duration => instruments.OnOutboxDeliveryDuration(duration, grainType); + record(TimeSpan.FromMilliseconds(12.5)); + record(TimeSpan.FromMilliseconds(-10)); + var values = probe.Read(instrumentName); + Assert.Equal(2, values.Length); + Assert.All(values, value => + { + Assert.Equal("ms", value.Unit); + Assert.Equal(new[] { new KeyValuePair("grain_type", grainType) }, value.Tags); + }); + Assert.Equal(12.5, values[0].Value); + Assert.Equal(0, values[1].Value); + } + + [Fact] + public void DepthInstruments_RecordNonZeroDeltasTaggedByGrainType() + { + using var factory = new TestMeterFactory(); + var instruments = new Orleans.Messaging.MessagingInstruments(new OrleansInstruments(factory)); + using var probe = new MessageMetricListener(factory.Meter); + instruments.OnInboxDepthChanged("instrument-test", 3); + instruments.OnInboxDepthChanged("instrument-test", 0); + instruments.OnInboxDepthChanged("instrument-test", -1); + instruments.OnOutboxDepthChanged("instrument-test", 4); + var inboxDepth = probe.Read("inbox-depth"); + var outboxDepth = Assert.Single(probe.Read("outbox-depth")); + Assert.Equal([3d, -1d], inboxDepth.Select(static row => row.Value)); + Assert.Equal(4, outboxDepth.Value); + Assert.All(inboxDepth.Append(outboxDepth), row => + { + Assert.Null(row.Unit); + Assert.Equal(new[] { new KeyValuePair("grain_type", "instrument-test") }, row.Tags); + }); + } + private sealed class TestMeterFactory : IMeterFactory + { + public Meter Meter { get; private set; } = null!; + public Meter Create(MeterOptions options) => Meter = new Meter(options); + public void Dispose() => Meter.Dispose(); + } +} + +internal sealed class MessageMetricListener : IDisposable +{ + internal const string Prefix = "orleans-messaging-"; + private readonly MeterListener _listener; + private readonly object _lock = new(); + private readonly List _measurements = []; + private readonly List<(string Name, int Count, TaskCompletionSource Completion)> _waiters = []; + + public MessageMetricListener(Meter meter) + { + _listener = new MeterListener + { + InstrumentPublished = (instrument, listener) => + { + if (ReferenceEquals(instrument.Meter, meter) && instrument.Name.StartsWith(Prefix, StringComparison.Ordinal)) + listener.EnableMeasurementEvents(instrument); + } + }; + _listener.SetMeasurementEventCallback((instrument, value, tags, _) => Record(instrument, value, tags)); + _listener.SetMeasurementEventCallback((instrument, value, tags, _) => Record(instrument, value, tags)); + _listener.Start(); + } + + private void Record(Instrument instrument, double value, ReadOnlySpan> tags) + { + lock (_lock) + { + _measurements.Add(new(instrument.Name, instrument.Unit, value, tags.ToArray())); + foreach (var waiter in _waiters.ToArray()) + { + if (_measurements.Count(row => row.Name == waiter.Name) >= waiter.Count) + { + _waiters.Remove(waiter); + waiter.Completion.TrySetResult(); + } + } + } + } + public Task WaitForCountAsync(string name, int count) + { + lock (_lock) + { + if (_measurements.Count(row => row.Name == Prefix + name) >= count) return Task.CompletedTask; + var completion = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + _waiters.Add((Prefix + name, count, completion)); + return completion.Task.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + } + } + public Measurement[] Read(string name) + { + lock (_lock) return _measurements.Where(row => row.Name == Prefix + name).ToArray(); + } + public void Dispose() => _listener.Dispose(); + internal sealed record Measurement(string Name, string? Unit, double Value, KeyValuePair[] Tags) + { + public string Series => string.Join(";", Tags.Select(tag => $"{tag.Key}={tag.Value}")); + public void AssertCounter(string grainType, string? status = null) + { + Assert.Equal(1, Value); + Assert.Null(Unit); + KeyValuePair[] expected = status is null + ? [new("grain_type", grainType)] + : [new("grain_type", grainType), new("status", status)]; + Assert.Equal(expected, Tags); + } + } +} diff --git a/test/Orleans.Messaging.Tests/Functional/MessagingProviderCutoverTests.cs b/test/Orleans.Messaging.Tests/Functional/MessagingProviderCutoverTests.cs new file mode 100644 index 00000000000..53d58160bcf --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/MessagingProviderCutoverTests.cs @@ -0,0 +1,87 @@ +using System.Collections.Immutable; +using System.Net; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; +using Microsoft.Extensions.Time.Testing; +using NSubstitute; +using Orleans.DurableJobs; +using Orleans.Hosting; +using Orleans.Journaling; +using Orleans.Messaging.Tests.Support; +using Orleans.Runtime; +using Orleans.Serialization; +using Orleans.Serialization.Session; +using Orleans.TestingHost; +using Orleans.Timers; +using Xunit; + +namespace Orleans.Messaging.Tests.Functional; + +[Collection(MessagingClusterCollection.Name)] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class MessagingProviderCutoverTests +{ + private const string JournalFormatKey = "orleans-binary"; + private static CancellationToken Token => TestContext.Current.CancellationToken; + + [Fact] + public async Task PublicHosting_NamedFactories_CommitsCompositeEndpointsAndRecoversFreshActivation() + { + var state = new ControlledJournalStorageProvider(); + var snapshots = new NamedFactoryMessagingProbe(); + var builder = new InProcessTestClusterBuilder(1); + builder.ConfigureSilo((_, silo) => + { + silo.AddVolatileJournalStorage(); + silo.AddJournalStorage("state", services => + { + state.Configure(services.GetRequiredService>()); + return state; + }); + silo.AddVolatileJournalStorage("jobs"); + silo.UseJournaledDurableJobs(options => options.ActiveProviderName = "jobs"); + silo.Services.Configure(options => options.JournalFormatKey = JournalFormatKey); + silo.AddMessaging(); + silo.Services.AddSingleton(snapshots); + silo.Services.AddScoped(services => + { + var context = services.GetRequiredService(); + var manager = services.GetRequiredKeyedService("state") + .CreateStandalone(JournalId.FromGrainId(context.GrainId)); + ((ILifecycleParticipant)manager).Participate(context.ObservableLifecycle); + return manager; + }); + }); + await using var cluster = builder.Build(); + await cluster.DeployAsync(Token); + var sender = cluster.Client.GetGrain(Guid.NewGuid()); + var receiver = cluster.Client.GetGrain(Guid.NewGuid()); + var sink = cluster.Client.GetGrain(Guid.NewGuid()); + var logicalId = MessageId.Create("cutover", Guid.NewGuid().ToString("N")); + var receivedTask = snapshots.WaitAsync(receiver.GetGrainId(), Token); + var forwardedTask = snapshots.WaitAsync(sink.GetGrainId(), Token); + await sender.SendAsync(receiver.GetGrainId(), "messages/record", new TestMessage(logicalId, 1, "composite", sink.GetGrainId())); + var received = await receivedTask; + var forwarded = await forwardedTask; + Assert.Equal(0, received.OutboxCount); + Assert.Equal(0, forwarded.OutboxCount); + Assert.Equal(new MessageEffect(logicalId, 1, 1, "composite"), Assert.Single(received.Effects)); + Assert.Equal(Assert.Single(received.Effects), Assert.Single(forwarded.Effects)); + Assert.True(cluster.TryGetGrainContext(receiver.GetGrainId(), out var oldContext)); + var oldManager = oldContext.ActivationServices.GetRequiredService(); + Assert.False(oldManager is IDurableStateManager); + await receiver.RequestDeactivationAsync(); + await oldContext.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), Token); + var recovered = await receiver.GetSnapshotAsync(); + Assert.NotEqual(received.ActivationId, recovered.ActivationId); + Assert.Equal(received.Effects, recovered.Effects); + Assert.True(cluster.TryGetGrainContext(receiver.GetGrainId(), out var newContext)); + Assert.NotSame(oldManager, newContext.ActivationServices.GetRequiredService()); + var journal = JournalId.FromGrainId(receiver.GetGrainId()); + Assert.NotNull(await state.CreateStorage(journal).GetMetadataAsync(Token)); + Assert.Null(await cluster.Silos[0].ServiceProvider.GetRequiredService().CreateStorage(journal).GetMetadataAsync(Token)); + } +} diff --git a/test/Orleans.Messaging.Tests/Functional/MessagingRoutingAndDeadLetterTests.cs b/test/Orleans.Messaging.Tests/Functional/MessagingRoutingAndDeadLetterTests.cs new file mode 100644 index 00000000000..cffdbb8193a --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/MessagingRoutingAndDeadLetterTests.cs @@ -0,0 +1,147 @@ +using Microsoft.Extensions.DependencyInjection; +using Orleans.DurableJobs; +using Orleans.Journaling; +using Orleans.Messaging.Tests.Support; +using Orleans.Runtime; +using Orleans.Serialization.Session; +using Xunit; + +namespace Orleans.Messaging.Tests.Functional; + +[Collection(MessagingClusterCollection.Name)] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class MessagingRoutingAndDeadLetterTests : MessagingBehaviorTestBase +{ + [Fact] + public async Task InboxDeadLettersRetainNewestEntriesWithinConfiguredCapacity() + { + var receiver = NewGrain(); + var messageIds = new List(); + for (var sequence = 0; sequence < 3; sequence++) + { + var envelope = CreateEnvelope( + receiver, + new TestMessage(TestApplicationProtocol.NewMessageId(), + 20 + sequence, + $"dead-letter-{sequence}", + ThrowDuringPreparation: true)); + messageIds.Add(envelope.Envelope.MessageId); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + await Fixture.SnapshotProbe.WaitAsync( + receiver.GetGrainId(), + snapshot => snapshot.InboxDeadLetters.Any(entry => entry.MessageId == envelope.Envelope.MessageId)); + } + + var state = await receiver.GetSnapshotAsync(); + Assert.Equal(2, state.InboxDeadLetters.Count); + Assert.DoesNotContain(state.InboxDeadLetters, entry => entry.MessageId == messageIds[0]); + Assert.Contains(state.InboxDeadLetters, entry => entry.MessageId == messageIds[1]); + Assert.Contains(state.InboxDeadLetters, entry => entry.MessageId == messageIds[2]); + } + + [Fact] + public async Task ActivationRemovesExpiredInboxDeadLetters() + { + var receiver = NewGrain(); + var envelope = CreateEnvelope( + receiver, + new TestMessage(TestApplicationProtocol.NewMessageId(), + 30, + "expired-dead-letter", + ThrowDuringPreparation: true)); + + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + var before = await Fixture.WaitForDeadLetterCountAsync(receiver, 1); + + Fixture.Clock.Advance(TimeSpan.FromHours(2)); + await ReactivateAsync(receiver); + var after = await receiver.GetSnapshotAsync(); + Assert.NotEqual(before.ActivationId, after.ActivationId); + Assert.Empty(after.InboxDeadLetters); + + await ReactivateAsync(receiver); + Assert.Empty((await receiver.GetSnapshotAsync()).InboxDeadLetters); + } + + [Fact] + public async Task MalformedTypedBody_DeadLettersAndDoesNotBlockLaterValidMessage() + { + var receiver = NewGrain(); + var malformed = CreateEnvelope(receiver, "wrong-body", "typed"); + + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, malformed)).Status); + var poisoned = await Fixture.WaitForDeadLetterCountAsync(receiver, 1); + Assert.Empty(poisoned.Effects); + var deadLetter = Assert.Single(poisoned.InboxDeadLetters); + Assert.Equal(malformed.Envelope.MessageId, deadLetter.MessageId); + Assert.Contains(nameof(TestMessage), deadLetter.Reason, StringComparison.Ordinal); + + var valid = CreateEnvelope(receiver, NewMessage(41, "valid-after-poison"), "typed"); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, valid)).Status); + var recovered = await Fixture.WaitForEffectCountAsync(receiver, 1); + Assert.Equal("valid-after-poison", Assert.Single(recovered.Effects).Value); + Assert.Single(recovered.InboxDeadLetters); + } + + [Fact] + public async Task InboxDeadLetterRemoval_IsDurable() + { + var receiver = NewGrain(); + var malformed = CreateEnvelope(receiver, "wrong-body", "typed"); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, malformed)).Status); + _ = await Fixture.WaitForDeadLetterCountAsync(receiver, 1); + + Assert.True(await receiver.RemoveInboxDeadLetterAsync( + malformed.Envelope.MessageId)); + Assert.Empty((await receiver.GetSnapshotAsync()).InboxDeadLetters); + Assert.False(await receiver.RemoveInboxDeadLetterAsync( + malformed.Envelope.MessageId)); + + await receiver.RequestDeactivationAsync(); + Assert.Empty((await receiver.GetSnapshotAsync()).InboxDeadLetters); + } + + [Fact] + public async Task HandlerNotFound_IsRejectedWithoutInboxPersistence() + { + var receiver = NewGrain(); + await receiver.ConfigureHandlerAsync(false); + await ReactivateAsync(receiver); + var envelope = CreateEnvelope(receiver, NewMessage(71, "missing"), "unknown/route"); + + var result = await DeliverAsync(receiver, envelope); + + Assert.Equal(DeliveryStatus.HandlerNotFound, result.Status); + Assert.Equal("No inbox handler is registered.", result.Message); + var state = await receiver.GetSnapshotAsync(); + Assert.Equal(0, state.InboxCount); + Assert.Empty(state.Effects); + } + + [Fact] + public async Task Deliver_ImplicitReceiverNamespacesTheSameIncomingCommand() + { + var receiver = NewGrain(); + var declaredReceiver = NewGrain(); + var envelope = CreateEnvelope( + declaredReceiver, + NewMessage(74, "wrong-receiver")); + + Assert.Null(typeof(InboxMessage).GetProperty("ReceiverId")); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + var first = await Fixture.WaitForEffectCountAsync(receiver, 1); + Assert.Equal(envelope.Envelope.MessageId, Assert.Single(first.Effects).LogicalId); + Assert.Equal(1, Assert.Single(first.Effects).Count); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(declaredReceiver, envelope)).Status); + var delivered = await Fixture.WaitForEffectCountAsync(declaredReceiver, 1); + Assert.Equal(envelope.Envelope.MessageId, Assert.Single(delivered.Effects).LogicalId); + Assert.Equal(1, Assert.Single(delivered.Effects).Count); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(receiver, envelope)).Status); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(declaredReceiver, envelope)).Status); + Assert.Equal(1, (await receiver.GetSnapshotAsync()).ProcessedMessageCount); + Assert.Equal(1, (await declaredReceiver.GetSnapshotAsync()).ProcessedMessageCount); + } + +} diff --git a/test/Orleans.Messaging.Tests/Functional/MultiSiloMessagingFailoverTests.cs b/test/Orleans.Messaging.Tests/Functional/MultiSiloMessagingFailoverTests.cs new file mode 100644 index 00000000000..a1e8bb36b81 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/MultiSiloMessagingFailoverTests.cs @@ -0,0 +1,64 @@ +using Orleans.Messaging.Tests.Support; +using Xunit; + +namespace Orleans.Messaging.Tests.Functional; + +[CollectionDefinition(Name, DisableParallelization = true)] +public sealed class MultiSiloMessagingCollection : ICollectionFixture +{ + public const string Name = "Messaging multi-silo cluster"; +} + +[Collection(MultiSiloMessagingCollection.Name)] +[TestSuite("Functional")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class MultiSiloMessagingFailoverTests(MultiSiloMessagingClusterFixture fixture) +{ + [Fact] + public async Task ReceiverSiloStops_DuringBlockedHandler_NewActivationRecoversFromJournalAndProcessesOnce() + { + var receiver = fixture.Client.GetGrain(Guid.NewGuid()); + using var barrier = fixture.HandlerProbe.Arm(receiver.GetGrainId(), "messages/failover"); + var sender = fixture.Client.GetGrain(Guid.NewGuid()); + var logicalId = MessageId.Create("failover", Guid.NewGuid().ToString("N")); + fixture.JobManagerProbe.DuplicateNext(MessagingPump.JobName, receiver.GetGrainId()); + + await sender.SendAsync( + receiver.GetGrainId(), + "messages/failover", + new TestMessage(logicalId, 81, "failover")); + await barrier.WaitUntilEnteredAsync(); + Assert.True(fixture.JobManagerProbe.GetScheduledJobs(MessagingPump.JobName, receiver.GetGrainId()).Count >= 2); + var before = fixture.GetSnapshot(receiver); + var owner = fixture.Cluster.Silos.Single( + silo => silo.SiloAddress.ToParsableString() == before.SiloAddress); + + await fixture.Cluster.KillSiloAsync(owner, TestContext.Current.CancellationToken); + await fixture.Cluster.WaitForLivenessToStabilizeAsync(); + var activationRequest = receiver.GetSnapshotAsync(); + barrier.Release(); + var reactivated = await activationRequest; + Assert.NotEqual(before.ActivationId, reactivated.ActivationId); + EndpointSnapshot recovered; + try + { + recovered = await fixture.WaitForEffectCountAsync(receiver, 1); + } + catch (TimeoutException exception) + { + var snapshot = await receiver.GetSnapshotAsync(); + throw new TimeoutException( + $"Recovery did not complete. Activation={snapshot.ActivationId}, silo={snapshot.SiloAddress}, inbox={snapshot.InboxCount}, effects={snapshot.Effects.Count}, deadLetters={snapshot.InboxDeadLetters.Count}.", + exception); + } + + Assert.Equal(reactivated.ActivationId, recovered.ActivationId); + Assert.NotEqual(before.SiloAddress, recovered.SiloAddress); + var effect = Assert.Single(recovered.Effects); + Assert.Equal(logicalId, effect.LogicalId); + Assert.Equal(1, effect.Count); + Assert.Equal(0, recovered.InboxCount); + Assert.Single(fixture.Cluster.Silos); + } +} diff --git a/test/Orleans.Messaging.Tests/Functional/OutboxDeliveryBehaviorTests.cs b/test/Orleans.Messaging.Tests/Functional/OutboxDeliveryBehaviorTests.cs new file mode 100644 index 00000000000..38c8ec04e8b --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/OutboxDeliveryBehaviorTests.cs @@ -0,0 +1,257 @@ +using Microsoft.Extensions.DependencyInjection; +using Orleans.DurableJobs; +using Orleans.Journaling; +using Orleans.Messaging.Tests.Support; +using Orleans.Runtime; +using Orleans.Serialization.Session; +using Xunit; + +namespace Orleans.Messaging.Tests.Functional; + +[Collection(MessagingClusterCollection.Name)] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class OutboxDeliveryBehaviorTests : MessagingBehaviorTestBase +{ + public OutboxDeliveryBehaviorTests() : base(receiverOnly: false) + { + } + + [Fact] + public async Task OutboxSends_ReportTheirStagedChanges() + { + // The outbox journals every send when it is staged, so a failed handler attempt can tell whether it sent + // anything, and an unrelated failure doesn't cost the activation. + var sender = NewGrain(); + _ = await sender.GetSnapshotAsync(); + var pump = Fixture.GetGrainContext(sender).ActivationServices.GetRequiredService(); + var before = await Fixture.OnGrainAsync(sender, () => pump.StagedChangeCount); + + _ = await sender.StageWithoutCommitAsync(NewGrain().GetGrainId(), "messages/staged-change", NewMessage(150, "staged-change")); + + Assert.True(before >= 0); + Assert.True(await Fixture.OnGrainAsync(sender, () => pump.StagedChangeCount) > before); + } + + [Fact] + public async Task StagedOutboxWithoutCommit_IsRemovedOnReactivationAndNeverDispatched() + { + var sender = NewGrain(); + var receiver = NewGrain(); + var message = NewMessage(51, "uncommitted"); + + await sender.StageWithoutCommitAsync(receiver.GetGrainId(), "messages/uncommitted", message); + Assert.Equal(1, (await sender.GetSnapshotAsync()).OutboxCount); + await sender.RequestDeactivationAsync(); + var reactivated = await sender.GetSnapshotAsync(); + + Assert.Equal(0, reactivated.OutboxCount); + Assert.Empty((await receiver.GetSnapshotAsync()).Effects); + } + + [Fact] + public async Task DuplicateOutboxEnqueue_UsesOneDurableWakeup() + { + var sender = NewGrain(); + var receiver = NewGrain(); + + await sender.SendDuplicateAsync( + receiver.GetGrainId(), + "messages/duplicate-outbox-enqueue", + NewMessage(54, "duplicate-enqueue")); + var delivered = await Fixture.WaitForEffectCountAsync(receiver, 1); + + Assert.Equal(1, Assert.Single(delivered.Effects).Count); + Assert.Equal( + 1, + Fixture.JobManagerProbe.GetSuccessCount( + MessagingPump.JobName, + sender.GetGrainId())); + } + + [Fact] + public async Task CommittedOutbox_ToSameGrain_DeliversLocally() + { + var grain = NewGrain(); + + await grain.SendAsync( + grain.GetGrainId(), + "messages/loopback", + NewMessage(82, "loopback")); + var delivered = await Fixture.WaitForEffectCountAsync(grain, 1); + delivered = await Fixture.WaitForOutboxCountAsync(grain, 0); + + Assert.Equal("loopback", Assert.Single(delivered.Effects).Value); + Assert.Equal(0, delivered.InboxCount); + Assert.Empty(delivered.InboxDeadLetters); + Assert.Empty(delivered.OutboxDeadLetters); + } + + [Fact] + public async Task ReciprocalOutboxPumps_DoNotBlockInboxDelivery() + { + var first = NewGrain(); + var second = NewGrain(); + _ = await first.GetSnapshotAsync(); + _ = await second.GetSnapshotAsync(); + + var firstSend = first.SendAsync( + second.GetGrainId(), + "messages/reciprocal", + NewMessage(83, "first-to-second")); + var secondSend = second.SendAsync( + first.GetGrainId(), + "messages/reciprocal", + NewMessage(84, "second-to-first")); + await Task.WhenAll(firstSend, secondSend); + + var completed = await Task.WhenAll( + Fixture.SnapshotProbe.WaitAsync( + first.GetGrainId(), + static snapshot => snapshot.Effects.Any(effect => effect.Value == "second-to-first"), + TimeSpan.FromSeconds(10)), + Fixture.SnapshotProbe.WaitAsync( + second.GetGrainId(), + static snapshot => snapshot.Effects.Any(effect => effect.Value == "first-to-second"), + TimeSpan.FromSeconds(10))); + await Task.WhenAll( + Fixture.WaitForOutboxCountAsync(first, 0), + Fixture.WaitForOutboxCountAsync(second, 0)); + + Assert.Contains(completed[0].Effects, effect => effect.Value == "second-to-first"); + Assert.Contains(completed[1].Effects, effect => effect.Value == "first-to-second"); + } + + [Fact] + public async Task OutboxSchedulingPrerequisiteFailure_KeepsActivationHealthyAndRetryEstablishesWakeup() + { + var sender = NewGrain(); + var receiver = NewGrain(); + var before = await sender.GetSnapshotAsync(); + var oldContext = Fixture.GetGrainContext(sender); + var oldManager = oldContext.ActivationServices.GetRequiredService(); + var message = NewMessage(55, "schedule-retry"); + Fixture.JobManagerProbe.FailAfterNext(MessagingPump.JobName); + + var prerequisite = await Assert.ThrowsAsync(() => sender.SendAsync(receiver.GetGrainId(), "messages/schedule-retry", message)); + Assert.IsType(prerequisite.InnerException); + Assert.False(oldContext.Deactivated.IsCompleted); + Assert.Empty((await receiver.GetSnapshotAsync()).Effects); + var healthy = await sender.GetSnapshotAsync(); + Assert.Equal(before.ActivationId, healthy.ActivationId); + Assert.Same(oldManager, Fixture.GetGrainContext(sender).ActivationServices.GetRequiredService()); + Assert.Equal(1, healthy.OutboxCount); + Assert.Single(Fixture.JobManagerProbe.GetScheduledJobs(MessagingPump.JobName, sender.GetGrainId())); + + var receiverWrite = Fixture.Storage.BlockWrite(JournalId.FromGrainId(receiver.GetGrainId())); + try + { + await sender.RetryWriteStateAsync(); + await receiverWrite.WaitUntilEnteredAsync(); + var committed = Fixture.GetSnapshot(sender); + var jobs = Fixture.JobManagerProbe.GetScheduledJobs(MessagingPump.JobName, sender.GetGrainId()); + Assert.NotEmpty(jobs); + Assert.All(jobs, job => + { + Assert.Null(job.Metadata); + Assert.Equal(sender.GetGrainId(), job.TargetGrainId); + }); + Assert.Equal(1, committed.OutboxCount); + } + finally + { + receiverWrite.Release(); + } + var delivered = await Fixture.WaitForEffectCountAsync(receiver, 1); + Assert.Equal("schedule-retry", Assert.Single(delivered.Effects).Value); + Assert.Equal(1, Assert.Single(delivered.Effects).Count); + Assert.True(Fixture.JobManagerProbe.GetAttemptCount(MessagingPump.JobName, sender.GetGrainId()) >= 1); + } + + [Fact] + public async Task FailedAtomicWrite_FencesOldObjectsAndFreshReplayOmitsEffectAndOutgoingMessage() + { + var sender = NewGrain(); + var receiver = NewGrain(); + var before = await sender.GetSnapshotAsync(); + var oldContext = Fixture.GetGrainContext(sender); + var oldManager = oldContext.ActivationServices.GetRequiredService(); + var oldGrain = Assert.IsType(oldContext.GrainInstance); + await sender.StageEffectAsync(new MessageEffect(MessageId.Create("outbox", "failed-write"), 1, 53, "failed-write")); + Fixture.Storage.FailWrite(JournalId.FromGrainId(sender.GetGrainId())); + + var failure = await Assert.ThrowsAsync(() => sender.SendAsync( + receiver.GetGrainId(), "messages/write-failure", NewMessage(53, "failed-write"))); + await oldContext.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + Assert.Equal(failure.Message, Assert.IsType( + await oldGrain.DeactivationFailure.Task.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken)).Message); + await Assert.ThrowsAsync(() => oldManager.WriteStateAsync(CancellationToken.None).AsTask()); + var recovered = await sender.GetSnapshotAsync(); + + Assert.NotEqual(before.ActivationId, recovered.ActivationId); + Assert.NotSame(oldManager, Fixture.GetGrainContext(sender).ActivationServices.GetRequiredService()); + Assert.Equal(0, recovered.OutboxCount); + Assert.Empty(recovered.Effects); + Assert.Empty((await receiver.GetSnapshotAsync()).Effects); + Assert.Single(Fixture.JobManagerProbe.GetScheduledJobs(MessagingPump.JobName, sender.GetGrainId())); + } + + [Fact] + public async Task StopDeleteAndDeactivate_DiscardsUncommittedOutboxAndFreshOwnerCanSend() + { + var sender = NewGrain(); + var receiver = NewGrain(); + await sender.StageWithoutCommitAsync( + receiver.GetGrainId(), + "messages/delete-uncommitted", + NewMessage(75, "deleted-intent")); + + var before = await sender.GetSnapshotAsync(); + var oldContext = Fixture.GetGrainContext(sender); + Assert.Equal(1, before.OutboxCount); + await sender.DeleteStateAndDeactivateAsync(); + await oldContext.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + var fresh = await sender.GetSnapshotAsync(); + + Assert.NotEqual(before.ActivationId, fresh.ActivationId); + Assert.Equal(0, fresh.OutboxCount); + Assert.Empty((await receiver.GetSnapshotAsync()).Effects); + + var effect = new MessageEffect(MessageId.Create("outbox", "fresh-owner"), 1, 76, "fresh-owner"); + await sender.StageEffectAsync(effect); + var drained = Fixture.SnapshotProbe.WaitAsync( + sender.GetGrainId(), + snapshot => snapshot.ActivationId == fresh.ActivationId + && snapshot.OutboxCount == 0 + && snapshot.Effects.Contains(effect)); + await sender.SendAsync(receiver.GetGrainId(), "messages/after-delete", NewMessage(76, "after-delete")); + var delivered = await Fixture.WaitForEffectCountAsync(receiver, 1); + await drained; + Assert.Equal(0, (await sender.GetSnapshotAsync()).OutboxCount); + Assert.Equal("after-delete", Assert.Single(delivered.Effects).Value); + } + + [Fact] + public async Task BlockedInboxHandler_DoesNotStopIndependentOutboxAndInboxPumps() + { + var blocked = NewGrain(); + var independentSender = NewGrain(); + var independentReceiver = NewGrain(); + using var barrier = Fixture.HandlerProbe.Arm(blocked.GetGrainId(), "messages/blocked-pump"); + var blockedEnvelope = CreateEnvelope(blocked, NewMessage(61, "blocked"), "messages/blocked-pump"); + + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(blocked, blockedEnvelope)).Status); + await barrier.WaitUntilEnteredAsync(); + await independentSender.SendAsync( + independentReceiver.GetGrainId(), + "messages/independent", + NewMessage(62, "independent")); + var independent = await Fixture.WaitForEffectCountAsync(independentReceiver, 1); + + Assert.Equal("independent", Assert.Single(independent.Effects).Value); + Assert.Empty(Fixture.GetSnapshot(blocked).Effects); + barrier.Release(); + Assert.Equal("blocked", Assert.Single((await Fixture.WaitForEffectCountAsync(blocked, 1)).Effects).Value); + } +} diff --git a/test/Orleans.Messaging.Tests/Functional/OutboxJournalRecoveryTests.cs b/test/Orleans.Messaging.Tests/Functional/OutboxJournalRecoveryTests.cs new file mode 100644 index 00000000000..8e54453978d --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/OutboxJournalRecoveryTests.cs @@ -0,0 +1,207 @@ +using Microsoft.Extensions.DependencyInjection; +using NSubstitute; +using Orleans.DurableJobs; +using Orleans.Journaling; +using Orleans.Messaging.Tests.Support; +using Xunit; + +namespace Orleans.Messaging.Tests.Functional; + +[Collection(MessagingClusterCollection.Name)] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class OutboxJournalRecoveryTests : MessagingBehaviorTestBase +{ + public OutboxJournalRecoveryTests() : base(receiverOnly: false) + { + } + + [Theory] + [InlineData(false, false)] + [InlineData(false, true)] + [InlineData(true, false)] + [InlineData(true, true)] + public async Task RecoveredSnapshot_EveryPendingEntryRequiresItsSentAt(bool reference, bool hasSentAt) + { + var sender = NewGrain(); + var receiver = NewGrain(); + _ = await sender.GetSnapshotAsync(); + var first = TestApplicationProtocol.Create(Sessions, sender.GetGrainId(), receiver.GetGrainId(), + "messages/recovered-expiration", NewMessage(63, "first-valid")); + var target = TestApplicationProtocol.Create(Sessions, sender.GetGrainId(), receiver.GetGrainId(), + "messages/recovered-expiration", NewMessage(64, "second-validated")); + var sentAt = Fixture.JobNow; + var outbox = Fixture.GetGrainContext(sender).ActivationServices.GetRequiredService(); + await Fixture.OnGrainAsync(sender, () => + { + if (reference) outbox.Register(Substitute.For()); + ((IDurableValueCommandHandler)outbox).ApplySet(new() + { + Kind = OutboxCommandKind.Snapshot, + Snapshot = new() + { + Messages = + [ + new() { Message = first, State = new() { SentAt = sentAt } }, + new() + { + Message = reference ? null : target, + Reference = reference ? new() { MessageId = target.Envelope.MessageId } : null, + State = new() { SentAt = hasSentAt ? sentAt : null } + } + ] + } + }); + + if (hasSentAt) + { + ((IStateMachine)outbox).OnRecoveryCompleted(); + } + else + { + var error = Assert.Throws(() => ((IStateMachine)outbox).OnRecoveryCompleted()); + Assert.Equal("A recovered outbox entry requires its send time.", error.Message); + } + + Assert.Equal(2, outbox.MessageState.Count); + Assert.Equal(sentAt, outbox.MessageState[first.Envelope.MessageId].State.SentAt); + var recovered = outbox.MessageState[target.Envelope.MessageId]; + Assert.Equal(target.Envelope.MessageId, recovered.MessageId); + Assert.Equal(reference, recovered.Reference is not null); + Assert.Equal(hasSentAt ? sentAt : (DateTimeOffset?)null, recovered.State.SentAt); + }); + } + + [Fact] + public async Task CommittedOutbox_DeactivationBeforeLocalFollowUp_RecoversPendingWork() + { + var sender = NewGrain(); + var receiver = NewGrain(); + var before = await sender.GetSnapshotAsync(); + _ = await receiver.GetSnapshotAsync(); + var receiverWrite = Fixture.Storage.BlockWrite(JournalId.FromGrainId(receiver.GetGrainId())); + + await sender.SendAndDeactivateAsync( + receiver.GetGrainId(), + "messages/outbox-crash-window", + NewMessage(52, "durable-wakeup")); + await receiverWrite.WaitUntilEnteredAsync(); + + // Deactivation cancels the in-flight attempt rather than waiting for the receiver, so the sender's journal and + // its durable wake-up carry the message. + Assert.NotEmpty(Fixture.JobManagerProbe.GetScheduledJobs(MessagingPump.JobName, sender.GetGrainId())); + + receiverWrite.Release(); + var delivered = await Fixture.WaitForEffectCountAsync(receiver, 1); + var recovered = await Fixture.SnapshotProbe.WaitAsync( + sender.GetGrainId(), + snapshot => snapshot.ActivationId != before.ActivationId && snapshot.OutboxCount == 0); + + Assert.Equal("durable-wakeup", Assert.Single(delivered.Effects).Value); + Assert.NotEqual(before.ActivationId, recovered.ActivationId); + } + + [Fact] + public async Task OutboxRemovalWriteFailure_FencesOldManagerAndFreshActivationCleansUp() + { + var sender = NewGrain(); + var receiver = NewGrain(); + var before = await sender.GetSnapshotAsync(); + var oldContext = Fixture.GetGrainContext(sender); + var oldManager = oldContext.ActivationServices.GetRequiredService(); + var oldGrain = Assert.IsType(oldContext.GrainInstance); + Fixture.Storage.FailWrite(JournalId.FromGrainId(sender.GetGrainId()), matchingWrite: 2); + + await sender.SendAsync(receiver.GetGrainId(), "messages/outbox-clear-retry", NewMessage(56, "outbox-clear-retry")); + _ = await Fixture.WaitForEffectCountAsync(receiver, 1); + await oldContext.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + Assert.IsType(await oldGrain.DeactivationFailure.Task.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken)); + _ = await sender.GetSnapshotAsync(); + var cleaned = await Fixture.SnapshotProbe.WaitAsync(sender.GetGrainId(), + snapshot => snapshot.ActivationId != before.ActivationId && snapshot.OutboxCount == 0); + + Assert.NotSame(oldManager, Fixture.GetGrainContext(sender).ActivationServices.GetRequiredService()); + Assert.Equal(1, Assert.Single((await receiver.GetSnapshotAsync()).Effects).Count); + await sender.RequestDeactivationAsync(); + Assert.Equal(0, (await sender.GetSnapshotAsync()).OutboxCount); + } + + [Fact] + public async Task Outbox_SpuriousWakeupPreservesBlockedPendingWork() + { + var sender = NewGrain(); + var receiver = NewGrain(); + _ = await sender.GetSnapshotAsync(); + _ = await receiver.GetSnapshotAsync(); + var receiverWrite = Fixture.Storage.BlockWrite(JournalId.FromGrainId(receiver.GetGrainId())); + + await sender.SendAsync(receiver.GetGrainId(), "messages/spurious-outbox-wakeup", NewMessage(61, "spurious-wakeup")); + await receiverWrite.WaitUntilEnteredAsync(); + Assert.Equal(1, Fixture.GetSnapshot(sender).OutboxCount); + var attemptBaseline = Fixture.Metrics.GetCount("orleans-durablejobs-job-attempts-started"); + var manager = Fixture.Cluster.Silos[0].ServiceProvider.GetRequiredService(); + + try + { + await manager.ScheduleJobAsync( + new ScheduleJobRequest + { + Target = sender.GetGrainId(), + JobName = MessagingPump.JobName, + DueTime = DateTimeOffset.UtcNow + }, + TestContext.Current.CancellationToken); + await Fixture.Metrics.WaitForCountAsync("orleans-durablejobs-job-attempts-started", attemptBaseline + 1); + + Assert.Equal(1, Fixture.GetSnapshot(sender).OutboxCount); + } + finally + { + receiverWrite.Release(); + } + + Assert.Equal("spurious-wakeup", Assert.Single((await Fixture.WaitForEffectCountAsync(receiver, 1)).Effects).Value); + Assert.Equal(0, (await Fixture.WaitForOutboxCountAsync(sender, 0)).OutboxCount); + } + + [Fact] + public async Task Outbox_AmbiguousCommit_FreshReplayRestoresMessagesBeforeDelivery() + { + var sender = NewGrain(); + var receiver = NewGrain(); + var before = await sender.GetSnapshotAsync(); + _ = await receiver.GetSnapshotAsync(); + var oldContext = Fixture.GetGrainContext(sender); + var journal = JournalId.FromGrainId(sender.GetGrainId()); + var recoveryRead = Fixture.Storage.BlockRead(journal); + var receiverWrite = Fixture.Storage.BlockWrite(JournalId.FromGrainId(receiver.GetGrainId())); + Fixture.Storage.FailAfterWrite(journal); + + await Assert.ThrowsAsync(() => sender.SendAsync( + receiver.GetGrainId(), "messages/recovery-visibility", NewMessage(62, "recovery-visibility"))); + await oldContext.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + var activation = sender.GetSnapshotAsync(); + try + { + await recoveryRead.WaitUntilEnteredAsync(); + Assert.Empty((await receiver.GetSnapshotAsync()).Effects); + recoveryRead.Release(); + await receiverWrite.WaitUntilEnteredAsync(); + var recovered = await activation; + + Assert.NotEqual(before.ActivationId, recovered.ActivationId); + Assert.Equal(1, recovered.OutboxCount); + } + finally + { + recoveryRead.Release(); + receiverWrite.Release(); + } + + var delivered = await Fixture.WaitForEffectCountAsync(receiver, 1); + Assert.Equal("recovery-visibility", Assert.Single(delivered.Effects).Value); + Assert.Equal(1, Assert.Single(delivered.Effects).Count); + Assert.Equal(0, (await Fixture.WaitForOutboxCountAsync(sender, 0)).OutboxCount); + } +} diff --git a/test/Orleans.Messaging.Tests/Functional/ProcessedRetentionMaintenanceTests.cs b/test/Orleans.Messaging.Tests/Functional/ProcessedRetentionMaintenanceTests.cs new file mode 100644 index 00000000000..73596f1ab83 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/ProcessedRetentionMaintenanceTests.cs @@ -0,0 +1,142 @@ +using Orleans.Journaling; +using Orleans.Messaging.Configuration; +using Orleans.Messaging.Tests.Support; +using Orleans.Runtime; +using Xunit; + +namespace Orleans.Messaging.Tests.Functional; + +[Collection(MessagingClusterCollection.Name)] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class ProcessedRetentionMaintenanceTests : MessagingBehaviorTestBase +{ + [Fact] + public async Task SnapshotWrite_PrunesExpiredProcessedIdsAndRecoveryRetainsCurrentOnes() + { + var receiver = NewGrain(); + var old = CreateEnvelope(receiver, NewMessage(120, "old")); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, old)).Status); + await Fixture.WaitForEffectCountAsync(receiver, 1); + Fixture.Clock.Advance(TimeSpan.FromMinutes(6)); + var current = CreateEnvelope(receiver, NewMessage(121, "current")); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, current)).Status); + await Fixture.WaitForEffectCountAsync(receiver, 2); + Fixture.Clock.Advance(TimeSpan.FromMinutes(5)); + + Fixture.Storage.RequestSnapshot(JournalId.FromGrainId(receiver.GetGrainId())); + var fresh = CreateEnvelope(receiver, NewMessage(122, "fresh")); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, fresh)).Status); + await Fixture.WaitForEffectCountAsync(receiver, 3); + + var processed = await GetProcessedAsync(receiver); + Assert.DoesNotContain(old.Envelope.MessageId, processed); + Assert.Contains(current.Envelope.MessageId, processed); + Assert.Contains(fresh.Envelope.MessageId, processed); + + await ReactivateAsync(receiver); + var recovered = await GetProcessedAsync(receiver); + Assert.True(processed.SetEquals(recovered)); + Assert.Equal(DeliveryStatus.Expired, (await DeliverAsync(receiver, old)).Status); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(receiver, current)).Status); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(receiver, fresh)).Status); + Assert.All((await receiver.GetSnapshotAsync()).Effects, static effect => Assert.Equal(1, effect.Count)); + } + + [Fact] + public async Task SnapshotCompaction_RemovesExpiredRecordsInterleavedWithRetainedOnesInOnePass() + { + // Compaction removes processed records while enumerating them, which Dictionary supports on the frameworks + // Messaging targets. Completion order places expired records first and between retained ones. + var receiver = NewGrain(); + var first = CreateEnvelope(receiver, NewMessage(140, "first-expired")); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, first)).Status); + await Fixture.WaitForEffectCountAsync(receiver, 1); + Fixture.Clock.Advance(TimeSpan.FromMinutes(6)); + var kept = CreateEnvelope(receiver, NewMessage(141, "kept")); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, kept)).Status); + await Fixture.WaitForEffectCountAsync(receiver, 2); + var middle = CreateEnvelope(receiver, NewMessage(142, "middle-expired")) with { SentAt = first.SentAt }; + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, middle)).Status); + await Fixture.WaitForEffectCountAsync(receiver, 3); + var alsoKept = CreateEnvelope(receiver, NewMessage(143, "also-kept")); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, alsoKept)).Status); + await Fixture.WaitForEffectCountAsync(receiver, 4); + Assert.Equal(4, (await GetProcessedAsync(receiver)).Count); + Fixture.Clock.Advance(TimeSpan.FromMinutes(5)); + + Fixture.Storage.RequestSnapshot(JournalId.FromGrainId(receiver.GetGrainId())); + var fresh = CreateEnvelope(receiver, NewMessage(144, "fresh")); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, fresh)).Status); + await Fixture.WaitForEffectCountAsync(receiver, 5); + + var processed = await GetProcessedAsync(receiver); + Assert.True(processed.SetEquals([kept.Envelope.MessageId, alsoKept.Envelope.MessageId, fresh.Envelope.MessageId])); + var snapshot = await receiver.GetSnapshotAsync(); + Assert.Equal(3, snapshot.ProcessedMessageCount); + Assert.Equal(DeliveryStatus.Expired, (await DeliverAsync(receiver, middle)).Status); + } + + [Fact] + public async Task SentAt_ExpiresOnlyAfterSnapshotAndRecoveryPreserveTheFloor() + { + var receiver = NewGrain(); + var message = CreateEnvelope(receiver, NewMessage(130, "expiring")); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, message)).Status); + await Fixture.WaitForEffectCountAsync(receiver, 1); + + Fixture.Clock.Advance(TimeSpan.FromMinutes(10) - TimeSpan.FromTicks(1)); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(receiver, message)).Status); + + Fixture.Clock.Advance(TimeSpan.FromTicks(1)); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(receiver, message)).Status); + Fixture.Storage.RequestSnapshot(JournalId.FromGrainId(receiver.GetGrainId())); + await receiver.RetryWriteStateAsync(); + await ReactivateAsync(receiver); + Assert.Equal(DeliveryStatus.Expired, (await DeliverAsync(receiver, message)).Status); + Assert.Equal(1, Assert.Single((await receiver.GetSnapshotAsync()).Effects).Count); + } + + private Task> GetProcessedAsync(IMessagingTestGrain receiver) + { + var inbox = Fixture.GetInbox(receiver); + return Fixture.OnGrainAsync(receiver, () => InboxTestAccess.GetProcessedKeys(inbox)); + } +} + +[Collection(MessagingClusterCollection.Name)] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class MaximumDeduplicationWindowTests() : MessagingBehaviorTestBase(new MaximumWindowClusterFixture()) +{ + [Fact] + public async Task MaximumWindow_ClampsTheAdmissionFloorThroughDeliveryAndSnapshotCompaction() + { + // Validation accepts any positive window, so the largest one is a valid configuration: the floor computation + // subtracts it from the clock and clamps at the minimum time instead of overflowing. + var receiver = NewGrain(); + var message = CreateEnvelope(receiver, NewMessage(160, "maximum-window")); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, message)).Status); + await Fixture.WaitForEffectCountAsync(receiver, 1); + + Fixture.Storage.RequestSnapshot(JournalId.FromGrainId(receiver.GetGrainId())); + await receiver.RetryWriteStateAsync(); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(receiver, message)).Status); + + var inbox = Fixture.GetInbox(receiver); + var floor = await Fixture.OnGrainAsync(receiver, () => InboxTestAccess.DurableAdmissionFloor(inbox)); + Assert.Equal(DateTimeOffset.MinValue, floor); + Assert.Contains(message.Envelope.MessageId, await Fixture.OnGrainAsync(receiver, () => InboxTestAccess.GetProcessedKeys(inbox))); + } + + private sealed class MaximumWindowClusterFixture : MessagingClusterFixture + { + protected override void ConfigureOptions(DurableMessagingOptions options) + { + base.ConfigureOptions(options); + options.DeduplicationWindow = TimeSpan.MaxValue; + } + } +} diff --git a/test/Orleans.Messaging.Tests/Functional/PublicInboxHandlerTransactionTests.cs b/test/Orleans.Messaging.Tests/Functional/PublicInboxHandlerTransactionTests.cs new file mode 100644 index 00000000000..e4813480988 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/PublicInboxHandlerTransactionTests.cs @@ -0,0 +1,319 @@ +using Microsoft.Extensions.DependencyInjection; +using Orleans.DurableJobs; +using Orleans.Journaling; +using Orleans.Messaging.Tests.Support; +using Orleans.Runtime; +using Orleans.Serialization.Session; +using Xunit; + +namespace Orleans.Messaging.Tests.Functional; + +[Collection(MessagingClusterCollection.Name)] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class PublicInboxHandlerTransactionTests : MessagingBehaviorTestBase +{ + public PublicInboxHandlerTransactionTests() : base(receiverOnly: false) + { + } + + [Theory] + [InlineData(1)] + [InlineData(2)] + public async Task ArbitraryRawPayload_AcceptanceReplayForwardAndBusinessCompletionRequireNoApplicationCodec(int sends) + { + var receiver = Fixture.Client.GetGrain(Guid.NewGuid()); + var sink = Fixture.Client.GetGrain(Guid.NewGuid()); + await receiver.ConfigureForwardAsync(sink.GetGrainId(), sends); + _ = await sink.GetSnapshotAsync(); + Assert.True(Fixture.Cluster.TryGetGrainContext(receiver.GetGrainId(), out var original)); + Assert.True(Fixture.Cluster.TryGetGrainContext(sink.GetGrainId(), out var sinkContext)); + var sinkGrain = Assert.IsType(sinkContext.GrainInstance); + sinkGrain.ExpectedAcknowledgedEffects = sends; + var bytes = System.Text.Encoding.UTF8.GetBytes("raw application bytes\0\u03c0/forward").Concat(new byte[] { 0xff, 0x80, 0x01 }).ToArray(); + var envelope = new Envelope + { + MessageId = MessageId.Create("raw", Guid.NewGuid().ToString("N")), + Payload = bytes, + Headers = new Dictionary(StringComparer.Ordinal) { ["attachment"] = [0xff, 0x80, 0x01] } + }; + var sentAt = Fixture.JobNow; + // Deactivation cancels any handler the original activation starts, leaving the message to recovery. + using (Fixture.HandlerProbe.Arm(receiver.GetGrainId(), RawPayloadTestGrain.HandlerBarrier)) + { + Assert.Equal(DeliveryStatus.Accepted, (await receiver.AcceptAndDeactivateAsync( + new InboxMessage { Envelope = envelope, SentAt = sentAt })).Status); + await original.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + } + var accepted = Assert.IsType(Assert.IsType(original.GrainInstance).AcceptedSnapshot); + Assert.Equal(1, accepted.InboxCount); + Assert.Empty(accepted.Effects); + + var journal = JournalId.FromGrainId(receiver.GetGrainId()); + using var replay = Fixture.Storage.BlockRead(journal); + using var handler = Fixture.HandlerProbe.Arm(receiver.GetGrainId(), RawPayloadTestGrain.HandlerBarrier); + var activation = receiver.GetSnapshotAsync(); + await replay.WaitUntilEnteredAsync(); + replay.Release(); + await handler.WaitUntilEnteredAsync(); + Assert.True(Fixture.Cluster.TryGetGrainContext(receiver.GetGrainId(), out var current)); + Assert.NotSame(original, current); + var grain = Assert.IsType(current.GrainInstance); + var pending = Assert.Single(current.ActivationServices.GetRequiredService().Messages()); + Assert.Equal(bytes, pending.Envelope.Payload.ToArray()); + Assert.Null(pending.Envelope.Subject); + Assert.False(Assert.IsType>(pending.Envelope.Headers).ContainsKey("sender")); + Assert.Empty(grain.GetSnapshotForTest().Effects); + using var completion = Fixture.Storage.BlockWrite(journal); + handler.Release(); + await completion.WaitUntilEnteredAsync(); + var staged = grain.GetSnapshotForTest(); + Assert.Equal(0, staged.InboxCount); + Assert.Equal(sends, staged.OutboxCount); + Assert.Equal(1, staged.ProcessedCount); + Assert.Equal(new RawPayloadEffect(envelope.MessageId, Convert.ToBase64String(bytes), 1), Assert.Single(staged.Effects)); + var forwarded = current.ActivationServices.GetRequiredService().Messages().ToArray(); + Assert.Equal(sends, forwarded.Length); + Assert.Equal(sends, forwarded.Select(output => output.Envelope.MessageId).Distinct().Count()); + Assert.Contains(forwarded, output => output.Envelope.MessageId == envelope.MessageId); + Assert.All(forwarded, output => + { + Assert.Equal(sink.GetGrainId(), output.ReceiverId); + Assert.Equal(bytes, output.Envelope.Payload.ToArray()); + var headers = Assert.IsType>(output.Envelope.Headers); + Assert.Equal(new byte[] { 0xff, 0x80, 0x01 }, headers["attachment"]); + Assert.False(headers.ContainsKey("sender")); + Assert.Null(output.Envelope.Subject); + if (output.Envelope.MessageId == envelope.MessageId) + { + Assert.Same(pending.Envelope.Payload, output.Envelope.Payload); + Assert.Same(pending.Envelope.Headers, output.Envelope.Headers); + } + }); + Assert.False(grain.Acknowledged.IsCompleted); + Assert.False(sinkGrain.Acknowledged.IsCompleted); + completion.Release(); + await activation; + var delivered = await sinkGrain.Acknowledged.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + var acknowledged = await grain.Acknowledged.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + Assert.Equal(Assert.Single(acknowledged.Effects), Assert.Single(delivered.Effects, effect => effect.MessageId == envelope.MessageId)); + Assert.Equal(sends, delivered.Effects.Length); + Assert.All(delivered.Effects, effect => + { + Assert.Equal(Convert.ToBase64String(bytes), effect.Bytes); + Assert.Equal(1, effect.Count); + }); + Assert.Equal(0, acknowledged.OutboxCount); + Assert.Equal(sends, delivered.ProcessedCount); + Assert.Empty(current.ActivationServices.GetRequiredService().InboxDeadLetters); + Assert.Empty(current.ActivationServices.GetRequiredService().OutboxDeadLetters); + await receiver.RequestDeactivationAsync(); + await current.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + var recovered = await receiver.GetSnapshotAsync(); + Assert.NotEqual(acknowledged.ActivationId, recovered.ActivationId); + Assert.Equal(acknowledged.Effects, recovered.Effects); + Assert.Equal(1, recovered.ProcessedCount); + Assert.Equal(0, recovered.OutboxCount); + Assert.Equal(DeliveryStatus.Duplicate, (await receiver.AsReference().DeliverAsync( + new InboxMessage { Envelope = envelope, SentAt = sentAt }, TestContext.Current.CancellationToken)).Status); + Assert.Equal(1, Assert.Single((await receiver.GetSnapshotAsync()).Effects).Count); + var sinkState = await sink.GetSnapshotAsync(); + Assert.Equal(sends, sinkState.Effects.Length); + Assert.All(sinkState.Effects, effect => Assert.Equal(1, effect.Count)); + } + + [Fact] + public async Task CaptureProbeSnapshot_RemainsStableAcrossLaterWrites() + { + var receiver = NewGrain(); + var first = new MessageEffect(MessageId.Create("capture", "first"), 1, 1, "first"); + var second = new MessageEffect(MessageId.Create("capture", "second"), 1, 2, "second"); + await receiver.StageEffectAsync(first); + await receiver.RetryWriteStateAsync(); + var grain = Assert.IsType(Fixture.GetGrainContext(receiver).GrainInstance); + var captures = grain.Captures; + var originalCount = captures.Count; + using var enumerator = captures.GetEnumerator(); + Assert.True(enumerator.MoveNext()); + var observed = new List { enumerator.Current }; + + await receiver.StageEffectAsync(second); + await receiver.RetryWriteStateAsync(); + + while (enumerator.MoveNext()) + { + observed.Add(enumerator.Current); + } + + Assert.Equal(originalCount, captures.Count); + Assert.Equal(originalCount, observed.Count); + Assert.Equal(captures, observed); + Assert.DoesNotContain(captures, capture => capture.Effects.Contains(second)); + Assert.Contains(grain.Captures, capture => capture.Effects.Contains(second)); + } + + [Fact] + public async Task HandlerSuccess_CommitsEffectCompletionDedupeAndOutgoingAtomically() + { + var receiver = NewGrain(); + var sink = NewGrain(); + var logicalId = MessageId.Create("atomic", Guid.NewGuid().ToString("N")); + var envelope = CreateEnvelope( + receiver, + new TestMessage(logicalId, 7, "atomic", sink.GetGrainId())); + + var result = await DeliverAsync(receiver, envelope); + var receiverState = await Fixture.WaitForEffectCountAsync(receiver, 1); + var sinkState = await Fixture.WaitForEffectCountAsync(sink, 1); + receiverState = await Fixture.WaitForOutboxCountAsync(receiver, 0); + + Assert.Equal(DeliveryStatus.Accepted, result.Status); + var effect = Assert.Single(receiverState.Effects); + Assert.Equal(new MessageEffect(logicalId, 1, 7, "atomic"), effect); + Assert.Equal(0, receiverState.InboxCount); + Assert.Equal(0, receiverState.OutboxCount); + Assert.Equal(effect, Assert.Single(sinkState.Effects)); + + var duplicate = await DeliverAsync(receiver, envelope); + Assert.Equal(DeliveryStatus.Duplicate, duplicate.Status); + Assert.Equal(1, Assert.Single((await receiver.GetSnapshotAsync()).Effects).Count); + Assert.Equal(1, Assert.Single((await sink.GetSnapshotAsync()).Effects).Count); + } + + [Fact] + public async Task HandlerPreparationFailure_DeadLettersWithoutEffectsOrSinkDelivery() + { + var receiver = NewGrain(); + var sink = NewGrain(); + var envelope = CreateEnvelope( + receiver, + new TestMessage(MessageId.Create("preparation", "failure"), 9, "preparation-failure", sink.GetGrainId(), ThrowDuringPreparation: true)); + + var accepted = await DeliverAsync(receiver, envelope); + var state = await Fixture.WaitForDeadLetterCountAsync(receiver, 1); + + Assert.Equal(DeliveryStatus.Accepted, accepted.Status); + Assert.Empty(state.Effects); + Assert.Equal(0, state.InboxCount); + Assert.Equal(0, state.OutboxCount); + var deadLetter = Assert.Single(state.InboxDeadLetters); + Assert.Equal(envelope.Envelope.MessageId, deadLetter.MessageId); + Assert.Equal(1, deadLetter.AttemptCount); + Assert.Contains("Injected handler preparation failure", deadLetter.Reason, StringComparison.Ordinal); + Assert.Empty((await sink.GetSnapshotAsync()).Effects); + } + [Fact] + public async Task HandlerApplicationPreparation_AllowsIndependentWritesAndPreservesCancelledWriteWait() + { + var receiver = NewGrain(); + var sink = NewGrain(); + _ = await receiver.GetSnapshotAsync(); + var context = Fixture.GetGrainContext(receiver); + var grain = Assert.IsType(context.GrainInstance); + var manager = context.ActivationServices.GetRequiredService(); + using var handler = Fixture.HandlerProbe.Arm(receiver.GetGrainId(), "messages/admitted-outgoing"); + var envelope = CreateEnvelope(receiver, + new TestMessage(MessageId.Create("preparation", "admitted"), 91, "admitted", sink.GetGrainId()), "messages/admitted-outgoing"); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + await handler.WaitUntilEnteredAsync(); + var localPreparation = grain.GetSnapshotForTest(); + Assert.Empty(localPreparation.Effects); + Assert.Equal(1, localPreparation.InboxCount); + Assert.Equal(0, localPreparation.ProcessedMessageCount); + Assert.Equal(0, localPreparation.OutboxCount); + using var applicationPreparation = Fixture.HandlerProbe.Arm(receiver.GetGrainId(), "messages/admitted-outgoing/application-preparation"); + handler.Release(); + await applicationPreparation.WaitUntilEnteredAsync(); + var preparing = grain.GetSnapshotForTest(); + Assert.Empty(preparing.Effects); + Assert.Equal(1, preparing.InboxCount); + Assert.Equal(0, preparing.ProcessedMessageCount); + Assert.Equal(0, preparing.OutboxCount); + await manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask() + .WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + Assert.DoesNotContain(grain.Captures, snapshot => snapshot.Effects.Count > 0 || snapshot.OutboxCount > 0); + + var writes = Fixture.Storage.GetSuccessfulWriteCount(JournalId.FromGrainId(receiver.GetGrainId())); + using var cancellation = new CancellationTokenSource(); + var storage = Fixture.Storage.BlockWrite(JournalId.FromGrainId(receiver.GetGrainId())); + Task completedWait; + try + { + applicationPreparation.Release(); + await storage.WaitUntilEnteredAsync(); + var captured = grain.GetSnapshotForTest(); + Assert.Equal(0, captured.InboxCount); + Assert.Equal(1, captured.ProcessedMessageCount); + Assert.Equal(1, captured.OutboxCount); + Assert.Single(captured.Effects); + Assert.All(Fixture.JobManagerProbe.GetScheduledJobs(MessagingPump.JobName, receiver.GetGrainId()), + job => Assert.Equal(receiver.GetGrainId(), job.TargetGrainId)); + var canceledWait = manager.WriteStateAsync(cancellation.Token).AsTask(); + completedWait = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + Assert.False(canceledWait.IsCompleted); + Assert.False(completedWait.IsCompleted); + cancellation.Cancel(); + await Assert.ThrowsAnyAsync(() => canceledWait); + Assert.False(completedWait.IsCompleted); + Assert.Equal(writes, Fixture.Storage.GetSuccessfulWriteCount(JournalId.FromGrainId(receiver.GetGrainId()))); + Assert.Empty((await sink.GetSnapshotAsync()).Effects); + } + finally + { + storage.Release(); + } + await completedWait; + var delivered = await Fixture.WaitForEffectCountAsync(sink, 1); + var completed = await Fixture.WaitForOutboxCountAsync(receiver, 0); + Assert.Equal(Assert.Single(completed.Effects), Assert.Single(delivered.Effects)); + Assert.DoesNotContain(grain.Captures, snapshot => snapshot.Effects.Count > 0 && snapshot.ProcessedMessageCount == 0); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(receiver, envelope)).Status); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task HandlerCompletionPersistenceFailure_FreshActivationConvergesEffectDedupeAndSink(bool storageCommitted) + { + var receiver = NewGrain(); + var sink = NewGrain(); + var before = await receiver.GetSnapshotAsync(); + var oldContext = Fixture.GetGrainContext(receiver); + var oldGrain = Assert.IsType(oldContext.GrainInstance); + using var handler = Fixture.HandlerProbe.Arm(receiver.GetGrainId(), "messages/completion-outcome"); + var message = new TestMessage(MessageId.Create("completion", "outcome"), 92, "completion-outcome", sink.GetGrainId()); + var envelope = CreateEnvelope(receiver, message, "messages/completion-outcome"); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + await handler.WaitUntilEnteredAsync(); + var journal = JournalId.FromGrainId(receiver.GetGrainId()); + if (storageCommitted) + { + Fixture.Storage.FailAfterWrite(journal); + } + else + { + Fixture.Storage.FailWrite(journal); + } + handler.Release(); + Assert.IsType(await oldGrain.DeactivationFailure.Task.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken)); + await oldContext.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + _ = await receiver.GetSnapshotAsync(); + var completed = await Fixture.SnapshotProbe.WaitAsync(receiver.GetGrainId(), + snapshot => snapshot.ActivationId != before.ActivationId && snapshot.Effects.Count == 1); + var delivered = await Fixture.WaitForEffectCountAsync(sink, 1); + completed = await Fixture.SnapshotProbe.WaitAsync(receiver.GetGrainId(), + snapshot => snapshot.ActivationId != before.ActivationId && snapshot.Effects.Count == 1 && snapshot.OutboxCount == 0); + + Assert.NotEqual(before.ActivationId, completed.ActivationId); + Assert.Equal(new MessageEffect(message.LogicalId, 1, message.Sequence, message.Value), Assert.Single(completed.Effects)); + Assert.Equal(Assert.Single(completed.Effects), Assert.Single(delivered.Effects)); + Assert.Equal(1, completed.ProcessedMessageCount); + Assert.Equal(0, completed.InboxCount); + Assert.Empty(completed.InboxDeadLetters); + Assert.Empty(completed.OutboxDeadLetters); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(receiver, envelope)).Status); + Assert.Equal(1, Assert.Single((await sink.GetSnapshotAsync()).Effects).Count); + } + +} diff --git a/test/Orleans.Messaging.Tests/Functional/PublicMessagingRoutingAndDeadLetterTests.cs b/test/Orleans.Messaging.Tests/Functional/PublicMessagingRoutingAndDeadLetterTests.cs new file mode 100644 index 00000000000..b2c01aded23 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/PublicMessagingRoutingAndDeadLetterTests.cs @@ -0,0 +1,67 @@ +using Microsoft.Extensions.DependencyInjection; +using Orleans.DurableJobs; +using Orleans.Journaling; +using Orleans.Messaging.Tests.Support; +using Orleans.Runtime; +using Orleans.Serialization.Session; +using Xunit; + +namespace Orleans.Messaging.Tests.Functional; + +[Collection(MessagingClusterCollection.Name)] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class PublicMessagingRoutingAndDeadLetterTests : MessagingBehaviorTestBase +{ + public PublicMessagingRoutingAndDeadLetterTests() : base(receiverOnly: false) + { + } + + [Fact] + public async Task HandlerNotFound_OutboxRetriesThenDeadLettersWithoutReceiverPersistence() + { + var sender = NewGrain(); + var receiver = NewGrain(); + await receiver.ConfigureHandlerAsync(false); + await ReactivateAsync(receiver); + + await sender.SendAsync( + receiver.GetGrainId(), + "unknown/outbox-route", + NewMessage(72, "undeliverable")); + var senderState = await Fixture.WaitForDeadLetterCountAsync(sender, 1); + + Assert.Equal(0, senderState.OutboxCount); + var deadLetter = Assert.Single(senderState.OutboxDeadLetters); + Assert.Equal("unknown/outbox-route", deadLetter.Route); + Assert.True(deadLetter.AttemptCount > 1); + Assert.Contains("No inbox handler", deadLetter.Reason, StringComparison.Ordinal); + var receiverState = await receiver.GetSnapshotAsync(); + Assert.Equal(0, receiverState.InboxCount); + Assert.Empty(receiverState.Effects); + Assert.Empty(receiverState.InboxDeadLetters); + } + + [Fact] + public async Task OutboxDeadLetterRemoval_IsDurable() + { + var sender = NewGrain(); + var receiver = NewGrain(); + await receiver.ConfigureHandlerAsync(false); + await ReactivateAsync(receiver); + var messageId = await sender.SendAsync( + receiver.GetGrainId(), + "unknown/removable-outbox-route", + NewMessage(73, "removable-undeliverable")); + _ = await Fixture.WaitForDeadLetterCountAsync(sender, 1); + + Assert.True(await sender.RemoveOutboxDeadLetterAsync(messageId)); + Assert.Empty((await sender.GetSnapshotAsync()).OutboxDeadLetters); + Assert.False(await sender.RemoveOutboxDeadLetterAsync(messageId)); + + await sender.RequestDeactivationAsync(); + Assert.Empty((await sender.GetSnapshotAsync()).OutboxDeadLetters); + } + +} diff --git a/test/Orleans.Messaging.Tests/Functional/PublicServiceCollectionMessagingCompositionTests.cs b/test/Orleans.Messaging.Tests/Functional/PublicServiceCollectionMessagingCompositionTests.cs new file mode 100644 index 00000000000..01664a96263 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/PublicServiceCollectionMessagingCompositionTests.cs @@ -0,0 +1,11 @@ +using Orleans.Messaging.Tests.Support; +using Xunit; + +namespace Orleans.Messaging.Tests.Functional; + +[Collection(MessagingClusterCollection.Name)] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class PublicServiceCollectionMessagingCompositionTests() + : MessagingGrainTypeConfiguratorTests(new BootstrapClusterFixture(useServiceCollection: true)); diff --git a/test/Orleans.Messaging.Tests/Functional/ScheduledDeliveryTests.cs b/test/Orleans.Messaging.Tests/Functional/ScheduledDeliveryTests.cs new file mode 100644 index 00000000000..fb1e38b3a2a --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/ScheduledDeliveryTests.cs @@ -0,0 +1,584 @@ +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; +using Orleans.DurableJobs; +using Orleans.Journaling; +using Orleans.Messaging.Configuration; +using Orleans.Messaging.Diagnostics; +using Orleans.Messaging.Tests.Support; +using Orleans.Runtime; +using Orleans.TestingHost.Diagnostics; +using Xunit; + +namespace Orleans.Messaging.Tests.Functional; + +[Collection(MessagingClusterCollection.Name)] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class ScheduledDeliveryTests() : MessagingBehaviorTestBase(new ScheduledDeliveryClusterFixture()) +{ + private const string HeldRoute = "messages/held"; + private static readonly TimeSpan Delay = TimeSpan.FromHours(1); + + private ManualJobClock JobClock => ((ScheduledDeliveryClusterFixture)Fixture).JobClock; + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task OutboxRetries_UseJobTimeAndDeadLettersUseGrainTime(bool snapshot) + { + Fixture.Clock.Advance(TimeSpan.FromDays(2)); + var sender = NewGrain(); + var receiver = Fixture.Client.GetGrain(Guid.NewGuid()); + await receiver.PingAsync(); + _ = await sender.GetSnapshotAsync(); + var options = Fixture.GetGrainContext(sender).ActivationServices + .GetRequiredService>().Value; + using var events = new DiagnosticEventCollector(MessagingEvents.ListenerName); + var firstRetry = WaitForRetryAsync(1); + var startedAt = JobClock.GetUtcNow(); + var expiresAt = startedAt + options.OutboxRetryTimeout; + + var messageId = await sender.SendAsync(receiver.GetGrainId(), "messages/unhandled", NewMessage(61, "dead-letter")); + await firstRetry; + await AssertWakeupAtAsync(sender, startedAt + options.RetryInitialInterval); + var state = await GetRetryStateAsync(); + Assert.Equal(1, state.AttemptCount); + Assert.Equal(startedAt, state.SentAt); + Assert.Equal(startedAt + options.RetryInitialInterval, state.NextAttemptAt); + + Fixture.Clock.Advance(TimeSpan.FromDays(1)); + await AssertWakeupAtAsync(sender, Assert.IsType(state.NextAttemptAt)); + Assert.Equal(1, (await GetRetryStateAsync()).AttemptCount); + + var secondRetry = WaitForRetryAsync(2); + JobClock.Advance(options.RetryInitialInterval); + _ = await Fixture.RunPumpAsync(sender); + await secondRetry; + state = await GetRetryStateAsync(); + Assert.Equal(startedAt, state.SentAt); + Assert.Equal(JobClock.GetUtcNow() + 2 * options.RetryInitialInterval, state.NextAttemptAt); + await AssertWakeupAtAsync(sender, Assert.IsType(state.NextAttemptAt)); + + if (snapshot) + { + Fixture.Storage.RequestSnapshot(JournalId.FromGrainId(sender.GetGrainId())); + await sender.RetryWriteStateAsync(); + } + await ReactivateAsync(sender); + var recovered = await GetRetryStateAsync(); + Assert.Equal(state.AttemptCount, recovered.AttemptCount); + Assert.Equal(state.SentAt, recovered.SentAt); + Assert.Equal(state.NextAttemptAt, recovered.NextAttemptAt); + await AssertWakeupAtAsync(sender, Assert.IsType(recovered.NextAttemptAt)); + + var thirdRetry = WaitForRetryAsync(3); + JobClock.Advance(expiresAt - JobClock.GetUtcNow() - TimeSpan.FromTicks(1)); + _ = await Fixture.RunPumpAsync(sender); + await thirdRetry; + await AssertWakeupAtAsync(sender, expiresAt); + JobClock.Advance(TimeSpan.FromTicks(1)); + _ = await Fixture.RunPumpAsync(sender); + var letter = Assert.Single((await Fixture.WaitForDeadLetterCountAsync(sender, 1)).OutboxDeadLetters); + Assert.Equal(messageId, letter.MessageId); + Assert.Equal(3, letter.AttemptCount); + Assert.Equal(Fixture.Clock.GetUtcNow(), letter.DeadLetteredAt); + + if (snapshot) + { + Fixture.Storage.RequestSnapshot(JournalId.FromGrainId(sender.GetGrainId())); + await sender.RetryWriteStateAsync(); + } + JobClock.Advance(options.DeadLetterRetentionPeriod); + await ReactivateAsync(sender); + Assert.Equal(letter, Assert.Single((await sender.GetSnapshotAsync()).OutboxDeadLetters)); + + Fixture.Clock.Advance(options.DeadLetterRetentionPeriod - TimeSpan.FromTicks(1)); + await ReactivateAsync(sender); + Assert.Equal(letter, Assert.Single((await sender.GetSnapshotAsync()).OutboxDeadLetters)); + Fixture.Clock.Advance(TimeSpan.FromTicks(1)); + await ReactivateAsync(sender); + Assert.Empty((await sender.GetSnapshotAsync()).OutboxDeadLetters); + await ReactivateAsync(sender); + Assert.Empty((await sender.GetSnapshotAsync()).OutboxDeadLetters); + + Task WaitForRetryAsync(int attempt) => events.WaitForEventAsync( + nameof(MessagingEvents.OutboxRetryScheduled), + entry => entry.Payload is MessagingEvents.OutboxRetryScheduled scheduled + && scheduled.GrainContext.GrainId == sender.GetGrainId() && scheduled.AttemptCount == attempt, + TimeSpan.FromSeconds(30), + TestContext.Current.CancellationToken); + + Task GetRetryStateAsync() => Fixture.OnGrainAsync(sender, () => + Fixture.GetGrainContext(sender).ActivationServices.GetRequiredService().MessageState[messageId].State); + } + + [Fact] + public async Task OutboxDeadLetterInsertion_CompactsOnGrainTime() + { + Fixture.Clock.Advance(TimeSpan.FromDays(2)); + var sender = NewGrain(); + var receiver = Fixture.Client.GetGrain(Guid.NewGuid()); + await receiver.PingAsync(); + _ = await sender.GetSnapshotAsync(); + var options = Fixture.GetGrainContext(sender).ActivationServices + .GetRequiredService>().Value; + + var first = Assert.Single((await DeadLetterAsync(62)).OutboxDeadLetters); + Assert.Equal(Fixture.Clock.GetUtcNow(), first.DeadLetteredAt); + JobClock.Advance(options.DeadLetterRetentionPeriod); + var retained = (await DeadLetterAsync(63)).OutboxDeadLetters; + Assert.Equal(2, retained.Count); + Assert.Contains(first, retained); + Assert.All(retained, letter => Assert.Equal(Fixture.Clock.GetUtcNow(), letter.DeadLetteredAt)); + + Fixture.Clock.Advance(options.DeadLetterRetentionPeriod); + var replacement = Assert.Single((await DeadLetterAsync(64)).OutboxDeadLetters); + Assert.Equal(Fixture.Clock.GetUtcNow(), replacement.DeadLetteredAt); + await ReactivateAsync(sender); + Assert.Equal(replacement, Assert.Single((await sender.GetSnapshotAsync()).OutboxDeadLetters)); + + async Task DeadLetterAsync(int sequence) + { + using var events = new DiagnosticEventCollector(MessagingEvents.ListenerName); + var retry = events.WaitForEventAsync( + nameof(MessagingEvents.OutboxRetryScheduled), + entry => entry.Payload is MessagingEvents.OutboxRetryScheduled scheduled + && scheduled.GrainContext.GrainId == sender.GetGrainId(), + TimeSpan.FromSeconds(30), + TestContext.Current.CancellationToken); + var messageId = await sender.SendAsync(receiver.GetGrainId(), "messages/unhandled", NewMessage(sequence, "dead-letter")); + await retry; + JobClock.Advance(options.OutboxRetryTimeout); + _ = await Fixture.RunPumpAsync(sender); + return await Fixture.SnapshotProbe.WaitAsync(sender.GetGrainId(), + state => state.OutboxCount == 0 && state.OutboxDeadLetters.Any(letter => letter.MessageId == messageId)); + } + } + + [Fact] + public async Task InboxWakeup_BareStagingWaitsForCommitAfterPriorWorkDrains() + { + var receiver = NewGrain(); + _ = await receiver.GetSnapshotAsync(); + Fixture.JobManagerProbe.SkipNext(MessagingPump.JobName); + Assert.Equal(DeliveryStatus.Accepted, + (await DeliverAsync(receiver, CreateEnvelope(receiver, NewMessage(70, "committed")))).Status); + _ = await Fixture.WaitForEffectCountAsync(receiver, 1); + var pump = Fixture.GetPump(receiver); + Assert.NotNull(await Fixture.OnGrainAsync(receiver, () => InboxTestAccess.WakeupJobId(pump))); + var attempts = Fixture.JobManagerProbe.GetAttemptCount(MessagingPump.JobName, receiver.GetGrainId()); + var staged = CreateEnvelope(receiver, NewMessage(71, "staged")); + await Fixture.OnGrainAsync(receiver, () => InboxTestAccess.StageAcceptance(Fixture.GetInbox(receiver), staged)); + + Assert.Equal(DurableJobRunStatus.Completed, (await Fixture.RunPumpAsync(receiver)).Status); + Assert.Equal(DurableJobRunStatus.Completed, (await Fixture.RunPumpAsync(receiver)).Status); + Assert.Equal(attempts, Fixture.JobManagerProbe.GetAttemptCount(MessagingPump.JobName, receiver.GetGrainId())); + var pending = await receiver.GetSnapshotAsync(); + Assert.Equal(1, pending.InboxCount); + Assert.Equal("committed", Assert.Single(pending.Effects).Value); + + Fixture.JobManagerProbe.SkipNext(MessagingPump.JobName); + await receiver.RetryWriteStateAsync(); + Assert.Equal(attempts + 1, Fixture.JobManagerProbe.GetAttemptCount(MessagingPump.JobName, receiver.GetGrainId())); + var completed = await Fixture.WaitForEffectCountAsync(receiver, 2); + Assert.Equal(new[] { "committed", "staged" }, completed.Effects.Select(effect => effect.Value)); + Assert.All(completed.Effects, effect => Assert.Equal(1, effect.Count)); + } + + [Fact] + public async Task Wakeup_RetriesActivationFailuresBeyondTheDefaultLimitAndDeliversWithoutAClientCall() + { + var receiver = NewGrain(); + _ = await receiver.GetSnapshotAsync(); + var journal = JournalId.FromGrainId(receiver.GetGrainId()); + var deliverAt = JobClock.GetUtcNow() + Delay; + Assert.Equal(DeliveryStatus.Accepted, + (await DeliverAsync(receiver, Schedule(CreateEnvelope(receiver, NewMessage(80, "outage")), deliverAt))).Status); + await Fixture.CollectAsync(Fixture.GetGrainContext(receiver)); + + // Durable Jobs gives up on other jobs after 5 failed runs; the pump job keeps retrying through a longer outage. + const int failures = 7; + Fixture.Storage.FailReads(journal, failures); + // Each retry becomes due on the Durable Jobs clock once Durable Jobs records the failed run. + var reads = Fixture.Storage.GetReadCount(journal); + await WaitUntilAsync(() => + { + if (Fixture.Storage.GetReadCount(journal) > reads + failures) return true; + JobClock.Advance(Delay); + return false; + }); + + var recovered = await Fixture.WaitForEffectCountAsync(receiver, 1); + Assert.Equal("outage", Assert.Single(recovered.Effects).Value); + Assert.Equal(0, recovered.InboxCount); + } + + [Fact] + public async Task Pump_StartsWhenActivationCompletesSoStagedWorkRunsWithoutAWakeup() + { + var receiver = NewGrain(); + _ = await receiver.GetSnapshotAsync(); + + // No messaging call has reached this activation, and its wake-up is suppressed. + Fixture.JobManagerProbe.SkipNext(MessagingPump.JobName); + var staged = CreateEnvelope(receiver, NewMessage(83, "staged")); + await Fixture.OnGrainAsync(receiver, () => InboxTestAccess.StageAcceptance(Fixture.GetInbox(receiver), staged)); + await receiver.RetryWriteStateAsync(); + + Assert.Equal("staged", Assert.Single((await Fixture.WaitForEffectCountAsync(receiver, 1)).Effects).Value); + } + [Fact] + public async Task Wakeup_ForDueWorkIsScheduledOncePerIdlePeriodWithinTheRecheckDelay() + { + var receiver = NewGrain(); + _ = await receiver.GetSnapshotAsync(); + var grainId = receiver.GetGrainId(); + var scheduled = Fixture.JobManagerProbe.GetScheduledJobs(MessagingPump.JobName, grainId).Count; + var now = JobClock.GetUtcNow(); + using (var held = Fixture.HandlerProbe.Arm(grainId, HeldRoute)) + { + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, CreateEnvelope(receiver, NewMessage(81, "first"), HeldRoute))).Status); + await WaitForBarrierAsync(receiver, held); + + // The live loop performs due work, so the wake-up is crash insurance due within RecheckDelay, and later + // writes reuse it. + var jobs = Fixture.JobManagerProbe.GetScheduledJobs(MessagingPump.JobName, grainId); + Assert.Equal(scheduled + 1, jobs.Count); + Assert.Equal(now + MessagingPump.RecheckDelay, jobs[^1].DueTime); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, CreateEnvelope(receiver, NewMessage(82, "second")))).Status); + Assert.Equal(scheduled + 1, Fixture.JobManagerProbe.GetScheduledJobs(MessagingPump.JobName, grainId).Count); + } + + var completed = await Fixture.WaitForEffectCountAsync(receiver, 2); + Assert.Equal(new[] { "first", "second" }, completed.Effects.Select(static effect => effect.Value)); + } + + [Theory] + [InlineData(false, false, false)] + [InlineData(false, false, true)] + [InlineData(false, true, false)] + [InlineData(false, true, true)] + [InlineData(true, false, false)] + [InlineData(true, false, true)] + [InlineData(true, true, false)] + [InlineData(true, true, true)] + public async Task InboxWakeup_CapturedWriteRetainsCoverageUntilAcknowledged(bool snapshot, bool stored, bool lateStaging) + { + var receiver = NewGrain(); + _ = await receiver.GetSnapshotAsync(); + var journal = JournalId.FromGrainId(receiver.GetGrainId()); + if (snapshot) Fixture.Storage.RequestSnapshot(journal); + var deliverAt = JobClock.GetUtcNow() + Delay; + var accepted = Schedule(CreateEnvelope(receiver, NewMessage(72, "captured")), deliverAt); + using var acknowledgement = stored ? Fixture.Storage.BlockAcknowledgement(journal) : Fixture.Storage.BlockWrite(journal); + var delivery = DeliverAsync(receiver, accepted); + await acknowledgement.WaitUntilEnteredAsync(); + Assert.False(delivery.IsCompleted); + + await AssertWakeupAtAsync(receiver, deliverAt); + if (lateStaging) + { + var staged = CreateEnvelope(receiver, NewMessage(73, "later-staging")); + await Fixture.OnGrainAsync(receiver, () => InboxTestAccess.StageAcceptance(Fixture.GetInbox(receiver), staged)); + await AssertWakeupAtAsync(receiver, deliverAt); + } + Assert.Empty((await Fixture.OnGrainAsync(receiver, () => Fixture.GetSnapshot(receiver))).Effects); + + acknowledgement.Release(); + Assert.Equal(DeliveryStatus.Accepted, (await delivery).Status); + await ReactivateAsync(receiver); + Assert.Equal(1, (await receiver.GetSnapshotAsync()).InboxCount); + await AssertWakeupAtAsync(receiver, deliverAt); + JobClock.Advance(Delay); + Assert.Equal(DurableJobRunStatus.Completed, (await Fixture.RunPumpAsync(receiver)).Status); + var completed = await Fixture.WaitForEffectCountAsync(receiver, 1); + Assert.Equal("captured", Assert.Single(completed.Effects).Value); + Assert.Equal(1, Assert.Single(completed.Effects).Count); + Assert.Equal(0, completed.InboxCount); + } + + [Fact] + public async Task InboxWakeup_CompletedJobDuringPreparationIsReplacedBeforeCapture() + { + var receiver = NewGrain(); + _ = await receiver.GetSnapshotAsync(); + var deliverAt = JobClock.GetUtcNow() + Delay; + Fixture.JobManagerProbe.SkipNext(MessagingPump.JobName); + using var scheduling = Fixture.JobManagerProbe.BlockNext(MessagingPump.JobName); + var delivery = DeliverAsync(receiver, Schedule(CreateEnvelope(receiver, NewMessage(74, "prepared")), deliverAt)); + await scheduling.WaitUntilEnteredAsync(); + + Assert.Equal(DurableJobRunStatus.Completed, (await Fixture.RunPumpAsync(receiver)).Status); + Assert.False(delivery.IsCompleted); + Assert.Equal(0, Fixture.Storage.GetSuccessfulWriteCount(JournalId.FromGrainId(receiver.GetGrainId()))); + + scheduling.Continue(); + Assert.Equal(DeliveryStatus.Accepted, (await delivery).Status); + Assert.Equal(2, Fixture.JobManagerProbe.GetAttemptCount(MessagingPump.JobName, receiver.GetGrainId())); + Assert.Equal(deliverAt, Assert.Single(Fixture.JobManagerProbe.GetScheduledJobs(MessagingPump.JobName, receiver.GetGrainId())).DueTime); + await AssertWakeupAtAsync(receiver, deliverAt); + JobClock.Advance(Delay); + Assert.Equal(DurableJobRunStatus.Completed, (await Fixture.RunPumpAsync(receiver)).Status); + Assert.Equal("prepared", Assert.Single((await Fixture.WaitForEffectCountAsync(receiver, 1)).Effects).Value); + } + + [Fact] + public async Task ScheduledMessage_IsHeldUntilDeliverAtThenProcessedOnce() + { + var receiver = NewGrain(); + _ = await receiver.GetSnapshotAsync(); + var deliverAt = JobClock.GetUtcNow() + Delay; + var envelope = Schedule(CreateEnvelope(receiver, NewMessage(1, "scheduled")), deliverAt); + + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + + // The acceptance is durable, but wake-ups before the due time leave the message pending and reschedule for it. + await AssertWakeupAtAsync(receiver, deliverAt); + await AssertWakeupAtAsync(receiver, deliverAt); + var pending = await receiver.GetSnapshotAsync(); + Assert.Equal(1, pending.InboxCount); + Assert.Empty(pending.Effects); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(receiver, envelope)).Status); + + JobClock.Advance(Delay - TimeSpan.FromTicks(1)); + await AssertWakeupAtAsync(receiver, deliverAt); + Assert.Empty((await receiver.GetSnapshotAsync()).Effects); + + JobClock.Advance(TimeSpan.FromTicks(1)); + Assert.Equal(DurableJobRunStatus.Completed, (await Fixture.RunPumpAsync(receiver)).Status); + var completed = await Fixture.WaitForEffectCountAsync(receiver, 1); + Assert.Equal("scheduled", Assert.Single(completed.Effects).Value); + Assert.Equal(1, Assert.Single(completed.Effects).Count); + Assert.Equal(0, completed.InboxCount); + + // Late and duplicate wake-ups and redeliveries after completion change nothing. + Assert.Equal(DurableJobRunStatus.Completed, (await Fixture.RunPumpAsync(receiver)).Status); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(receiver, envelope)).Status); + Assert.Equal(1, Assert.Single((await receiver.GetSnapshotAsync()).Effects).Count); + } + + [Fact] + public async Task EarlierScheduledMessage_SchedulesAnEarlierWakeupWhichSupersedesTheLaterOne() + { + var receiver = NewGrain(); + _ = await receiver.GetSnapshotAsync(); + var grainId = receiver.GetGrainId(); + var later = JobClock.GetUtcNow() + Delay; + var earlier = JobClock.GetUtcNow() + TimeSpan.FromMinutes(1); + + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, Schedule(CreateEnvelope(receiver, NewMessage(20, "later")), later))).Status); + var laterJob = Assert.Single(Fixture.JobManagerProbe.GetScheduledJobs(MessagingPump.JobName, grainId)); + Assert.Equal(later, laterJob.DueTime); + + // The later wake-up does not cover a message due first, so its acceptance schedules an earlier one. + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, Schedule(CreateEnvelope(receiver, NewMessage(21, "earlier")), earlier))).Status); + var jobs = Fixture.JobManagerProbe.GetScheduledJobs(MessagingPump.JobName, grainId); + Assert.Equal(2, jobs.Count); + Assert.Equal(earlier, jobs[1].DueTime); + + // Any other wake-up completes in favour of the earliest one, whose run reschedules for the remaining work. + Assert.Equal(DurableJobRunStatus.Completed, (await Fixture.RunDuplicatePumpAsync(receiver)).Status); + await AssertWakeupAtAsync(receiver, earlier); + JobClock.Advance(earlier - JobClock.GetUtcNow()); + await AssertWakeupAtAsync(receiver, later); + Assert.Equal("earlier", Assert.Single((await Fixture.WaitForEffectCountAsync(receiver, 1)).Effects).Value); + Assert.Equal(DurableJobRunStatus.Completed, (await Fixture.RunDuplicatePumpAsync(receiver)).Status); + + // Work due later than the earliest wake-up schedules nothing further. + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, Schedule(CreateEnvelope(receiver, NewMessage(22, "last")), later + Delay))).Status); + Assert.Equal(2, Fixture.JobManagerProbe.GetScheduledJobs(MessagingPump.JobName, grainId).Count); + await AssertWakeupAtAsync(receiver, later); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task ScheduledMessage_SurvivesReactivationAndRestoresCapacityAccounting(bool snapshot) + { + var receiver = NewGrain(); + _ = await receiver.GetSnapshotAsync(); + if (snapshot) + { + Fixture.Storage.RequestSnapshot(JournalId.FromGrainId(receiver.GetGrainId())); + } + + var deliverAt = JobClock.GetUtcNow() + Delay; + var envelope = Schedule(CreateEnvelope(receiver, NewMessage(2, "recovered")), deliverAt); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + + await ReactivateAsync(receiver); + + var recovered = await receiver.GetSnapshotAsync(); + Assert.Equal(1, recovered.InboxCount); + Assert.Empty(recovered.Effects); + var inbox = Fixture.GetInbox(receiver); + var recoveredDeliverAt = await Fixture.OnGrainAsync(receiver, + () => InboxTestAccess.GetPendingMessages(inbox)[envelope.Envelope.MessageId].Message.Envelope.DeliverAt); + Assert.Equal(deliverAt, recoveredDeliverAt); + await AssertWakeupAtAsync(receiver, deliverAt); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(receiver, envelope)).Status); + + // The recovered scheduled message counts toward MaxCapacity (2). + var second = CreateEnvelope(receiver, NewMessage(4, "second")); + using (var held = Fixture.HandlerProbe.Arm(receiver.GetGrainId(), HeldRoute)) + { + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, CreateEnvelope(receiver, NewMessage(3, "held"), HeldRoute))).Status); + await WaitForBarrierAsync(receiver, held); + Assert.Equal(DeliveryStatus.Backpressured, (await DeliverAsync(receiver, second)).Status); + } + + Assert.Equal("held", Assert.Single((await Fixture.WaitForEffectCountAsync(receiver, 1)).Effects).Value); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, second)).Status); + var unscheduled = await Fixture.WaitForEffectCountAsync(receiver, 2); + Assert.DoesNotContain(unscheduled.Effects, effect => effect.Value == "recovered"); + + JobClock.Advance(Delay); + _ = await Fixture.RunPumpAsync(receiver); + var completed = await Fixture.WaitForEffectCountAsync(receiver, 3); + Assert.Equal(new[] { "recovered", "held", "second" }, completed.Effects.Select(static effect => effect.Value)); + Assert.All(completed.Effects, static effect => Assert.Equal(1, effect.Count)); + Assert.Equal(0, completed.InboxCount); + } + + [Fact] + public async Task ScheduledMessages_CountTowardCapacityFromAcceptance() + { + var receiver = NewGrain(); + _ = await receiver.GetSnapshotAsync(); + var deliverAt = JobClock.GetUtcNow() + Delay; + + // MaxCapacity is 2, and scheduled messages hold capacity while they wait. + for (var sequence = 10; sequence < 12; sequence++) + { + var scheduled = Schedule(CreateEnvelope(receiver, NewMessage(sequence, $"scheduled-{sequence}")), deliverAt); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, scheduled)).Status); + } + + var laterScheduled = Schedule(CreateEnvelope(receiver, NewMessage(12, "scheduled-12")), deliverAt); + var unscheduled = CreateEnvelope(receiver, NewMessage(20, "unscheduled")); + var past = Schedule(CreateEnvelope(receiver, NewMessage(21, "past")), DateTimeOffset.MinValue); + Assert.Equal(DeliveryStatus.Backpressured, (await DeliverAsync(receiver, laterScheduled)).Status); + Assert.Equal(DeliveryStatus.Backpressured, (await DeliverAsync(receiver, unscheduled)).Status); + Assert.Equal(DeliveryStatus.Backpressured, (await DeliverAsync(receiver, past)).Status); + Assert.Equal(2, (await receiver.GetSnapshotAsync()).InboxCount); + + JobClock.Advance(Delay); + _ = await Fixture.RunPumpAsync(receiver); + Assert.Equal(0, (await Fixture.WaitForEffectCountAsync(receiver, 2)).InboxCount); + + Assert.Equal(DeliveryStatus.Expired, (await DeliverAsync(receiver, laterScheduled)).Status); + Assert.Equal(DeliveryStatus.Expired, (await DeliverAsync(receiver, unscheduled)).Status); + laterScheduled = laterScheduled with { SentAt = JobClock.GetUtcNow() }; + unscheduled = unscheduled with { SentAt = JobClock.GetUtcNow() }; + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, laterScheduled)).Status); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, unscheduled)).Status); + var all = await Fixture.WaitForEffectCountAsync(receiver, 4); + Assert.All(all.Effects, static effect => Assert.Equal(1, effect.Count)); + } + [Fact] + public async Task ScheduledMessage_RedeliveryCoalescesAndDifferentDeliverAtConflicts() + { + var receiver = NewGrain(); + _ = await receiver.GetSnapshotAsync(); + var deliverAt = JobClock.GetUtcNow() + Delay; + var envelope = Schedule(CreateEnvelope(receiver, NewMessage(30, "identity")), deliverAt); + + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(receiver, envelope)).Status); + Assert.Equal(DeliveryStatus.Duplicate, + (await DeliverAsync(receiver, Schedule(envelope, deliverAt.ToOffset(TimeSpan.FromHours(-7))))).Status); + await Assert.ThrowsAsync(() => DeliverAsync(receiver, Schedule(envelope, deliverAt + TimeSpan.FromSeconds(1)))); + await Assert.ThrowsAsync(() => DeliverAsync(receiver, Schedule(envelope, null))); + Assert.Equal(1, (await receiver.GetSnapshotAsync()).InboxCount); + var inbox = Fixture.GetInbox(receiver); + Assert.Equal(deliverAt, await Fixture.OnGrainAsync(receiver, + () => InboxTestAccess.GetPendingMessages(inbox)[envelope.Envelope.MessageId].Message.Envelope.DeliverAt)); + + JobClock.Advance(Delay); + Assert.Equal(DurableJobRunStatus.Completed, (await Fixture.RunPumpAsync(receiver)).Status); + Assert.Equal(1, Assert.Single((await Fixture.WaitForEffectCountAsync(receiver, 1)).Effects).Count); + + // Completed IDs are deduplicated regardless of content. + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(receiver, Schedule(envelope, deliverAt + Delay))).Status); + Assert.Equal(1, Assert.Single((await receiver.GetSnapshotAsync()).Effects).Count); + } + + [Fact] + public async Task SelfSentScheduledMessage_IsAcceptedWithItsWriteAndProcessedWhenDue() + { + var grain = NewGrain(); + _ = await grain.GetSnapshotAsync(); + var deliverAt = JobClock.GetUtcNow() + Delay; + var message = NewMessage(40, "self"); + var outgoing = TestApplicationProtocol.Create(Sessions, grain.GetGrainId(), grain.GetGrainId(), "messages/self", message); + outgoing = new OutboxMessage { ReceiverId = outgoing.ReceiverId, Envelope = outgoing.Envelope with { DeliverAt = deliverAt } }; + + await grain.StageOutputAsync(outgoing); + await grain.RetryWriteStateAsync(); + + // The grain's own inbox accepted the message in that write, without waiting for the due time. + var delivered = await Fixture.SnapshotProbe.WaitAsync( + grain.GetGrainId(), static snapshot => snapshot.InboxCount == 1 && snapshot.OutboxCount == 0); + Assert.Empty(delivered.Effects); + await AssertWakeupAtAsync(grain, deliverAt); + Assert.Empty((await grain.GetSnapshotAsync()).Effects); + + JobClock.Advance(Delay); + Assert.Equal(DurableJobRunStatus.Completed, (await Fixture.RunPumpAsync(grain)).Status); + var completed = await Fixture.WaitForEffectCountAsync(grain, 1); + Assert.Equal("self", Assert.Single(completed.Effects).Value); + Assert.Equal(0, completed.InboxCount); + Assert.Equal(0, completed.OutboxCount); + } + + [Fact] + public async Task ScheduledMessage_LeavesActivationCollectableUntilDue() + { + var receiver = NewGrain(); + var before = await receiver.GetSnapshotAsync(); + var deliverAt = JobClock.GetUtcNow() + Delay; + var envelope = Schedule(CreateEnvelope(receiver, NewMessage(50, "collected")), deliverAt); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + await AssertWakeupAtAsync(receiver, deliverAt); + + // Work which is due later does not keep the activation alive; the durable wake-up reactivates it. + var context = Fixture.GetGrainContext(receiver); + await Fixture.CollectAsync(context); + + var reactivated = await receiver.GetSnapshotAsync(); + Assert.NotEqual(before.ActivationId, reactivated.ActivationId); + Assert.Equal(1, reactivated.InboxCount); + JobClock.Advance(Delay); + Assert.Equal(DurableJobRunStatus.Completed, (await Fixture.RunPumpAsync(receiver)).Status); + Assert.Equal("collected", Assert.Single((await Fixture.WaitForEffectCountAsync(receiver, 1)).Effects).Value); + } + + private async Task AssertWakeupAtAsync(IMessagingTestGrain grain, DateTimeOffset deliverAt) + { + var result = await Fixture.RunPumpAsync(grain); + Assert.Equal(DurableJobRunStatus.RescheduleRequested, result.Status); + Assert.Equal(deliverAt, result.RescheduleTime); + } + + private static InboxMessage Schedule(InboxMessage message, DateTimeOffset? deliverAt) => + message with { Envelope = message.Envelope with { DeliverAt = deliverAt } }; + + private static async Task WaitUntilAsync(Func condition) + { + var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(30); + while (!condition()) + { + if (DateTime.UtcNow > deadline) throw new TimeoutException("The condition was not met."); + await Task.Delay(10, TestContext.Current.CancellationToken); + } + } + + private sealed class ScheduledDeliveryClusterFixture() : MessagingClusterFixture(1, receiverOnly: false) + { + public ManualJobClock JobClock { get; } = new(DateTimeOffset.UtcNow); + + protected override void ConfigureServices(IServiceCollection services) => + services.AddKeyedSingleton(DurableJobTimeProviderNames.DurableJobs, JobClock); + } +} diff --git a/test/Orleans.Messaging.Tests/Functional/SelfAddressedMessageTests.cs b/test/Orleans.Messaging.Tests/Functional/SelfAddressedMessageTests.cs new file mode 100644 index 00000000000..15a03be0998 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/SelfAddressedMessageTests.cs @@ -0,0 +1,395 @@ +using Microsoft.Extensions.DependencyInjection; +using Orleans.DurableJobs; +using Orleans.Journaling; +using Orleans.Messaging.Tests.Support; +using Orleans.Runtime; +using Xunit; + +namespace Orleans.Messaging.Tests.Functional; + +/// +/// Messages a grain sends to itself are accepted by its own inbox in the caller's journal write, without an outbox entry. +/// +[Collection(MessagingClusterCollection.Name)] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class SelfAddressedMessageTests() : MessagingBehaviorTestBase(new SelfAddressedClusterFixture()) +{ + private static readonly TimeSpan Delay = TimeSpan.FromHours(1); + + private ManualJobClock JobClock => ((SelfAddressedClusterFixture)Fixture).JobClock; + + [Fact] + public async Task SelfSend_IsAcceptedWithoutOutboxEntryInTheCallersSingleWrite() + { + var grain = NewGrain(); + _ = await grain.GetSnapshotAsync(); + var instance = GetInstance(grain); + var journal = JournalId.FromGrainId(grain.GetGrainId()); + const string route = "messages/self-single-write"; + using var held = Fixture.HandlerProbe.Arm(grain.GetGrainId(), route); + var writes = Fixture.Storage.GetSuccessfulWriteCount(journal); + var effect = new MessageEffect(TestApplicationProtocol.NewMessageId(), 1, 0, "caller-state"); + + await grain.StageEffectAsync(effect); + await grain.StageOutputAsync(SelfMessage(grain, NewMessage(1, "self"), route)); + var staged = await grain.GetSnapshotAsync(); + Assert.Equal(1, staged.InboxCount); + Assert.Equal(0, staged.OutboxCount); + Assert.Equal(writes, Fixture.Storage.GetSuccessfulWriteCount(journal)); + + instance.ClearCaptures(); + await grain.RetryWriteStateAsync(); + var written = Assert.Single(instance.Captures); + Assert.Equal(1, written.InboxCount); + Assert.Equal(0, written.OutboxCount); + Assert.Equal(effect, Assert.Single(written.Effects)); + Assert.Empty(instance.OutputCaptures[^1]); + + // Processing starts after that one write; the handler is held before it stages any further write. + await WaitForBarrierAsync(grain, held); + Assert.Equal(writes + 1, Fixture.Storage.GetSuccessfulWriteCount(journal)); + held.Release(); + var completed = await Fixture.WaitForEffectCountAsync(grain, 2); + Assert.Equal(0, completed.InboxCount); + Assert.Equal(0, completed.OutboxCount); + Assert.Equal(1, completed.ProcessedMessageCount); + Assert.Equal(writes + 2, Fixture.Storage.GetSuccessfulWriteCount(journal)); + } + + [Fact] + public async Task SelfSend_IsProcessedOnlyAfterItsWriteIsDurable() + { + var grain = NewGrain(); + _ = await grain.GetSnapshotAsync(); + var instance = GetInstance(grain); + var storage = Fixture.Storage.BlockWrite(JournalId.FromGrainId(grain.GetGrainId())); + + var send = grain.SendAsync(grain.GetGrainId(), "messages/self-blocked", NewMessage(2, "blocked")); + await storage.WaitUntilEnteredAsync(); + + // A full pump pass while the write is in flight leaves the accepted message pending. + Assert.Equal(DurableJobRunStatus.RescheduleRequested, (await Fixture.RunPumpAsync(grain)).Status); + var pending = await Fixture.OnGrainAsync(grain, () => instance.GetSnapshotForTest()); + Assert.Equal(1, pending.InboxCount); + Assert.Equal(0, pending.OutboxCount); + Assert.Empty(pending.Effects); + + storage.Release(); + await send; + var completed = await Fixture.WaitForEffectCountAsync(grain, 1); + Assert.Equal("blocked", Assert.Single(completed.Effects).Value); + Assert.Equal(0, completed.InboxCount); + } + + [Fact] + public async Task SelfSend_IsDiscardedWhenItsWriteFails() + { + var grain = NewGrain(); + var initial = await grain.GetSnapshotAsync(); + var context = Fixture.GetGrainContext(grain); + Fixture.Storage.FailWrite(JournalId.FromGrainId(grain.GetGrainId())); + + await Assert.ThrowsAnyAsync(() => grain.SendAsync(grain.GetGrainId(), "messages/self-failed", NewMessage(3, "failed"))); + + await context.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + var recovered = await grain.GetSnapshotAsync(); + Assert.NotEqual(initial.ActivationId, recovered.ActivationId); + Assert.Equal(0, recovered.InboxCount); + Assert.Equal(0, recovered.OutboxCount); + Assert.Equal(DurableJobRunStatus.Completed, (await Fixture.RunPumpAsync(grain)).Status); + var after = await grain.GetSnapshotAsync(); + Assert.Empty(after.Effects); + Assert.Equal(0, after.ProcessedMessageCount); + } + + [Fact] + public async Task SelfSend_FollowsInboxIdentityRules() + { + var grain = NewGrain(); + _ = await grain.GetSnapshotAsync(); + const string route = "messages/self-identity"; + var message = SelfMessage(grain, NewMessage(4, "identity"), route); + var differentBody = SelfMessage(grain, NewMessage(5, "different"), route, messageId: message.Envelope.MessageId); + var differentDeliverAt = WithDeliverAt(message, JobClock.GetUtcNow() + Delay); + + using (var held = Fixture.HandlerProbe.Arm(grain.GetGrainId(), route)) + { + await grain.StageOutputAsync(message); + await grain.StageOutputAsync(message); + await Assert.ThrowsAsync(() => grain.StageOutputAsync(differentBody)); + await Assert.ThrowsAsync(() => grain.StageOutputAsync(differentDeliverAt)); + Assert.Equal(1, (await grain.GetSnapshotAsync()).InboxCount); + + // The outbox keys pending messages by ID, so the ID cannot address another grain while it is pending there. + var other = SelfMessage(grain, NewMessage(6, "other"), route); + var otherReceiver = NewGrain().GetGrainId(); + await grain.StageOutputAsync(new OutboxMessage { ReceiverId = otherReceiver, Envelope = other.Envelope }); + await Assert.ThrowsAsync(() => grain.StageOutputAsync(other)); + + await grain.RetryWriteStateAsync(); + await WaitForBarrierAsync(grain, held); + } + + var completed = await Fixture.WaitForEffectCountAsync(grain, 1); + Assert.Equal(1, completed.ProcessedMessageCount); + + // A completed ID is a duplicate regardless of content while its completion record is retained. + await grain.StageOutputAsync(message); + await grain.StageOutputAsync(differentBody); + var repeated = await grain.GetSnapshotAsync(); + Assert.Equal(0, repeated.InboxCount); + await grain.RetryWriteStateAsync(); + _ = await Fixture.RunPumpAsync(grain); + var final = await grain.GetSnapshotAsync(); + Assert.Equal(1, Assert.Single(final.Effects).Count); + Assert.Empty(final.InboxDeadLetters); + } + + [Fact] + public async Task SelfSend_IsAdmittedBeyondCapacityAndWithoutHandlers() + { + var grain = NewGrain(); + _ = await grain.GetSnapshotAsync(); + await grain.ConfigureHandlerAsync(false); + await ReactivateAsync(grain); + + // MaxCapacity is 2 and no handler is registered; a synchronous send admits every message. + for (var sequence = 10; sequence < 13; sequence++) + { + await grain.StageOutputAsync(SelfMessage(grain, NewMessage(sequence, $"self-{sequence}"), "messages/self-capacity")); + } + + await grain.RetryWriteStateAsync(); + var admitted = await grain.GetSnapshotAsync(); + Assert.Equal(3, admitted.InboxCount); + Assert.Equal(0, admitted.OutboxCount); + + // Unscheduled self-sent messages occupy capacity for deliveries from other grains. + var external = CreateEnvelope(grain, NewMessage(20, "external")); + Assert.Equal(DeliveryStatus.Backpressured, (await DeliverAsync(grain, external)).Status); + Assert.Equal(DurableJobRunStatus.Completed, (await Fixture.RunPumpAsync(grain)).Status); + var waiting = await grain.GetSnapshotAsync(); + Assert.Equal(3, waiting.InboxCount); + Assert.Empty(waiting.Effects); + Assert.Empty(waiting.InboxDeadLetters); + + await grain.ConfigureHandlerAsync(true); + var completed = await Fixture.WaitForEffectCountAsync(grain, 3); + Assert.Equal(0, completed.InboxCount); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(grain, external)).Status); + await Fixture.WaitForEffectCountAsync(grain, 4); + } + + [Fact] + public async Task SelfSendFromHandler_IsProcessedInALaterBatch() + { + var grain = NewGrain(); + _ = await grain.GetSnapshotAsync(); + var instance = GetInstance(grain); + instance.ClearCaptures(); + var message = NewMessage(30, "forwarding") with { ForwardTo = grain.GetGrainId() }; + + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(grain, CreateEnvelope(grain, message))).Status); + + var completed = await Fixture.WaitForEffectCountAsync(grain, 2); + Assert.Equal(2, Assert.Single(completed.Effects).Count); + Assert.Equal(0, completed.InboxCount); + Assert.Equal(0, completed.OutboxCount); + Assert.Equal(2, completed.ProcessedMessageCount); + + // The batch which processed the original also accepted the forwarded message, which a later batch processed. + var captures = instance.Captures; + var original = Assert.Single(captures, static capture => capture.Effects.Sum(static effect => effect.Count) == 1); + Assert.Equal(1, original.InboxCount); + Assert.Equal(0, original.OutboxCount); + Assert.Equal(1, original.ProcessedMessageCount); + Assert.All(instance.OutputCaptures, static output => Assert.Empty(output)); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task ScheduledSelfSend_SurvivesRecoveryAndIsProcessedWhenDue(bool snapshot) + { + var grain = NewGrain(); + _ = await grain.GetSnapshotAsync(); + var journal = JournalId.FromGrainId(grain.GetGrainId()); + if (snapshot) + { + Fixture.Storage.RequestSnapshot(journal); + } + + var deliverAt = JobClock.GetUtcNow() + Delay; + var message = WithDeliverAt(SelfMessage(grain, NewMessage(40, "recovered"), "messages/self-recovered"), deliverAt); + await grain.StageOutputAsync(message); + await grain.RetryWriteStateAsync(); + + await ReactivateAsync(grain); + var recovered = await grain.GetSnapshotAsync(); + Assert.Equal(1, recovered.InboxCount); + Assert.Equal(0, recovered.OutboxCount); + Assert.Empty(recovered.Effects); + var inbox = Fixture.GetInbox(grain); + Assert.Equal(deliverAt, await Fixture.OnGrainAsync(grain, + () => InboxTestAccess.GetPendingMessages(inbox)[message.Envelope.MessageId].Message.Envelope.DeliverAt)); + + var held = await Fixture.RunPumpAsync(grain); + Assert.Equal(DurableJobRunStatus.RescheduleRequested, held.Status); + Assert.Equal(deliverAt, held.RescheduleTime); + + // The recovered pending message deduplicates a repeated send. + await grain.StageOutputAsync(message); + Assert.Equal(1, (await grain.GetSnapshotAsync()).InboxCount); + + JobClock.Advance(Delay); + Assert.Equal(DurableJobRunStatus.Completed, (await Fixture.RunPumpAsync(grain)).Status); + var completed = await Fixture.WaitForEffectCountAsync(grain, 1); + Assert.Equal("recovered", Assert.Single(completed.Effects).Value); + Assert.Equal(0, completed.InboxCount); + Assert.Equal(0, completed.OutboxCount); + } + + [Fact] + public async Task SelfSend_RecordsSendAndReceiptWithoutOutboxMetrics() + { + var grain = NewGrain(); + _ = await grain.GetSnapshotAsync(); + var services = Fixture.GetGrainContext(grain).ActivationServices; + using var probe = new MessageMetricListener(services.GetRequiredService().Meter); + var grainType = grain.GetGrainId().Type.ToString(); + const string route = "messages/self-metrics"; + var message = SelfMessage(grain, NewMessage(50, "metrics"), route); + + using (var held = Fixture.HandlerProbe.Arm(grain.GetGrainId(), route)) + { + await grain.StageOutputAsync(message); + await grain.StageOutputAsync(message); + await grain.RetryWriteStateAsync(); + await WaitForBarrierAsync(grain, held); + } + + await Fixture.WaitForEffectCountAsync(grain, 1); + var sent = probe.Read("outbox-messages-sent"); + Assert.Equal(2, sent.Length); + Assert.All(sent, row => row.AssertCounter(grainType)); + var received = probe.Read("inbox-messages-received"); + Assert.Equal(2, received.Length); + received[0].AssertCounter(grainType, "accepted"); + received[1].AssertCounter(grainType, "duplicate"); + Assert.Equal([1d, -1d], probe.Read("inbox-depth").Select(static row => row.Value)); + Assert.Empty(probe.Read("outbox-depth")); + Assert.Empty(probe.Read("outbox-messages-delivered")); + Assert.Empty(probe.Read("outbox-delivery-duration")); + } + + [Fact] + public async Task RecoveredOutboxEntryAddressedToItsOwnGrain_IsRejected() + { + var grain = NewGrain(); + _ = await grain.GetSnapshotAsync(); + var outbox = Fixture.GetGrainContext(grain).ActivationServices.GetRequiredService(); + var message = SelfMessage(grain, NewMessage(60, "outbox-entry"), "messages/self-outbox-entry"); + + // Replays an outbox entry which a correct journal never contains, then validates the recovered state. + var error = await Assert.ThrowsAsync(() => Fixture.OnGrainAsync(grain, () => + { + ((IDurableValueCommandHandler)outbox).ApplySet(new OutboxCommand + { + Kind = OutboxCommandKind.Enqueue, + Message = message, + State = new() { SentAt = JobClock.GetUtcNow() } + }); + ((IStateMachine)outbox).OnRecoveryCompleted(); + })); + Assert.Contains("addressed to its own grain", error.Message, StringComparison.Ordinal); + } + + [Fact] + public async Task SelfSend_StagesAndExtendsSentAtAcrossClockMotionSnapshotAndRecovery() + { + var grain = NewGrain(); + var started = JobClock.GetUtcNow(); + var due = started + Delay; + var output = WithDeliverAt(SelfMessage(grain, NewMessage(61, "local-sent-time"), "messages/self-sent-time"), due); + await grain.StageOutputAsync(output); + await grain.RetryWriteStateAsync(); + + var inbox = Fixture.GetInbox(grain); + Assert.Equal(started, await Fixture.OnGrainAsync(grain, () => + InboxTestAccess.GetPendingMessages(inbox)[output.Envelope.MessageId].Message.SentAt)); + JobClock.Advance(TimeSpan.FromMinutes(1)); + await grain.StageOutputAsync(output); + await grain.RetryWriteStateAsync(); + var sentAt = JobClock.GetUtcNow(); + Assert.Equal(sentAt, await Fixture.OnGrainAsync(grain, () => + InboxTestAccess.GetPendingMessages(inbox)[output.Envelope.MessageId].Message.SentAt)); + + JobClock.Advance(-TimeSpan.FromDays(2)); + await grain.StageOutputAsync(output); + Fixture.Storage.RequestSnapshot(JournalId.FromGrainId(grain.GetGrainId())); + await grain.RetryWriteStateAsync(); + await ReactivateAsync(grain); + inbox = Fixture.GetInbox(grain); + var recovered = Assert.Single(await Fixture.OnGrainAsync(grain, () => InboxTestAccess.GetPendingMessages(inbox).Values.ToArray())); + Assert.Equal(sentAt, recovered.Message.SentAt); + Assert.Equal(due, recovered.Message.Envelope.DeliverAt); + Assert.Equal(0, (await grain.GetSnapshotAsync()).OutboxCount); + await grain.StageOutputAsync(output); + await grain.RetryWriteStateAsync(); + JobClock.Advance(due - JobClock.GetUtcNow()); + await Fixture.RunPumpAsync(grain); + await Fixture.WaitForInboxCountAsync(grain, 0); + var completed = await grain.GetSnapshotAsync(); + Assert.Equal("local-sent-time", Assert.Single(completed.Effects).Value); + Assert.Equal(1, Assert.Single(completed.Effects).Count); + Assert.Equal(0, completed.OutboxCount); + Fixture.Storage.RequestSnapshot(JournalId.FromGrainId(grain.GetGrainId())); + await grain.RetryWriteStateAsync(); + var replay = output.ToInboxMessage(sentAt); + Assert.Equal(DeliveryStatus.Expired, (await DeliverAsync(grain, replay)).Status); + } + + [Fact] + public async Task SelfSend_AfterClockRollback_StampsSentAtAboveTheDurableFloor() + { + var grain = NewGrain(); + var original = JobClock.GetUtcNow(); + JobClock.Advance(TimeSpan.FromDays(2)); + var floor = JobClock.GetUtcNow() - TimeSpan.FromMinutes(10); + var stale = CreateEnvelope(grain, NewMessage(62, "floor")) with { SentAt = floor }; + Assert.Equal(DeliveryStatus.Expired, (await DeliverAsync(grain, stale)).Status); + JobClock.Advance(original - JobClock.GetUtcNow()); + var output = WithDeliverAt(SelfMessage(grain, NewMessage(63, "after-rollback"), "messages/self-above-floor"), original + Delay); + await grain.StageOutputAsync(output); + var inbox = Fixture.GetInbox(grain); + var sentAt = await Fixture.OnGrainAsync(grain, () => + InboxTestAccess.GetPendingMessages(inbox)[output.Envelope.MessageId].Message.SentAt); + Assert.True(sentAt > floor); + Assert.Equal(0, (await grain.GetSnapshotAsync()).OutboxCount); + await grain.StageOutputAsync(output); + await grain.RetryWriteStateAsync(); + JobClock.Advance(Delay); + await Fixture.RunPumpAsync(grain); + await Fixture.WaitForInboxCountAsync(grain, 0); + Assert.Equal(1, Assert.Single((await grain.GetSnapshotAsync()).Effects).Count); + } + + private MessagingTestGrain GetInstance(IMessagingTestGrain grain) => + Assert.IsType(Fixture.GetGrainContext(grain).GrainInstance); + + private OutboxMessage SelfMessage(IMessagingTestGrain grain, TestMessage message, string route, MessageId messageId = default) => + TestApplicationProtocol.Create(Sessions, grain.GetGrainId(), grain.GetGrainId(), route, message, messageId); + + private static OutboxMessage WithDeliverAt(OutboxMessage message, DateTimeOffset? deliverAt) => + new() { ReceiverId = message.ReceiverId, Envelope = message.Envelope with { DeliverAt = deliverAt } }; + + private sealed class SelfAddressedClusterFixture() : MessagingClusterFixture(1, receiverOnly: false) + { + public ManualJobClock JobClock { get; } = new(DateTimeOffset.UtcNow); + + protected override void ConfigureServices(IServiceCollection services) => + services.AddKeyedSingleton(DurableJobTimeProviderNames.DurableJobs, JobClock); + } +} diff --git a/test/Orleans.Messaging.Tests/Functional/StandardStateJsonContext.cs b/test/Orleans.Messaging.Tests/Functional/StandardStateJsonContext.cs new file mode 100644 index 00000000000..a8bc38600e9 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/StandardStateJsonContext.cs @@ -0,0 +1,9 @@ +using System.Text.Json.Serialization; + +namespace Orleans.Messaging.Tests.Functional; + +[JsonSerializable(typeof(string))] +[JsonSerializable(typeof(int))] +[JsonSerializable(typeof(uint))] +[JsonSerializable(typeof(DateTime))] +internal partial class StandardStateJsonContext : JsonSerializerContext; diff --git a/test/Orleans.Messaging.Tests/Hosting/PublicMessagingRegistrationTests.cs b/test/Orleans.Messaging.Tests/Hosting/PublicMessagingRegistrationTests.cs new file mode 100644 index 00000000000..abe5a402e12 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Hosting/PublicMessagingRegistrationTests.cs @@ -0,0 +1,208 @@ +using System.Diagnostics.CodeAnalysis; +using System.Runtime.CompilerServices; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Options; +using Orleans.DurableJobs; +using NSubstitute; +using Orleans.Hosting; +using Orleans.Journaling; +using Orleans.Messaging.Configuration; +using Orleans.Messaging.Tests.Support; +using Orleans.Runtime; +using Xunit; + +namespace Orleans.Messaging.Tests.Hosting; + +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class PublicMessagingRegistrationTests +{ + [Theory] + [InlineData(false)] + [InlineData(true)] + public void AddMessaging_RepeatedPublicRegistrationRetainsOneConfiguratorAndScopedBindings(bool useSiloBuilder) + { + var builder = new TestSiloBuilder(); + builder.AddJournaling(); + var services = builder.Services; + for (var invocation = 0; invocation < 2; invocation++) + { + if (useSiloBuilder) + { + Assert.Same(builder, builder.AddMessaging()); + } + else + { + Assert.Same(services, services.AddMessaging()); + } + } + + var configurator = Assert.Single(services, descriptor => + descriptor.ServiceType == typeof(IConfigureGrainTypeComponents) + && descriptor.ImplementationType == typeof(MessagingGrainTypeConfigurator)); + Assert.Equal(ServiceLifetime.Singleton, configurator.Lifetime); + foreach (var contract in new[] + { + typeof(MessagingPump), typeof(Inbox), typeof(Outbox), typeof(IInbox), typeof(IOutbox), typeof(IMessagingDiagnostics) + }) + { + var descriptor = Assert.Single(services, descriptor => descriptor.ServiceType == contract && !descriptor.IsKeyedService); + Assert.Equal(ServiceLifetime.Scoped, descriptor.Lifetime); + } + + // The configurator binds the messaging extension at activation; the only keyed grain extension is the durable-jobs receiver. + var extension = Assert.Single(services, descriptor => descriptor.ServiceType == typeof(IGrainExtension)); + Assert.Equal(typeof(Orleans.DurableJobs.DurableJob).Assembly, Assert.IsAssignableFrom(extension.ServiceKey).Assembly); + } + + [Fact] + public void AddMessaging_PreservesCallerSuppliedTimeProviderAndAppliesOptions() + { + var expectedTime = new FixedTimeProvider(new DateTimeOffset(2040, 2, 3, 4, 5, 6, TimeSpan.Zero)); + var services = new ServiceCollection(); + services.AddSingleton(expectedTime); + + services.AddMessaging(options => + { + options.MaxCapacity = 17; + options.InboxBatchSize = 4; + options.OutboxBatchSize = 5; + }); + using var provider = services.BuildServiceProvider(); + + Assert.Same(expectedTime, provider.GetRequiredService()); + var configured = provider.GetRequiredService>().Value; + Assert.Equal(17, configured.MaxCapacity); + Assert.Equal(4, configured.InboxBatchSize); + Assert.Equal(5, configured.OutboxBatchSize); + } + + [Theory] + [InlineData("json")] + [InlineData("orleans-binary")] + public void AddMessaging_PreservesConfiguredJournalFormat(string format) + { + var services = new ServiceCollection(); + services.Configure(options => options.JournalFormatKey = format); + services.AddMessaging(); + using var provider = services.BuildServiceProvider(); + + Assert.Equal(format, provider.GetRequiredService>().Value.JournalFormatKey); + } + + [Theory] + [InlineData(1)] + [InlineData(5)] + [InlineData(50)] + public void AddMessaging_RetriesThePumpJobIndefinitelyOnTheDurableJobsClockAndKeepsOtherJobsPolicy(int dequeueCount) + { + var clock = new ManualJobClock(new DateTimeOffset(2026, 1, 1, 0, 0, 0, TimeSpan.Zero)); + var services = new ServiceCollection(); + services.AddMessaging(options => options.RetryInitialInterval = TimeSpan.FromSeconds(2)); + services.AddKeyedSingleton(DurableJobTimeProviderNames.DurableJobs, clock); + using var provider = services.BuildServiceProvider(); + var options = provider.GetRequiredService>().Value; + var error = new IOException("The journal is unavailable."); + + var retryAt = options.ShouldRetry(new RunContext(MessagingPump.JobName, dequeueCount), error); + + var multiplier = 1L << Math.Min(dequeueCount - 1, DurableMessagingOptions.MaximumBackoffExponent); + Assert.Equal(clock.GetUtcNow() + TimeSpan.FromSeconds(2 * multiplier), retryAt); + var defaultPolicy = new DurableJobsOptions().ShouldRetry(new RunContext("other", dequeueCount), error); + Assert.Equal(defaultPolicy is null, options.ShouldRetry(new RunContext("other", dequeueCount), error) is null); + } + + private sealed class RunContext(string name, int dequeueCount) : IJobRunContext + { + public DurableJob Job { get; } = new() { Id = "job", Name = name, ShardId = "shard", TargetGrainId = GrainId.Create("test", "1"), DueTime = DateTimeOffset.UnixEpoch }; + public string RunId => "run"; + public int DequeueCount => dequeueCount; + } + [Fact] + public void AddMessaging_InvalidOptionsFailThroughOptionsContract() + { + var services = new ServiceCollection(); + services.AddMessaging(options => options.MaxCapacity = 0); + using var provider = services.BuildServiceProvider(); + + var exception = Assert.Throws( + () => provider.GetRequiredService>().Value); + + Assert.Contains("DurableMessagingOptions validation failed", exception.Message, StringComparison.Ordinal); + } + + + [Fact] + public async Task AddMessaging_BindsInboxContractToDeliveredInboxOverEarlierRegistration() + { + var services = new ServiceCollection(); + services.AddScoped(); + services.AddMessaging(); + + // Inbox construction needs an activation, so substitute an uninitialized scoped instance to observe the binding. + services.Replace(ServiceDescriptor.Scoped(static _ => (Inbox)RuntimeHelpers.GetUninitializedObject(typeof(Inbox)))); + await using var provider = services.BuildServiceProvider(); + await using var scope = provider.CreateAsyncScope(); + + Assert.Single(services, static descriptor => descriptor.ServiceType == typeof(IInbox)); + Assert.Same(scope.ServiceProvider.GetRequiredService(), scope.ServiceProvider.GetRequiredService()); + } + + [Fact] + public async Task AddMessaging_BindsOutboxContractToDeliveredOutboxOverEarlierRegistration() + { + var services = new ServiceCollection(); + services.AddScoped(static _ => Substitute.For()); + services.AddMessaging(); + + // Outbox construction needs an activation, so substitute an uninitialized scoped instance to observe the binding. + services.Replace(ServiceDescriptor.Scoped(static _ => (Outbox)RuntimeHelpers.GetUninitializedObject(typeof(Outbox)))); + await using var provider = services.BuildServiceProvider(); + await using var scope = provider.CreateAsyncScope(); + + Assert.Single(services, static descriptor => descriptor.ServiceType == typeof(IOutbox)); + Assert.Same(scope.ServiceProvider.GetRequiredService(), scope.ServiceProvider.GetRequiredService()); + } + + [Fact] + public async Task AddMessaging_DoesNotReplaceUnrelatedConstructionErrors() + { + var services = new ServiceCollection(); + services.AddMessaging(); + services.AddScoped(); + await using var provider = services.BuildServiceProvider(); + await using var scope = provider.CreateAsyncScope(); + + var exception = Assert.Throws(() => scope.ServiceProvider.GetRequiredService()); + + Assert.Contains(nameof(IGrainContext), exception.Message, StringComparison.Ordinal); + } + + private sealed class FixedTimeProvider(DateTimeOffset utcNow) : TimeProvider + { + public override DateTimeOffset GetUtcNow() => utcNow; + } + + private sealed class TestSiloBuilder : ISiloBuilder + { + public IServiceCollection Services { get; } = new ServiceCollection(); + public IConfiguration Configuration { get; } = new ConfigurationBuilder().Build(); + } + + private sealed class ConstructionTestStateManager : IJournaledStateManager + { + public ValueTask InitializeAsync(CancellationToken cancellationToken) => default; + public void RegisterStateMachine(string name, IStateMachine state) { } + public bool TryGetStateMachine(string name, [NotNullWhen(true)] out IStateMachine? state) + { + state = null; + return false; + } + + public ValueTask WriteStateAsync(CancellationToken cancellationToken) => default; + public ValueTask DeleteStateAsync(CancellationToken cancellationToken) => default; + } +} diff --git a/test/Orleans.Messaging.Tests/Orleans.Messaging.Tests.csproj b/test/Orleans.Messaging.Tests/Orleans.Messaging.Tests.csproj new file mode 100644 index 00000000000..152a1b16ed0 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Orleans.Messaging.Tests.csproj @@ -0,0 +1,22 @@ + + + $(TestTargetFrameworks) + true + enable + enable + $(NoWarn);ORLEANSEXP005 + + + + + + + + + + + + + + + diff --git a/test/Orleans.Messaging.Tests/Support/ActivationValidationTestGrains.cs b/test/Orleans.Messaging.Tests/Support/ActivationValidationTestGrains.cs new file mode 100644 index 00000000000..47e0d937673 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Support/ActivationValidationTestGrains.cs @@ -0,0 +1,137 @@ +using System.Collections.Concurrent; +using Microsoft.Extensions.DependencyInjection; +using Orleans.Concurrency; +using Orleans.Journaling; +using Orleans.Metadata; +using Orleans.Runtime; +using Orleans.Serialization.Invocation; + +namespace Orleans.Messaging.Tests.Support; + +public sealed class ActivationValidationProbe +{ + private readonly ConcurrentDictionary> _observations = new(); + public Observation[] Get(GrainId grainId) => _observations.TryGetValue(grainId, out var values) ? values.ToArray() : []; + public Observation Track(IGrainContext context) + { + var observation = new Observation(context); + _observations.GetOrAdd(context.GrainId, static _ => new()).Enqueue(observation); + return observation; + } + + public sealed class Observation(IGrainContext context) + { + public IGrainContext Context { get; } = context; + public bool InstanceAvailableDuringConstruction { get; } = context.GrainInstance is not null; + public int Activated; + public int Calls; + } +} + +public interface IActivationValidationTestGrain : IGrainWithGuidKey +{ + Task IncrementAsync(); + Task DeactivateAsync(); +} + +public interface IAlwaysInterleaveValidationTestGrain : IActivationValidationTestGrain +{ + [AlwaysInterleave] + Task InterleaveAsync(); +} + +public abstract class ActivationValidationTestGrain : DurableGrain, IActivationValidationTestGrain +{ + private readonly IDurableValue _value; + private readonly ActivationValidationProbe.Observation _observation; + + protected ActivationValidationTestGrain(IGrainContext context, ActivationValidationProbe probe, + IDurableValue value) + { + _value = value; + _observation = probe.Track(context); + } + + public override Task OnActivateAsync(CancellationToken cancellationToken) + { + Interlocked.Increment(ref _observation.Activated); + return base.OnActivateAsync(cancellationToken); + } + + public async Task IncrementAsync() + { + Interlocked.Increment(ref _observation.Calls); + _value.Value++; + await WriteStateAsync(); + return _value.Value; + } + + public Task DeactivateAsync() + { + DeactivateOnIdle(); + return Task.CompletedTask; + } + +} + +public sealed class SupportedActivationValidationTestGrain( + IGrainContext context, ActivationValidationProbe probe, + [FromKeyedServices("activation-validation")] IDurableValue value) + : ActivationValidationTestGrain(context, probe, value); + +[Reentrant] +public sealed class ReentrantActivationValidationTestGrain( + IGrainContext context, ActivationValidationProbe probe, + [FromKeyedServices("activation-validation")] IDurableValue value) + : ActivationValidationTestGrain(context, probe, value); + +[StatelessWorker] +public sealed class StatelessActivationValidationTestGrain( + IGrainContext context, ActivationValidationProbe probe, + [FromKeyedServices("activation-validation")] IDurableValue value) + : ActivationValidationTestGrain(context, probe, value); + +[MayInterleave(nameof(Interleave))] +public sealed class MayInterleaveActivationValidationTestGrain( + IGrainContext context, ActivationValidationProbe probe, + [FromKeyedServices("activation-validation")] IDurableValue value) + : ActivationValidationTestGrain(context, probe, value) +{ + public static bool Interleave(IInvokable request) => true; +} + +public sealed class AlwaysInterleaveActivationValidationTestGrain( + IGrainContext context, ActivationValidationProbe probe, + [FromKeyedServices("activation-validation")] IDurableValue value) + : ActivationValidationTestGrain(context, probe, value), IAlwaysInterleaveValidationTestGrain +{ + public Task InterleaveAsync() => Task.CompletedTask; +} + +[AttributeUsage(AttributeTargets.Class, AllowMultiple = true)] +public sealed class ExecutionPropertyAttribute(string key, string value) : Attribute, IGrainPropertiesProviderAttribute +{ + public void Populate(IServiceProvider services, Type grainClass, GrainType grainType, Dictionary properties) => + properties[key] = value; +} + +[ExecutionProperty(WellKnownGrainTypeProperties.Reentrant, "TrUe")] +public sealed class MetadataReentrantActivationValidationGrain( + IGrainContext context, ActivationValidationProbe probe, + [FromKeyedServices("activation-validation")] IDurableValue value) + : ActivationValidationTestGrain(context, probe, value); + +[ExecutionProperty(WellKnownGrainTypeProperties.MayInterleavePredicate, nameof(Interleave))] +public sealed class MetadataMayInterleaveActivationValidationGrain( + IGrainContext context, ActivationValidationProbe probe, + [FromKeyedServices("activation-validation")] IDurableValue value) + : ActivationValidationTestGrain(context, probe, value) +{ + public static bool Interleave(IInvokable request) => true; +} + +[ExecutionProperty(WellKnownGrainTypeProperties.Reentrant, "false")] +public sealed class MetadataNonReentrantActivationValidationGrain( + IGrainContext context, ActivationValidationProbe probe, + [FromKeyedServices("activation-validation")] IDurableValue value) + : ActivationValidationTestGrain(context, probe, value); diff --git a/test/Orleans.Messaging.Tests/Support/BootstrapOutboxServices.cs b/test/Orleans.Messaging.Tests/Support/BootstrapOutboxServices.cs new file mode 100644 index 00000000000..85f74157330 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Support/BootstrapOutboxServices.cs @@ -0,0 +1,133 @@ +using System.Collections.Concurrent; +using Microsoft.Extensions.DependencyInjection; +using Orleans.Journaling; +using Orleans.Messaging.Diagnostics; +using Orleans.Runtime; + +namespace Orleans.Messaging.Tests.Support; + +public sealed class BootstrapDeliveryProbe : IOutgoingGrainCallFilter, IObserver, IDisposable +{ + private readonly TaskCompletionSource _release = new(TaskCreationOptions.RunContinuationsAsynchronously); + private readonly ConcurrentDictionary> _outputs = new(); + + private readonly ConcurrentDictionary _drains = new(); + + // The event stream is process-wide, so the fixture disposes the subscription with its cluster. + private readonly IDisposable _subscription; + + public BootstrapDeliveryProbe() => _subscription = MessagingEvents.AllEvents.Subscribe(this); + + public void Dispose() => _subscription.Dispose(); + + /// Waits until the sender's outbox retires a delivered message. + public Task WaitForDrainAsync(GrainId sender) => + _drains.GetOrAdd(sender, static _ => new(TaskCreationOptions.RunContinuationsAsynchronously)).Task.WaitAsync(TimeSpan.FromSeconds(30)); + + void IObserver.OnNext(MessagingEvents.MessagingEvent value) + { + if (value is MessagingEvents.OutboxMessageRetired { Retirement.Status: OutboxMessageRetirementStatus.Delivered } retired) + { + _drains.GetOrAdd(retired.GrainContext.GrainId, static _ => new(TaskCreationOptions.RunContinuationsAsynchronously)).TrySetResult(); + } + } + + void IObserver.OnCompleted() { } + + void IObserver.OnError(Exception error) { } + public async Task Invoke(IOutgoingGrainCallContext context) + { + if (context.TargetId == BootstrapState.OutputTarget && context.MethodName == nameof(IMessagingExtension.DeliverAsync)) + { + await _release.Task.WaitAsync(context.Request.GetCancellationToken()); + } + await context.Invoke(); + } + + public void Release() => _release.TrySetResult(); + public Task<(MessageId MessageId, int Value)> WaitForOutputAsync(GrainId sender) => + GetOutput(sender).Task.WaitAsync(TimeSpan.FromSeconds(30)); + public void OnOutput(GrainId sender, MessageId messageId, int value) => GetOutput(sender).TrySetResult((messageId, value)); + private TaskCompletionSource<(MessageId MessageId, int Value)> GetOutput(GrainId sender) => + _outputs.GetOrAdd(sender, static _ => new(TaskCreationOptions.RunContinuationsAsynchronously)); +} + +public interface IBootstrapOutputGrain : IGrainWithStringKey +{ + Task GetMessageCountAsync(); +} + +[GrainType("bootstrap-output")] +public sealed class BootstrapOutputGrain : Grain, IBootstrapOutputGrain, IMessagingGrain, IInboxHandler, + IStateMachine, IDurableDictionaryCommandHandler +{ + private readonly Dictionary _values = []; + private readonly List<(GrainId Sender, MessageId MessageId, int Value)> _pending = []; + private (GrainId Sender, MessageId MessageId, int Value)[] _captured = []; + private readonly BootstrapDeliveryProbe _probe; + private readonly Orleans.Serialization.Session.SerializerSessionPool _sessions; + private readonly Orleans.Serialization.Serializer _senderSerializer; + private readonly IDurableDictionaryCommandCodec _codec; + + public BootstrapOutputGrain(IJournaledStateManager manager, IInbox inbox, BootstrapDeliveryProbe probe, + IDurableDictionaryCommandCodec codec, Orleans.Serialization.Session.SerializerSessionPool sessions, Orleans.Serialization.Serializer senderSerializer) + { + _sessions = sessions; + _senderSerializer = senderSerializer; + _probe = probe; + _codec = codec; + manager.RegisterStateMachine("bootstrap-output-values", this); + inbox.Register(this); + } + + public bool CanHandle(InboxMessage message) => true; + + + public ValueTask HandleAsync(IInboxHandlerContext context, CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + var value = (int)(TestApplicationProtocol.Read(_sessions, context.Message).Body + ?? throw new InvalidOperationException("The bootstrap output must contain an integer.")); + if (context.Message.Envelope.Headers is not { } headers || !headers.TryGetValue("sender", out var senderBytes)) + { + throw new InvalidOperationException("The bootstrap output must supply its sender."); + } + var sender = _senderSerializer.Deserialize(senderBytes); + _values.Add(context.Message.Envelope.MessageId, value); + _pending.Add((sender, context.Message.Envelope.MessageId, value)); + context.Complete(); + return ValueTask.CompletedTask; + } + + public Task GetMessageCountAsync() => Task.FromResult(_values.Count); + public void WritePendingEntries(JournalStreamWriter writer) + { + _captured = _pending.ToArray(); + _pending.Clear(); + foreach (var item in _captured) _codec.WriteSet(item.MessageId, item.Value, writer); + } + public void WriteSnapshot(JournalStreamWriter writer) + { + _captured = _pending.ToArray(); + _pending.Clear(); + _codec.WriteSnapshot(_values, writer); + } + public void OnWriteCompleted() + { + foreach (var output in _captured) _probe.OnOutput(output.Sender, output.MessageId, output.Value); + _captured = []; + } + public void Reset(JournalStreamWriter writer) + { + _values.Clear(); + _pending.Clear(); + _captured = []; + } + public void ReplayEntry(JournalEntry entry, JournalReplayContext context) => + context.GetRequiredCommandCodec(entry.FormatKey, _codec).Apply(entry.Reader, this); + + public void ApplySet(MessageId key, int value) => _values[key] = value; + public void ApplyRemove(MessageId key) => _values.Remove(key); + public void ApplyClear() => _values.Clear(); + public void Reset(int capacityHint) => _values.Clear(); +} diff --git a/test/Orleans.Messaging.Tests/Support/BootstrapTestGrains.cs b/test/Orleans.Messaging.Tests/Support/BootstrapTestGrains.cs new file mode 100644 index 00000000000..0cd7f70094a --- /dev/null +++ b/test/Orleans.Messaging.Tests/Support/BootstrapTestGrains.cs @@ -0,0 +1,327 @@ +using System.Collections.Concurrent; +using System.Globalization; +using System.Reflection; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Logging; +using Orleans.Concurrency; +using Orleans.Journaling; +using Orleans.Metadata; +using Orleans.Runtime; +using Orleans.Serialization.Invocation; + +namespace Orleans.Messaging.Tests.Support; + +public sealed class BootstrapProbe +{ + private readonly ConcurrentDictionary> _activations = new(); + public void Add(BootstrapObservation observation) => + _activations.GetOrAdd(observation.Context.GrainId, static _ => new()).Enqueue(observation); + public BootstrapObservation[] Get(GrainId id) => _activations[id].ToArray(); +} + +public sealed class BootstrapObservation : IDisposable +{ + public BootstrapObservation(IGrainContext context, BootstrapProbe probe) + { + Context = context; + probe.Add(this); + } + + public IGrainContext Context { get; } + public object? ConstructedGrain { get; private set; } + public bool InstanceAvailableInConstructor { get; private set; } + public IJournaledStateManager? Manager { get; set; } + public IDurableValue? Value { get; set; } + internal Inbox? Inbox { get; set; } + public IOutbox? Outbox { get; set; } + public Exception? ExpectedFailure { get; set; } + public int Activations { get; set; } + public int GrainDisposals { get; set; } + public int Disposals { get; private set; } + + public void Constructed(object grain) + { + ConstructedGrain = grain; + InstanceAvailableInConstructor = Context.GrainInstance is not null; + } + + public void Dispose() => Disposals++; +} + +public sealed class BootstrapState : IInboxHandler, IDisposable +{ + public const string Route = "bootstrap"; + public static GrainId OutputTarget { get; } = GrainId.Create("bootstrap-output", "capture"); + private readonly HandlerProbe _handlers; + public BootstrapState(BootstrapObservation observation, IJournaledStateManager manager, HandlerProbe handlers, + [FromKeyedServices("bootstrap-value")] IDurableValue value, IGrainContext context, IOutbox outbox) + { + _handlers = handlers; + Observation = observation; + observation.Manager = manager; + observation.Value = value; + observation.Inbox = context.ActivationServices.GetRequiredService(); + observation.Outbox = outbox; + observation.Inbox.Register(this); + } + + public BootstrapObservation Observation { get; } + public int MessagingStateCountAtActivation { get; private set; } + public object? GrainAtActivation { get; private set; } + public int ActivationValue { get; private set; } + public int HandlerCalls { get; private set; } + internal IInboxHandler? HandlerOverride { get; set; } + public int Disposals { get; private set; } + public Task Read() => Task.FromResult(Observation.Value!.Value); + public async Task Set(int value) + { + Observation.Value!.Value = value; + await Observation.Manager!.WriteStateAsync(CancellationToken.None); + } + public async Task SendValue(int value) + { + var context = Observation.Context; + var envelope = OutboxTestMessages.CreateApplication(context.ActivationServices, + MessageId.Create("bootstrap", context.GrainId.ToString(), "value", value.ToString(CultureInfo.InvariantCulture)), + context.GrainId, OutputTarget, "output", value); + Observation.Value!.Value = value; + Observation.Outbox!.Send(envelope); + await Observation.Manager!.WriteStateAsync(CancellationToken.None); + } + public async Task SendSynchronousValue(int value) + { + var context = Observation.Context; + var envelope = OutboxTestMessages.CreateApplication(context.ActivationServices, + MessageId.Create("bootstrap", context.GrainId.ToString(), "value", value.ToString(CultureInfo.InvariantCulture)), + context.GrainId, OutputTarget, "output", value); + Observation.Value!.Value = value; + Observation.Outbox!.Send(envelope); + await Observation.Manager!.WriteStateAsync(CancellationToken.None); + } + public Task Persist() => Observation.Manager!.WriteStateAsync(CancellationToken.None).AsTask(); + public Task Activate() + { + Observation.Activations++; + GrainAtActivation = Observation.Context.GrainInstance; + MessagingStateCountAtActivation = ReadMessagingStates(Observation.Manager!).Count(); + ActivationValue = Observation.Value!.Value; + return Task.CompletedTask; + } + + public bool CanHandle(InboxMessage message) => true; + + + public async ValueTask HandleAsync(IInboxHandlerContext context, CancellationToken cancellationToken) + { + if (HandlerOverride is { } handler) + { + await handler.HandleAsync(context, cancellationToken); + return; + } + if (_handlers.TryGet(Observation.Context.GrainId, Route, out var barrier)) + { + barrier.Entered.TrySetResult(); + await barrier.Continue.Task.WaitAsync(cancellationToken); + } + var value = Observation.Value!.Value + 1; + var outgoing = OutboxTestMessages.CreateApplication(Observation.Context.ActivationServices, + new MessageId(context.Message.Envelope.MessageId.Key.CreateChildKey("output")), Observation.Context.GrainId, OutputTarget, "output", value); + Observation.Value.Value = value; + Observation.Outbox!.Send(outgoing); + HandlerCalls++; + context.Complete(); + } + public void Dispose() => Disposals++; + public static IEnumerable ReadMessagingStates(IJournaledStateManager manager) + { + foreach (var name in new[] { MessagingStateNames.Inbox, MessagingStateNames.Outbox }) + { + if (manager.TryGetStateMachine(name, out var state)) yield return state; + } + } + +} + +public interface IBootstrapTestGrain : IGrainWithGuidKey +{ + Task GetValueAsync(); + Task SetValueAsync(int value); + Task SendValueAsync(int value); + Task SendSynchronousValueAsync(int value); + Task PersistAsync(); + Task DeactivateAsync(); +} +public interface IMarkedBootstrapTestGrain : IBootstrapTestGrain, IMessagingGrain; +public interface IGenericBootstrapTestGrain : IBootstrapTestGrain; + +public abstract class BootstrapGrainBase : Grain, IBootstrapTestGrain, IDisposable +{ + private readonly BootstrapState _state; + protected BootstrapGrainBase(BootstrapState state) + { + _state = state; + state.Observation.Constructed(this); + } + public override Task OnActivateAsync(CancellationToken cancellationToken) => _state.Activate(); + public Task GetValueAsync() => _state.Read(); + public Task SetValueAsync(int value) => _state.Set(value); + public Task SendValueAsync(int value) => _state.SendValue(value); + public Task SendSynchronousValueAsync(int value) => _state.SendSynchronousValue(value); + public Task PersistAsync() => _state.Persist(); + public Task DeactivateAsync() { DeactivateOnIdle(); return Task.CompletedTask; } + public void Dispose() => _state.Observation.GrainDisposals++; +} +public sealed class PlainBootstrapGrain(BootstrapState state) : BootstrapGrainBase(state), IMessagingGrain; +public abstract class ApplicationBootstrapBase(BootstrapState state) : BootstrapGrainBase(state), IMessagingGrain; +public sealed class ApplicationBootstrapGrain(BootstrapState state) : ApplicationBootstrapBase(state); +public sealed class InterfaceBootstrapGrain(BootstrapState state) : BootstrapGrainBase(state), IMarkedBootstrapTestGrain; +public sealed class GenericBootstrapGrain(BootstrapState state) : BootstrapGrainBase(state), IGenericBootstrapTestGrain, IMessagingGrain; + +public class DurableBootstrapGrain : DurableGrain, IBootstrapTestGrain, IDisposable +{ + private readonly BootstrapState _state; + public DurableBootstrapGrain(BootstrapState state) + { + _state = state; + state.Observation.Constructed(this); + } + public override Task OnActivateAsync(CancellationToken cancellationToken) => _state.Activate(); + public Task GetValueAsync() => _state.Read(); + public Task SetValueAsync(int value) => _state.Set(value); + public Task SendValueAsync(int value) => _state.SendValue(value); + public Task SendSynchronousValueAsync(int value) => _state.SendSynchronousValue(value); + public Task PersistAsync() => _state.Persist(); + public Task DeactivateAsync() { DeactivateOnIdle(); return Task.CompletedTask; } + public void Dispose() => _state.Observation.GrainDisposals++; +} + +public sealed class MarkedDurableBootstrapGrain(BootstrapState state) : DurableBootstrapGrain(state), IMessagingGrain; + +public interface IBootstrapControlGrain : IGrainWithGuidKey { Task PingAsync(); } +public interface IInterleavingBootstrapControlGrain : IBootstrapControlGrain +{ + [AlwaysInterleave] Task InterleaveAsync(); +} +public abstract class BootstrapControlGrain : Grain, IBootstrapControlGrain, IDisposable +{ + protected BootstrapObservation Observation { get; } + protected BootstrapControlGrain(BootstrapObservation observation) + { + Observation = observation; + observation.Constructed(this); + } + public override Task OnActivateAsync(CancellationToken cancellationToken) + { + Observation.Activations++; + return Task.CompletedTask; + } + public Task PingAsync() => Task.CompletedTask; + public void Dispose() => Observation.GrainDisposals++; +} +public sealed class EagerBootstrapGrain(BootstrapObservation observation) : BootstrapControlGrain(observation), IMessagingGrain; +public sealed class UnselectedBootstrapGrain(BootstrapObservation observation) : BootstrapControlGrain(observation); +public sealed class FailingBootstrapGrain(BootstrapObservation observation) : BootstrapControlGrain(observation), IMessagingGrain; +public sealed class UnselectedEndpointBootstrapGrain : BootstrapControlGrain +{ + public UnselectedEndpointBootstrapGrain(BootstrapObservation observation, IGrainContext context) : base(observation) => + observation.Inbox = context.ActivationServices.GetRequiredService(); +} +public sealed class JournalOnlyBootstrapGrain : BootstrapControlGrain +{ + public JournalOnlyBootstrapGrain(BootstrapObservation observation, IJournaledStateManager manager, + [FromKeyedServices("bootstrap-journal-only")] IDurableValue value) : base(observation) + { + observation.Manager = manager; + observation.Value = value; + } +} +[Reentrant] +public sealed class ReentrantBootstrapGrain(BootstrapObservation observation) : BootstrapControlGrain(observation), IMessagingGrain; +[StatelessWorker] +public sealed class StatelessBootstrapGrain(BootstrapObservation observation) : BootstrapControlGrain(observation), IMessagingGrain; +[MayInterleave(nameof(Interleave))] +public sealed class MayInterleaveBootstrapGrain(BootstrapObservation observation) : BootstrapControlGrain(observation), IMessagingGrain +{ + public static bool Interleave(IInvokable request) => true; +} +public sealed class AlwaysInterleaveBootstrapGrain(BootstrapObservation observation) + : BootstrapControlGrain(observation), IInterleavingBootstrapControlGrain, IMessagingGrain +{ + public Task InterleaveAsync() => Task.CompletedTask; +} + +[ExecutionProperty(WellKnownGrainTypeProperties.Reentrant, "true")] +public sealed class MetadataReentrantBootstrapGrain(BootstrapObservation observation) + : BootstrapControlGrain(observation), IMessagingGrain; + +[ExecutionProperty(WellKnownGrainTypeProperties.MayInterleavePredicate, nameof(Interleave))] +public sealed class MetadataMayInterleaveBootstrapGrain(BootstrapObservation observation) + : BootstrapControlGrain(observation), IMessagingGrain +{ + public static bool Interleave(IInvokable request) => true; +} + +[ExecutionProperty(WellKnownGrainTypeProperties.PlacementStrategy, "StatelessWorkerPlacement")] +public sealed class MetadataStatelessPlacementBootstrapGrain(BootstrapObservation observation) + : BootstrapControlGrain(observation), IMessagingGrain; + +[ExecutionProperty(WellKnownGrainTypeProperties.PlacementStrategy, BootstrapClusterFixture.StatelessPlacementAlias)] +public sealed class AliasedStatelessPlacementBootstrapGrain(BootstrapObservation observation) + : BootstrapControlGrain(observation), IMessagingGrain; + +[ExecutionProperty(WellKnownGrainTypeProperties.PlacementStrategy, nameof(RandomPlacement))] +public sealed class MetadataOrdinaryPlacementBootstrapGrain(BootstrapObservation observation) + : BootstrapControlGrain(observation), IMessagingGrain; + +[ExecutionProperty(WellKnownGrainTypeProperties.PlacementStrategy, BootstrapClusterFixture.OrdinaryPlacementAlias)] +public sealed class AliasedOrdinaryPlacementBootstrapGrain(BootstrapObservation observation) + : BootstrapControlGrain(observation), IMessagingGrain; + +public sealed class BootstrapClusterFixture : MessagingClusterFixture +{ + public const string StatelessPlacementAlias = "bootstrap-worker-alias"; + public const string OrdinaryPlacementAlias = "bootstrap-directory-alias"; + + public BootstrapClusterFixture(bool useServiceCollection = false) + : base(1, receiverOnly: false, useServiceCollection: useServiceCollection) + { + } + + public BootstrapProbe Probe { get; } = new(); + public BootstrapDeliveryProbe Delivery { get; } = new(); + + public override async ValueTask DisposeAsync() + { + Delivery.Dispose(); + await base.DisposeAsync(); + } + + protected override void ConfigureServices(IServiceCollection services) + { + services.AddScoped(provider => provider.GetRequiredKeyedService>("orleans-binary")); + services.AddKeyedSingleton(StatelessPlacementAlias, new StatelessWorkerAttribute(1).PlacementStrategy); + services.AddKeyedSingleton(OrdinaryPlacementAlias, new RandomPlacement()); + services.AddSingleton(Probe); + services.AddSingleton(Delivery); + services.AddSingleton(Delivery); + services.AddScoped(); + services.AddScoped(); + services.RemoveAll(); + services.AddScoped(provider => + { + var inbox = ActivatorUtilities.CreateInstance(provider); + if (provider.GetRequiredService().GrainInstance is FailingBootstrapGrain) + { + var observation = provider.GetRequiredService(); + observation.Manager = provider.GetRequiredService(); + observation.Inbox = inbox; + observation.Outbox = provider.GetRequiredService(); + observation.ExpectedFailure = new IOException("Expected bootstrap runtime construction failure."); + throw observation.ExpectedFailure; + } + + return inbox; + }); + } +} diff --git a/test/Orleans.Messaging.Tests/Support/ControlledDurableJobManager.cs b/test/Orleans.Messaging.Tests/Support/ControlledDurableJobManager.cs new file mode 100644 index 00000000000..d1831ad66eb --- /dev/null +++ b/test/Orleans.Messaging.Tests/Support/ControlledDurableJobManager.cs @@ -0,0 +1,172 @@ +using System.Collections.Concurrent; +using Microsoft.Extensions.DependencyInjection; +using Orleans.DurableJobs; +using Orleans.Runtime; + +namespace Orleans.Messaging.Tests.Support; + +public sealed class DurableJobManagerProbe +{ + private readonly ConcurrentDictionary<(string JobName, GrainId Target), int> _attempts = []; + private readonly ConcurrentDictionary<(string JobName, GrainId Target), int> _successes = []; + private readonly ConcurrentDictionary _failures = new(StringComparer.Ordinal); + private readonly ConcurrentDictionary _postScheduleFailures = new(StringComparer.Ordinal); + private readonly ConcurrentDictionary<(string JobName, GrainId Target), int> _duplicates = []; + private readonly ConcurrentDictionary _skips = new(StringComparer.Ordinal); + private readonly ConcurrentDictionary<(string JobName, GrainId Target), ConcurrentQueue> _scheduledJobs = []; + private readonly ConcurrentDictionary _scheduleBarriers = new(StringComparer.Ordinal); + + public void FailNext(string jobName) => + _failures.AddOrUpdate(jobName, 1, static (_, count) => count + 1); + + public void FailAfterNext(string jobName) => + _postScheduleFailures.AddOrUpdate(jobName, 1, static (_, count) => count + 1); + + /// Returns the next schedule without storing its job, modeling a job which has already run. + public void SkipNext(string jobName) => + _skips.AddOrUpdate(jobName, 1, static (_, count) => count + 1); + + public void DuplicateNext(string jobName, GrainId target) => + _duplicates.AddOrUpdate((jobName, target), 1, static (_, count) => count + 1); + + public int GetAttemptCount(string jobName, GrainId target) => + _attempts.TryGetValue((jobName, target), out var count) ? count : 0; + + public int GetSuccessCount(string jobName, GrainId target) => + _successes.TryGetValue((jobName, target), out var count) ? count : 0; + + public IReadOnlyList GetScheduledJobs(string jobName, GrainId target) => + _scheduledJobs.TryGetValue((jobName, target), out var jobs) ? jobs.ToArray() : []; + + public ScheduleBarrier BlockNext(string jobName) + { + var barrier = new ScheduleBarrier(); + if (!_scheduleBarriers.TryAdd(jobName, barrier)) + { + throw new InvalidOperationException($"A scheduling barrier is already armed for '{jobName}'."); + } + + return barrier; + } + + internal void OnAttempt(ScheduleJobRequest request) => + _attempts.AddOrUpdate((request.JobName, request.Target), 1, static (_, count) => count + 1); + + internal void OnSuccess(ScheduleJobRequest request) => + _successes.AddOrUpdate((request.JobName, request.Target), 1, static (_, count) => count + 1); + + internal void OnScheduled(ScheduleJobRequest request, DurableJob job) => + _scheduledJobs.GetOrAdd((request.JobName, request.Target), static _ => new()).Enqueue(job); + + internal bool ShouldFail(string jobName) + => TryConsumeFailure(_failures, jobName); + + internal bool ShouldFailAfterSchedule(string jobName) + => TryConsumeFailure(_postScheduleFailures, jobName); + + internal bool ShouldSkip(string jobName) + => TryConsumeFailure(_skips, jobName); + + internal bool ShouldDuplicate(ScheduleJobRequest request) + => TryConsumeFailure(_duplicates, (request.JobName, request.Target)); + + internal async Task WaitIfBlockedAsync(string jobName, CancellationToken cancellationToken) + { + if (!_scheduleBarriers.TryRemove(jobName, out var barrier)) + { + return; + } + + barrier.Entered.TrySetResult(); + await barrier.Release.Task.WaitAsync(cancellationToken); + } + + private static bool TryConsumeFailure( + ConcurrentDictionary failures, + TKey jobName) + where TKey : notnull + { + while (failures.TryGetValue(jobName, out var remaining) && remaining > 0) + { + if (failures.TryUpdate(jobName, remaining - 1, remaining)) + { + return true; + } + } + + return false; + } + + public sealed class ScheduleBarrier : IDisposable + { + internal TaskCompletionSource Entered { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + internal TaskCompletionSource Release { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public Task WaitUntilEnteredAsync() => Entered.Task.WaitAsync(TimeSpan.FromSeconds(30)); + + public void Continue() => Release.TrySetResult(); + + public void Dispose() => Continue(); + } +} + +internal sealed class ControlledDurableJobManager( + ILocalDurableJobManager inner, + DurableJobManagerProbe probe) : ILocalDurableJobManager +{ + public async Task ScheduleJobAsync( + ScheduleJobRequest request, + CancellationToken cancellationToken) + { + probe.OnAttempt(request); + if (probe.ShouldFail(request.JobName)) + { + throw new IOException($"Injected durable job scheduling failure for '{request.JobName}'."); + } + + await probe.WaitIfBlockedAsync(request.JobName, cancellationToken); + if (probe.ShouldSkip(request.JobName)) + { + return new DurableJob + { + Id = Guid.NewGuid().ToString("N"), + Name = request.JobName, + ShardId = "skipped", + TargetGrainId = request.Target, + DueTime = request.DueTime + }; + } + + var result = await inner.ScheduleJobAsync(request, cancellationToken); + probe.OnScheduled(request, result); + if (probe.ShouldDuplicate(request)) + { + var duplicate = await inner.ScheduleJobAsync(request, cancellationToken); + probe.OnScheduled(request, duplicate); + } + + probe.OnSuccess(request); + if (probe.ShouldFailAfterSchedule(request.JobName)) + { + throw new IOException( + $"Injected durable job scheduling response failure for '{request.JobName}'."); + } + + return result; + } + + public Task CancelAsync(DurableJob job, CancellationToken cancellationToken) => + inner.CancelAsync(job, cancellationToken); + + public static void Decorate(IServiceCollection services, DurableJobManagerProbe probe) + { + var descriptor = services.Last(service => service.ServiceType == typeof(ILocalDurableJobManager)); + var factory = descriptor.ImplementationFactory + ?? throw new InvalidOperationException("The durable job manager registration must use an implementation factory."); + services.Remove(descriptor); + services.AddSingleton( + serviceProvider => new ControlledDurableJobManager( + (ILocalDurableJobManager)factory(serviceProvider), + probe)); + } +} diff --git a/test/Orleans.Messaging.Tests/Support/ControlledJournalStorageProvider.cs b/test/Orleans.Messaging.Tests/Support/ControlledJournalStorageProvider.cs new file mode 100644 index 00000000000..365e251508f --- /dev/null +++ b/test/Orleans.Messaging.Tests/Support/ControlledJournalStorageProvider.cs @@ -0,0 +1,308 @@ +using System.Buffers; +using System.Collections.Concurrent; +using Microsoft.Extensions.Options; +using Orleans.Journaling; + +namespace Orleans.Messaging.Tests.Support; + +public sealed class ControlledJournalStorageProvider : IJournalStorageProvider, IJournalStorageCatalog +{ + private VolatileJournalStorageProvider? _inner; + private readonly ConcurrentDictionary _readPlans = new(); + private readonly ConcurrentDictionary _deletePlans = new(); + private readonly ConcurrentDictionary _writePlans = new(); + private readonly ConcurrentDictionary _postWritePlans = new(); + private readonly ConcurrentDictionary _postDeleteFailures = new(); + private readonly ConcurrentDictionary _successfulWrites = new(); + private readonly ConcurrentDictionary _successfulAppends = new(); + private readonly ConcurrentDictionary _successfulSnapshots = new(); + private readonly ConcurrentDictionary _reads = new(); + private readonly ConcurrentDictionary _readFailures = new(); + private readonly ConcurrentDictionary _creations = new(); + private readonly ConcurrentDictionary _initializations = new(); + + private readonly ConcurrentDictionary _snapshots = new(); + public void RequestSnapshot(JournalId journalId) => _snapshots[journalId] = 0; + + public string? JournalFormatKey { get; private set; } + + public void Configure(IOptions options) + { + ArgumentNullException.ThrowIfNull(options); + JournalFormatKey = options.Value.JournalFormatKey; + _inner ??= new VolatileJournalStorageProvider(options); + } + + public IJournalStorage CreateStorage(JournalId journalId) + { + _creations.AddOrUpdate(journalId, 1, static (_, count) => count + 1); + return new ControlledJournalStorage(this, journalId, Inner.CreateStorage(journalId)); + } + + public IAsyncEnumerable ListAsync( + JournalCatalogListOptions? options = null, + CancellationToken cancellationToken = default) => + Inner.ListAsync(options, cancellationToken); + + private VolatileJournalStorageProvider Inner => + _inner ?? throw new InvalidOperationException("The controlled journal storage provider has not been configured."); + + public WriteBarrier BlockWrite(JournalId journalId, int matchingWrite = 1) + { + ArgumentOutOfRangeException.ThrowIfNegativeOrZero(matchingWrite); + var plan = new WritePlan(matchingWrite, fail: false); + if (!_writePlans.TryAdd(journalId, plan)) + { + throw new InvalidOperationException($"A write plan is already armed for journal '{journalId}'."); + } + + return new WriteBarrier(plan); + } + + public WriteBarrier BlockDelete(JournalId journalId) + { + var plan = new WritePlan(1, fail: false); + if (!_deletePlans.TryAdd(journalId, plan)) + { + throw new InvalidOperationException($"A delete plan is already armed for journal '{journalId}'."); + } + return new WriteBarrier(plan); + } + + public WriteBarrier BlockRead(JournalId journalId, int matchingRead = 1) + { + ArgumentOutOfRangeException.ThrowIfNegativeOrZero(matchingRead); + var plan = new WritePlan(matchingRead, fail: false); + if (!_readPlans.TryAdd(journalId, plan)) + { + throw new InvalidOperationException($"A read plan is already armed for journal '{journalId}'."); + } + + return new WriteBarrier(plan); + } + + /// Fails the next reads of the journal, as a storage outage would. + public void FailReads(JournalId journalId, int count) + { + ArgumentOutOfRangeException.ThrowIfNegativeOrZero(count); + _readFailures[journalId] = count; + } + + public void FailWrite(JournalId journalId, int matchingWrite = 1) + { + ArgumentOutOfRangeException.ThrowIfNegativeOrZero(matchingWrite); + if (!_writePlans.TryAdd(journalId, new WritePlan(matchingWrite, fail: true))) + { + throw new InvalidOperationException($"A write plan is already armed for journal '{journalId}'."); + } + } + + public void FailAfterWrite(JournalId journalId, int matchingWrite = 1) + { + ArgumentOutOfRangeException.ThrowIfNegativeOrZero(matchingWrite); + if (!_postWritePlans.TryAdd(journalId, new WritePlan(matchingWrite, fail: true))) + { + throw new InvalidOperationException($"A post-write plan is already armed for journal '{journalId}'."); + } + } + + public WriteBarrier BlockAcknowledgement(JournalId journalId) + { + var plan = new WritePlan(1, fail: false); + if (!_postWritePlans.TryAdd(journalId, plan)) + { + throw new InvalidOperationException($"A post-write plan is already armed for journal '{journalId}'."); + } + return new WriteBarrier(plan); + } + + public void FailAfterDelete(JournalId journalId) + { + if (!_postDeleteFailures.TryAdd(journalId, 0)) + { + throw new InvalidOperationException($"A post-delete failure is already armed for journal '{journalId}'."); + } + } + + public int GetSuccessfulWriteCount(JournalId journalId) => + _successfulWrites.TryGetValue(journalId, out var count) ? count : 0; + + public int GetSuccessfulAppendCount(JournalId journalId) => + _successfulAppends.TryGetValue(journalId, out var count) ? count : 0; + + public int GetSuccessfulSnapshotCount(JournalId journalId) => + _successfulSnapshots.TryGetValue(journalId, out var count) ? count : 0; + + public int GetCreationCount(JournalId journalId) => _creations.TryGetValue(journalId, out var count) ? count : 0; + public int GetReadCount(JournalId journalId) => _reads.TryGetValue(journalId, out var count) ? count : 0; + public int GetInitializationCount(JournalId journalId) => _initializations.TryGetValue(journalId, out var count) ? count : 0; + + private async ValueTask BeforeWriteAsync(JournalId journalId, CancellationToken cancellationToken) + { + if (!_writePlans.TryGetValue(journalId, out var plan) + || Interlocked.Increment(ref plan.Seen) != plan.Target) + { + return; + } + + _writePlans.TryRemove(new KeyValuePair(journalId, plan)); + plan.EntryScheduler = TaskScheduler.Current; + plan.EntryContext = ReceiverTestServices.CurrentGrainContext; + plan.Entered.TrySetResult(); + if (plan.Fail) + { + throw new IOException($"Injected journal write failure for '{journalId}'."); + } + + await plan.Release.Task.WaitAsync(cancellationToken).ConfigureAwait(false); + } + + private async ValueTask BeforeReadAsync(JournalId journalId, CancellationToken cancellationToken) + { + if (_readFailures.TryGetValue(journalId, out var remaining) && remaining > 0 + && _readFailures.TryUpdate(journalId, remaining - 1, remaining)) + { + throw new IOException("Injected journal read failure."); + } + + if (!_readPlans.TryGetValue(journalId, out var plan) + || Interlocked.Increment(ref plan.Seen) != plan.Target) + { + return; + } + + _readPlans.TryRemove(new KeyValuePair(journalId, plan)); + plan.Entered.TrySetResult(); + await plan.Release.Task.WaitAsync(cancellationToken).ConfigureAwait(false); + } + + private void OnWriteSucceeded(JournalId journalId, bool snapshot) + { + _successfulWrites.AddOrUpdate(journalId, 1, static (_, count) => count + 1); + var counts = snapshot ? _successfulSnapshots : _successfulAppends; + counts.AddOrUpdate(journalId, 1, static (_, count) => count + 1); + } + + private async ValueTask AfterWriteAsync(JournalId journalId, CancellationToken cancellationToken) + { + if (!_postWritePlans.TryGetValue(journalId, out var plan) + || Interlocked.Increment(ref plan.Seen) != plan.Target) + { + return; + } + + _postWritePlans.TryRemove(new KeyValuePair(journalId, plan)); + if (plan.Fail) + { + throw new IOException($"Injected post-commit journal response failure for '{journalId}'."); + } + plan.Entered.TrySetResult(); + await plan.Release.Task.WaitAsync(cancellationToken).ConfigureAwait(false); + } + + internal sealed class WritePlan(int target, bool fail) + { + public int Target { get; } = target; + public bool Fail { get; } = fail; + public int Seen; + public TaskScheduler? EntryScheduler; + public IGrainContext? EntryContext; + public TaskCompletionSource Entered { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + public TaskCompletionSource Release { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + } + + public sealed class WriteBarrier : IDisposable + { + private readonly WritePlan _plan; + + internal WriteBarrier(WritePlan plan) => _plan = plan; + + public TaskScheduler? EntryScheduler => _plan.EntryScheduler; + public IGrainContext? EntryContext => _plan.EntryContext; + public Task WaitUntilEnteredAsync() => _plan.Entered.Task.WaitAsync(TimeSpan.FromSeconds(30)); + public void Release() => _plan.Release.TrySetResult(); + public void Dispose() => Release(); + public void Fail() => Fail(new IOException("Injected blocked journal write failure.")); + public void Fail(Exception exception) => _plan.Release.TrySetException(exception); + } + + private sealed class ControlledJournalStorage( + ControlledJournalStorageProvider owner, + JournalId journalId, + IJournalStorage inner) : IJournalStorage + { + public bool IsCompactionRequested => owner._snapshots.ContainsKey(journalId) || inner.IsCompactionRequested; + + public async ValueTask ReadAsync(IJournalStorageConsumer consumer, CancellationToken cancellationToken) + { + var grain = ReceiverTestServices.CurrentGrainContext?.GrainInstance as MessagingTestGrain; + owner._reads.AddOrUpdate(journalId, 1, static (_, count) => count + 1); + await owner.BeforeReadAsync(journalId, cancellationToken).ConfigureAwait(false); + await inner.ReadAsync(consumer, cancellationToken).ConfigureAwait(false); + grain?.CaptureStorageRead(); + } + + public ValueTask CreateIfNotExistsAsync( + IReadOnlyDictionary? metadata = null, + CancellationToken cancellationToken = default) + { + owner._initializations.AddOrUpdate(journalId, 1, static (_, count) => count + 1); + return inner.CreateIfNotExistsAsync(metadata, cancellationToken); + } + + public ValueTask GetMetadataAsync(CancellationToken cancellationToken = default) => + inner.GetMetadataAsync(cancellationToken); + + public ValueTask UpdateMetadataAsync( + IReadOnlyDictionary? set = null, + IEnumerable? remove = null, + string? expectedETag = null, + CancellationToken cancellationToken = default) => + inner.UpdateMetadataAsync(set, remove, expectedETag, cancellationToken); + + public async ValueTask ReplaceAsync(ReadOnlySequence value, CancellationToken cancellationToken) + { + var instance = ReceiverTestServices.CurrentGrainContext?.GrainInstance; + var grain = instance as MessagingTestGrain; + var named = instance as NamedFactoryMessagingGrain; + var captured = grain?.CaptureStorageWrite(); + var namedSnapshot = named?.CaptureStorageWrite(); + await owner.BeforeWriteAsync(journalId, cancellationToken).ConfigureAwait(false); + await inner.ReplaceAsync(value, cancellationToken).ConfigureAwait(false); + owner._snapshots.TryRemove(journalId, out _); + owner.OnWriteSucceeded(journalId, snapshot: true); + await owner.AfterWriteAsync(journalId, cancellationToken).ConfigureAwait(false); + if (captured is not null) grain!.PublishStoredSnapshot(captured); + if (namedSnapshot is not null) named!.PublishStoredSnapshot(namedSnapshot); + } + + public async ValueTask AppendAsync(ReadOnlySequence value, CancellationToken cancellationToken) + { + var instance = ReceiverTestServices.CurrentGrainContext?.GrainInstance; + var grain = instance as MessagingTestGrain; + var named = instance as NamedFactoryMessagingGrain; + var captured = grain?.CaptureStorageWrite(); + var namedSnapshot = named?.CaptureStorageWrite(); + await owner.BeforeWriteAsync(journalId, cancellationToken).ConfigureAwait(false); + await inner.AppendAsync(value, cancellationToken).ConfigureAwait(false); + owner.OnWriteSucceeded(journalId, snapshot: false); + await owner.AfterWriteAsync(journalId, cancellationToken).ConfigureAwait(false); + if (captured is not null) grain!.PublishStoredSnapshot(captured); + if (namedSnapshot is not null) named!.PublishStoredSnapshot(namedSnapshot); + } + + public async ValueTask DeleteAsync(CancellationToken cancellationToken) + { + if (owner._deletePlans.TryRemove(journalId, out var plan)) + { + plan.Entered.TrySetResult(); + await plan.Release.Task.WaitAsync(cancellationToken).ConfigureAwait(false); + } + await inner.DeleteAsync(cancellationToken).ConfigureAwait(false); + if (owner._postDeleteFailures.TryRemove(journalId, out _)) + { + throw new IOException($"Injected post-delete journal response failure for '{journalId}'."); + } + } + } +} diff --git a/test/Orleans.Messaging.Tests/Support/DeliveryFaultInjector.cs b/test/Orleans.Messaging.Tests/Support/DeliveryFaultInjector.cs new file mode 100644 index 00000000000..4e0df8f42b8 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Support/DeliveryFaultInjector.cs @@ -0,0 +1,334 @@ +using System.Collections.Concurrent; +using Orleans.Runtime; + +namespace Orleans.Messaging.Tests.Support; + +/// +/// Injects scripted faults into the outbox's call to the receiver by decorating each silo's outbox transport. +/// +/// +/// +/// Each delivery try takes the next scripted ; once the script is exhausted, tries take a +/// fault from when one is set, or else call the receiver unchanged. Faults never wait for time to +/// pass: a timeout is reported at once, and holds wait for an explicit release or the delivery's cancellation. +/// +/// +/// The injector records the receiver's own answers, including answers which a fault hides from the sender, so tests +/// can assert what the receiver decided independently of what the sender observed. +/// +/// +public sealed class DeliveryFaultInjector +{ + private readonly ConcurrentQueue _script = new(); + private readonly CountSignal _executions = new(); + private readonly CountSignal _receiverCalls = new(); + private readonly CountSignal _answerCount = new(); + private readonly List _answers = []; + private readonly List _receiverFailures = []; + private readonly List _sentTimes = []; + private Func? _repeat; + + /// Gets the number of delivery tries which reached the injector. + public int Executions => _executions.Value; + + /// Gets the number of tries which called the receiver. + public int ReceiverCalls => _receiverCalls.Value; + + /// Gets the statuses which the receiver answered, in completion order, whether or not the sender saw them. + public IReadOnlyList Answers + { + get + { + lock (_answers) return [.. _answers]; + } + } + + /// Gets the logical send time which the sender stamped on each try, in the order the tries started. + public IReadOnlyList SentTimes + { + get + { + lock (_sentTimes) return [.. _sentTimes]; + } + } + + /// Gets failures thrown by actual receiver calls, not failures fabricated by the scripted transport. + public IReadOnlyList ReceiverFailures + { + get + { + lock (_receiverFailures) return [.. _receiverFailures]; + } + } + + /// Returns a transport which injects this injector's faults into calls through . + internal IOutboxTransport Wrap(IOutboxTransport inner) => new Transport(this, inner); + + /// Scripts for the next delivery tries, in order. + public void Enqueue(params DeliveryFault[] faults) + { + foreach (var fault in faults) _script.Enqueue(fault); + } + + /// Applies a new fault from to every try once the script is exhausted. + public void Repeat(Func create) => _repeat = create; + + /// Lets tries call the receiver unchanged once the script is exhausted. + public void StopRepeating() => _repeat = null; + + /// Discards scripted faults which no try has taken, and stops repeating. + public void Clear() + { + _script.Clear(); + _repeat = null; + } + + /// Waits until tries have reached the injector. + public Task WaitForExecutionsAsync(int count) => _executions.WaitAsync(count); + + /// Waits until tries have called the receiver. + public Task WaitForReceiverCallsAsync(int count) => _receiverCalls.WaitAsync(count); + + /// Waits until the receiver has answered calls, and returns all answers so far. + public async Task> WaitForAnswersAsync(int count) + { + await _answerCount.WaitAsync(count); + return Answers; + } + + // Awaits continue on the sending activation's scheduler, so every receiver call is made from the sending grain. + private ValueTask DeliverAsync(IOutboxTransport inner, GrainId receiver, InboxMessage message, CancellationToken cancellationToken) + { + lock (_sentTimes) _sentTimes.Add(message.SentAt); + _executions.Increment(); + var fault = _script.TryDequeue(out var scripted) ? scripted : _repeat?.Invoke(); + return fault is null + ? CallReceiverAsync(inner, receiver, message, cancellationToken, null) + : PlayAsync(fault, inner, receiver, message, cancellationToken); + } + + private async ValueTask PlayAsync( + DeliveryFault fault, IOutboxTransport inner, GrainId receiver, InboxMessage message, CancellationToken cancellationToken) + { + if (fault.Kind != DeliveryFaultKind.HoldAfterCall) fault.OnEntered(); + switch (fault.Kind) + { + case DeliveryFaultKind.Throw: + throw fault.Exception!; + case DeliveryFaultKind.TimeOut: + throw new TimeoutException("Injected delivery timeout before the receiver was called."); + case DeliveryFaultKind.Answer: + return fault.Result!.Value; + case DeliveryFaultKind.TimeOutInFlight: + // The abandoned call continues to the receiver; its answer is observable through ReceiverAnswer only. + CallReceiverAsync(inner, receiver, message, cancellationToken, fault).AsTask().Ignore(); + throw new TimeoutException("Injected delivery timeout while the receiver call was in flight."); + case DeliveryFaultKind.LoseResponse: + try + { + await CallReceiverAsync(inner, receiver, message, cancellationToken, fault); + } + catch + { + } + + throw new TimeoutException("Injected loss of the receiver's response."); + case DeliveryFaultKind.HoldBeforeCall: + await fault.WaitForReleaseAsync(cancellationToken); + return await CallReceiverAsync(inner, receiver, message, cancellationToken, fault); + case DeliveryFaultKind.HoldAfterCall: + var answer = await CallReceiverAsync(inner, receiver, message, cancellationToken, fault); + fault.OnEntered(); + await fault.WaitForReleaseAsync(cancellationToken); + return answer; + default: + throw new InvalidOperationException($"Unknown delivery fault '{fault.Kind}'."); + } + } + + private async ValueTask CallReceiverAsync( + IOutboxTransport inner, GrainId receiver, InboxMessage message, CancellationToken cancellationToken, DeliveryFault? fault) + { + _receiverCalls.Increment(); + DeliveryResult result; + try + { + result = await inner.DeliverAsync(receiver, message, cancellationToken); + } + catch (Exception exception) + { + lock (_receiverFailures) _receiverFailures.Add(exception); + fault?.OnAnswered(exception); + throw; + } + + lock (_answers) _answers.Add(result.Status); + _answerCount.Increment(); + fault?.OnAnswered(result); + return result; + } + + private sealed class Transport(DeliveryFaultInjector owner, IOutboxTransport inner) : IOutboxTransport + { + public ValueTask DeliverAsync(GrainId receiver, InboxMessage message, CancellationToken cancellationToken) => + owner.DeliverAsync(inner, receiver, message, cancellationToken); + } + /// A counter whose waiters complete once it reaches their target. + private sealed class CountSignal + { + private readonly List<(int Target, TaskCompletionSource Signal)> _waiters = []; + private int _value; + + public int Value + { + get + { + lock (_waiters) return _value; + } + } + + public void Increment() + { + List? ready = null; + lock (_waiters) + { + _value++; + for (var index = _waiters.Count - 1; index >= 0; index--) + { + if (_waiters[index].Target > _value) continue; + (ready ??= []).Add(_waiters[index].Signal); + _waiters.RemoveAt(index); + } + } + + ready?.ForEach(static signal => signal.TrySetResult()); + } + + public Task WaitAsync(int target) + { + lock (_waiters) + { + if (_value >= target) return Task.CompletedTask; + var signal = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + _waiters.Add((target, signal)); + return signal.Task.WaitAsync(TimeSpan.FromSeconds(30)); + } + } + } +} + +/// The kinds of fault which injects. +public enum DeliveryFaultKind +{ + /// The try fails with an exception, without calling the receiver. + Throw, + + /// The try times out without calling the receiver. + TimeOut, + + /// The try returns an answer without calling the receiver. + Answer, + + /// The try calls the receiver and times out at once, while the call continues. + TimeOutInFlight, + + /// The try calls the receiver, then hides its answer behind a timeout. + LoseResponse, + + /// The try waits for a release before it calls the receiver. + HoldBeforeCall, + + /// The try calls the receiver, then waits for a release before it returns the answer. + HoldAfterCall +} + +/// +/// One single-use fault for a delivery try, and the handle through which a test observes and controls it. +/// +public sealed class DeliveryFault +{ + private readonly TaskCompletionSource _entered = new(TaskCreationOptions.RunContinuationsAsynchronously); + private readonly TaskCompletionSource _released = new(TaskCreationOptions.RunContinuationsAsynchronously); + private readonly TaskCompletionSource _answer = new(TaskCreationOptions.RunContinuationsAsynchronously); + private int _used; + + private DeliveryFault(DeliveryFaultKind kind, Exception? exception = null, DeliveryResult? result = null) + { + Kind = kind; + Exception = exception; + Result = result; + } + + /// Gets the kind of this fault. + public DeliveryFaultKind Kind { get; } + + internal Exception? Exception { get; } + internal DeliveryResult? Result { get; } + + /// + /// Gets the receiver's answer to the call which this fault made, or its failure. It completes only for faults which + /// call the receiver. + /// + public Task ReceiverAnswer => _answer.Task.WaitAsync(TimeSpan.FromSeconds(30)); + + /// Fails the try with , or an by default. + public static DeliveryFault Throw(Exception? exception = null) => + new(DeliveryFaultKind.Throw, exception ?? new IOException("Injected transient delivery failure.")); + + /// Times the try out with a , without calling the receiver. + public static DeliveryFault TimeOut() => new(DeliveryFaultKind.TimeOut); + + /// Answers the try with , without calling the receiver. + public static DeliveryFault Answer(DeliveryResult result) => new(DeliveryFaultKind.Answer, result: result); + + /// + /// Calls the receiver and times the try out at once, while the call continues; reports + /// the abandoned call's answer. + /// + public static DeliveryFault TimeOutInFlight() => new(DeliveryFaultKind.TimeOutInFlight); + + /// + /// Calls the receiver and lets it decide, then fails the try with a , as when + /// the response is lost after the receiver committed. + /// + public static DeliveryFault LoseResponse() => new(DeliveryFaultKind.LoseResponse); + + /// + /// Holds the try before it calls the receiver until or , or until the delivery + /// is canceled. + /// + public static DeliveryFault HoldBeforeCall() => new(DeliveryFaultKind.HoldBeforeCall); + + /// + /// Calls the receiver, then holds its answer until or , or until the delivery + /// is canceled, which models a sender that stops after the receiver committed. + /// + public static DeliveryFault HoldAfterCall() => new(DeliveryFaultKind.HoldAfterCall); + + /// + /// Waits until a try takes this fault; for , until the receiver answered. + /// + public Task WaitUntilEnteredAsync() => _entered.Task.WaitAsync(TimeSpan.FromSeconds(30)); + + /// Releases a held try to continue. + public void Release() => _released.TrySetResult(); + + /// Releases a held try by failing it with , without the receiver's answer. + public void Fail(Exception exception) => _released.TrySetException(exception); + + internal void OnEntered() + { + if (Interlocked.Exchange(ref _used, 1) != 0) + { + throw new InvalidOperationException("A delivery fault applies to one try only."); + } + + _entered.TrySetResult(); + } + + internal Task WaitForReleaseAsync(CancellationToken cancellationToken) => _released.Task.WaitAsync(cancellationToken); + + internal void OnAnswered(DeliveryResult result) => _answer.TrySetResult(result); + + internal void OnAnswered(Exception exception) => _answer.TrySetException(exception); +} diff --git a/test/Orleans.Messaging.Tests/Support/FaultInjectionClusterFixture.cs b/test/Orleans.Messaging.Tests/Support/FaultInjectionClusterFixture.cs new file mode 100644 index 00000000000..389f63ed832 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Support/FaultInjectionClusterFixture.cs @@ -0,0 +1,96 @@ +using Microsoft.Extensions.DependencyInjection; +using Orleans.DurableJobs; +using Orleans.Journaling; +using Orleans.Journaling.Json; +using Orleans.Messaging.Configuration; +using Orleans.Runtime; +using Xunit; + +namespace Orleans.Messaging.Tests.Support; + +/// +/// A messaging cluster whose delivery calls, journal writes, Durable Jobs scheduling, and Durable Jobs clock are all +/// under test control, for deterministic fault injection at every boundary of the delivery chain. +/// +/// +/// +/// injects faults into the outbox's call to the receiver; +/// fails or holds journal writes before or after they become durable; +/// fails or holds wake-up scheduling. drives the outbox's retry backoff and expiry and scheduled +/// delivery times, and moves only when a test advances it. +/// +/// +/// Durable Jobs fires a wake-up which is due when it is scheduled at once, but a wake-up which is due later waits in real +/// time; is long enough that such wake-ups never fire during a test. After advancing +/// , tests run the pump explicitly through . +/// +/// +/// stays zero, so a handler canceled by +/// deactivation which were wrongly counted as a failure would dead-letter at once. +/// +/// +public class FaultInjectionClusterFixture(string journalFormatKey = "orleans-binary") : MessagingClusterFixture(1, receiverOnly: false) +{ + /// The first outbox retry backoff, which doubles with each further attempt. + public static readonly TimeSpan RetryInterval = TimeSpan.FromMinutes(10); + + /// How long the outbox retries a message before it dead-letters the message. + public static readonly TimeSpan OutboxRetryTimeout = TimeSpan.FromHours(1); + + /// Gets the Durable Jobs clock, which the outbox and scheduled deliveries use. + public ManualJobClock JobClock { get; } = new(DateTimeOffset.UtcNow); + + /// Gets the fault injector for the outbox's calls to receivers. + public DeliveryFaultInjector Deliveries { get; } = new(); + + /// Gets the probe for activations. + public Functional.SourceSinkProbe SourceSinkProbe { get; } = new(); + + /// Advances by , then runs one pump pass for . + public async Task AdvanceAndRunPumpAsync(IAddressable grain, TimeSpan delta) + { + JobClock.Advance(delta); + return await RunPumpAsync(grain); + } + + /// Waits until the outbox of has recorded failed attempts for . + internal Task WaitForFailedAttemptsAsync(IAddressable sender, MessageId messageId, int attempts) => + WaitUntilAsync(sender, () => + GetOutboxState(sender) is var state + && state.AttemptKeys.Contains(messageId) + && state.Attempt(messageId).AttemptCount >= attempts); + + /// Gets test access to the outbox state of the active . + internal OutboxStateTestAccess GetOutboxState(IAddressable grain) => + new(GetGrainContext(grain).ActivationServices.GetRequiredService()); + + /// + /// Deactivates as a messaging failure does, which cancels its in-flight deliveries and + /// handlers, and waits until the activation is gone. + /// + public async Task CrashAsync(IAddressable grain) + { + var context = GetGrainContext(grain); + await OnGrainAsync(grain, () => context.Deactivate( + new DeactivationReason(DeactivationReasonCode.ApplicationError, "Injected activation crash."))); + await context.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + } + + protected override void ConfigureServices(IServiceCollection services) + { + services.AddKeyedSingleton(DurableJobTimeProviderNames.DurableJobs, JobClock); + services.AddSingleton(SourceSinkProbe); + services.Configure(options => options.JournalFormatKey = journalFormatKey); + services.Configure(options => options.AddTypeInfoResolver(Functional.SourceSinkJsonContext.Default)); + services.AddSingleton(sp => Deliveries.Wrap(new GrainOutboxTransport(sp.GetRequiredService()))); + } + + protected override void ConfigureOptions(DurableMessagingOptions options) + { + base.ConfigureOptions(options); + options.MaxCapacity = 8; + options.DeduplicationWindow = TimeSpan.FromDays(1); + options.OutboxRetryTimeout = OutboxRetryTimeout; + options.RetryInitialInterval = RetryInterval; + } +} diff --git a/test/Orleans.Messaging.Tests/Support/FaultingInboxCodecFixture.cs b/test/Orleans.Messaging.Tests/Support/FaultingInboxCodecFixture.cs new file mode 100644 index 00000000000..02441689ec6 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Support/FaultingInboxCodecFixture.cs @@ -0,0 +1,38 @@ +using Microsoft.Extensions.DependencyInjection; +using Orleans.Journaling; + +namespace Orleans.Messaging.Tests.Support; + +public sealed class FaultingInboxCodecFixture : MessagingClusterFixture +{ + public Exception? NextFailure { get; set; } + public bool FailOnSnapshot { get; set; } + public bool? FailedSnapshot { get; private set; } + protected override void ConfigureServices(IServiceCollection services) + { + var descriptor = services.Last(entry => entry.ServiceType == typeof(IDurableValueCommandCodec<>) + && Equals(entry.ServiceKey, "orleans-binary")); + var implementation = descriptor.KeyedImplementationType!.MakeGenericType(typeof(InboxCommand)); + services.AddKeyedSingleton>("orleans-binary", (sp, _) => + new FaultingCodec(this, (IDurableValueCommandCodec)ActivatorUtilities.CreateInstance(sp, implementation))); + } + + private sealed class FaultingCodec(FaultingInboxCodecFixture owner, IDurableValueCommandCodec inner) + : IDurableValueCommandCodec + { + public void WriteSet(InboxCommand value, JournalStreamWriter writer) + { + var snapshot = value.Kind == InboxCommandKind.Snapshot; + if (snapshot == owner.FailOnSnapshot && owner.NextFailure is { } failure) + { + owner.NextFailure = null; + owner.FailedSnapshot = snapshot; + throw failure; + } + + inner.WriteSet(value, writer); + } + + public void Apply(JournalBufferReader input, IDurableValueCommandHandler consumer) => inner.Apply(input, consumer); + } +} diff --git a/test/Orleans.Messaging.Tests/Support/HandlerProbe.cs b/test/Orleans.Messaging.Tests/Support/HandlerProbe.cs new file mode 100644 index 00000000000..63e30f2c8da --- /dev/null +++ b/test/Orleans.Messaging.Tests/Support/HandlerProbe.cs @@ -0,0 +1,59 @@ +using System.Collections.Concurrent; +using System.Diagnostics.CodeAnalysis; +using Orleans.Runtime; + +namespace Orleans.Messaging.Tests.Support; + +public sealed class HandlerProbe +{ + private readonly ConcurrentDictionary<(GrainId GrainId, string Route), Barrier> _barriers = new(); + + public Barrier Arm(GrainId grainId, string route, bool observeCancellation = true) + { + var barrier = new Barrier(this, (grainId, route), observeCancellation); + if (!_barriers.TryAdd((grainId, route), barrier)) + { + throw new InvalidOperationException($"A handler barrier is already armed for '{grainId}' and route '{route}'."); + } + + return barrier; + } + + public bool TryGet( + GrainId grainId, + string route, + [NotNullWhen(true)] out Barrier? barrier) => + _barriers.TryGetValue((grainId, route), out barrier); + + public sealed class Barrier : IDisposable + { + private readonly HandlerProbe _owner; + private readonly (GrainId GrainId, string Route) _key; + private readonly bool _observeCancellation; + + internal Barrier(HandlerProbe owner, (GrainId GrainId, string Route) key, bool observeCancellation) + { + _owner = owner; + _key = key; + _observeCancellation = observeCancellation; + } + + internal TaskCompletionSource Entered { get; } = + new(TaskCreationOptions.RunContinuationsAsynchronously); + internal TaskCompletionSource Continue { get; } = + new(TaskCreationOptions.RunContinuationsAsynchronously); + + public Task WaitUntilEnteredAsync() => Entered.Task.WaitAsync(TimeSpan.FromSeconds(30)); + public void Release() => Continue.TrySetResult(); + + // A barrier that ignores cancellation holds the handler until Release, even after shutdown. + internal Task WaitAsync(CancellationToken cancellationToken) => + _observeCancellation ? Continue.Task.WaitAsync(cancellationToken) : Continue.Task; + + public void Dispose() + { + Release(); + _owner._barriers.TryRemove(new KeyValuePair<(GrainId GrainId, string Route), Barrier>(_key, this)); + } + } +} diff --git a/test/Orleans.Messaging.Tests/Support/HierarchicalKeyTestExtensions.cs b/test/Orleans.Messaging.Tests/Support/HierarchicalKeyTestExtensions.cs new file mode 100644 index 00000000000..b1a0fd1b284 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Support/HierarchicalKeyTestExtensions.cs @@ -0,0 +1,15 @@ +namespace Orleans.Messaging.Tests.Support; + +internal static class HierarchicalKeyTestExtensions +{ + public static int CountSegments(this HierarchicalKey key) + { + var count = 0; + foreach (var _ in key) + { + count++; + } + + return count; + } +} diff --git a/test/Orleans.Messaging.Tests/Support/InboxTestAccess.cs b/test/Orleans.Messaging.Tests/Support/InboxTestAccess.cs new file mode 100644 index 00000000000..016a73a24e9 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Support/InboxTestAccess.cs @@ -0,0 +1,59 @@ +using System.Runtime.CompilerServices; + +namespace Orleans.Messaging.Tests.Support; + +internal static class InboxTestAccess +{ + + public static IEnumerable Messages(this Inbox inbox) => inbox.MessageState.Values.Select(static entry => entry.Message); + + public static int Count(this IOutbox outbox) => outbox switch + { + Outbox value => value.Count, + JournaledTestOutbox value => value.Count, + _ => throw new NotSupportedException(outbox.GetType().FullName) + }; + + public static IEnumerable Messages(this IOutbox outbox) => outbox switch + { + Outbox value => value.MessageState.Values.Where(static entry => entry.Message is not null).Select(static entry => entry.Message!.Value), + JournaledTestOutbox value => value.Messages, + _ => throw new NotSupportedException(outbox.GetType().FullName) + }; + + public static int GetProcessedCount(Inbox inbox) => GetProcessed(inbox).Count; + + public static HashSet GetProcessedKeys(Inbox inbox) => [.. GetProcessed(inbox).Keys]; + + public static void AddProcessed(Inbox inbox, MessageId messageId, DateTimeOffset sentAt) => + GetProcessed(inbox).Add(messageId, new() { MessageId = messageId, SentAt = sentAt }); + + public static IReadOnlyDictionary GetPendingMessages(Inbox inbox) => inbox.MessageState; + + public static DateTimeOffset GetProcessedSentAt(Inbox inbox, MessageId messageId) => + GetProcessed(inbox)[messageId].SentAt; + + [UnsafeAccessor(UnsafeAccessorKind.Field, Name = "_admissionFloor")] + public static extern ref DateTimeOffset AdmissionFloor(Inbox inbox); + + [UnsafeAccessor(UnsafeAccessorKind.Field, Name = "_durableAdmissionFloor")] + public static extern ref DateTimeOffset DurableAdmissionFloor(Inbox inbox); + + public static InboxMessageState GetMessageState(Inbox inbox, MessageId messageId) => + inbox.MessageState[messageId].State; + + public static void StageAcceptance(Inbox inbox, InboxMessage message) => + Emit(inbox, new() { Kind = InboxCommandKind.Accept, Message = message }); + + [UnsafeAccessor(UnsafeAccessorKind.Method, Name = "Emit")] + private static extern void Emit(Inbox inbox, InboxCommand command); + + [UnsafeAccessor(UnsafeAccessorKind.Field, Name = "_wakeupJobId")] + public static extern ref string? WakeupJobId(MessagingPump pump); + + [UnsafeAccessor(UnsafeAccessorKind.Field, Name = "_messages")] + private static extern ref Dictionary GetMessages(Inbox inbox); + + [UnsafeAccessor(UnsafeAccessorKind.Field, Name = "_processed")] + private static extern ref Dictionary GetProcessed(Inbox inbox); +} diff --git a/test/Orleans.Messaging.Tests/Support/IndependentJobClockFixture.cs b/test/Orleans.Messaging.Tests/Support/IndependentJobClockFixture.cs new file mode 100644 index 00000000000..f9e1eade400 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Support/IndependentJobClockFixture.cs @@ -0,0 +1,35 @@ +using System.Collections.Concurrent; +using Microsoft.Extensions.DependencyInjection; +using Orleans.DurableJobs; +using Orleans.Runtime; + +namespace Orleans.Messaging.Tests.Support; + +/// Separates actor job clocks without advancing the background scheduler's clock or timers. +internal sealed class IndependentJobClockFixture : FaultInjectionClusterFixture +{ + public IndependentJobClockFixture() => Clocks = new(JobClock); + + public PerGrainJobClock Clocks { get; } + + protected override void ConfigureServices(IServiceCollection services) + { + base.ConfigureServices(services); + services.AddKeyedSingleton(DurableJobTimeProviderNames.DurableJobs, Clocks); + } +} + +internal sealed class PerGrainJobClock(TimeProvider backgroundClock) : TimeProvider +{ + private readonly ConcurrentDictionary _clocks = new(); + + public void Set(IAddressable grain, TimeProvider clock) => _clocks[grain.GetGrainId()] = clock; + + public override DateTimeOffset GetUtcNow() => + ReceiverTestServices.CurrentGrainContext is { } context && _clocks.TryGetValue(context.GrainId, out var clock) + ? clock.GetUtcNow() + : backgroundClock.GetUtcNow(); + + public override ITimer CreateTimer(TimerCallback callback, object? state, TimeSpan dueTime, TimeSpan period) => + System.CreateTimer(callback, state, dueTime, period); +} diff --git a/test/Orleans.Messaging.Tests/Support/JournalProbeState.cs b/test/Orleans.Messaging.Tests/Support/JournalProbeState.cs new file mode 100644 index 00000000000..46feb316bf4 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Support/JournalProbeState.cs @@ -0,0 +1,25 @@ +using Orleans.Journaling; + +namespace Orleans.Messaging.Tests.Support; + +// An empty journal state which lets tests fail write preparation or observe write acknowledgement. It never stages a +// change, so it doesn't hide handler changes from failure detection. +internal sealed class JournalProbeState : IStateMachine, IReportsStagedChanges +{ + public const string Name = "test-journal-probe"; + + public JournalProbeState(IJournaledStateManager manager) => manager.RegisterStateMachine(Name, this); + + public Exception? PreparationFailure { get; set; } + public Action? WriteCompleted { get; set; } + + public ValueTask PrepareWriteAsync(CancellationToken cancellationToken = default) => + PreparationFailure is { } failure ? ValueTask.FromException(failure) : default; + + public void OnWriteCompleted() => WriteCompleted?.Invoke(); + public void ReplayEntry(JournalEntry entry, JournalReplayContext context) => + throw new NotSupportedException("The journal probe has no entries."); + public void Reset(JournalStreamWriter writer) { } + public void WritePendingEntries(JournalStreamWriter writer) { } + public void WriteSnapshot(JournalStreamWriter writer) { } +} diff --git a/test/Orleans.Messaging.Tests/Support/JournaledTestOutbox.cs b/test/Orleans.Messaging.Tests/Support/JournaledTestOutbox.cs new file mode 100644 index 00000000000..a24c13530be --- /dev/null +++ b/test/Orleans.Messaging.Tests/Support/JournaledTestOutbox.cs @@ -0,0 +1,69 @@ +using System.Collections; +using System.Diagnostics.CodeAnalysis; +using Microsoft.Extensions.DependencyInjection; +using Orleans.Journaling; +using Orleans.Runtime; +using Orleans.Serialization; +using Orleans.Serialization.Session; + +namespace Orleans.Messaging.Tests.Support; + +// Captures opaque handler output in the same journal as inbox effects. Dispatch belongs to the outbox layer. +internal sealed class JournaledTestOutbox(IDurableDictionary messages) + : IOutbox, ILifecycleObserver, IEnumerable> +{ + private bool _stopped; + private readonly TaskCompletionSource _stopping = new(TaskCreationOptions.RunContinuationsAsynchronously); + public Task Stopping => _stopping.Task; + public int SendCalls { get; private set; } + public Exception? NextSendFailure { get; set; } + public IDurableDictionary StoredMessages { get; } = messages; + public int Count => StoredMessages.Count; + public IEnumerable Messages => StoredMessages.Values; + public IEnumerator> GetEnumerator() => StoredMessages.GetEnumerator(); + IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); + + public void Send(OutboxMessage envelope) + { + SendCalls++; + if (_stopped) throw new InvalidOperationException("The test outbox owner is stopped."); + if (envelope.Envelope.MessageId.IsDefault || envelope.ReceiverId.IsDefault + || envelope.Envelope.Payload is null) + { + throw new ArgumentException("An outgoing envelope requires identities and opaque payload bytes.", nameof(envelope)); + } + if (TryGetMessage(envelope.Envelope.MessageId, out var existing)) + { + if (!EnvelopeEquivalence.AreEquivalent(existing, envelope)) + { + throw new InvalidOperationException($"The outbox already contains a different envelope with message ID '{envelope.Envelope.MessageId}'."); + } + return; + } + if (NextSendFailure is { } failure) + { + NextSendFailure = null; + throw failure; + } + StoredMessages.Add(envelope.Envelope.MessageId, envelope); + } + + public void Send(MessageId messageId) => + throw new NotSupportedException("The test outbox stores complete envelopes only."); + + public bool Withdraw(MessageId messageId) => + throw new NotSupportedException("The test outbox stores complete envelopes only."); + + public IOutbox Register(IOutboxMessageSource source) => + throw new NotSupportedException("The test outbox stores complete envelopes only."); + + public bool TryGetMessage(MessageId messageId, [MaybeNullWhen(false)] out OutboxMessage envelope) => + StoredMessages.TryGetValue(messageId, out envelope); + public Task OnStart(CancellationToken cancellationToken) => Task.CompletedTask; + public Task OnStop(CancellationToken cancellationToken) + { + _stopped = true; + _stopping.TrySetResult(); + return Task.CompletedTask; + } +} diff --git a/test/Orleans.Messaging.Tests/Support/ManualJobClock.cs b/test/Orleans.Messaging.Tests/Support/ManualJobClock.cs new file mode 100644 index 00000000000..ae123397e30 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Support/ManualJobClock.cs @@ -0,0 +1,33 @@ +namespace Orleans.Messaging.Tests.Support; + +/// +/// A Durable Jobs clock whose current time moves only when a test advances it. +/// +/// +/// Timers follow real time and fire at least every , so Durable Jobs re-reads the clock, +/// starts shards, and runs wake-ups promptly once a test advances it, while tests decide exactly when work becomes due. +/// +public sealed class ManualJobClock(DateTimeOffset start) : TimeProvider +{ + private static readonly TimeSpan PollInterval = TimeSpan.FromMilliseconds(100); + private long _utcTicks = start.UtcTicks; + + public override ITimer CreateTimer(TimerCallback callback, object? state, TimeSpan dueTime, TimeSpan period) => + new PollingTimer(base.CreateTimer(callback, state, Clamp(dueTime), Clamp(period))); + + public override DateTimeOffset GetUtcNow() => new(Interlocked.Read(ref _utcTicks), TimeSpan.Zero); + + public void Advance(TimeSpan delta) => Interlocked.Add(ref _utcTicks, delta.Ticks); + + private static TimeSpan Clamp(TimeSpan dueTime) => dueTime > PollInterval ? PollInterval : dueTime; + + // Task.Delay creates its timer disarmed and arms it through Change, so both paths clamp. + private sealed class PollingTimer(ITimer timer) : ITimer + { + public bool Change(TimeSpan dueTime, TimeSpan period) => timer.Change(Clamp(dueTime), Clamp(period)); + + public void Dispose() => timer.Dispose(); + + public ValueTask DisposeAsync() => timer.DisposeAsync(); + } +} diff --git a/test/Orleans.Messaging.Tests/Support/MessagingBehaviorTestBase.cs b/test/Orleans.Messaging.Tests/Support/MessagingBehaviorTestBase.cs new file mode 100644 index 00000000000..fdc54566a80 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Support/MessagingBehaviorTestBase.cs @@ -0,0 +1,83 @@ +using Microsoft.Extensions.DependencyInjection; +using Orleans.DurableJobs; +using Orleans.Journaling; +using Orleans.Runtime; +using Orleans.Serialization.Session; +using Xunit; + +namespace Orleans.Messaging.Tests.Support; + +public abstract class MessagingBehaviorTestBase : IAsyncLifetime +{ + protected MessagingBehaviorTestBase(bool receiverOnly = true) + : this(new MessagingClusterFixture(receiverOnly)) + { + } + + protected MessagingBehaviorTestBase(MessagingClusterFixture fixture) => Fixture = fixture; + + protected MessagingClusterFixture Fixture { get; } + protected SerializerSessionPool Sessions => Fixture.Client.ServiceProvider.GetRequiredService(); + + public ValueTask InitializeAsync() => Fixture.InitializeAsync(); + public ValueTask DisposeAsync() => Fixture.DisposeAsync(); + + protected IMessagingTestGrain NewGrain() => + Fixture.Client.GetGrain(Guid.NewGuid()); + + protected static TestMessage NewMessage(int sequence, string value) => + new(TestApplicationProtocol.NewMessageId(), sequence, value); + + protected async Task ReactivateAsync(IMessagingTestGrain receiver) + { + var previous = Fixture.GetGrainContext(receiver); + await receiver.RequestDeactivationAsync(); + await previous.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + _ = await receiver.GetSnapshotAsync(); + Assert.NotSame(previous, Fixture.GetGrainContext(receiver)); + } + + protected static Task DeliverAsync( + IMessagingTestGrain receiver, + InboxMessage envelope) => + DeliverWithCancellationAsync(receiver, envelope, TestContext.Current.CancellationToken); + + protected static async Task DeliverWithCancellationAsync( + IMessagingTestGrain receiver, + InboxMessage envelope, + CancellationToken cancellationToken) => + await receiver.AsReference().DeliverAsync(envelope, cancellationToken); + + protected static async Task WaitForBarrierAsync( + IMessagingTestGrain receiver, + HandlerProbe.Barrier barrier) + { + try + { + await barrier.WaitUntilEnteredAsync(); + } + catch (TimeoutException exception) + { + var snapshot = await receiver.GetSnapshotAsync(); + throw new TimeoutException( + $"Handler did not start. Inbox={snapshot.InboxCount}, effects={snapshot.Effects.Count}, maxHandlers={snapshot.MaxConcurrentHandlers}, deadLetters={string.Join(" | ", snapshot.InboxDeadLetters.Select(static item => item.Reason))}.", + exception); + } + } + + protected InboxMessage CreateEnvelope( + IMessagingTestGrain receiver, + TestMessage message, + string route = "messages/record") => + CreateEnvelope(receiver, (object)message, route); + + protected InboxMessage CreateEnvelope( + IMessagingTestGrain receiver, + object body, + string route) + { + var sessions = Fixture.Client.ServiceProvider.GetRequiredService(); + var sender = GrainId.Create("external-test-sender", Guid.NewGuid().ToString("N")); + return TestApplicationProtocol.CreateInbox(sessions, sender, route, body, Fixture.JobNow); + } +} diff --git a/test/Orleans.Messaging.Tests/Support/MessagingClusterFixture.cs b/test/Orleans.Messaging.Tests/Support/MessagingClusterFixture.cs new file mode 100644 index 00000000000..de8e28db255 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Support/MessagingClusterFixture.cs @@ -0,0 +1,373 @@ +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Options; +using Microsoft.Extensions.Time.Testing; +using Orleans.Configuration; +using Orleans.DurableJobs; +using Orleans.Hosting; +using Orleans.Journaling; +using Orleans.Messaging.Configuration; +using Orleans.TestingHost; +using Xunit; + +namespace Orleans.Messaging.Tests.Support; + +/// +/// Hosts an in-process messaging test cluster. +/// +/// +/// The fixture builds and deploys the cluster in , after every derived constructor has run, so +/// and overrides observe fully constructed derived state. +/// +public class MessagingClusterFixture : IAsyncLifetime +{ + private readonly int _initialSilos; + private readonly bool _receiverOnly; + private readonly bool _useServiceCollection; + private InProcessTestCluster? _cluster; + + public MessagingClusterFixture() : this(receiverOnly: true) + { + } + + public MessagingClusterFixture(bool receiverOnly) : this(1, receiverOnly) + { + } + + protected MessagingClusterFixture(int initialSilos, bool receiverOnly = false, bool useServiceCollection = false) + { + Clock = new FakeTimeProvider(DateTimeOffset.UtcNow); + Storage = new ControlledJournalStorageProvider(); + Metrics = new MessagingMetricProbe(); + JobManagerProbe = new DurableJobManagerProbe(); + HandlerProbe = new HandlerProbe(); + SnapshotProbe = new SnapshotProbe(); + ActivationProbe = new ActivationValidationProbe(); + _initialSilos = initialSilos; + _receiverOnly = receiverOnly; + _useServiceCollection = useServiceCollection; + } + + public InProcessTestCluster Cluster => + _cluster ?? throw new InvalidOperationException($"The cluster is available after {nameof(InitializeAsync)} builds it."); + + public IClusterClient Client => Cluster.Client!; + public DateTimeOffset JobNow => Cluster.Silos[0].ServiceProvider + .GetRequiredKeyedService(DurableJobTimeProviderNames.DurableJobs).GetUtcNow(); + public FakeTimeProvider Clock { get; } + public ControlledJournalStorageProvider Storage { get; } + public MessagingMetricProbe Metrics { get; } + public DurableJobManagerProbe JobManagerProbe { get; } + public HandlerProbe HandlerProbe { get; } + public SnapshotProbe SnapshotProbe { get; } + public ActivationValidationProbe ActivationProbe { get; } + + private InProcessTestCluster BuildCluster() + { + var clusterId = $"messaging-{Guid.NewGuid():N}"; + var serviceId = $"messaging-service-{Guid.NewGuid():N}"; + var builder = new InProcessTestClusterBuilder((short)_initialSilos); + builder.ConfigureClient(clientBuilder => + clientBuilder.Configure(options => + { + options.ClusterId = clusterId; + options.ServiceId = serviceId; + })); + builder.ConfigureSilo((_, siloBuilder) => + { + siloBuilder.Configure(options => + { + options.ClusterId = clusterId; + options.ServiceId = serviceId; + }); + siloBuilder.Services.AddSingleton(Clock); + siloBuilder.Services.UseTimeProviderForBackgroundAreas(TimeProvider.System); + // Advancing the test clock also advances the Durable Jobs clock, which inbox retry deadlines follow. + siloBuilder.Services.AddKeyedSingleton( + DurableJobTimeProviderNames.DurableJobs, + new FollowingJobClock(Clock, Clock.GetUtcNow())); + siloBuilder.Services.AddSingleton(HandlerProbe); + siloBuilder.Services.AddSingleton(new TestHandlerConfiguration()); + siloBuilder.Services.AddSingleton(SnapshotProbe); + siloBuilder.Services.AddSingleton(ActivationProbe); + siloBuilder.UseInMemoryDurableJobs(); + // The test grains journal Orleans-serialized types. + siloBuilder.Services.Configure(options => options.JournalFormatKey = "orleans-binary"); + if (_receiverOnly) + { + ReceiverTestServices.Add(siloBuilder.Services, ConfigureOptions); + } + else if (_useServiceCollection) + { + siloBuilder.Services.AddMessaging(ConfigureOptions); + } + else + { + siloBuilder.AddMessaging(ConfigureOptions); + } + siloBuilder.ConfigureServices(services => + ControlledDurableJobManager.Decorate(services, JobManagerProbe)); + siloBuilder.Services.RemoveAll(); + siloBuilder.Services.RemoveAll(); + siloBuilder.Services.AddSingleton(Storage); + siloBuilder.Services.AddSingleton(serviceProvider => + { + Storage.Configure(serviceProvider.GetRequiredService>()); + return Storage; + }); + siloBuilder.Services.AddSingleton( + serviceProvider => (IJournalStorageCatalog)serviceProvider.GetRequiredService()); + ConfigureServices(siloBuilder.Services); + }); + return builder.Build(); + } + + /// + /// Makes the Durable Jobs clock read , so retry deadlines are exact test-clock times. Durable Jobs + /// timers keep following real time and run each wake-up once the test clock reaches its due time. + /// + protected void UseTestClockForDurableJobs(IServiceCollection services) => + services.AddKeyedSingleton(DurableJobTimeProviderNames.DurableJobs, new TestJobClock(Clock)); + + /// A Durable Jobs clock which follows real time plus the test clock's advance since the fixture started. + internal sealed class FollowingJobClock(TimeProvider clock, DateTimeOffset clockStart) : TimeProvider + { + public override DateTimeOffset GetUtcNow() => System.GetUtcNow() + (clock.GetUtcNow() - clockStart); + } + + private sealed class TestJobClock(TimeProvider clock) : TimeProvider + { + public override DateTimeOffset GetUtcNow() => clock.GetUtcNow(); + } + + public Task WaitForEffectCountAsync(IMessagingTestGrain grain, int expected) => + SnapshotProbe.WaitAsync( + grain.GetGrainId(), + snapshot => snapshot.Effects.Sum(static effect => effect.Count) >= expected); + + public Task WaitForInboxCountAsync(IMessagingTestGrain grain, int expected) => + SnapshotProbe.WaitAsync(grain.GetGrainId(), snapshot => snapshot.InboxCount == expected); + + public Task WaitForDeadLetterCountAsync(IMessagingTestGrain grain, int expected) => + SnapshotProbe.WaitAsync( + grain.GetGrainId(), + snapshot => snapshot.InboxDeadLetters.Count + snapshot.OutboxDeadLetters.Count >= expected); + + public Task WaitForOutboxCountAsync(IMessagingTestGrain grain, int expected) => + SnapshotProbe.WaitAsync(grain.GetGrainId(), snapshot => snapshot.OutboxCount == expected); + + /// Gets the outbox depth which the cluster's silos report through the outbox depth metric. + public long GetOutboxDepth() => + Cluster.Silos.Sum(silo => Metrics.GetDepth( + silo.ServiceProvider.GetRequiredService().Meter, + "orleans-messaging-outbox-depth")); + + /// Waits until the cluster's silos report an outbox depth of . + public Task WaitForOutboxDepthAsync(long expected) => + Metrics.WaitUntilAsync(() => GetOutboxDepth() == expected, TestContext.Current.CancellationToken); + + public EndpointSnapshot GetSnapshot(IMessagingTestGrain grain) => + GetGrainInstance(grain).GetSnapshotForTest(); + + public OutboxMessage[] GetStagedOutput(IMessagingTestGrain grain) => + GetGrainContext(grain).ActivationServices.GetRequiredService().Messages().ToArray(); + + public ValueTask WriteStateAsync(IMessagingTestGrain grain) => + GetGrainContext(grain).ActivationServices + .GetRequiredService() + .WriteStateAsync(TestContext.Current.CancellationToken); + + internal Inbox GetInbox(IAddressable grain) => + GetGrainContext(grain).ActivationServices.GetRequiredService(); + + internal MessagingPump GetPump(IAddressable grain) => + GetGrainContext(grain).ActivationServices.GetRequiredService(); + + /// + /// Schedules an additional pump wake-up. Wake-ups are at-least-once hints, so extra ones must be harmless. + /// + public async Task WakeAsync(IAddressable grain) + { + var services = Cluster.TryGetGrainContext(grain.GetGrainId(), out var context) + ? context.ActivationServices + : Cluster.Silos[0].ServiceProvider; + await services.GetRequiredService().ScheduleJobAsync( + new ScheduleJobRequest + { + Target = grain.GetGrainId(), + JobName = MessagingPump.JobName, + DueTime = DateTimeOffset.UtcNow + }, + TestContext.Current.CancellationToken); + } + + /// + /// Executes the pump's durable job callback directly as its current wake-up job, as an early or spurious firing of that + /// job would. + /// + internal Task RunPumpAsync(IAddressable grain) => + RunCancelablePumpAsync(grain, TestContext.Current.CancellationToken); + + /// Executes the pump's durable job callback directly as a job other than its current wake-up job. + internal Task RunDuplicatePumpAsync(IAddressable grain) => + RunPumpAsync(grain, TestContext.Current.CancellationToken, duplicate: true); + + /// + /// Executes the pump's durable job callback directly with as the run's token, so a + /// test can cancel the run as Durable Jobs does. + /// + internal Task RunCancelablePumpAsync(IAddressable grain, CancellationToken runCancellation) => + RunPumpAsync(grain, runCancellation, duplicate: false); + + private async Task RunPumpAsync(IAddressable grain, CancellationToken runCancellation, bool duplicate) + { + var pump = GetPump(grain); + var run = await OnGrainAsync(grain, () => + { + var job = new DurableJob + { + Id = (duplicate ? null : InboxTestAccess.WakeupJobId(pump)) ?? Guid.NewGuid().ToString("N"), + Name = MessagingPump.JobName, + ShardId = "test", + TargetGrainId = grain.GetGrainId(), + DueTime = DateTimeOffset.UtcNow + }; + return pump.ExecuteJobAsync(new PumpRunContext(job), runCancellation).AsTask(); + }); + return await run.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + } + + /// Runs on the grain's activation scheduler. + internal Task OnGrainAsync(IAddressable grain, Func func) + { + var done = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + GetGrainContext(grain).Scheduler.QueueAction(() => + { + try + { + done.SetResult(func()); + } + catch (Exception exception) + { + done.SetException(exception); + } + }); + return done.Task; + } + + internal Task OnGrainAsync(IAddressable grain, Action action) => + OnGrainAsync(grain, () => + { + action(); + return true; + }); + + /// Polls on the grain's scheduler until it holds. + internal async Task WaitUntilAsync(IAddressable grain, Func predicate) + { + var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(30); + while (!await OnGrainAsync(grain, predicate)) + { + if (DateTime.UtcNow > deadline) + { + throw new TimeoutException($"Condition was not met for grain '{grain.GetGrainId()}'."); + } + + await Task.Delay(10, TestContext.Current.CancellationToken); + } + } + + /// Forces idle activation collection until deactivates. + internal async Task CollectAsync(IGrainContext context) + { + var management = Client.GetGrain(0); + var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(30); + while (!context.Deactivated.IsCompleted) + { + if (DateTime.UtcNow > deadline) + { + throw new TimeoutException($"Activation '{context.GrainId}' was not collected."); + } + + await management.ForceActivationCollection(TimeSpan.Zero, TestContext.Current.CancellationToken); + await Task.Delay(50, TestContext.Current.CancellationToken); + } + } + + internal IGrainContext GetGrainContext(IAddressable grain) + { + if (!Cluster.TryGetGrainContext(grain.GetGrainId(), out var context)) + { + throw new InvalidOperationException($"Grain '{grain.GetGrainId()}' is not active."); + } + + return context; + } + + private MessagingTestGrain GetGrainInstance(IMessagingTestGrain grain) => + GetGrainContext(grain).GrainInstance as MessagingTestGrain + ?? throw new InvalidOperationException($"Grain '{grain.GetGrainId()}' has an unexpected implementation."); + + private sealed class PumpRunContext(DurableJob job) : IJobRunContext + { + public DurableJob Job { get; } = job; + public string RunId { get; } = Guid.NewGuid().ToString("N"); + public int DequeueCount => 1; + } + + protected virtual void ConfigureServices(IServiceCollection services) { } + + protected virtual void ConfigureOptions(DurableMessagingOptions options) + { + options.MaxCapacity = 2; + options.DeduplicationWindow = TimeSpan.FromMinutes(10); + options.OutboxRetryTimeout = TimeSpan.FromSeconds(3); + options.InboxRetryTimeout = TimeSpan.Zero; + options.MaxRetainedDeadLetters = 2; + options.DeadLetterRetentionPeriod = TimeSpan.FromHours(1); + options.RetryInitialInterval = TimeSpan.FromMilliseconds(25); + options.InboxBatchSize = 8; + options.OutboxBatchSize = 8; + } + + public ValueTask InitializeAsync() + { + if (_cluster is not null) throw new InvalidOperationException("The cluster is already initialized."); + _cluster = BuildCluster(); + return new(_cluster.DeployAsync()); + } + + public virtual async ValueTask DisposeAsync() + { + if (_cluster is not null) await _cluster.DisposeAsync(); + Metrics.Dispose(); + } +} + +public sealed class MultiSiloMessagingClusterFixture : MessagingClusterFixture +{ + public MultiSiloMessagingClusterFixture() : base(2) + { + } +} + +public sealed class DedupeExpiryClusterFixture : MessagingClusterFixture +{ +} + +public sealed class InboxCapacityClusterFixture : MessagingClusterFixture +{ + protected override void ConfigureOptions(DurableMessagingOptions options) + { + base.ConfigureOptions(options); + options.MaxCapacity = 1; + options.InboxRetryTimeout = TimeSpan.FromHours(1); + options.RetryInitialInterval = TimeSpan.FromHours(1); + } + + protected override void ConfigureServices(IServiceCollection services) + { + base.ConfigureServices(services); + UseTestClockForDurableJobs(services); + } +} diff --git a/test/Orleans.Messaging.Tests/Support/MessagingMetricProbe.cs b/test/Orleans.Messaging.Tests/Support/MessagingMetricProbe.cs new file mode 100644 index 00000000000..7b06473c373 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Support/MessagingMetricProbe.cs @@ -0,0 +1,130 @@ +using System.Collections.Concurrent; +using System.Diagnostics.Metrics; +using Xunit; + +namespace Orleans.Messaging.Tests.Support; + +public sealed class MessagingMetricProbe : IDisposable +{ + private readonly ConcurrentDictionary<(string Instrument, string TagValue), long> _measurements = []; + private readonly ConcurrentDictionary<(Meter Meter, string Instrument), long> _depths = []; + private readonly object _lock = new(); + private TaskCompletionSource _changed = CreateSignal(); + private readonly MeterListener _listener; + + public MessagingMetricProbe() + { + _listener = new MeterListener + { + InstrumentPublished = static (instrument, listener) => + { + if (instrument.Meter.Name == "Microsoft.Orleans" + && instrument.Name is "orleans-messaging-orphaned-jobs-reclaimed" + or "orleans-messaging-inbox-messages-processed" + or "orleans-durablejobs-job-attempts-started" + or "orleans-durablejobs-handler-executions-started" + or "orleans-durablejobs-jobs-completed" + or "orleans-durablejobs-jobs-rescheduled" + or "orleans-messaging-inbox-depth" + or "orleans-messaging-outbox-depth") + { + listener.EnableMeasurementEvents(instrument); + } + } + }; + _listener.SetMeasurementEventCallback(OnMeasurement); + _listener.Start(); + } + + public long GetCount(string instrument, string tagValue = "") => + _measurements.TryGetValue((instrument, tagValue), out var count) ? count : 0; + + /// Gets the current value of a depth instrument created by . + /// Each silo creates its own meter, so a meter identifies one silo's depth. + public long GetDepth(Meter meter, string instrument) => _depths.TryGetValue((meter, instrument), out var value) ? value : 0; + + public Task WaitForCountAsync( + string instrument, + long expected, + string tagValue = "") => + WaitForCountWithCancellationAsync( + instrument, + expected, + tagValue, + TestContext.Current.CancellationToken); + + public Task WaitForCountWithCancellationAsync( + string instrument, + long expected, + string tagValue, + CancellationToken cancellationToken) => + WaitUntilAsync(() => GetCount(instrument, tagValue) >= expected, cancellationToken); + + /// Waits until , which reads this probe's measurements, holds. + public async Task WaitUntilAsync(Func condition, CancellationToken cancellationToken) + { + using var timeout = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + timeout.CancelAfter(TimeSpan.FromSeconds(30)); + while (!condition()) + { + Task changed; + lock (_lock) + { + if (condition()) + { + return; + } + + changed = _changed.Task; + } + + await changed.WaitAsync(timeout.Token); + } + } + + public void Dispose() => _listener.Dispose(); + + private void OnMeasurement( + Instrument instrument, + long measurement, + ReadOnlySpan> tags, + object? state) + { + if (instrument.Name is "orleans-messaging-inbox-depth" or "orleans-messaging-outbox-depth") + { + // The depth instruments are UpDownCounters, so each measurement is a delta. + _depths.AddOrUpdate((instrument.Meter, instrument.Name), measurement, (_, current) => current + measurement); + SignalChanged(); + return; + } + + var tagValue = ""; + var tagName = instrument.Name == "orleans-messaging-inbox-messages-processed" ? "status" : "job_name"; + foreach (var tag in tags) + { + if (tag.Key == tagName) + { + tagValue = tag.Value as string ?? ""; + break; + } + } + + _measurements.AddOrUpdate( + (instrument.Name, tagValue), + measurement, + (_, current) => current + measurement); + SignalChanged(); + } + + private void SignalChanged() + { + lock (_lock) + { + _changed.TrySetResult(); + _changed = CreateSignal(); + } + } + + private static TaskCompletionSource CreateSignal() => + new(TaskCreationOptions.RunContinuationsAsynchronously); +} diff --git a/test/Orleans.Messaging.Tests/Support/MessagingTestGrains.cs b/test/Orleans.Messaging.Tests/Support/MessagingTestGrains.cs new file mode 100644 index 00000000000..5b8f9ea718e --- /dev/null +++ b/test/Orleans.Messaging.Tests/Support/MessagingTestGrains.cs @@ -0,0 +1,497 @@ +using System.Collections.Concurrent; +using Microsoft.Extensions.DependencyInjection; +using Orleans.Concurrency; +using Orleans.DurableJobs; +using Orleans.Journaling; +using Orleans.Messaging; +using Orleans.Runtime; +using Orleans.Runtime.Diagnostics; +using Orleans.Serialization; +using Orleans.Serialization.Session; + +namespace Orleans.Messaging.Tests.Support; + +public interface IMessagingTestGrain : IGrainWithGuidKey +{ + Task SendAsync(GrainId target, string route, TestMessage message); + Task SendDuplicateAsync(GrainId target, string route, TestMessage message); + Task SendAndDeactivateAsync(GrainId target, string route, TestMessage message); + Task StageWithoutCommitAsync(GrainId target, string route, TestMessage message); + Task RetryWriteStateAsync(); + Task StageEffectAsync(MessageEffect effect); + Task StageOutputAsync(OutboxMessage envelope); + Task AcceptAndDeactivateAsync(InboxMessage envelope); + Task ConfigureHandlerAsync(bool enabled); + Task RemoveInboxDeadLetterAsync(MessageId messageId); + Task RemoveOutboxDeadLetterAsync(MessageId messageId); + Task GetSnapshotAsync(); + Task RequestDeactivationAsync(); + Task SetControlEnvelopeAsync(InboxMessage envelope); + Task DeleteStateAndDeactivateAsync(); + Task HoldPumpTurnAsync(string barrierRoute, bool deactivate); + Task HoldPumpTurnAsync(string barrierRoute, InboxMessage replacement, bool deactivate); +} + +[GenerateSerializer, Immutable] +public sealed record TestMessage( + [property: Id(0)] MessageId LogicalId, + [property: Id(1)] int Sequence, + [property: Id(2)] string Value, + [property: Id(3)] GrainId? ForwardTo = null, + [property: Id(8)] bool ThrowDuringPreparation = false, + [property: Id(5)] bool CommitDuringHandling = false, + [property: Id(6)] bool DeleteDuringHandling = false, + [property: Id(9)] bool ThrowOnceDuringPreparation = false, + [property: Id(10)] int ThrowForAttempts = 0, + [property: Id(11)] bool ThrowAfterStaging = false); + +[GenerateSerializer, Immutable] +public sealed record MessageEffect( + [property: Id(0)] MessageId LogicalId, + [property: Id(1)] int Count, + [property: Id(2)] int Sequence, + [property: Id(3)] string Value); + +[GenerateSerializer, Immutable] +public sealed record EndpointSnapshot( + [property: Id(0)] Guid ActivationId, + [property: Id(1)] string SiloAddress, + [property: Id(2)] int InboxCount, + [property: Id(3)] int OutboxCount, + [property: Id(4)] int MaxConcurrentHandlers, + [property: Id(5)] IReadOnlyList Effects, + [property: Id(6)] IReadOnlyList InboxDeadLetters, + [property: Id(7)] IReadOnlyList OutboxDeadLetters, + [property: Id(9)] int ProcessedMessageCount); + +[GenerateSerializer, Immutable] +public sealed record DeadLetterSnapshot( + [property: Id(0)] MessageId MessageId, + [property: Id(1)] string Route, + [property: Id(2)] string Reason, + [property: Id(3)] int AttemptCount, + [property: Id(4)] DateTimeOffset DeadLetteredAt); + +[GrainType("messaging-inbox-test")] +public sealed class MessagingTestGrain : DurableGrain, IMessagingTestGrain, IDurableJobHandler, + IObserver, IDisposable +{ + private readonly IGrainContext _grainContext; + private readonly IDisposable _lifecycleSubscription; + private readonly IJournaledStateManager _journalOwner; + private readonly IOutbox _outbox; + private readonly IMessagingDiagnostics _diagnostics; + private readonly IDurableDictionary _effects; + private readonly Inbox _inboxState; + private readonly IInbox _inbox; + private readonly SerializerSessionPool _sessions; + private readonly TestHandlerConfiguration _handlerConfiguration; + internal IInboxHandler? HandlerOverride { get; set; } + private readonly JournalProbeState _journalProbe; + internal Exception? JournalPreparationFailure + { + get => _journalProbe.PreparationFailure; + set => _journalProbe.PreparationFailure = value; + } + private readonly ILocalSiloDetails _siloDetails; + private readonly HandlerProbe _handlerProbe; + private readonly SnapshotProbe _snapshotProbe; + private readonly Guid _activationId = Guid.NewGuid(); + private int _activeHandlers; + private int _maxConcurrentHandlers; + private bool _handlerRegistered; + // Spans activations so a replayed message does not fail again after recovery. + private static readonly ConcurrentDictionary FailedOnce = new(); + private static readonly ConcurrentDictionary FailedAttempts = new(); + + public MessagingTestGrain( + IGrainContext grainContext, + IJournaledStateManager journalOwner, + IInbox inbox, + IOutbox outbox, + IMessagingDiagnostics diagnostics, + [FromKeyedServices("test-effects")] IDurableDictionary effects, + [FromKeyedServices("inbox")] IDurableValue applicationInboxState, + + + SerializerSessionPool sessions, + TestHandlerConfiguration handlerConfiguration, + ILocalSiloDetails siloDetails, + HandlerProbe handlerProbe, + SnapshotProbe snapshotProbe) + { + _grainContext = grainContext; + _journalOwner = journalOwner; + _inbox = inbox; + _outbox = outbox; + _diagnostics = diagnostics; + _effects = effects; + ArgumentNullException.ThrowIfNull(applicationInboxState); + + _inboxState = grainContext.ActivationServices.GetRequiredService(); + + _sessions = sessions; + _handlerConfiguration = handlerConfiguration; + _siloDetails = siloDetails; + _handlerProbe = handlerProbe; + _snapshotProbe = snapshotProbe; + _lifecycleSubscription = GrainLifecycleEvents.AllEvents.Subscribe(this); + _journalProbe = new JournalProbeState(journalOwner); + } + + public override async Task OnActivateAsync(CancellationToken cancellationToken) + { + if (_handlerConfiguration.IsEnabled(this.GetGrainId())) + { + RegisterHandler(); + } + await base.OnActivateAsync(cancellationToken); + _snapshotProbe.Publish(this.GetGrainId(), CreateSnapshot()); + } + + public async Task SendAsync(GrainId target, string route, TestMessage message) + { + var envelope = CreateEnvelope(target, route, message); + _outbox.Send(envelope); + await WriteStateAsync(); + return envelope.Envelope.MessageId; + } + + public async Task SendDuplicateAsync(GrainId target, string route, TestMessage message) + { + var envelope = CreateEnvelope(target, route, message); + _outbox.Send(envelope); + _outbox.Send(envelope); + await WriteStateAsync(); + return envelope.Envelope.MessageId; + } + + public async Task SendAndDeactivateAsync(GrainId target, string route, TestMessage message) + { + var messageId = await SendAsync(target, route, message); + DeactivateOnIdle(); + return messageId; + } + + public Task StageWithoutCommitAsync(GrainId target, string route, TestMessage message) + { + var envelope = CreateEnvelope(target, route, message); + _outbox.Send(envelope); + return Task.FromResult(envelope.Envelope.MessageId); + } + + public async Task RetryWriteStateAsync() => await WriteStateAsync(); + + public async Task AcceptAndDeactivateAsync(InboxMessage envelope) + { + var extension = _inboxState; + var result = await extension.DeliverAsync(envelope); + AcceptedSnapshot = CreateSnapshot(); + DeactivateOnIdle(); + return result; + } + + public Task StageOutputAsync(OutboxMessage envelope) + { + _outbox.Send(envelope); + return Task.CompletedTask; + } + + public Task StageEffectAsync(MessageEffect effect) + { + _effects[effect.LogicalId] = effect; + return Task.CompletedTask; + } + + public Task ConfigureHandlerAsync(bool enabled) + { + _handlerConfiguration.Set(this.GetGrainId(), enabled); + if (enabled && !_handlerRegistered) RegisterHandler(); + return Task.CompletedTask; + } + + private void RegisterHandler() + { + _inbox.Register(new ApplicationDispatcher(this)); + _handlerRegistered = true; + } + + public async Task RemoveInboxDeadLetterAsync(MessageId messageId) + { + if (!_diagnostics.RemoveInboxDeadLetter(messageId)) + { + return false; + } + + await WriteStateAsync(); + return true; + } + + public async Task RemoveOutboxDeadLetterAsync(MessageId messageId) + { + if (!_diagnostics.RemoveOutboxDeadLetter(messageId)) + { + return false; + } + + await WriteStateAsync(); + return true; + } + + public Task GetSnapshotAsync() => Task.FromResult(CreateSnapshot()); + + internal EndpointSnapshot GetSnapshotForTest() => CreateSnapshot(); + + public Task RequestDeactivationAsync() + { + DeactivateOnIdle(); + return Task.CompletedTask; + } + + public async Task DeleteStateAndDeactivateAsync() + { + try + { + await ((ILifecycleObserver)ServiceProvider.GetRequiredService()).OnStop(CancellationToken.None); + if (_outbox is ILifecycleObserver testOutbox) + { + await testOutbox.OnStop(CancellationToken.None); + } + + await _journalOwner.DeleteStateAsync(CancellationToken.None); + } + finally + { + DeactivateOnIdle(); + } + } + + internal EndpointSnapshot? AcceptedSnapshot { get; private set; } + + private InboxMessage? _controlEnvelope; + internal TaskCompletionSource ControlDeliveryEntered { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public Task SetControlEnvelopeAsync(InboxMessage envelope) + { + _controlEnvelope = envelope; + return Task.CompletedTask; + } + + internal TaskScheduler? JobScheduler { get; private set; } + internal IGrainContext? JobGrainContext { get; private set; } + + public async Task ExecuteJobAsync(IJobRunContext context, CancellationToken attemptCancellationToken) + { + JobScheduler = TaskScheduler.Current; + JobGrainContext = ReceiverTestServices.CurrentGrainContext; + switch (context.Job.Name) + { + case "test/deliver-envelope": + ControlDeliveryEntered.TrySetResult(); + var extension = _inboxState; + await extension.DeliverAsync(_controlEnvelope ?? throw new InvalidOperationException("A control envelope must be configured."), attemptCancellationToken); + break; + case "test/write-journal": + await StateManager.WriteStateAsync(attemptCancellationToken); + break; + case "test/probe-scheduler": + break; + default: + throw new NotSupportedException($"Unknown test job '{context.Job.Name}'."); + } + } + + public Task HoldPumpTurnAsync(string barrierRoute, bool deactivate) => + HoldPumpTurnCoreAsync(barrierRoute, null, deactivate); + + public Task HoldPumpTurnAsync(string barrierRoute, InboxMessage replacement, bool deactivate) => + HoldPumpTurnCoreAsync(barrierRoute, replacement, deactivate); + + private async Task HoldPumpTurnCoreAsync(string barrierRoute, InboxMessage? replacement, bool deactivate) + { + if (!_handlerProbe.TryGet(this.GetGrainId(), barrierRoute, out var barrier)) + { + throw new InvalidOperationException("The pump-turn barrier must be armed."); + } + barrier.Entered.TrySetResult(); + await barrier.Continue.Task; + if (replacement is { } envelope) + { + var extension = _inboxState; + await extension.DeliverAsync(envelope); + } + if (deactivate) + { + DeactivateOnIdle(); + } + } + + internal TaskCompletionSource DeactivationFailure { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + void IObserver.OnNext(GrainLifecycleEvents.LifecycleEvent value) + { + if (value is GrainLifecycleEvents.Deactivating { Reason.Exception: { } exception } deactivating + && ReferenceEquals(deactivating.GrainContext, _grainContext)) + { + DeactivationFailure.TrySetResult(exception); + } + } + + void IObserver.OnError(Exception error) => DeactivationFailure.TrySetException(error); + void IObserver.OnCompleted() { } + public void Dispose() + { + _controlEnvelope = null; + _lifecycleSubscription.Dispose(); + } + + private readonly ConcurrentQueue _captures = new(); + private readonly ConcurrentQueue _outputCaptures = new(); + internal IReadOnlyList Captures => _captures.ToArray(); + internal IReadOnlyList OutputCaptures => _outputCaptures.ToArray(); + internal Exception? NextApplyFailure { get; set; } + internal TaskCompletionSource ApplyAttempted { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + internal void ClearCaptures() => _captures.Clear(); + + internal EndpointSnapshot CaptureStorageWrite() + { + var snapshot = CreateSnapshot(); + _captures.Enqueue(snapshot); + _outputCaptures.Enqueue(_outbox.Messages().Select(static envelope => envelope.Envelope.MessageId).ToArray()); + return snapshot; + } + + internal void PublishStoredSnapshot(EndpointSnapshot snapshot) => _snapshotProbe.Publish(this.GetGrainId(), snapshot); + internal void CaptureStorageRead() => ReplayedSnapshot = CreateSnapshot(); + internal EndpointSnapshot? ReplayedSnapshot { get; private set; } + + private OutboxMessage CreateEnvelope(GrainId target, string route, TestMessage message) => + TestApplicationProtocol.Create(_sessions, this.GetGrainId(), target, route, message); + + private async ValueTask HandleAsync( + TestMessage message, + IInboxHandlerContext context, + CancellationToken cancellationToken) + { + var active = Interlocked.Increment(ref _activeHandlers); + _maxConcurrentHandlers = Math.Max(_maxConcurrentHandlers, active); + try + { + if (_handlerProbe.TryGet(this.GetGrainId(), TestApplicationProtocol.Read(_sessions, context.Message).Route, out var gate)) + { + gate.Entered.TrySetResult(); + await gate.WaitAsync(cancellationToken); + } + + if (_handlerProbe.TryGet(this.GetGrainId(), TestApplicationProtocol.Read(_sessions, context.Message).Route + "/application-preparation", out var preparation)) + { + preparation.Entered.TrySetResult(); + await preparation.Continue.Task.WaitAsync(cancellationToken); + } + cancellationToken.ThrowIfCancellationRequested(); + if (message.CommitDuringHandling) + { + await WriteStateAsync(cancellationToken); + } + if (message.ThrowDuringPreparation + || (message.ThrowOnceDuringPreparation && FailedOnce.TryAdd(message.LogicalId, 0)) + || (message.ThrowForAttempts > 0 && FailedAttempts.AddOrUpdate(message.LogicalId, 1, static (_, count) => count + 1) <= message.ThrowForAttempts)) + { + throw new InvalidOperationException($"Injected handler preparation failure for {message.LogicalId}."); + } + + var outgoing = message.ForwardTo is { } destination + ? TestApplicationProtocol.Create(_sessions, this.GetGrainId(), destination, "messages/forwarded", + message with { ForwardTo = null, ThrowDuringPreparation = false }, + new MessageId(context.Message.Envelope.MessageId.Key.CreateChildKey("forwarded"))) + : (OutboxMessage?)null; + if (NextApplyFailure is { } failure) + { + NextApplyFailure = null; + throw failure; + } + _effects.TryGetValue(message.LogicalId, out var prior); + _effects[message.LogicalId] = new MessageEffect(message.LogicalId, (prior?.Count ?? 0) + 1, message.Sequence, message.Value); + if (message.ThrowAfterStaging) + { + throw new InvalidOperationException($"Injected handler failure after staging for {message.LogicalId}."); + } + + ApplyAttempted.TrySetResult(); + if (outgoing is { } output) + { + _outbox.Send(output); + if (TestApplicationProtocol.Read(_sessions, context.Message).Route == "messages/duplicate-output") + { + _outbox.Send(output); + } + } + context.Complete(); + } + finally + { + Interlocked.Decrement(ref _activeHandlers); + } + } + + private void PublishSnapshot() => _snapshotProbe.Publish(this.GetGrainId(), CreateSnapshot()); + + private EndpointSnapshot CreateSnapshot() => + new( + _activationId, + _siloDetails.SiloAddress.ToParsableString(), + _inboxState.Count, + _outbox.Count(), + _maxConcurrentHandlers, + _effects.Values.OrderBy(static effect => effect.Sequence).ToArray(), + _diagnostics.InboxDeadLetters.Select(ToSnapshot).ToArray(), + _diagnostics.OutboxDeadLetters.Select(ToSnapshot).ToArray(), + InboxTestAccess.GetProcessedCount(_inboxState)); + + private DeadLetterSnapshot ToSnapshot(DeadLetter deadLetter) => + new( + deadLetter.Message.Envelope.MessageId, + TestApplicationProtocol.Read(_sessions, deadLetter.Message).Route, + deadLetter.Reason, + deadLetter.AttemptCount, + deadLetter.DeadLetteredAt); + + private DeadLetterSnapshot ToSnapshot(DeadLetter deadLetter) => + new( + deadLetter.Message.Envelope.MessageId, + TestApplicationProtocol.Read(_sessions, deadLetter.Message).Route, + deadLetter.Reason, + deadLetter.AttemptCount, + deadLetter.DeadLetteredAt); + + private sealed class ApplicationDispatcher(MessagingTestGrain owner) : IInboxHandler + { + public bool CanHandle(InboxMessage message) => true; + + public ValueTask HandleAsync(IInboxHandlerContext context, CancellationToken cancellationToken) + { + if (owner.HandlerOverride is { } handler) + { + return handler.HandleAsync(context, cancellationToken); + } + + var application = TestApplicationProtocol.Read(owner._sessions, context.Message); + if (!application.Route.StartsWith("messages/", StringComparison.Ordinal) && application.Route != "typed") + { + throw new InvalidOperationException($"Unknown application route '{application.Route}'."); + } + return owner.HandleAsync(application.Body as TestMessage + ?? throw new InvalidOperationException($"Expected {nameof(TestMessage)} application payload."), + context, cancellationToken); + } + } +} + +// Tests can change registration between real activations without adding a production registry API. +public sealed class TestHandlerConfiguration +{ + private readonly ConcurrentDictionary _enabled = new(); + public bool IsEnabled(GrainId id) => !_enabled.TryGetValue(id, out var enabled) || enabled; + public void Set(GrainId id, bool enabled) => _enabled[id] = enabled; +} diff --git a/test/Orleans.Messaging.Tests/Support/NamedFactoryMessagingGrain.cs b/test/Orleans.Messaging.Tests/Support/NamedFactoryMessagingGrain.cs new file mode 100644 index 00000000000..6f20494c9b2 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Support/NamedFactoryMessagingGrain.cs @@ -0,0 +1,111 @@ +using System.Collections.Concurrent; +using Microsoft.Extensions.DependencyInjection; +using Orleans.Journaling; +using Orleans.Runtime; +using Orleans.Serialization.Session; +using Xunit; + +namespace Orleans.Messaging.Tests.Support; + +public interface INamedFactoryMessagingGrain : IGrainWithGuidKey +{ + Task SendAsync(GrainId target, string route, TestMessage message); + Task GetSnapshotAsync(); + Task RequestDeactivationAsync(); +} + +[GenerateSerializer] +public sealed record NamedFactoryMessagingSnapshot( + [property: Id(0)] Guid ActivationId, + [property: Id(1)] int OutboxCount, + [property: Id(2)] MessageEffect[] Effects); + +public sealed class NamedFactoryMessagingProbe +{ + private readonly ConcurrentDictionary> _completed = new(); + + public Task WaitAsync(GrainId grainId, CancellationToken cancellationToken) => + GetCompletion(grainId).Task.WaitAsync(TimeSpan.FromSeconds(30), cancellationToken); + + public void Publish(GrainId grainId, NamedFactoryMessagingSnapshot snapshot) + { + if (snapshot.Effects.Length > 0 && snapshot.OutboxCount == 0) + { + GetCompletion(grainId).TrySetResult(snapshot); + } + } + + private TaskCompletionSource GetCompletion(GrainId grainId) => + _completed.GetOrAdd(grainId, static _ => new(TaskCreationOptions.RunContinuationsAsynchronously)); +} + +public sealed class NamedFactoryMessagingGrain : Grain, INamedFactoryMessagingGrain, IMessagingGrain, IInboxHandler +{ + private readonly IJournaledStateManager _owner; + private readonly IOutbox _outbox; + private readonly IDurableDictionary _effects; + private readonly SerializerSessionPool _sessions; + private readonly NamedFactoryMessagingProbe _probe; + private readonly Guid _activationId = Guid.NewGuid(); + + public NamedFactoryMessagingGrain( + IJournaledStateManager owner, + IInbox inbox, + IOutbox outbox, + [FromKeyedServices("cutover-effects")] IDurableDictionary effects, + SerializerSessionPool sessions, + NamedFactoryMessagingProbe probe) + { + _owner = owner; + _outbox = outbox; + _effects = effects; + _sessions = sessions; + _probe = probe; + inbox.Register(this); + } + + public async Task SendAsync(GrainId target, string route, TestMessage message) + { + var envelope = TestApplicationProtocol.Create(_sessions, this.GetGrainId(), target, route, message); + _outbox.Send(envelope); + await _owner.WriteStateAsync(); + } + + public Task GetSnapshotAsync() => Task.FromResult(CreateSnapshot()); + + internal NamedFactoryMessagingSnapshot CaptureStorageWrite() => CreateSnapshot(); + internal void PublishStoredSnapshot(NamedFactoryMessagingSnapshot snapshot) => _probe.Publish(this.GetGrainId(), snapshot); + + public Task RequestDeactivationAsync() + { + DeactivateOnIdle(); + return Task.CompletedTask; + } + + + public bool CanHandle(InboxMessage message) => true; + + + + public async ValueTask HandleAsync(IInboxHandlerContext context, CancellationToken cancellationToken) + { + var message = Assert.IsType(TestApplicationProtocol.Read(_sessions, context.Message).Body); + var outgoing = message.ForwardTo is { } target + ? TestApplicationProtocol.Create(_sessions, this.GetGrainId(), target, "messages/forwarded", + message with { ForwardTo = null }, new MessageId(context.Message.Envelope.MessageId.Key.CreateChildKey("forwarded"))) + : (OutboxMessage?)null; + + cancellationToken.ThrowIfCancellationRequested(); + _effects.TryGetValue(message.LogicalId, out var prior); + var effect = new MessageEffect(message.LogicalId, (prior?.Count ?? 0) + 1, message.Sequence, message.Value); + _effects[message.LogicalId] = effect; + if (outgoing is { } batch) + { + _outbox.Send(batch); + } + context.Complete(); + } + + private NamedFactoryMessagingSnapshot CreateSnapshot() => + new(_activationId, _outbox.Count(), _effects.Values.OrderBy(static effect => effect.Sequence).ToArray()); +} diff --git a/test/Orleans.Messaging.Tests/Support/OutboxStateTestAccess.cs b/test/Orleans.Messaging.Tests/Support/OutboxStateTestAccess.cs new file mode 100644 index 00000000000..a4411446cab --- /dev/null +++ b/test/Orleans.Messaging.Tests/Support/OutboxStateTestAccess.cs @@ -0,0 +1,11 @@ +namespace Orleans.Messaging.Tests.Support; + +internal sealed class OutboxStateTestAccess(IOutbox outbox) +{ + private readonly Outbox _outbox = (Outbox)outbox; + + public IEnumerable AttemptKeys => _outbox.MessageState.Keys; + public IEnumerable DeadLetterKeys => _outbox.DeadLetterState.Keys; + public OutboxMessageState Attempt(MessageId id) => _outbox.MessageState[id].State; + public OutboxDeadLetter DeadLetter(MessageId id) => _outbox.DeadLetterState[id]; +} diff --git a/test/Orleans.Messaging.Tests/Support/OutboxTestMessages.cs b/test/Orleans.Messaging.Tests/Support/OutboxTestMessages.cs new file mode 100644 index 00000000000..55e35f9431d --- /dev/null +++ b/test/Orleans.Messaging.Tests/Support/OutboxTestMessages.cs @@ -0,0 +1,31 @@ +using Microsoft.Extensions.DependencyInjection; +using Orleans.Runtime; +using Orleans.Serialization; +using Orleans.Serialization.Session; + +namespace Orleans.Messaging.Tests.Support; + +internal static class OutboxTestMessages +{ + public static OutboxMessage Create(MessageId messageId, GrainId receiver, ReadOnlyMemory payload, + GrainId? sender, string? subject, Serializer senderSerializer) => new() + { + ReceiverId = receiver, + Envelope = new Envelope + { + MessageId = messageId, + Payload = payload.ToArray(), + Subject = subject, + Headers = sender is { } senderId + ? new Dictionary(StringComparer.Ordinal) { ["sender"] = senderSerializer.SerializeToArray(senderId) } + : null + } + }; + + public static OutboxMessage CreateApplication(IServiceProvider services, MessageId messageId, + GrainId sender, GrainId receiver, string subject, object? body) => Create(messageId, receiver, + TestApplicationProtocol.Encode(services.GetRequiredService(), new TestApplicationMessage(subject, body)), + sender, subject, services.GetRequiredService>()); + + public static string? Subject(Envelope envelope) => envelope.Subject; +} diff --git a/test/Orleans.Messaging.Tests/Support/PersistentStateMessagingGrain.cs b/test/Orleans.Messaging.Tests/Support/PersistentStateMessagingGrain.cs new file mode 100644 index 00000000000..a5c2a833837 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Support/PersistentStateMessagingGrain.cs @@ -0,0 +1,51 @@ +using System.Collections.Concurrent; +using Microsoft.Extensions.DependencyInjection; +using Orleans.Runtime; + +namespace Orleans.Messaging.Tests.Support; + +public interface IPersistentStateMessagingGrain : IGrainWithGuidKey +{ + Task GetSnapshotAsync(); +} + +[GenerateSerializer] +public sealed record PersistentStateMessagingSnapshot( + [property: Id(0)] int InboxCount, + [property: Id(1)] int Handled, + [property: Id(2)] string[] DeadLetterReasons, + [property: Id(3)] int[] DeadLetterAttempts); + +// A messaging grain which also holds journaled IPersistentState, whose in-place changes the journal can't observe. +public sealed class PersistentStateMessagingGrain : Grain, IPersistentStateMessagingGrain, IMessagingGrain, IInboxHandler +{ + private static readonly ConcurrentDictionary Handled = new(); + private readonly IInbox _inbox; + private readonly IMessagingDiagnostics _diagnostics; + + public PersistentStateMessagingGrain( + IInbox inbox, + IMessagingDiagnostics diagnostics, + [FromKeyedServices("ledger")] IPersistentState> ledger) + { + _inbox = inbox; + _diagnostics = diagnostics; + _ = ledger; + inbox.Register(this); + } + + public Task GetSnapshotAsync() => Task.FromResult(new PersistentStateMessagingSnapshot( + ((Inbox)_inbox).Count, + Handled.GetValueOrDefault(this.GetGrainId()), + _diagnostics.InboxDeadLetters.Select(static letter => letter.Reason).ToArray(), + _diagnostics.InboxDeadLetters.Select(static letter => letter.AttemptCount).ToArray())); + + public bool CanHandle(InboxMessage message) => true; + + // Fails before touching any state, which the inbox can't distinguish from a failure after an in-place change. + public ValueTask HandleAsync(IInboxHandlerContext context, CancellationToken cancellationToken) + { + Handled.AddOrUpdate(this.GetGrainId(), 1, static (_, count) => count + 1); + throw new InvalidOperationException("Injected handler failure before any state change."); + } +} diff --git a/test/Orleans.Messaging.Tests/Support/RawPayloadTestGrain.cs b/test/Orleans.Messaging.Tests/Support/RawPayloadTestGrain.cs new file mode 100644 index 00000000000..b01d529ec97 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Support/RawPayloadTestGrain.cs @@ -0,0 +1,115 @@ +using System.Globalization; +using Microsoft.Extensions.DependencyInjection; +using Orleans.Journaling; +using Orleans.Runtime; + +namespace Orleans.Messaging.Tests.Support; + +public interface IRawPayloadTestGrain : IGrainWithGuidKey +{ + Task ConfigureForwardAsync(GrainId target, int copies = 1); + Task AcceptAndDeactivateAsync(InboxMessage message); + Task GetSnapshotAsync(); + Task RequestDeactivationAsync(); +} + +[GenerateSerializer] +public sealed record RawPayloadEffect([property: Id(0)] MessageId MessageId, [property: Id(1)] string Bytes, [property: Id(2)] int Count); + +[GenerateSerializer] +public sealed record RawPayloadSnapshot([property: Id(0)] Guid ActivationId, [property: Id(1)] int InboxCount, + [property: Id(2)] int OutboxCount, [property: Id(3)] int ProcessedCount, [property: Id(4)] RawPayloadEffect[] Effects); + +// The application protocol is arbitrary bytes. Orleans codecs are only used for journal commands, +// business state and RPC results, never to encode/decode the messaging payload. +public sealed class RawPayloadTestGrain : DurableGrain, IRawPayloadTestGrain, IInboxHandler +{ + public const string HandlerBarrier = "raw-payload"; + private readonly Inbox _inbox; + private readonly IOutbox _outbox; + private readonly IDurableDictionary _effects; + private readonly IDurableValue _forward; + private readonly IDurableValue _forwardCopies; + internal int ExpectedAcknowledgedEffects { get; set; } = 1; + private readonly HandlerProbe _handlers; + private readonly Guid _activationId = Guid.NewGuid(); + private readonly TaskCompletionSource _acknowledged = new(TaskCreationOptions.RunContinuationsAsynchronously); + internal Task Acknowledged => _acknowledged.Task; + + public RawPayloadTestGrain(IGrainContext context, IOutbox outbox, IJournaledStateManager manager, + [FromKeyedServices("raw-effects")] IDurableDictionary effects, + [FromKeyedServices("raw-forward")] IDurableValue forward, + [FromKeyedServices("raw-forward-copies")] IDurableValue forwardCopies, + HandlerProbe handlers) + { + _inbox = context.ActivationServices.GetRequiredService(); + _outbox = outbox; + _effects = effects; + _forward = forward; + _forwardCopies = forwardCopies; + _handlers = handlers; + _inbox.Register(this); + new JournalProbeState(manager).WriteCompleted = () => + { + if (_inbox.Count == 0 && _outbox.Count() == 0 && _effects.Count == ExpectedAcknowledgedEffects) + { + _acknowledged.TrySetResult(GetSnapshotForTest()); + } + }; + } + + public async Task ConfigureForwardAsync(GrainId target, int copies = 1) + { + _forward.Value = target; + _forwardCopies.Value = copies; + await WriteStateAsync(); + } + + public async Task AcceptAndDeactivateAsync(InboxMessage message) + { + var result = await ((IMessagingExtension)_inbox).DeliverAsync(message); + AcceptedSnapshot = GetSnapshotForTest(); + DeactivateOnIdle(); + return result; + } + + public bool CanHandle(InboxMessage message) => true; + + public async ValueTask HandleAsync(IInboxHandlerContext context, CancellationToken cancellationToken) + { + if (_handlers.TryGet(this.GetGrainId(), HandlerBarrier, out var barrier)) + { + barrier.Entered.TrySetResult(); + await barrier.Continue.Task.WaitAsync(cancellationToken); + } + cancellationToken.ThrowIfCancellationRequested(); + var input = context.Message.Envelope; + var bytes = Convert.ToBase64String(input.Payload); + var outgoing = _forward.Value.IsDefault + ? [] + : Enumerable.Range(0, _forwardCopies.Value).Select(index => new OutboxMessage + { + Envelope = index == 0 + ? input + : input with { MessageId = new MessageId(input.MessageId.Key.CreateChildKey(index.ToString(CultureInfo.InvariantCulture))) }, + ReceiverId = _forward.Value + }).ToArray(); + _effects.TryGetValue(input.MessageId, out var prior); + _effects[input.MessageId] = new RawPayloadEffect(input.MessageId, bytes, (prior?.Count ?? 0) + 1); + foreach (var envelope in outgoing) + { + _outbox.Send(envelope); + } + context.Complete(); + } + + internal RawPayloadSnapshot? AcceptedSnapshot { get; private set; } + + public Task GetSnapshotAsync() => Task.FromResult(GetSnapshotForTest()); + internal RawPayloadSnapshot GetSnapshotForTest() => new(_activationId, _inbox.Count, _outbox.Count(), InboxTestAccess.GetProcessedCount(_inbox), _effects.Values.ToArray()); + public Task RequestDeactivationAsync() + { + DeactivateOnIdle(); + return Task.CompletedTask; + } +} diff --git a/test/Orleans.Messaging.Tests/Support/ReceiverTestServices.cs b/test/Orleans.Messaging.Tests/Support/ReceiverTestServices.cs new file mode 100644 index 00000000000..99eb91692e2 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Support/ReceiverTestServices.cs @@ -0,0 +1,27 @@ +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Orleans.Hosting; +using Orleans.Journaling; +using Orleans.Messaging.Configuration; +using Orleans.Runtime; + +namespace Orleans.Messaging.Tests.Support; + +internal static class ReceiverTestServices +{ + private static readonly Func ReadCurrentContext = typeof(IGrainContext).Assembly + .GetType("Orleans.Runtime.RuntimeContext", throwOnError: true)! + .GetProperty("Current")!.GetMethod!.CreateDelegate>(); + + public static IGrainContext? CurrentGrainContext => ReadCurrentContext(); + + // Uses the production registration, replacing the application-facing outbox with a journaled capture. + public static void Add(IServiceCollection services, Action configure) + { + services.AddMessaging(configure); + services.RemoveAll(); + services.AddScoped(static sp => + new JournaledTestOutbox( + sp.GetRequiredKeyedService>("test-handler-output"))); + } +} diff --git a/test/Orleans.Messaging.Tests/Support/RecordingJournalStorageProvider.cs b/test/Orleans.Messaging.Tests/Support/RecordingJournalStorageProvider.cs new file mode 100644 index 00000000000..e49e1885191 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Support/RecordingJournalStorageProvider.cs @@ -0,0 +1,111 @@ +using System.Buffers; +using System.Collections.Concurrent; +using System.Text; +using Microsoft.Extensions.Options; +using Orleans.Journaling; + +namespace Orleans.Messaging.Tests.Support; + +/// Stores journals in memory and records the bytes of every successful journal write. +public sealed class RecordingJournalStorageProvider : IJournalStorageProvider, IJournalStorageCatalog +{ + private readonly ConcurrentDictionary> _writes = new(); + private readonly ConcurrentDictionary _snapshots = new(); + private VolatileJournalStorageProvider? _inner; + + public void Configure(IOptions options) => _inner ??= new VolatileJournalStorageProvider(options); + + /// Requests that the journal's next write replaces it with a snapshot. + public void RequestSnapshot(JournalId journalId) => _snapshots[journalId] = 0; + + public IJournalStorage CreateStorage(JournalId journalId) => + new RecordingJournalStorage(this, journalId, _writes.GetOrAdd(journalId, static _ => new()), Inner.CreateStorage(journalId)); + + public IAsyncEnumerable ListAsync( + JournalCatalogListOptions? options = null, + CancellationToken cancellationToken = default) => + Inner.ListAsync(options, cancellationToken); + + /// Gets the bytes of each successful write to a journal, in write order. + public byte[][] GetWrites(JournalId journalId) => _writes.TryGetValue(journalId, out var writes) ? [.. writes] : []; + + /// + /// Counts the occurrences of across a journal's writes, as raw bytes in the Orleans binary + /// format or as the base64 string which the JSON format writes for a byte array. + /// + public int CountPayloadOccurrences(JournalId journalId, byte[] payload) + { + var base64 = Encoding.UTF8.GetBytes(Convert.ToBase64String(payload)); + var count = 0; + foreach (var write in GetWrites(journalId)) + { + count += CountOccurrences(write, payload) + CountOccurrences(write, base64); + } + + return count; + } + + private static int CountOccurrences(ReadOnlySpan data, ReadOnlySpan value) + { + if (value.IsEmpty) + { + return 0; + } + + var count = 0; + for (var index = data.IndexOf(value); index >= 0; index = data.IndexOf(value)) + { + count++; + data = data[(index + value.Length)..]; + } + + return count; + } + + private VolatileJournalStorageProvider Inner => + _inner ?? throw new InvalidOperationException("The recording journal storage provider has not been configured."); + + private sealed class RecordingJournalStorage( + RecordingJournalStorageProvider owner, + JournalId journalId, + ConcurrentQueue writes, + IJournalStorage inner) : IJournalStorage + { + public bool IsCompactionRequested => owner._snapshots.ContainsKey(journalId) || inner.IsCompactionRequested; + + public ValueTask ReadAsync(IJournalStorageConsumer consumer, CancellationToken cancellationToken) => + inner.ReadAsync(consumer, cancellationToken); + + public ValueTask CreateIfNotExistsAsync( + IReadOnlyDictionary? metadata = null, + CancellationToken cancellationToken = default) => + inner.CreateIfNotExistsAsync(metadata, cancellationToken); + + public ValueTask GetMetadataAsync(CancellationToken cancellationToken = default) => + inner.GetMetadataAsync(cancellationToken); + + public ValueTask UpdateMetadataAsync( + IReadOnlyDictionary? set = null, + IEnumerable? remove = null, + string? expectedETag = null, + CancellationToken cancellationToken = default) => + inner.UpdateMetadataAsync(set, remove, expectedETag, cancellationToken); + + public async ValueTask ReplaceAsync(ReadOnlySequence value, CancellationToken cancellationToken) + { + var bytes = value.ToArray(); + await inner.ReplaceAsync(value, cancellationToken).ConfigureAwait(false); + owner._snapshots.TryRemove(journalId, out _); + writes.Enqueue(bytes); + } + + public async ValueTask AppendAsync(ReadOnlySequence value, CancellationToken cancellationToken) + { + var bytes = value.ToArray(); + await inner.AppendAsync(value, cancellationToken).ConfigureAwait(false); + writes.Enqueue(bytes); + } + + public ValueTask DeleteAsync(CancellationToken cancellationToken) => inner.DeleteAsync(cancellationToken); + } +} diff --git a/test/Orleans.Messaging.Tests/Support/SnapshotProbe.cs b/test/Orleans.Messaging.Tests/Support/SnapshotProbe.cs new file mode 100644 index 00000000000..7550f59ff90 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Support/SnapshotProbe.cs @@ -0,0 +1,124 @@ +using System.Collections.Concurrent; +using Orleans.Runtime; + +namespace Orleans.Messaging.Tests.Support; + +public sealed class SnapshotProbe +{ + private readonly ConcurrentDictionary _latest = new(); + private readonly ConcurrentDictionary> _history = new(); + private readonly ConcurrentDictionary> _waiters = new(); + + internal int WaiterListCount => _waiters.Count; + + public async Task WaitAsync( + GrainId grainId, + Func predicate, + TimeSpan? timeout = null) + { + if (_latest.TryGetValue(grainId, out var current) && predicate(current)) + { + return current; + } + + var waiter = new Waiter(predicate); + List waiters; + while (true) + { + waiters = _waiters.GetOrAdd(grainId, static _ => []); + lock (waiters) + { + if (!_waiters.TryGetValue(grainId, out var currentWaiters) + || !ReferenceEquals(waiters, currentWaiters)) + { + continue; + } + + if (_latest.TryGetValue(grainId, out current) && predicate(current)) + { + RemoveWaiterListIfEmpty(grainId, waiters); + return current; + } + + waiters.Add(waiter); + break; + } + } + + try + { + return await waiter.Completion.Task.WaitAsync(timeout ?? TimeSpan.FromSeconds(30)); + } + catch (TimeoutException exception) + { + var published = _history.TryGetValue(grainId, out var history) ? string.Join(" | ", history) : "none"; + throw new TimeoutException($"Snapshot wait for '{grainId}' timed out. Published (oldest first): {published}", exception); + } + finally + { + lock (waiters) + { + waiters.Remove(waiter); + RemoveWaiterListIfEmpty(grainId, waiters); + } + } + } + + public void Publish(GrainId grainId, EndpointSnapshot snapshot) + { + _latest[grainId] = snapshot; + var history = _history.GetOrAdd(grainId, static _ => new()); + history.Enqueue(Describe(snapshot)); + while (history.Count > 8) history.TryDequeue(out _); + if (!_waiters.TryGetValue(grainId, out var waiters)) + { + return; + } + + lock (waiters) + { + foreach (var waiter in waiters.ToArray()) + { + bool matched; + try + { + matched = waiter.Predicate(snapshot); + } + catch (Exception exception) + { + waiters.Remove(waiter); + waiter.Completion.TrySetException(exception); + continue; + } + + if (matched) + { + waiters.Remove(waiter); + waiter.Completion.TrySetResult(snapshot); + } + } + + RemoveWaiterListIfEmpty(grainId, waiters); + } + } + + internal static string Describe(EndpointSnapshot snapshot) => + $"activation={snapshot.ActivationId}, inbox={snapshot.InboxCount}, outbox={snapshot.OutboxCount}, " + + $"effects={snapshot.Effects.Sum(static effect => effect.Count)}, processed={snapshot.ProcessedMessageCount}, " + + $"inboxDeadLetters={snapshot.InboxDeadLetters.Count}, outboxDeadLetters={snapshot.OutboxDeadLetters.Count}"; + + private void RemoveWaiterListIfEmpty(GrainId grainId, List waiters) + { + if (waiters.Count == 0) + { + _waiters.TryRemove(new KeyValuePair>(grainId, waiters)); + } + } + + private sealed class Waiter(Func predicate) + { + public Func Predicate { get; } = predicate; + public TaskCompletionSource Completion { get; } = + new(TaskCreationOptions.RunContinuationsAsynchronously); + } +} diff --git a/test/Orleans.Messaging.Tests/Support/SnapshotProbeTests.cs b/test/Orleans.Messaging.Tests/Support/SnapshotProbeTests.cs new file mode 100644 index 00000000000..e796c52a077 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Support/SnapshotProbeTests.cs @@ -0,0 +1,82 @@ +using Orleans.Runtime; +using Xunit; + +namespace Orleans.Messaging.Tests.Support; + +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public class SnapshotProbeTests +{ + [Fact] + public async Task WaitAsync_TimeoutRemovesWaiter() + { + var probe = new SnapshotProbe(); + var grainId = GrainId.Create("snapshot-probe", "timeout"); + var predicateCalls = 0; + + await Assert.ThrowsAsync( + () => probe.WaitAsync( + grainId, + _ => + { + predicateCalls++; + return false; + }, + TimeSpan.Zero)); + + Assert.Equal(0, probe.WaiterListCount); + + probe.Publish( + grainId, + CreateSnapshot(inboxCount: 0)); + + Assert.Equal(0, predicateCalls); + } + + [Fact] + public async Task WaitAsync_SuccessRemovesWaiterList() + { + var probe = new SnapshotProbe(); + var grainId = GrainId.Create("snapshot-probe", "success"); + var wait = probe.WaitAsync(grainId, static snapshot => snapshot.InboxCount == 1); + var snapshot = CreateSnapshot(inboxCount: 1); + + probe.Publish(grainId, snapshot); + + Assert.Equal(0, probe.WaiterListCount); + Assert.Same(snapshot, await wait); + } + + [Fact] + public async Task WaitAsync_TimeoutRetainsListWithActiveWaiter() + { + var probe = new SnapshotProbe(); + var grainId = GrainId.Create("snapshot-probe", "shared-list"); + var activeWait = probe.WaitAsync(grainId, static snapshot => snapshot.InboxCount == 1); + + await Assert.ThrowsAsync( + () => probe.WaitAsync(grainId, static _ => false, TimeSpan.Zero)); + + Assert.Equal(1, probe.WaiterListCount); + + probe.Publish( + grainId, + CreateSnapshot(inboxCount: 1)); + + await activeWait; + Assert.Equal(0, probe.WaiterListCount); + } + + private static EndpointSnapshot CreateSnapshot(int inboxCount) => + new( + Guid.Empty, + string.Empty, + inboxCount, + 0, + 0, + [], + [], + [], + 0); +} diff --git a/test/Orleans.Messaging.Tests/Support/TestApplicationProtocol.cs b/test/Orleans.Messaging.Tests/Support/TestApplicationProtocol.cs new file mode 100644 index 00000000000..f381a4293dd --- /dev/null +++ b/test/Orleans.Messaging.Tests/Support/TestApplicationProtocol.cs @@ -0,0 +1,106 @@ +using System.Buffers; +using Orleans.Runtime; +using Orleans.Serialization; +using Orleans.Serialization.Buffers; +using Orleans.Serialization.Codecs; +using Orleans.Serialization.Session; +using Orleans.Serialization.WireProtocol; + +namespace Orleans.Messaging.Tests.Support; + +// Routes and typed values belong to this test application, not to the messaging transport. +[GenerateSerializer] +public sealed record TestApplicationMessage( + [property: Id(0)] string Route, + [property: Id(1)] object? Body); + +internal static class TestApplicationProtocol +{ + public static MessageId NewMessageId() => MessageId.Create("tests", Guid.NewGuid().ToString("N")); + + public static OutboxMessage Create(SerializerSessionPool sessions, GrainId sender, GrainId receiver, + string route, object? body, MessageId messageId = default) => + new() + { + ReceiverId = receiver, + Envelope = CreateEnvelope(sessions, sender, route, body, messageId) + }; + + public static InboxMessage CreateInbox(SerializerSessionPool sessions, GrainId sender, + string route, object? body, DateTimeOffset sentAt, MessageId messageId = default) => + new() { Envelope = CreateEnvelope(sessions, sender, route, body, messageId), SentAt = sentAt }; + + private static Envelope CreateEnvelope(SerializerSessionPool sessions, GrainId sender, + string route, object? body, MessageId messageId) + { + var key = messageId.IsDefault + ? body is TestMessage message ? message.LogicalId : NewMessageId() + : messageId; + return new() + { + MessageId = key, + Payload = Encode(sessions, new TestApplicationMessage(route, body)), + Subject = route, + Headers = new(StringComparer.Ordinal) + { + ["sender"] = new Serializer(sessions).SerializeToArray(sender) + } + }; + } + + public static byte[] Encode(SerializerSessionPool sessions, T value) + { + var output = new ArrayBufferWriter(); + using var session = sessions.GetSession(); + var writer = Writer.Create(output, session); + sessions.CodecProvider.GetCodec().WriteField(ref writer, 0, typeof(T), value); + writer.Commit(); + return output.WrittenSpan.ToArray(); + } + + public static T Decode(SerializerSessionPool sessions, byte[] payload) + { + using var session = sessions.GetSession(); + var reader = Reader.Create(payload, session); + var field = reader.ReadFieldHeader(); + return sessions.CodecProvider.GetCodec().ReadValue(ref reader, field)!; + } + + public static TestApplicationMessage Read(SerializerSessionPool sessions, InboxMessage message) => + Decode(sessions, message.Envelope.Payload); + + public static TestApplicationMessage Read(SerializerSessionPool sessions, OutboxMessage message) => + Decode(sessions, message.Envelope.Payload); + + public static GrainId Sender(SerializerSessionPool sessions, Envelope envelope) => + envelope.Headers is { } headers && headers.TryGetValue("sender", out var sender) + ? new Serializer(sessions).Deserialize(sender) : default; + + public static string? Subject(Envelope envelope) => + envelope.Subject; + + public static InboxMessage With(InboxMessage message, SerializerSessionPool sessions, + MessageId? messageId = null, GrainId? sender = null, string? subject = null, + byte[]? payload = null, bool removeSender = false, bool removeSubject = false) + { + var headers = message.Envelope.Headers is { } existing + ? new Dictionary(existing, StringComparer.Ordinal) : null; + if (removeSender) headers?.Remove("sender"); + if (sender is { } value) + { + (headers ??= new(StringComparer.Ordinal))["sender"] = new Serializer(sessions).SerializeToArray(value); + } + return new() + { + Envelope = new() + { + MessageId = messageId ?? message.Envelope.MessageId, + Payload = payload ?? message.Envelope.Payload, + Subject = removeSubject ? null : subject ?? message.Envelope.Subject, + Headers = headers, + DeliverAt = message.Envelope.DeliverAt + }, + SentAt = message.SentAt + }; + } +} diff --git a/test/Orleans.Messaging.Tests/Support/TestInbox.cs b/test/Orleans.Messaging.Tests/Support/TestInbox.cs new file mode 100644 index 00000000000..2714042b8e0 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Support/TestInbox.cs @@ -0,0 +1,28 @@ +namespace Orleans.Messaging.Tests.Support; + +/// An which dispatches directly to its registered handlers. +internal sealed class TestInbox : IInbox +{ + private readonly InboxDispatcher _dispatcher = new([]); + + public bool IsEmpty => _dispatcher.IsEmpty; + + public IInbox Register(IInboxHandler handler) + { + _dispatcher.Add(handler); + return this; + } + + public IInbox Register(IInboxMessageSink sink) => throw new NotSupportedException("The test inbox dispatches to handlers only."); + + public IInbox Register(IInboxWorkSource source) => throw new NotSupportedException("The test inbox dispatches to handlers only."); + + public ValueTask HandleAsync(IInboxHandlerContext context, CancellationToken cancellationToken) => + _dispatcher.HandleAsync(context, cancellationToken); +} + +internal static class TestInboxExtensions +{ + public static ValueTask HandleAsync(this IInbox inbox, IInboxHandlerContext context, CancellationToken cancellationToken) => + ((TestInbox)inbox).HandleAsync(context, cancellationToken); +}