File: /home/xedaptot/ai.naniguide.com/tests/Unit/AbTestLifecycleTest.php
<?php
/**
* AbTestLifecycleTest — full state-machine coverage for the A/B testing pipeline.
*
* ─── Coverage ──────────────────────────────────────────────────────────────────
*
* ✅ Full lifecycle (test_percentage=100):
* NEW → QUEUED → SENDING → AWAITING_WINNER → DONE
* AbTest status helpers: isDraft → isRunning → isAwaitingWinner → isCompleted
*
* ✅ Full lifecycle with winner rollout (test_percentage=50):
* NEW → SENDING → AWAITING_WINNER → SENDING_WINNER → DONE
* Remaining subscribers receive winner variant email.
*
* ✅ Partial SendMessage failure (non-critical exception):
* Some messages throw during WarmupQuotaService::send() but are caught
* internally → tracked as 'failed' → campaign still transitions to
* AWAITING_WINNER → winner evaluation uses only 'sent' messages.
*
* ✅ Critical SendMessage failure (OutOfCredits):
* Exception causes batch to fail → campaign transitions to ERROR.
* AbTest has no winner. Campaign has last_error set.
*
* ─── Infrastructure notes ──────────────────────────────────────────────────────
*
* - QUEUE_CONNECTION=sync → all jobs inline, deterministic.
* - Reuses abTestFixture() and abTestCampaign() from AbTestFlowTest.
* - WarmupQuotaService is mocked per test.
*/
use App\Jobs\EvaluateAbTestWinner;
use App\Jobs\PopulateAbTestAssignments;
use App\Library\Exception\OutOfCredits;
use App\Model\AbTest;
use App\Model\AbTestAssignment;
use App\Model\AbTestVariant;
use App\Model\Campaign;
use App\Model\Contact;
use App\Model\Customer;
use App\Model\Email;
use App\Model\MailList;
use App\Model\OpenLog;
use App\Model\Plan;
use App\Model\SendingServer;
use App\Model\Subscriber;
use App\Model\Subscription;
use App\Model\Template;
use App\Model\TrackingLog;
use App\Services\Plans\Credits\CreditKey;
use App\Services\Plans\Entitlements\EntitlementKey;
use App\SendingServers\Drivers\DeliveryStatus;
use App\SendingServers\Drivers\SendResult;
use App\Services\Warmup\WarmupQuotaService;
use Illuminate\Support\Facades\DB;
use Tests\TestCase;
uses(TestCase::class);
beforeEach(function () {
// Clean rate limit tracker files — stale data from prior runs causes false throttling
array_map('unlink', glob(storage_path('app/quota/server-send-email-rate-tracking-log-*')));
});
afterEach(function () {
$testCustomerIds = DB::table('customers')->where('uid', 'like', 'ablc_%')->pluck('id');
if ($testCustomerIds->isEmpty()) {
return;
}
$msgIds = DB::table('tracking_logs')->whereIn('customer_id', $testCustomerIds)->pluck('message_id');
DB::table('open_logs')->whereIn('message_id', $msgIds)->delete();
DB::table('click_logs')->whereIn('message_id', $msgIds)->delete();
DB::table('tracking_logs')->whereIn('customer_id', $testCustomerIds)->delete();
$abCampaignIds = DB::table('campaigns')->where('uid', 'like', '%ablc%')->pluck('id');
$abTestIds = DB::table('ab_tests')->whereIn('campaign_id', $abCampaignIds)->pluck('id');
DB::table('ab_test_assignments')->whereIn('ab_test_id', $abTestIds)->delete();
DB::table('ab_test_variants')->whereIn('ab_test_id', $abTestIds)->delete();
DB::table('ab_tests')->whereIn('id', $abTestIds)->delete();
DB::table('campaigns_lists_segments')->whereIn('campaign_id', $abCampaignIds)->delete();
DB::table('campaigns')->whereIn('id', $abCampaignIds)->delete();
DB::table('automation_emails')->whereIn('customer_id', $testCustomerIds)->delete();
DB::table('emails')->whereIn('customer_id', $testCustomerIds)->delete();
DB::table('subscribers')->where('email', 'like', '%ablc%')->delete();
DB::table('mail_lists')->where('uid', 'like', 'ablclist_%')->delete();
DB::table('sending_servers')->where('uid', 'like', 'ablcsrv_%')->delete();
DB::table('templates')->where('uid', 'like', 'ablctpl_%')->delete();
DB::table('subscriptions')->whereIn('customer_id', $testCustomerIds)->delete();
$planIds = DB::table('plans')->where('uid', 'like', 'ablcplan_%')->pluck('id');
DB::table('plan_entitlements')->whereIn('plan_id', $planIds)->delete();
DB::table('plan_credits')->whereIn('plan_id', $planIds)->delete();
DB::table('plans')->whereIn('id', $planIds)->delete();
DB::table('customers')->whereIn('id', $testCustomerIds)->delete();
DB::table('contacts')->where('uid', 'like', 'ablccontact_%')->delete();
});
// ─── Helpers ──────────────────────────────────────────────────────────────────
function lcFixture(int $subscriberCount = 10): array
{
$contact = Contact::forceCreate([
'uid' => uniqid('ablccontact_'),
'first_name' => 'LC',
'last_name' => 'Tester',
'email' => uniqid() . '@ablc.invalid',
]);
$customer = Customer::forceCreate([
'uid' => uniqid('ablc_'),
'contact_id' => $contact->id,
'status' => 'active',
'timezone' => 'UTC',
]);
$plan = Plan::forceCreate([
'uid' => uniqid('ablcplan_'),
'currency_id' => 1,
'name' => 'LC Test Plan',
'price' => 0,
'frequency_amount' => 1,
'frequency_unit' => 'month',
'status' => 'active',
]);
DB::table('plan_entitlements')->insert([
'plan_id' => $plan->id,
'entitlement_key' => EntitlementKey::HAS_OWN_SENDING_SERVER->value,
'enabled' => true,
'created_at' => now(),
'updated_at' => now(),
]);
DB::table('plan_credits')->insert([
'plan_id' => $plan->id,
'credit_key' => CreditKey::SEND_EMAIL->value,
'amount_per_cycle' => null,
'created_at' => now(),
'updated_at' => now(),
]);
$subscription = Subscription::forceCreate([
'uid' => uniqid('ablcsub_'),
'customer_id' => $customer->id,
'plan_id' => $plan->id,
'status' => Subscription::STATUS_ACTIVE,
]);
app(\App\Services\Plans\Lifecycle\SubscriptionLifecycle::class)
->onSubscribe($subscription);
$list = MailList::forceCreate([
'uid' => uniqid('ablclist_'),
'name' => 'LC Test List',
'customer_id' => $customer->id,
'from_email' => '[email protected]',
'from_name' => 'LC Sender',
]);
$server = SendingServer::forceCreate([
'uid' => uniqid('ablcsrv_'),
'name' => 'LC SMTP',
'type' => 'smtp',
'customer_id' => $customer->id,
'status' => SendingServer::STATUS_ACTIVE,
'host' => '127.0.0.1',
'smtp_port' => 25,
'quota_value' => -1,
'quota_base' => 1,
'quota_unit' => 'day',
]);
// Customer pool (Case A) picks up the SendingServer created above.
$subscribers = [];
for ($i = 0; $i < $subscriberCount; $i++) {
$subscribers[] = Subscriber::forceCreate([
'uid' => uniqid('ablcsub_'),
'email' => "sub{$i}_" . uniqid() . '@ablc.invalid',
'status' => Subscriber::STATUS_SUBSCRIBED,
'mail_list_id' => $list->id,
'verification_status' => 'unverified',
'from' => 'manual',
'ip' => '127.0.0.1',
]);
}
$template = Template::forceCreate([
'uid' => uniqid('ablctpl_'),
'name' => 'LC Template',
'content' => '<html><body><p>{CONTENT}</p><a href="{UNSUBSCRIBE_URL}">Unsubscribe</a></body></html>',
'json' => \App\Model\Template::DEFAULT_BUILDER_JSON,
]);
$thumbDir = storage_path('app/public/templates/thumb');
if (!is_dir($thumbDir)) {
mkdir($thumbDir, 0755, true);
}
file_put_contents($thumbDir . '/' . $template->uid . '.jpg', '');
return compact('customer', 'list', 'server', 'subscribers', 'template');
}
function lcCampaign(array $fixture, int $testPercentage = 100): array
{
['customer' => $customer, 'list' => $list, 'template' => $template] = $fixture;
$campaign = $customer->local()->newDefaultCampaign();
$campaign->uid = uniqid('ablc_');
$campaign->saveFromArray(['type' => Email::TYPE_REGULAR]);
$campaign->setTemplate($template, 'LC campaign');
$campaign->subject = 'Default Subject';
$campaign->from_email = '[email protected]';
$campaign->from_name = 'LC Sender';
$campaign->reply_to = '[email protected]';
$campaign->save();
$campaign->addListSegment($list, null);
$campaign->default_mail_list_id = $list->id;
$campaign->save();
$campaign = $campaign->fresh(['email']);
$emailA = Email::newDefault();
$emailA->customer_id = $customer->id;
$emailA->type = Email::TYPE_REGULAR;
$emailA->subject = 'Variant A — Lifecycle Subject';
$emailA->from_email = '[email protected]';
$emailA->from_name = 'LC Sender';
$emailA->reply_to = '[email protected]';
$emailA->save();
$emailA->setTemplate($template, 'Variant A template');
$emailA->refresh();
$emailB = Email::newDefault();
$emailB->customer_id = $customer->id;
$emailB->type = Email::TYPE_REGULAR;
$emailB->subject = 'Variant B — Lifecycle Subject';
$emailB->from_email = '[email protected]';
$emailB->from_name = 'LC Sender';
$emailB->reply_to = '[email protected]';
$emailB->save();
$emailB->setTemplate($template, 'Variant B template');
$emailB->refresh();
$abTest = AbTest::forceCreate([
'uid' => uniqid('ablct_'),
'campaign_id' => $campaign->id,
'winner_criteria' => AbTest::CRITERIA_OPEN_RATE,
'test_percentage' => $testPercentage,
'winner_wait_hours' => null,
]);
$variantA = AbTestVariant::forceCreate([
'uid' => uniqid('ablcv_'),
'ab_test_id' => $abTest->id,
'name' => 'Variant A',
'email_id' => $emailA->id,
'split_percentage' => 50,
'is_control' => true,
]);
$variantB = AbTestVariant::forceCreate([
'uid' => uniqid('ablcv_'),
'ab_test_id' => $abTest->id,
'name' => 'Variant B',
'email_id' => $emailB->id,
'split_percentage' => 50,
'is_control' => false,
]);
return compact('campaign', 'abTest', 'variantA', 'variantB');
}
// ─── Tests ────────────────────────────────────────────────────────────────────
test('lifecycle 100%: NEW → SENDING → AWAITING_WINNER → evaluate → DONE', function () {
$fixture = lcFixture(subscriberCount: 10);
['campaign' => $campaign, 'abTest' => $abTest, 'variantA' => $variantA, 'variantB' => $variantB] = lcCampaign($fixture);
// ── Phase 0: initial state ──────────────────────────────────────────────
expect($campaign->status)->toBe(Campaign::STATUS_NEW);
expect($abTest->isDraft())->toBeTrue();
expect($abTest->isActive())->toBeFalse();
expect($abTest->isCompleted())->toBeFalse();
expect($abTest->winner_variant_id)->toBeNull();
// ── Phase 1: execute → assignments populated → batch runs → AWAITING_WINNER
$this->mock(WarmupQuotaService::class, function ($mock) {
$mock->shouldReceive('send')
->andReturnUsing(fn () => new SendResult(runtimeMessageId: uniqid('msg_'), status: DeliveryStatus::SENT));
});
$campaign->execute();
$campaign->refresh();
$abTest->refresh();
expect($campaign->status)->toBe(Campaign::STATUS_AWAITING_WINNER);
expect($abTest->isAwaitingWinner())->toBeTrue();
expect($abTest->isRunning())->toBeFalse();
expect($abTest->isCompleted())->toBeFalse();
expect($abTest->winner_variant_id)->toBeNull();
// All 10 subscribers sent
$sentCount = TrackingLog::where('campaign_id', $campaign->id)->where('status', 'sent')->count();
expect($sentCount)->toBe(10);
// All tracking logs have ab_test_variant_id
$logsWithoutVariant = TrackingLog::where('campaign_id', $campaign->id)
->whereNull('ab_test_variant_id')
->count();
expect($logsWithoutVariant)->toBe(0);
// ── Phase 2: simulate opens for variant A → evaluate winner → DONE ──────
$variantALogs = TrackingLog::where('campaign_id', $campaign->id)
->where('ab_test_variant_id', $variantA->id)
->get();
foreach ($variantALogs as $log) {
OpenLog::forceCreate([
'message_id' => $log->message_id,
'ip_address' => '1.2.3.4',
'user_agent' => 'TestAgent',
]);
}
(new EvaluateAbTestWinner($abTest->id))->handle();
$campaign->refresh();
$abTest->refresh();
expect($campaign->status)->toBe(Campaign::STATUS_DONE);
expect($abTest->isCompleted())->toBeTrue();
expect($abTest->isActive())->toBeFalse();
expect($abTest->winner_variant_id)->toBe($variantA->id);
// test_percentage=100 → no winner_sent_at (no remaining phase)
expect($abTest->winner_sent_at)->toBeNull();
});
test('lifecycle 50%: NEW → SENDING → AWAITING_WINNER → SENDING_WINNER → DONE', function () {
$fixture = lcFixture(subscriberCount: 20);
['campaign' => $campaign, 'abTest' => $abTest, 'variantA' => $variantA, 'variantB' => $variantB] = lcCampaign($fixture, testPercentage: 50);
// ── Phase 0: initial state ──────────────────────────────────────────────
expect($campaign->status)->toBe(Campaign::STATUS_NEW);
expect($abTest->isDraft())->toBeTrue();
expect($abTest->hasRemainingPhase())->toBeTrue();
// ── Phase 1: execute → test phase sends to 50% of subscribers ───────────
$this->mock(WarmupQuotaService::class, function ($mock) {
$mock->shouldReceive('send')
->andReturnUsing(fn () => new SendResult(runtimeMessageId: uniqid('msg_'), status: DeliveryStatus::SENT));
});
$campaign->execute();
$campaign->refresh();
$abTest->refresh();
expect($campaign->status)->toBe(Campaign::STATUS_AWAITING_WINNER);
expect($abTest->isAwaitingWinner())->toBeTrue();
// ExactSplitStrategy: floor(20 * 50/100) = 10 subscribers in test pool
$phase1Sent = TrackingLog::where('campaign_id', $campaign->id)->where('status', 'sent')->count();
expect($phase1Sent)->toBe(10);
$phase1SubscriberIds = TrackingLog::where('campaign_id', $campaign->id)
->where('status', 'sent')
->pluck('subscriber_id');
// ── Phase 2: simulate opens for variant B → evaluate winner ─────────────
$variantBLogs = TrackingLog::where('campaign_id', $campaign->id)
->where('ab_test_variant_id', $variantB->id)
->get();
foreach ($variantBLogs as $log) {
OpenLog::forceCreate([
'message_id' => $log->message_id,
'ip_address' => '1.2.3.4',
'user_agent' => 'TestAgent',
]);
}
(new EvaluateAbTestWinner($abTest->id))->handle();
$campaign->refresh();
$abTest->refresh();
// Winner declared → remaining subscribers sent → batch finishes → DONE
// (declareWinner → SENDING_WINNER → sendWinnerToRemaining → run() → batch → finally → setDone)
expect($campaign->status)->toBe(Campaign::STATUS_DONE);
expect($abTest->isCompleted())->toBeTrue();
expect($abTest->winner_variant_id)->toBe($variantB->id);
expect($abTest->winner_sent_at)->not->toBeNull();
// All 20 subscribers now have tracking logs
$totalSent = TrackingLog::where('campaign_id', $campaign->id)->where('status', 'sent')->count();
expect($totalSent)->toBe(20);
// Remaining 10 subscribers all received winner variant B
$remainingLogs = TrackingLog::where('campaign_id', $campaign->id)
->where('status', 'sent')
->whereNotIn('subscriber_id', $phase1SubscriberIds)
->get();
expect($remainingLogs)->toHaveCount(10);
$remainingLogs->each(function ($log) use ($variantB) {
expect($log->ab_test_variant_id)->toBe($variantB->id);
});
});
test('partial SendMessage failure: failed messages tracked, campaign still reaches AWAITING_WINNER', function () {
$fixture = lcFixture(subscriberCount: 10);
['campaign' => $campaign, 'abTest' => $abTest, 'variantA' => $variantA, 'variantB' => $variantB] = lcCampaign($fixture);
// skip_failed_message=true → stopOnError()=false → non-critical exceptions
// are caught and tracked as 'failed' rather than aborting the batch.
$campaign->skip_failed_message = true;
$campaign->save();
// Make the first 3 subscribers' sends throw a generic exception.
// SendMessage catches it (non-OutOfCredits, selectedServer is set) → tracks as 'failed'.
$callCount = 0;
$this->mock(WarmupQuotaService::class, function ($mock) use (&$callCount) {
$mock->shouldReceive('send')
->andReturnUsing(function () use (&$callCount) {
$callCount++;
if ($callCount <= 3) {
throw new \RuntimeException('Simulated SMTP connection timeout');
}
return new SendResult(runtimeMessageId: uniqid('msg_'), status: DeliveryStatus::SENT);
});
});
$campaign->execute();
$campaign->refresh();
$abTest->refresh();
// Campaign should NOT be in error — individual failures are swallowed
expect($campaign->status)->toBe(Campaign::STATUS_AWAITING_WINNER);
expect($abTest->isAwaitingWinner())->toBeTrue();
expect($campaign->last_error)->toBeNull();
// 3 failed + 7 sent = 10 tracking logs total
$sentLogs = TrackingLog::where('campaign_id', $campaign->id)->where('status', 'sent')->count();
$failedLogs = TrackingLog::where('campaign_id', $campaign->id)->where('status', 'failed')->count();
expect($sentLogs)->toBe(7);
expect($failedLogs)->toBe(3);
// All tracking logs still have ab_test_variant_id set (even failed ones)
$logsWithoutVariant = TrackingLog::where('campaign_id', $campaign->id)
->whereNull('ab_test_variant_id')
->count();
expect($logsWithoutVariant)->toBe(0);
// ── Evaluate winner: only 'sent' messages count ─────────────────────────
// Simulate opens for all sent variant A messages
$sentVariantALogs = TrackingLog::where('campaign_id', $campaign->id)
->where('ab_test_variant_id', $variantA->id)
->where('status', 'sent')
->get();
foreach ($sentVariantALogs as $log) {
OpenLog::forceCreate([
'message_id' => $log->message_id,
'ip_address' => '1.2.3.4',
'user_agent' => 'TestAgent',
]);
}
(new EvaluateAbTestWinner($abTest->id))->handle();
$campaign->refresh();
$abTest->refresh();
expect($campaign->status)->toBe(Campaign::STATUS_DONE);
expect($abTest->isCompleted())->toBeTrue();
expect($abTest->winner_variant_id)->toBe($variantA->id);
});
test('OutOfCredits during send: campaign transitions to ERROR, no winner declared', function () {
$fixture = lcFixture(subscriberCount: 6);
['campaign' => $campaign, 'abTest' => $abTest] = lcCampaign($fixture);
// OutOfCredits is in $forceEndCampaignExceptions → re-thrown → batch fails → setError()
$this->mock(WarmupQuotaService::class, function ($mock) {
$mock->shouldReceive('send')
->andThrow(new OutOfCredits('No sending credits remaining'));
});
// execute() catches the error internally and calls setError()
$campaign->execute();
$campaign->refresh();
$abTest->refresh();
// With sync queue driver: execute() wraps everything in DB::transaction().
// When OutOfCredits bubbles up, the transaction rolls back (undoing setQueued
// and setSending), then the outer catch calls setError() on status='new'.
// In production (async driver), setSending would commit before the batch runs,
// so status would be 'sending'. The important invariant is: is_error=true.
expect($campaign->is_error)->toBeTrue();
expect($campaign->isError())->toBeTrue();
expect($campaign->last_error)->not->toBeNull();
expect($campaign->last_error)->toContain('No sending credits remaining');
// AbTest should NOT have a winner
expect($abTest->winner_variant_id)->toBeNull();
expect($abTest->isCompleted())->toBeFalse();
});
test('lifecycle with winner_wait_hours: delayed EvaluateAbTestWinner auto-fires after test phase', function () {
$fixture = lcFixture(subscriberCount: 10);
['campaign' => $campaign, 'abTest' => $abTest, 'variantA' => $variantA] = lcCampaign($fixture);
// Set winner_wait_hours — the finally callback dispatches EvaluateAbTestWinner
// with a delay. With sync queue the delayed job runs immediately.
$abTest->winner_wait_hours = 2;
$abTest->save();
$this->mock(WarmupQuotaService::class, function ($mock) {
$mock->shouldReceive('send')
->andReturnUsing(fn () => new SendResult(runtimeMessageId: uniqid('msg_'), status: DeliveryStatus::SENT));
});
$campaign->execute();
$campaign->refresh();
$abTest->refresh();
// With sync queue: finally callback dispatches EvaluateAbTestWinner which runs
// immediately. Both variants have 0 opens at this point → winner picked by
// sort order (first variant). Campaign goes straight to DONE.
expect($campaign->status)->toBe(Campaign::STATUS_DONE);
expect($abTest->isCompleted())->toBeTrue();
expect($abTest->winner_variant_id)->not->toBeNull();
// All 10 subscribers sent during test phase
$sentCount = TrackingLog::where('campaign_id', $campaign->id)->where('status', 'sent')->count();
expect($sentCount)->toBe(10);
});
test('CRITERIA_MANUAL: test phase ends at AWAITING_WINNER, EvaluateAbTestWinner is no-op, manual declareWinner works', function () {
$fixture = lcFixture(subscriberCount: 10);
['campaign' => $campaign, 'abTest' => $abTest, 'variantA' => $variantA, 'variantB' => $variantB] = lcCampaign($fixture);
$abTest->winner_criteria = AbTest::CRITERIA_MANUAL;
$abTest->save();
$this->mock(WarmupQuotaService::class, function ($mock) {
$mock->shouldReceive('send')
->andReturnUsing(fn () => new SendResult(runtimeMessageId: uniqid('msg_'), status: DeliveryStatus::SENT));
});
$campaign->execute();
$campaign->refresh();
$abTest->refresh();
expect($campaign->status)->toBe(Campaign::STATUS_AWAITING_WINNER);
// EvaluateAbTestWinner should be a no-op for MANUAL criteria
(new EvaluateAbTestWinner($abTest->id))->handle();
$campaign->refresh();
$abTest->refresh();
// Still awaiting — no automatic winner pick
expect($campaign->status)->toBe(Campaign::STATUS_AWAITING_WINNER);
expect($abTest->winner_variant_id)->toBeNull();
// Simulate admin manually declaring winner (same as controller would call)
$abTest->declareWinner($variantB);
$campaign->refresh();
$abTest->refresh();
// test_percentage=100 → no remaining phase → DONE directly
expect($campaign->status)->toBe(Campaign::STATUS_DONE);
expect($abTest->isCompleted())->toBeTrue();
expect($abTest->winner_variant_id)->toBe($variantB->id);
});
test('CRITERIA_MANUAL with 50% split: manual declareWinner triggers SENDING_WINNER → DONE', function () {
$fixture = lcFixture(subscriberCount: 20);
['campaign' => $campaign, 'abTest' => $abTest, 'variantA' => $variantA, 'variantB' => $variantB] = lcCampaign($fixture, testPercentage: 50);
$abTest->winner_criteria = AbTest::CRITERIA_MANUAL;
$abTest->save();
$this->mock(WarmupQuotaService::class, function ($mock) {
$mock->shouldReceive('send')
->andReturnUsing(fn () => new SendResult(runtimeMessageId: uniqid('msg_'), status: DeliveryStatus::SENT));
});
$campaign->execute();
$campaign->refresh();
$abTest->refresh();
expect($campaign->status)->toBe(Campaign::STATUS_AWAITING_WINNER);
$phase1Sent = TrackingLog::where('campaign_id', $campaign->id)->where('status', 'sent')->count();
expect($phase1Sent)->toBe(10);
// Admin picks variant A as winner
$abTest->declareWinner($variantA);
$campaign->refresh();
$abTest->refresh();
// declareWinner with hasRemainingPhase → SENDING_WINNER → sendWinnerToRemaining → DONE
expect($campaign->status)->toBe(Campaign::STATUS_DONE);
expect($abTest->isCompleted())->toBeTrue();
expect($abTest->winner_variant_id)->toBe($variantA->id);
expect($abTest->winner_sent_at)->not->toBeNull();
// All 20 subscribers sent
$totalSent = TrackingLog::where('campaign_id', $campaign->id)->where('status', 'sent')->count();
expect($totalSent)->toBe(20);
// Remaining 10 all received variant A (winner)
$phase1SubscriberIds = AbTestAssignment::where('ab_test_id', $abTest->id)
->whereIn('variant_id', [$variantA->id, $variantB->id])
->pluck('subscriber_id')
->unique();
// Winner phase logs: subscribers not in original test pool got variant A
$winnerPhaseLogs = TrackingLog::where('campaign_id', $campaign->id)
->where('status', 'sent')
->where('ab_test_variant_id', $variantA->id)
->get();
// At least the remaining 10 got variant A (plus the original 5 from test phase)
expect($winnerPhaseLogs->count())->toBeGreaterThanOrEqual(10);
});
test('all messages fail with non-critical exception: campaign still reaches AWAITING_WINNER with zero sent', function () {
$fixture = lcFixture(subscriberCount: 6);
['campaign' => $campaign, 'abTest' => $abTest, 'variantA' => $variantA, 'variantB' => $variantB] = lcCampaign($fixture);
// skip_failed_message=true → exceptions tracked as 'failed', batch continues
$campaign->skip_failed_message = true;
$campaign->save();
// Every send throws a non-critical exception → tracked as 'failed', batch continues
$this->mock(WarmupQuotaService::class, function ($mock) {
$mock->shouldReceive('send')
->andThrow(new \RuntimeException('Connection refused'));
});
$campaign->execute();
$campaign->refresh();
$abTest->refresh();
// Campaign didn't error — individual failures are swallowed
expect($campaign->status)->toBe(Campaign::STATUS_AWAITING_WINNER);
expect($abTest->isAwaitingWinner())->toBeTrue();
// All 6 tracked as 'failed'
$failedCount = TrackingLog::where('campaign_id', $campaign->id)->where('status', 'failed')->count();
$sentCount = TrackingLog::where('campaign_id', $campaign->id)->where('status', 'sent')->count();
expect($failedCount)->toBe(6);
expect($sentCount)->toBe(0);
// Evaluate winner: all variants have 0 sent → rate = 0.0 → first variant wins (tiebreaker)
(new EvaluateAbTestWinner($abTest->id))->handle();
$abTest->refresh();
$campaign->refresh();
expect($campaign->status)->toBe(Campaign::STATUS_DONE);
expect($abTest->isCompleted())->toBeTrue();
// Winner is the first variant (sorted by id) since all rates are 0.0
expect($abTest->winner_variant_id)->not->toBeNull();
});
// ─── Flag behavior tests ─────────────────────────────────────────────────────
test('setError sets is_error flag without changing lifecycle status', function () {
$fixture = lcFixture(subscriberCount: 4);
['campaign' => $campaign] = lcCampaign($fixture);
// Manually set to SENDING to simulate mid-send
$campaign->status = Campaign::STATUS_SENDING;
$campaign->save();
$campaign->setError('Something went wrong');
$campaign->refresh();
// Status stays at lifecycle phase
expect($campaign->status)->toBe(Campaign::STATUS_SENDING);
// Error flag is set
expect($campaign->is_error)->toBeTrue();
expect($campaign->isError())->toBeTrue();
// isSending() still true — both can be true simultaneously
expect($campaign->isSending())->toBeTrue();
expect($campaign->last_error)->toBe('Something went wrong');
});
test('setPaused sets is_paused flag without changing lifecycle status', function () {
$fixture = lcFixture(subscriberCount: 4);
['campaign' => $campaign] = lcCampaign($fixture);
$campaign->status = Campaign::STATUS_SENDING_WINNER;
$campaign->save();
$campaign->setPaused();
$campaign->refresh();
expect($campaign->status)->toBe(Campaign::STATUS_SENDING_WINNER);
expect($campaign->is_paused)->toBeTrue();
expect($campaign->isPaused())->toBeTrue();
expect($campaign->isSendingWinner())->toBeTrue();
});
test('setSending clears both error and paused flags', function () {
$fixture = lcFixture(subscriberCount: 4);
['campaign' => $campaign] = lcCampaign($fixture);
$campaign->status = Campaign::STATUS_SENDING;
$campaign->is_error = true;
$campaign->is_paused = true;
$campaign->last_error = 'old error';
$campaign->save();
$campaign->setSending();
$campaign->refresh();
expect($campaign->is_error)->toBeFalse();
expect($campaign->is_paused)->toBeFalse();
expect($campaign->status)->toBe(Campaign::STATUS_SENDING);
});
test('setDone clears both error and paused flags', function () {
$fixture = lcFixture(subscriberCount: 4);
['campaign' => $campaign] = lcCampaign($fixture);
$campaign->status = Campaign::STATUS_DONE;
$campaign->is_error = true;
$campaign->save();
$campaign->setDone();
$campaign->refresh();
expect($campaign->is_error)->toBeFalse();
expect($campaign->is_paused)->toBeFalse();
});
test('resume from error during SENDING: clears flags and re-executes', function () {
$fixture = lcFixture(subscriberCount: 6);
['campaign' => $campaign, 'abTest' => $abTest] = lcCampaign($fixture);
// Simulate error during SENDING phase
$campaign->status = Campaign::STATUS_SENDING;
$campaign->is_error = true;
$campaign->last_error = 'Connection timeout';
$campaign->save();
$this->mock(WarmupQuotaService::class, function ($mock) {
$mock->shouldReceive('send')
->andReturnUsing(fn () => new SendResult(runtimeMessageId: uniqid('msg_'), status: DeliveryStatus::SENT));
});
$campaign->resume(ACM_QUEUE_TYPE_BATCH);
$campaign->refresh();
$abTest->refresh();
// Flags cleared, campaign resumed and completed test phase
expect($campaign->is_error)->toBeFalse();
expect($campaign->is_paused)->toBeFalse();
expect($campaign->last_error)->toBeNull();
expect($campaign->status)->toBe(Campaign::STATUS_AWAITING_WINNER);
});
test('resume from error during SENDING_WINNER: restores SENDING_WINNER and completes', function () {
$fixture = lcFixture(subscriberCount: 20);
['campaign' => $campaign, 'abTest' => $abTest, 'variantA' => $variantA, 'variantB' => $variantB] = lcCampaign($fixture, testPercentage: 50);
$this->mock(WarmupQuotaService::class, function ($mock) {
$mock->shouldReceive('send')
->andReturnUsing(fn () => new SendResult(runtimeMessageId: uniqid('msg_'), status: DeliveryStatus::SENT));
});
// Run test phase normally
$campaign->execute();
$campaign->refresh();
expect($campaign->status)->toBe(Campaign::STATUS_AWAITING_WINNER);
// Simulate: winner declared, then error during winner rollout
$abTest->refresh();
$abTest->winner_variant_id = $variantA->id;
$abTest->save();
$campaign->status = Campaign::STATUS_SENDING_WINNER;
$campaign->is_error = true;
$campaign->last_error = 'SMTP timeout during winner rollout';
$campaign->save();
expect($abTest->isErrorDuringWinnerPhase())->toBeTrue();
// Resume should restore SENDING_WINNER and complete
$campaign->resume(ACM_QUEUE_TYPE_BATCH);
$campaign->refresh();
expect($campaign->is_error)->toBeFalse();
expect($campaign->status)->toBe(Campaign::STATUS_DONE);
});
test('resume from paused during AWAITING_WINNER: clears flag, stays in AWAITING_WINNER', function () {
$fixture = lcFixture(subscriberCount: 6);
['campaign' => $campaign, 'abTest' => $abTest] = lcCampaign($fixture);
// Simulate paused during awaiting winner
$campaign->status = Campaign::STATUS_AWAITING_WINNER;
$campaign->is_paused = true;
$campaign->save();
$campaign->resume(ACM_QUEUE_TYPE_BATCH);
$campaign->refresh();
// Flag cleared, stays in AWAITING_WINNER (no re-send needed)
expect($campaign->is_paused)->toBeFalse();
expect($campaign->status)->toBe(Campaign::STATUS_AWAITING_WINNER);
});