File: /home/xedaptot/ai.naniguide.com/tests/Unit/AbTestFlowTest.php
<?php
/**
* AbTestFlowTest — end-to-end integration test for the A/B testing pipeline.
*
* ─── Coverage ──────────────────────────────────────────────────────────────────
*
* ✅ Pre-assignment: PopulateAbTestAssignments assigns every eligible subscriber
* to exactly one variant via CRC32 bucket; no PHP memory for 1M IDs.
*
* ✅ Assignment coverage: all subscribers are assigned (none missed, none doubled).
*
* ✅ Split accuracy: variants receive roughly proportional subscriber counts.
*
* ✅ Sending: Campaign::run() branches into buildAbTestLoaders() when hasAbTest().
* Each subscriber receives the email of their assigned variant (different subject).
*
* ✅ Tracking: tracking_logs.ab_test_variant_id is set correctly per subscriber.
*
* ✅ Winner evaluation: EvaluateAbTestWinner picks the variant with higher open rate.
*
* ✅ Winner phase: remaining subscribers (test_percentage < 100) are assigned to
* winner and sent winner's email in a second run.
*
* ✅ ContinueCampaign compatibility: re-entering run() after rate limit still
* branches into A/B path correctly.
*
* ─── Infrastructure notes ──────────────────────────────────────────────────────
*
* - QUEUE_CONNECTION=sync (phpunit.xml) → all jobs inline, fully deterministic.
* - No DatabaseTransactions — Bus::batch uses DB::transaction internally.
* - WarmupQuotaService is mocked to avoid real SMTP.
* - OpenLog records are inserted directly to simulate opens for winner evaluation.
*/
use App\Jobs\EvaluateAbTestWinner;
use App\Jobs\PopulateAbTestAssignments;
use App\Model\AbTest;
use App\Model\AbTestAssignment;
use App\Model\AbTestVariant;
use App\Model\Campaign;
use App\Model\Contact;
use App\Model\Customer;
use App\Model\Email;
use App\Model\MailList;
use App\Model\OpenLog;
use App\Model\Plan;
use App\Model\SendingServer;
use App\Model\Subscriber;
use App\Model\Subscription;
use App\Model\Template;
use App\Model\TrackingLog;
use App\SendingServers\Drivers\DeliveryStatus;
use App\SendingServers\Drivers\SendResult;
use App\Services\Plans\Credits\CreditKey;
use App\Services\Plans\Entitlements\EntitlementKey;
use App\Services\Warmup\WarmupQuotaService;
use Illuminate\Support\Facades\DB;
use Tests\TestCase;
uses(TestCase::class);
beforeEach(function () {
// Clean rate limit tracker files — stale data from prior runs causes false throttling
array_map('unlink', glob(storage_path('app/quota/server-send-email-rate-tracking-log-*')));
});
afterEach(function () {
// Scope all cleanup to customers created by this test suite only
$testCustomerIds = DB::table('customers')->where('uid', 'like', 'abcust_%')->pluck('id');
if ($testCustomerIds->isEmpty()) {
return;
}
$msgIds = DB::table('tracking_logs')->whereIn('customer_id', $testCustomerIds)->pluck('message_id');
DB::table('open_logs')->whereIn('message_id', $msgIds)->delete();
DB::table('click_logs')->whereIn('message_id', $msgIds)->delete();
DB::table('tracking_logs')->whereIn('customer_id', $testCustomerIds)->delete();
$abCampaignIds = DB::table('campaigns')->where('uid', 'like', '%abtest%')->pluck('id');
$abTestIds = DB::table('ab_tests')->whereIn('campaign_id', $abCampaignIds)->pluck('id');
DB::table('ab_test_assignments')->whereIn('ab_test_id', $abTestIds)->delete();
DB::table('ab_test_variants')->whereIn('ab_test_id', $abTestIds)->delete();
DB::table('ab_tests')->whereIn('id', $abTestIds)->delete();
DB::table('campaigns_lists_segments')->whereIn('campaign_id', $abCampaignIds)->delete();
// Delete campaigns before emails (campaigns.email_id FK)
DB::table('campaigns')->whereIn('id', $abCampaignIds)->delete();
// Delete automation_emails before emails (automation_emails.email_id FK)
DB::table('automation_emails')->whereIn('customer_id', $testCustomerIds)->delete();
DB::table('emails')->whereIn('customer_id', $testCustomerIds)->delete();
DB::table('subscribers')->where('email', 'like', '%abtest%')->delete();
DB::table('mail_lists')->where('uid', 'like', 'ablist_%')->delete();
DB::table('sending_servers')->where('uid', 'like', 'absrv_%')->delete();
DB::table('templates')->where('uid', 'like', 'abtpl_%')->delete();
DB::table('subscriptions')->whereIn('customer_id', $testCustomerIds)->delete();
$planIds = DB::table('plans')->where('uid', 'like', 'abplan_%')->pluck('id');
DB::table('plan_entitlements')->whereIn('plan_id', $planIds)->delete();
DB::table('plan_credits')->whereIn('plan_id', $planIds)->delete();
DB::table('plans')->whereIn('id', $planIds)->delete();
DB::table('customers')->whereIn('id', $testCustomerIds)->delete();
DB::table('contacts')->where('uid', 'like', 'abcontact_%')->delete();
});
// ─── Helpers ──────────────────────────────────────────────────────────────────
function abTestFixture(int $subscriberCount = 10): array
{
$contact = Contact::forceCreate([
'uid' => uniqid('abcontact_'),
'first_name' => 'AB',
'last_name' => 'Tester',
'email' => uniqid() . '@abtest.invalid',
]);
$customer = Customer::forceCreate([
'uid' => uniqid('abcust_'),
'contact_id' => $contact->id,
'status' => 'active',
'timezone' => 'UTC',
]);
// Plan + active subscription so the SaaS pipeline (sending-server lookup,
// rate-limit cleanup, footer-from-plan) finds a real plan instead of erroring.
$plan = Plan::forceCreate([
'uid' => uniqid('abplan_'),
'currency_id' => 1,
'name' => 'AB Test Plan',
'price' => 0,
'frequency_amount' => 1,
'frequency_unit' => 'month',
'status' => 'active',
]);
DB::table('plan_entitlements')->insert([
'plan_id' => $plan->id,
'entitlement_key' => EntitlementKey::HAS_OWN_SENDING_SERVER->value,
'enabled' => true,
'created_at' => now(),
'updated_at' => now(),
]);
DB::table('plan_credits')->insert([
'plan_id' => $plan->id,
'credit_key' => CreditKey::SEND_EMAIL->value,
'amount_per_cycle' => null, // unlimited
'created_at' => now(),
'updated_at' => now(),
]);
$subscription = Subscription::forceCreate([
'uid' => uniqid('absub_'),
'customer_id' => $customer->id,
'plan_id' => $plan->id,
'status' => Subscription::STATUS_ACTIVE,
]);
app(\App\Services\Plans\Lifecycle\SubscriptionLifecycle::class)
->onSubscribe($subscription);
$list = MailList::forceCreate([
'uid' => uniqid('ablist_'),
'name' => 'AB Test List',
'customer_id' => $customer->id,
'from_email' => '[email protected]',
'from_name' => 'AB Sender',
]);
$server = SendingServer::forceCreate([
'uid' => uniqid('absrv_'),
'name' => 'AB SMTP',
'type' => 'smtp',
'customer_id' => $customer->id,
'status' => SendingServer::STATUS_ACTIVE,
'host' => '127.0.0.1',
'smtp_port' => 25,
'quota_value' => -1,
'quota_base' => 1,
'quota_unit' => 'day',
]);
// Customer pool (Case A) picks up the SendingServer created above.
$subscribers = [];
for ($i = 0; $i < $subscriberCount; $i++) {
$subscribers[] = Subscriber::forceCreate([
'uid' => uniqid('absub_'),
'email' => "sub{$i}_" . uniqid() . '@abtest.invalid',
'status' => Subscriber::STATUS_SUBSCRIBED,
'mail_list_id' => $list->id,
'verification_status' => 'unverified',
'from' => 'manual',
'ip' => '127.0.0.1',
]);
}
$template = Template::forceCreate([
'uid' => uniqid('abtpl_'),
'name' => 'AB Template',
'content' => '<html><body><p>{CONTENT}</p><a href="{UNSUBSCRIBE_URL}">Unsubscribe</a></body></html>',
'json' => \App\Model\Template::DEFAULT_BUILDER_JSON,
]);
$thumbDir = storage_path('app/public/templates/thumb');
if (!is_dir($thumbDir)) {
mkdir($thumbDir, 0755, true);
}
file_put_contents($thumbDir . '/' . $template->uid . '.jpg', '');
return compact('customer', 'list', 'server', 'subscribers', 'template');
}
/**
* Create campaign + A/B test with 2 variants (different subjects).
* Returns ['campaign', 'abTest', 'variantA', 'variantB']
*/
function abTestCampaign(array $fixture, int $testPercentage = 100): array
{
['customer' => $customer, 'list' => $list, 'template' => $template] = $fixture;
// Main campaign — serves as the orchestration container
$campaign = $customer->local()->newDefaultCampaign();
$campaign->uid = uniqid('abtest_'); // mark for cleanup
$campaign->saveFromArray(['type' => Email::TYPE_REGULAR]);
$campaign->setTemplate($template, 'AB Test campaign');
$campaign->subject = 'Default Subject (not used directly in A/B)';
$campaign->from_email = '[email protected]';
$campaign->from_name = 'AB Sender';
$campaign->reply_to = '[email protected]';
$campaign->save();
$campaign->addListSegment($list, null);
$campaign->default_mail_list_id = $list->id;
$campaign->save();
$campaign = $campaign->fresh(['email']);
// Variant A email — built like readyCampaign() so template is set correctly
$emailA = \App\Model\Email::newDefault();
$emailA->customer_id = $customer->id;
$emailA->type = Email::TYPE_REGULAR;
$emailA->subject = 'Subject A — Special Offer Inside';
$emailA->from_email = '[email protected]';
$emailA->from_name = 'AB Sender';
$emailA->reply_to = '[email protected]';
$emailA->save();
$emailA->setTemplate($template, 'Variant A template');
$emailA->refresh();
// Variant B email — built like readyCampaign() so template is set correctly
$emailB = \App\Model\Email::newDefault();
$emailB->customer_id = $customer->id;
$emailB->type = Email::TYPE_REGULAR;
$emailB->subject = 'Subject B — You Have a Message';
$emailB->from_email = '[email protected]';
$emailB->from_name = 'AB Sender';
$emailB->reply_to = '[email protected]';
$emailB->save();
$emailB->setTemplate($template, 'Variant B template');
$emailB->refresh();
// A/B test definition
$abTest = AbTest::forceCreate([
'uid' => uniqid('abt_'),
'campaign_id' => $campaign->id,
'winner_criteria' => AbTest::CRITERIA_OPEN_RATE,
'test_percentage' => $testPercentage,
'winner_wait_hours' => null, // manual in most tests; set explicitly when needed
]);
$variantA = AbTestVariant::forceCreate([
'uid' => uniqid('abv_'),
'ab_test_id' => $abTest->id,
'name' => 'Variant A',
'email_id' => $emailA->id,
'split_percentage' => 50,
'is_control' => true,
]);
$variantB = AbTestVariant::forceCreate([
'uid' => uniqid('abv_'),
'ab_test_id' => $abTest->id,
'name' => 'Variant B',
'email_id' => $emailB->id,
'split_percentage' => 50,
'is_control' => false,
]);
return compact('campaign', 'abTest', 'variantA', 'variantB');
}
/** Run campaign inline, wrapping in the same error guard as CampaignFlowTest */
function runCampaign(Campaign $campaign): void
{
try {
$campaign->execute();
} catch (\Throwable $e) {
throw $e;
}
}
// ─── Tests ────────────────────────────────────────────────────────────────────
test('PopulateAbTestAssignments assigns all subscribers exactly once', function () {
$fixture = abTestFixture(subscriberCount: 20);
['campaign' => $campaign, 'abTest' => $abTest] = abTestCampaign($fixture);
(new PopulateAbTestAssignments($abTest->id))->handle();
$allSubIds = collect($fixture['subscribers'])->pluck('id');
$assignedIds = AbTestAssignment::where('ab_test_id', $abTest->id)->pluck('subscriber_id');
// Every subscriber assigned
expect($assignedIds->count())->toBe($allSubIds->count());
// No subscriber assigned twice
expect($assignedIds->unique()->count())->toBe($assignedIds->count());
// Each assignment references a known variant
$variantIds = AbTestVariant::where('ab_test_id', $abTest->id)->pluck('id');
AbTestAssignment::where('ab_test_id', $abTest->id)->get()->each(function ($a) use ($variantIds) {
expect($variantIds)->toContain($a->variant_id);
});
});
test('assignments are split across variants (both variants receive subscribers)', function () {
$fixture = abTestFixture(subscriberCount: 50);
['campaign' => $campaign, 'abTest' => $abTest, 'variantA' => $variantA, 'variantB' => $variantB] = abTestCampaign($fixture);
(new PopulateAbTestAssignments($abTest->id))->handle();
$countA = AbTestAssignment::where('ab_test_id', $abTest->id)->where('variant_id', $variantA->id)->count();
$countB = AbTestAssignment::where('ab_test_id', $abTest->id)->where('variant_id', $variantB->id)->count();
// ExactSplitStrategy: round(50 × 50/100) = 25 each
expect($countA)->toBe(25);
expect($countB)->toBe(25);
});
test('PopulateAbTestAssignments is idempotent — re-running does not duplicate assignments', function () {
$fixture = abTestFixture(subscriberCount: 10);
['abTest' => $abTest] = abTestCampaign($fixture);
(new PopulateAbTestAssignments($abTest->id))->handle();
$countAfterFirst = AbTestAssignment::where('ab_test_id', $abTest->id)->count();
(new PopulateAbTestAssignments($abTest->id))->handle();
$countAfterSecond = AbTestAssignment::where('ab_test_id', $abTest->id)->count();
expect($countAfterFirst)->toBe($countAfterSecond);
});
test('campaign run() branches into A/B path: each subscriber receives their variant email', function () {
$fixture = abTestFixture(subscriberCount: 10);
['campaign' => $campaign, 'abTest' => $abTest, 'variantA' => $variantA, 'variantB' => $variantB] = abTestCampaign($fixture);
// Populate assignments first (normally done when admin starts the A/B test)
(new PopulateAbTestAssignments($abTest->id))->handle();
// Track which subject was used per subscriber during send
$sentSubjects = [];
$this->mock(WarmupQuotaService::class, function ($mock) use (&$sentSubjects) {
$mock->shouldReceive('send')
->andReturnUsing(function ($server, $message, $context) use (&$sentSubjects) {
// $message is a SwiftMessage — capture subject for assertion
$sentSubjects[$context['subscriber_id']] = $message->getSubject();
return new SendResult(runtimeMessageId: uniqid('msg_'), status: DeliveryStatus::SENT);
});
});
runCampaign($campaign);
$campaign->refresh();
// A/B test campaign: after test phase, status is AWAITING_WINNER (not DONE)
// — winner must be evaluated before campaign can complete.
expect($campaign->status)->toBe(Campaign::STATUS_AWAITING_WINNER);
// All 10 subscribers got an email
$logs = TrackingLog::where('campaign_id', $campaign->id)->get();
expect($logs)->toHaveCount(10);
expect($logs->every(fn ($l) => $l->status === 'sent'))->toBeTrue();
// Each tracking log has ab_test_variant_id set
expect($logs->every(fn ($l) => !is_null($l->ab_test_variant_id)))->toBeTrue();
// Subscribers assigned to variant A got Subject A; variant B got Subject B
$emailASubject = $variantA->email()->value('subject');
$emailBSubject = $variantB->email()->value('subject');
$logs->each(function ($log) use ($variantA, $variantB, $emailASubject, $emailBSubject) {
$assignment = AbTestAssignment::where('subscriber_id', $log->subscriber_id)
->where('ab_test_id', $variantA->ab_test_id)
->first();
expect($assignment)->not->toBeNull();
if ($assignment->variant_id === $variantA->id) {
expect($log->ab_test_variant_id)->toBe($variantA->id);
} else {
expect($log->ab_test_variant_id)->toBe($variantB->id);
}
});
});
test('no subscriber is sent to twice even when run() is called again (resume scenario)', function () {
$fixture = abTestFixture(subscriberCount: 6);
['campaign' => $campaign, 'abTest' => $abTest] = abTestCampaign($fixture);
(new PopulateAbTestAssignments($abTest->id))->handle();
$this->mock(WarmupQuotaService::class, function ($mock) {
$mock->shouldReceive('send')
->andReturnUsing(fn () => new SendResult(runtimeMessageId: uniqid('msg_'), status: DeliveryStatus::SENT));
});
runCampaign($campaign);
// Simulate resume (e.g. after rate limit): call run() again
$campaign->run($check = false);
$logs = TrackingLog::where('campaign_id', $campaign->id)
->where('status', 'sent')
->get();
// Each subscriber should appear exactly once
expect($logs->pluck('subscriber_id')->unique()->count())->toBe($logs->count());
expect($logs->count())->toBe(6);
});
test('EvaluateAbTestWinner picks variant with higher open rate', function () {
$fixture = abTestFixture(subscriberCount: 20);
['campaign' => $campaign, 'abTest' => $abTest, 'variantA' => $variantA, 'variantB' => $variantB] = abTestCampaign($fixture);
(new PopulateAbTestAssignments($abTest->id))->handle();
$this->mock(WarmupQuotaService::class, function ($mock) {
$mock->shouldReceive('send')
->andReturnUsing(fn () => new SendResult(runtimeMessageId: uniqid('msg_'), status: DeliveryStatus::SENT));
});
runCampaign($campaign);
// Simulate opens: all variant A subscribers open, zero variant B opens
$variantALogs = TrackingLog::where('campaign_id', $campaign->id)
->where('ab_test_variant_id', $variantA->id)
->get();
foreach ($variantALogs as $log) {
OpenLog::forceCreate([
'message_id' => $log->message_id,
'ip_address' => '1.2.3.4',
'user_agent' => 'TestAgent',
]);
}
// Run winner evaluation
(new EvaluateAbTestWinner($abTest->id))->handle();
$abTest->refresh();
expect($abTest->isCompleted())->toBeTrue();
expect($abTest->winner_variant_id)->toBe($variantA->id);
});
test('EvaluateAbTestWinner picks variant B when it has higher click rate', function () {
$fixture = abTestFixture(subscriberCount: 20);
['campaign' => $campaign, 'abTest' => $abTest, 'variantA' => $variantA, 'variantB' => $variantB] = abTestCampaign($fixture);
// Use click_rate criteria
$abTest->winner_criteria = AbTest::CRITERIA_CLICK_RATE;
$abTest->save();
(new PopulateAbTestAssignments($abTest->id))->handle();
$this->mock(WarmupQuotaService::class, function ($mock) {
$mock->shouldReceive('send')
->andReturnUsing(fn () => new SendResult(runtimeMessageId: uniqid('msg_'), status: DeliveryStatus::SENT));
});
runCampaign($campaign);
// Simulate clicks: only variant B subscribers click
$variantBLogs = TrackingLog::where('campaign_id', $campaign->id)
->where('ab_test_variant_id', $variantB->id)
->get();
foreach ($variantBLogs as $log) {
\App\Model\ClickLog::forceCreate([
'message_id' => $log->message_id,
'url' => 'https://example.com/promo',
'ip_address' => '1.2.3.4',
'user_agent' => 'TestAgent',
]);
}
(new EvaluateAbTestWinner($abTest->id))->handle();
$abTest->refresh();
expect($abTest->winner_variant_id)->toBe($variantB->id);
});
test('EvaluateAbTestWinner with CRITERIA_MANUAL does nothing', function () {
$fixture = abTestFixture(subscriberCount: 6);
['abTest' => $abTest] = abTestCampaign($fixture);
$abTest->winner_criteria = AbTest::CRITERIA_MANUAL;
$abTest->save();
$campaign = $abTest->campaign;
$campaign->status = Campaign::STATUS_AWAITING_WINNER;
$campaign->save();
(new EvaluateAbTestWinner($abTest->id))->handle();
$abTest->refresh();
$campaign->refresh();
expect($abTest->winner_variant_id)->toBeNull();
expect($campaign->isAwaitingWinner())->toBeTrue();
});
test('winner phase: remaining subscribers (test_percentage=50) receive winner email', function () {
// 50% test → half subscribers get A or B; other half waits for winner
$fixture = abTestFixture(subscriberCount: 20);
['campaign' => $campaign, 'abTest' => $abTest, 'variantA' => $variantA, 'variantB' => $variantB] = abTestCampaign($fixture, testPercentage: 50);
(new PopulateAbTestAssignments($abTest->id))->handle();
// Only bucket 0-49 gets assigned initially
$phase1SubscriberIds = AbTestAssignment::where('ab_test_id', $abTest->id)->pluck('subscriber_id');
$assignedCount = $phase1SubscriberIds->count();
// ExactSplitStrategy: test pool = floor(20 × 50/100) = 10 subscribers
expect($assignedCount)->toBe(10);
$this->mock(WarmupQuotaService::class, function ($mock) {
$mock->shouldReceive('send')
->andReturnUsing(fn () => new SendResult(runtimeMessageId: uniqid('msg_'), status: DeliveryStatus::SENT));
});
runCampaign($campaign);
// Only assigned subscribers got emails
$sentAfterPhase1 = TrackingLog::where('campaign_id', $campaign->id)->where('status', 'sent')->count();
expect($sentAfterPhase1)->toBe($assignedCount);
// Simulate: variant A wins (all A subscribers open)
$variantALogs = TrackingLog::where('campaign_id', $campaign->id)
->where('ab_test_variant_id', $variantA->id)
->get();
foreach ($variantALogs as $log) {
OpenLog::forceCreate([
'message_id' => $log->message_id,
'ip_address' => '1.2.3.4',
'user_agent' => 'TestAgent',
]);
}
// Evaluate winner → sends to remaining
(new EvaluateAbTestWinner($abTest->id))->handle();
$abTest->refresh();
expect($abTest->winner_variant_id)->toBe($variantA->id);
expect($abTest->winner_sent_at)->not->toBeNull();
// All 20 subscribers should now have been sent an email
$totalSent = TrackingLog::where('campaign_id', $campaign->id)->where('status', 'sent')->count();
expect($totalSent)->toBe(20);
// Remaining subscribers (those NOT in phase-1 assignments) all received winner variant A
$remainingLogs = TrackingLog::where('campaign_id', $campaign->id)
->where('status', 'sent')
->whereNotIn('subscriber_id', $phase1SubscriberIds)
->get();
// All remaining logs use winner variant A
$remainingLogs->each(function ($log) use ($variantA) {
expect($log->ab_test_variant_id)->toBe($variantA->id);
});
});
test('A/B test with 3 variants splits subscribers across all three', function () {
$fixture = abTestFixture(subscriberCount: 30);
['customer' => $customer, 'list' => $list, 'template' => $template] = $fixture;
$campaign = $customer->local()->newDefaultCampaign();
$campaign->uid = uniqid('abtest_');
$campaign->saveFromArray(['type' => Email::TYPE_REGULAR]);
$campaign->setTemplate($template, 'Three-variant test');
$campaign->subject = 'Default';
$campaign->from_email = '[email protected]';
$campaign->from_name = 'Sender';
$campaign->reply_to = '[email protected]';
$campaign->save();
$campaign->addListSegment($list, null);
$campaign->default_mail_list_id = $list->id;
$campaign->save();
$abTest = AbTest::forceCreate([
'uid' => uniqid('abt_'),
'campaign_id' => $campaign->id,
'winner_criteria' => AbTest::CRITERIA_OPEN_RATE,
'test_percentage' => 100,
]);
// Create 3 emails and 3 variants (33/33/34 split)
$variants = [];
foreach (['C', 'D', 'E'] as $i => $letter) {
$email = \App\Model\Email::newDefault();
$email->customer_id = $customer->id;
$email->type = Email::TYPE_REGULAR;
$email->subject = "Subject {$letter}";
$email->from_email = '[email protected]';
$email->from_name = 'Sender';
$email->reply_to = '[email protected]';
$email->save();
$email->setTemplate($template, "Variant {$letter} template");
$email->refresh();
$variants[] = AbTestVariant::forceCreate([
'uid' => uniqid('abv_'),
'ab_test_id' => $abTest->id,
'name' => "Variant {$letter}",
'email_id' => $email->id,
'split_percentage' => $letter === 'E' ? 34 : 33,
]);
}
(new PopulateAbTestAssignments($abTest->id))->handle();
// ExactSplitStrategy: round(30 × 33/100) = 10 for C and D; last variant E = 30 − 10 − 10 = 10
foreach ($variants as $variant) {
$count = AbTestAssignment::where('ab_test_id', $abTest->id)
->where('variant_id', $variant->id)
->count();
expect($count)->toBe(10);
}
$total = AbTestAssignment::where('ab_test_id', $abTest->id)->count();
expect($total)->toBe(30);
});
test('A/B test does not interfere with a regular campaign running simultaneously', function () {
// Create a regular campaign (no A/B) alongside an A/B campaign
$fixture = abTestFixture(subscriberCount: 6);
$fixture2 = abTestFixture(subscriberCount: 4); // separate fixture for regular campaign
['campaign' => $abCampaign, 'abTest' => $abTest] = abTestCampaign($fixture);
(new PopulateAbTestAssignments($abTest->id))->handle();
// Regular campaign (reuse campaignFixture pattern inline)
['customer' => $customer2, 'list' => $list2, 'template' => $template2] = $fixture2;
$regularCampaign = $customer2->local()->newDefaultCampaign();
$regularCampaign->uid = uniqid('abtest_regular_');
$regularCampaign->saveFromArray(['type' => Email::TYPE_REGULAR]);
$regularCampaign->setTemplate($template2, 'Regular campaign');
$regularCampaign->subject = 'Regular Subject';
$regularCampaign->from_email = '[email protected]';
$regularCampaign->from_name = 'Sender';
$regularCampaign->reply_to = '[email protected]';
$regularCampaign->save();
$regularCampaign->addListSegment($list2, null);
$regularCampaign->default_mail_list_id = $list2->id;
$regularCampaign->save();
$this->mock(WarmupQuotaService::class, function ($mock) {
$mock->shouldReceive('send')
->andReturnUsing(fn () => new SendResult(runtimeMessageId: uniqid('msg_'), status: DeliveryStatus::SENT));
});
runCampaign($abCampaign);
runCampaign($regularCampaign);
$abCampaign->refresh();
$regularCampaign->refresh();
// A/B campaign: test phase done → awaiting winner evaluation
expect($abCampaign->status)->toBe(Campaign::STATUS_AWAITING_WINNER);
// Regular campaign: no A/B → done immediately
expect($regularCampaign->status)->toBe(Campaign::STATUS_DONE);
// Regular campaign logs have no ab_test_variant_id
$regularLogs = TrackingLog::where('campaign_id', $regularCampaign->id)->get();
expect($regularLogs)->toHaveCount(4);
expect($regularLogs->every(fn ($l) => is_null($l->ab_test_variant_id)))->toBeTrue();
// A/B campaign logs all have ab_test_variant_id
$abLogs = TrackingLog::where('campaign_id', $abCampaign->id)->get();
expect($abLogs)->toHaveCount(6);
expect($abLogs->every(fn ($l) => !is_null($l->ab_test_variant_id)))->toBeTrue();
});
test('split_strategy=hash uses HashSplitStrategy: assigns all subscribers via CRC32 buckets', function () {
$fixture = abTestFixture(subscriberCount: 40);
['campaign' => $campaign, 'abTest' => $abTest, 'variantA' => $variantA, 'variantB' => $variantB] = abTestCampaign($fixture);
// Switch to hash strategy
$abTest->split_strategy = \App\Model\AbTest::STRATEGY_HASH;
$abTest->save();
(new PopulateAbTestAssignments($abTest->id))->handle();
$countA = AbTestAssignment::where('ab_test_id', $abTest->id)->where('variant_id', $variantA->id)->count();
$countB = AbTestAssignment::where('ab_test_id', $abTest->id)->where('variant_id', $variantB->id)->count();
// All 40 assigned (hash covers 0-99, test_percentage=100 → all pass CRC32 % 100 < 100)
expect($countA + $countB)->toBe(40);
// Both variants received subscribers
expect($countA)->toBeGreaterThan(0);
expect($countB)->toBeGreaterThan(0);
// No subscriber assigned twice
$assignedIds = AbTestAssignment::where('ab_test_id', $abTest->id)->pluck('subscriber_id');
expect($assignedIds->unique()->count())->toBe($assignedIds->count());
});
test('ExactSplitStrategy: incremental assignment preserves existing split when new subscribers added', function () {
// Start with 10 subscribers, 50/50 split
$fixture = abTestFixture(subscriberCount: 10);
['campaign' => $campaign, 'abTest' => $abTest, 'variantA' => $variantA, 'variantB' => $variantB] = abTestCampaign($fixture);
(new PopulateAbTestAssignments($abTest->id))->handle();
$countA_initial = AbTestAssignment::where('ab_test_id', $abTest->id)->where('variant_id', $variantA->id)->count();
$countB_initial = AbTestAssignment::where('ab_test_id', $abTest->id)->where('variant_id', $variantB->id)->count();
$initialAssignedIds = AbTestAssignment::where('ab_test_id', $abTest->id)->pluck('subscriber_id');
// ExactSplitStrategy: round(10 * 50/100) = 5 each
expect($countA_initial)->toBe(5);
expect($countB_initial)->toBe(5);
// Add 10 more subscribers to the same list
['list' => $list] = $fixture;
for ($i = 0; $i < 10; $i++) {
Subscriber::forceCreate([
'uid' => uniqid('absub_'),
'email' => "newsub{$i}_" . uniqid() . '@abtest.invalid',
'status' => Subscriber::STATUS_SUBSCRIBED,
'mail_list_id' => $list->id,
'verification_status' => 'unverified',
'from' => 'manual',
'ip' => '127.0.0.1',
]);
}
// Re-run assignment (simulate resume after new subscribers joined)
(new PopulateAbTestAssignments($abTest->id))->handle();
$countA_final = AbTestAssignment::where('ab_test_id', $abTest->id)->where('variant_id', $variantA->id)->count();
$countB_final = AbTestAssignment::where('ab_test_id', $abTest->id)->where('variant_id', $variantB->id)->count();
$finalAssignedIds = AbTestAssignment::where('ab_test_id', $abTest->id)->pluck('subscriber_id');
// ExactSplitStrategy: grand total 20, round(20 * 50/100) = 10 each
expect($countA_final)->toBe(10);
expect($countB_final)->toBe(10);
// Initial assignments unchanged (same subscriber_ids still present)
$initialAssignedIds->each(fn ($id) => expect($finalAssignedIds)->toContain($id));
// Each variant received exactly 5 new subscribers
expect($countA_final - $countA_initial)->toBe(5);
expect($countB_final - $countB_initial)->toBe(5);
// No duplicates
expect($finalAssignedIds->unique()->count())->toBe($finalAssignedIds->count());
});