Add provisioning failure retry logic with scheduled command

Retries failed provisioning every 30 minutes with max 3 attempts,
dispatches ProvisioningFailed event when max attempts reached.

Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com>

website/app/Console/Commands/RetryProvisioningCommand.php

@@ -0,0 +1,132 @@
+<?php
+
+declare(strict_types=1);
+
+namespace App\Console\Commands;
+
+use App\Events\ProvisioningFailed;
+use App\Models\ProvisioningLog;
+use App\Models\Service;
+use App\Services\Provisioning\ProvisioningFactory;
+use Illuminate\Console\Command;
+use Illuminate\Support\Facades\DB;
+use Illuminate\Support\Facades\Log;
+
+class RetryProvisioningCommand extends Command
+{
+    protected $signature = 'provisioning:retry
+        {--max-attempts=3 : Maximum number of retry attempts per service}';
+
+    protected $description = 'Retry failed provisioning attempts';
+
+    public function handle(ProvisioningFactory $provisioningFactory): int
+    {
+        $maxAttempts = (int) $this->option('max-attempts');
+
+        $failedServices = Service::query()
+            ->where('status', 'failed')
+            ->where('created_at', '>=', now()->subDays(7))
+            ->with(['subscription', 'plan'])
+            ->get();
+
+        if ($failedServices->isEmpty()) {
+            $this->info('No failed services found to retry.');
+
+            return self::SUCCESS;
+        }
+
+        $this->info("Found {$failedServices->count()} failed service(s) to evaluate.");
+
+        $retried = 0;
+        $skipped = 0;
+        $succeeded = 0;
+        $failed = 0;
+
+        foreach ($failedServices as $service) {
+            $failedAttempts = ProvisioningLog::query()
+                ->where('service_id', $service->id)
+                ->where('action', 'provision')
+                ->where('status', 'failed')
+                ->count();
+
+            if ($failedAttempts >= $maxAttempts) {
+                $this->warn("Service #{$service->id}: skipped (already at {$failedAttempts}/{$maxAttempts} attempts).");
+
+                if ($failedAttempts === $maxAttempts) {
+                    ProvisioningFailed::dispatch(
+                        $service,
+                        'Maximum retry attempts reached',
+                        $failedAttempts,
+                        true,
+                    );
+                }
+
+                $skipped++;
+
+                continue;
+            }
+
+            if (! $service->subscription) {
+                $this->warn("Service #{$service->id}: skipped (no associated subscription).");
+                $skipped++;
+
+                continue;
+            }
+
+            $attemptNumber = $failedAttempts + 1;
+            $this->info("Service #{$service->id}: retrying provisioning (attempt {$attemptNumber}/{$maxAttempts})...");
+
+            try {
+                $provisioningService = $provisioningFactory->make($service->service_type);
+
+                DB::transaction(function () use ($service) {
+                    $service->delete();
+                });
+
+                $newService = $provisioningService->provision($service->subscription);
+
+                $this->info("Service #{$newService->id}: provisioned successfully.");
+                Log::info("Provisioning retry succeeded for replaced service #{$service->id}", [
+                    'old_service_id' => $service->id,
+                    'new_service_id' => $newService->id,
+                    'service_type' => $service->service_type,
+                    'attempt' => $attemptNumber,
+                ]);
+
+                $retried++;
+                $succeeded++;
+            } catch (\Throwable $e) {
+                $this->error("Service #{$service->id}: provisioning failed — {$e->getMessage()}");
+                Log::error("Provisioning retry failed for service #{$service->id}", [
+                    'service_id' => $service->id,
+                    'service_type' => $service->service_type,
+                    'attempt' => $attemptNumber,
+                    'error' => $e->getMessage(),
+                ]);
+
+                $isMaxReached = $attemptNumber >= $maxAttempts;
+
+                $retryService = Service::query()
+                    ->where('subscription_id', $service->subscription_id)
+                    ->where('status', 'failed')
+                    ->latest()
+                    ->first();
+
+                ProvisioningFailed::dispatch(
+                    $retryService ?? $service,
+                    $e->getMessage(),
+                    $attemptNumber,
+                    $isMaxReached,
+                );
+
+                $retried++;
+                $failed++;
+            }
+        }
+
+        $this->newLine();
+        $this->info("Retry summary: {$retried} retried, {$succeeded} succeeded, {$failed} failed, {$skipped} skipped.");
+
+        return self::SUCCESS;
+    }
+}

website/app/Events/ProvisioningFailed.php

@@ -0,0 +1,21 @@
+<?php
+
+declare(strict_types=1);
+
+namespace App\Events;
+
+use App\Models\Service;
+use Illuminate\Foundation\Events\Dispatchable;
+use Illuminate\Queue\SerializesModels;
+
+class ProvisioningFailed
+{
+    use Dispatchable, SerializesModels;
+
+    public function __construct(
+        public Service $service,
+        public string $errorMessage,
+        public int $attemptNumber,
+        public bool $maxAttemptsReached,
+    ) {}
+}

website/app/Listeners/HandleProvisioningFailed.php

@@ -0,0 +1,36 @@
+<?php
+
+declare(strict_types=1);
+
+namespace App\Listeners;
+
+use App\Events\ProvisioningFailed;
+use Illuminate\Support\Facades\Log;
+
+class HandleProvisioningFailed
+{
+    public function handle(ProvisioningFailed $event): void
+    {
+        $service = $event->service;
+
+        Log::warning("Provisioning failed for service #{$service->id}", [
+            'service_id' => $service->id,
+            'user_id' => $service->user_id,
+            'service_type' => $service->service_type,
+            'platform' => $service->platform,
+            'error' => $event->errorMessage,
+            'attempt' => $event->attemptNumber,
+            'max_attempts_reached' => $event->maxAttemptsReached,
+        ]);
+
+        if ($event->maxAttemptsReached) {
+            Log::error("Provisioning permanently failed for service #{$service->id} after {$event->attemptNumber} attempts", [
+                'service_id' => $service->id,
+                'user_id' => $service->user_id,
+                'service_type' => $service->service_type,
+                'platform' => $service->platform,
+                'error' => $event->errorMessage,
+            ]);
+        }
+    }
+}

website/routes/console.php

@@ -10,3 +10,4 @@ Artisan::command('inspire', function () {
 
 Schedule::command('billing:process-dunning')->daily()->at('06:00');
 Schedule::command('tickets:process-emails')->everyTwoMinutes()->withoutOverlapping();
+Schedule::command('provisioning:retry')->everyThirtyMinutes();