Never Miss a Failed Cron Job: Heartbeat Monitoring with Jobably

using Noundry.Tuxedo.Contrib;
using Noundry.Tuxedo.Bowtie.Attributes;

namespace Jobably.Models;

[Table("monitors")]
public class JobMonitor
{
    [ExplicitKey]
    [Column("id")]
    public Guid Id { get; set; } = Guid.NewGuid();

    [Column("user_id")]
    [Index("IX_Monitors_UserId")]
    public Guid UserId { get; set; }

    [Column("name", MaxLength = 200)]
    public string Name { get; set; } = string.Empty;

    [Column("slug", MaxLength = 100)]
    [Index("IX_Monitors_Slug", IsUnique = true)]
    public string Slug { get; set; } = string.Empty;

    /// <summary>
    /// How often (in minutes) we expect a heartbeat ping.
    /// </summary>
    [Column("expected_interval")]
    [DefaultValue(60)]
    public int ExpectedInterval { get; set; } = 60;

    /// <summary>
    /// Extra minutes to wait before marking the monitor as DOWN.
    /// </summary>
    [Column("grace_period")]
    [DefaultValue(30)]
    public int GracePeriod { get; set; } = 30;

    /// <summary>
    /// Current status: new, up, down, paused.
    /// </summary>
    [Column("status", MaxLength = 20)]
    [DefaultValue("'new'")]
    public string Status { get; set; } = "new";

    [Column("last_ping_at")]
    public DateTime? LastPingAt { get; set; }

    [Column("uptime_percentage", Precision = 5, Scale = 2)]
    public decimal UptimePercentage { get; set; } = 100.0m;

    [Column("alert_email", MaxLength = 255)]
    public string? AlertEmail { get; set; }

    [Column("alert_slack_webhook", MaxLength = 500)]
    public string? AlertSlackWebhook { get; set; }

    [Column("alert_sms_phone", MaxLength = 20)]
    public string? AlertSmsPhone { get; set; }

    [Column("last_alerted_at")]
    public DateTime? LastAlertedAt { get; set; }

    /// <summary>
    /// Minimum minutes between repeated alerts to avoid spamming.
    /// </summary>
    [Column("alert_cooldown_minutes")]
    [DefaultValue(60)]
    public int AlertCooldownMinutes { get; set; } = 60;

    [Column("created_at")]
    [DefaultValue("CURRENT_TIMESTAMP", IsRawSql = true)]
    public DateTime CreatedAt { get; set; } = DateTime.UtcNow;

    [Column("updated_at")]
    public DateTime? UpdatedAt { get; set; }

    /// <summary>
    /// Returns the deadline: last ping + interval + grace. If no ping
    /// has ever been received, returns null (monitor is still "new").
    /// </summary>
    [Computed]
    public DateTime? Deadline => LastPingAt?
        .AddMinutes(ExpectedInterval + GracePeriod);

    [Computed]
    public bool IsOverdue => Deadline.HasValue && DateTime.UtcNow > Deadline.Value;
}

using Noundry.Tuxedo.Contrib;
using Noundry.Tuxedo.Bowtie.Attributes;

namespace Jobably.Models;

[Table("heartbeat_events")]
public class HeartbeatEvent
{
    [ExplicitKey]
    [Column("id")]
    public Guid Id { get; set; } = Guid.NewGuid();

    [Column("monitor_id")]
    [Index("IX_HeartbeatEvents_MonitorId")]
    public Guid MonitorId { get; set; }

    /// <summary>
    /// Event type: success, failed, error, timeout.
    /// </summary>
    [Column("event_type", MaxLength = 20)]
    public string EventType { get; set; } = "success";

    [Column("message", MaxLength = 2000)]
    public string? Message { get; set; }

    [Column("source_ip", MaxLength = 45)]
    public string? SourceIp { get; set; }

    [Column("created_at")]
    [DefaultValue("CURRENT_TIMESTAMP", IsRawSql = true)]
    public DateTime CreatedAt { get; set; } = DateTime.UtcNow;
}

using System.Data;
using Microsoft.AspNetCore.Mvc;
using Noundry.Tuxedo;
using Noundry.Tuxedo.Contrib;
using Jobably.Models;

namespace Jobably.Controllers;

[ApiController]
[Route("api/heartbeat")]
public class HeartbeatController : ControllerBase
{
    private readonly IDbConnection _db;
    private readonly ILogger<HeartbeatController> _logger;

    public HeartbeatController(IDbConnection db, ILogger<HeartbeatController> logger)
    {
        _db = db;
        _logger = logger;
    }

    /// <summary>
    /// Receive a success heartbeat ping.
    /// GET or POST /api/heartbeat/{slug}
    /// </summary>
    [HttpGet("{slug}")]
    [HttpPost("{slug}")]
    public async Task<IActionResult> Ping(string slug)
    {
        return await RecordEvent(slug, "success", null);
    }

    /// <summary>
    /// Record that the job ran but failed.
    /// GET or POST /api/heartbeat/{slug}/failed
    /// </summary>
    [HttpGet("{slug}/failed")]
    [HttpPost("{slug}/failed")]
    public async Task<IActionResult> PingFailed(string slug)
    {
        return await RecordEvent(slug, "failed", "Job reported failure.");
    }

    /// <summary>
    /// Record an error with a custom message.
    /// GET or POST /api/heartbeat/{slug}/error?message=...
    /// </summary>
    [HttpGet("{slug}/error")]
    [HttpPost("{slug}/error")]
    public async Task<IActionResult> PingError(string slug, [FromQuery] string? message)
    {
        return await RecordEvent(slug, "error", message ?? "Unknown error");
    }

    private async Task<IActionResult> RecordEvent(
        string slug, string eventType, string? message)
    {
        var monitor = await _db.QueryFirstOrDefaultAsync<JobMonitor>(
            "SELECT * FROM monitors WHERE slug = @Slug",
            new { Slug = slug });

        if (monitor == null)
            return NotFound(new { error = "Monitor not found." });

        // Record the heartbeat event
        var heartbeatEvent = new HeartbeatEvent
        {
            MonitorId = monitor.Id,
            EventType = eventType,
            Message = message,
            SourceIp = HttpContext.Connection.RemoteIpAddress?.ToString()
        };
        await _db.InsertAsync(heartbeatEvent);

        // Update the monitor: stamp last ping, flip status to "up"
        // (even on "failed" events, the job DID run and phone home)
        monitor.LastPingAt = DateTime.UtcNow;
        monitor.Status = eventType == "success" ? "up" : eventType;
        monitor.UpdatedAt = DateTime.UtcNow;
        await _db.UpdateAsync(monitor);

        _logger.LogInformation(
            "Heartbeat [{EventType}] received for monitor {Slug}",
            eventType, slug);

        return Ok(new { ok = true, monitor = monitor.Name, eventType });
    }
}

using System.Data;
using Noundry.Tuxedo;
using Noundry.Tuxedo.Contrib;
using Jobably.Models;

namespace Jobably.Services;

/// <summary>
/// Background service that checks every 60 seconds for overdue monitors
/// and fires alerts when a heartbeat is missed.
/// </summary>
public class MonitorCheckerService : BackgroundService
{
    private readonly IServiceProvider _serviceProvider;
    private readonly ILogger<MonitorCheckerService> _logger;
    private static readonly TimeSpan CheckInterval = TimeSpan.FromSeconds(60);

    public MonitorCheckerService(
        IServiceProvider serviceProvider,
        ILogger<MonitorCheckerService> logger)
    {
        _serviceProvider = serviceProvider;
        _logger = logger;
    }

    protected override async Task ExecuteAsync(CancellationToken stoppingToken)
    {
        _logger.LogInformation("MonitorCheckerService started.");

        while (!stoppingToken.IsCancellationRequested)
        {
            try
            {
                await CheckOverdueMonitorsAsync();
            }
            catch (Exception ex)
            {
                _logger.LogError(ex, "Error checking overdue monitors.");
            }

            await Task.Delay(CheckInterval, stoppingToken);
        }
    }

    private async Task CheckOverdueMonitorsAsync()
    {
        using var scope = _serviceProvider.CreateScope();
        var db = scope.ServiceProvider.GetRequiredService<IDbConnection>();

        // Find monitors that are "up" but haven't pinged within
        // their expected_interval + grace_period window
        var overdueMonitors = await db.QueryAsync<JobMonitor>(
            @"SELECT * FROM monitors
              WHERE status = 'up'
                AND last_ping_at IS NOT NULL
                AND last_ping_at + INTERVAL '1 minute'
                    * (expected_interval + grace_period) < NOW()");

        foreach (var monitor in overdueMonitors)
        {
            _logger.LogWarning(
                "Monitor '{Name}' (slug: {Slug}) is OVERDUE. " +
                "Last ping: {LastPing}. Marking as DOWN.",
                monitor.Name, monitor.Slug, monitor.LastPingAt);

            // Flip to DOWN
            monitor.Status = "down";
            monitor.UpdatedAt = DateTime.UtcNow;
            await db.UpdateAsync(monitor);

            // Record a timeout event
            var timeoutEvent = new HeartbeatEvent
            {
                MonitorId = monitor.Id,
                EventType = "timeout",
                Message = $"No heartbeat received within " +
                          $"{monitor.ExpectedInterval + monitor.GracePeriod} " +
                          $"minutes. Last ping: {monitor.LastPingAt:u}"
            };
            await db.InsertAsync(timeoutEvent);

            // Check alert cooldown before sending
            if (ShouldSendAlert(monitor))
            {
                await SendAlertsAsync(db, monitor);
            }
        }
    }

    private static bool ShouldSendAlert(JobMonitor monitor)
    {
        if (!monitor.LastAlertedAt.HasValue)
            return true;

        var minutesSinceLastAlert =
            (DateTime.UtcNow - monitor.LastAlertedAt.Value).TotalMinutes;

        return minutesSinceLastAlert >= monitor.AlertCooldownMinutes;
    }

    private async Task SendAlertsAsync(IDbConnection db, JobMonitor monitor)
    {
        // Email alert
        if (!string.IsNullOrWhiteSpace(monitor.AlertEmail))
        {
            _logger.LogInformation(
                "Sending email alert for '{Name}' to {Email}",
                monitor.Name, monitor.AlertEmail);
            // Integrate your SMTP service here
        }

        // Slack webhook alert
        if (!string.IsNullOrWhiteSpace(monitor.AlertSlackWebhook))
        {
            using var http = new HttpClient();
            var payload = new
            {
                text = $":rotating_light: *{monitor.Name}* is DOWN! " +
                       $"No heartbeat since {monitor.LastPingAt:u}."
            };
            await http.PostAsJsonAsync(monitor.AlertSlackWebhook, payload);

            _logger.LogInformation(
                "Sent Slack alert for '{Name}'.", monitor.Name);
        }

        // SMS alert (Twilio)
        if (!string.IsNullOrWhiteSpace(monitor.AlertSmsPhone))
        {
            _logger.LogInformation(
                "Sending SMS alert for '{Name}' to {Phone}",
                monitor.Name, monitor.AlertSmsPhone);
            // Integrate your Twilio service here
        }

        // Stamp the cooldown
        monitor.LastAlertedAt = DateTime.UtcNow;
        await db.UpdateAsync(monitor);
    }
}

using System.Data;
using Noundry.Jobs.Heartbeat;
using Noundry.Tuxedo;
using Noundry.Tuxedo.Contrib;

namespace MyApp.Jobs;

/// <summary>
/// Example: a nightly database cleanup job wrapped in an auto-managed
/// heartbeat. If the job completes, Jobably gets a success ping.
/// If it throws, Jobably gets an error ping with the exception message.
/// </summary>
public class NightlyCleanupJob : BackgroundService
{
    private readonly IServiceProvider _serviceProvider;
    private readonly ILogger<NightlyCleanupJob> _logger;

    // Your Jobably monitor URL
    private const string HeartbeatUrl =
        "https://jobably.com/api/heartbeat/nightly-cleanup-abc123";

    public NightlyCleanupJob(
        IServiceProvider serviceProvider,
        ILogger<NightlyCleanupJob> logger)
    {
        _serviceProvider = serviceProvider;
        _logger = logger;
    }

    protected override async Task ExecuteAsync(CancellationToken stoppingToken)
    {
        while (!stoppingToken.IsCancellationRequested)
        {
            // Wait until 2:00 AM UTC
            var now = DateTime.UtcNow;
            var nextRun = now.Date.AddDays(now.Hour >= 2 ? 1 : 0).AddHours(2);
            var delay = nextRun - now;
            await Task.Delay(delay, stoppingToken);

            // Auto-managed heartbeat: success on completion,
            // error on exception. No manual try/catch needed.
            await JobsHeartbeat.RunAsync(HeartbeatUrl, async () =>
            {
                using var scope = _serviceProvider.CreateScope();
                var db = scope.ServiceProvider
                    .GetRequiredService<IDbConnection>();

                // Delete records older than 90 days
                var deleted = await db.ExecuteAsync(
                    @"DELETE FROM audit_logs
                      WHERE created_at < NOW() - INTERVAL '90 days'");

                _logger.LogInformation(
                    "Nightly cleanup complete. Removed {Count} old records.",
                    deleted);

                // Delete expired sessions
                var expiredSessions = await db.ExecuteAsync(
                    @"DELETE FROM user_sessions
                      WHERE expires_at < NOW()");

                _logger.LogInformation(
                    "Removed {Count} expired sessions.", expiredSessions);
            });
        }
    }
}

using Noundry.Jobs.Heartbeat;

namespace MyApp.Jobs;

/// <summary>
/// Example: manual heartbeat control for jobs that need
/// fine-grained reporting of partial success or specific errors.
/// </summary>
public static class ManualHeartbeatExample
{
    private const string HeartbeatUrl =
        "https://jobably.com/api/heartbeat/data-sync-xyz789";

    public static async Task RunDataSyncAsync()
    {
        var processedCount = 0;
        var errorCount = 0;

        try
        {
            // Phase 1: Fetch new records from external API
            var records = await FetchExternalRecordsAsync();
            processedCount = records.Count;

            // Phase 2: Transform and save
            foreach (var record in records)
            {
                try
                {
                    await TransformAndSaveAsync(record);
                }
                catch
                {
                    errorCount++;
                }
            }

            // Decide what to report based on results
            if (errorCount == 0)
            {
                // All good - send success ping
                await JobsHeartbeat.SendAsync(HeartbeatUrl);
            }
            else if (errorCount < processedCount)
            {
                // Partial failure - send error with details
                await JobsHeartbeat.SendErrorAsync(HeartbeatUrl,
                    $"Partial sync: {processedCount - errorCount}/" +
                    $"{processedCount} succeeded, {errorCount} failed.");
            }
            else
            {
                // Total failure
                await JobsHeartbeat.SendFailedAsync(HeartbeatUrl);
            }
        }
        catch (Exception ex)
        {
            // Infrastructure-level failure
            await JobsHeartbeat.SendErrorAsync(HeartbeatUrl, ex.Message);
            throw;
        }
    }

    private static Task<List<object>> FetchExternalRecordsAsync()
        => Task.FromResult(new List<object>());

    private static Task TransformAndSaveAsync(object record)
        => Task.CompletedTask;
}

#!/bin/bash
# ============================================================
# Jobably heartbeat integration from any language via cURL
# ============================================================

MONITOR_SLUG="my-nightly-backup-abc123"
BASE_URL="https://jobably.com/api/heartbeat"

# ----- 1. Simple success ping (GET or POST) -----------------
# Call this at the end of your job:
curl -s "${BASE_URL}/${MONITOR_SLUG}"

# ----- 2. Report failure ------------------------------------
curl -s "${BASE_URL}/${MONITOR_SLUG}/failed"

# ----- 3. Report error with a message -----------------------
curl -s "${BASE_URL}/${MONITOR_SLUG}/error?message=Disk+space+full"

# ----- 4. Wrap a bash script with heartbeat -----------------
# Add this to the end of any cron job script:

run_with_heartbeat() {
    local slug="$1"
    shift

    if "$@"; then
        curl -s "${BASE_URL}/${slug}"
        echo "[Jobably] Success ping sent."
    else
        local exit_code=$?
        curl -s "${BASE_URL}/${slug}/error?message=Exit+code+${exit_code}"
        echo "[Jobably] Error ping sent (exit code: ${exit_code})."
    fi
}

# Usage in a crontab:
# 0 2 * * * /path/to/run_with_heartbeat my-backup-slug /path/to/backup.sh

# ----- 5. One-liner for crontab -----------------------------
# Append a success ping directly to your cron command:
# 0 * * * * /usr/bin/php /var/www/process.php && curl -s https://jobably.com/api/heartbeat/hourly-process-def456

using Jobably.Services;

var builder = WebApplication.CreateBuilder(args);

// Register the monitor checker as a hosted background service
builder.Services.AddHostedService<MonitorCheckerService>();

// Register your own jobs with heartbeat monitoring
// builder.Services.AddHostedService<NightlyCleanupJob>();

builder.Services.AddControllers();

var app = builder.Build();

app.MapControllers();

app.Run();