Add health checks and telemetry

AutoScan.cs

@@ -1,4 +1,3 @@
-using Microsoft.EntityFrameworkCore;
 using Quartz;
 
 namespace NejCommon;

Health.cs

@@ -0,0 +1,146 @@
+using System.Text.Json;
+using Microsoft.AspNetCore.Diagnostics.HealthChecks;
+using Microsoft.EntityFrameworkCore;
+using Microsoft.Extensions.Diagnostics.HealthChecks;
+using NejCommon.Utils;
+
+namespace NejCommon;
+
+public static class HealthUtils
+{
+    public static TBuilder AddDefaultHealthChecks<TBuilder>(this TBuilder builder) where TBuilder : IHostApplicationBuilder
+    {
+        builder.Services.AddHealthChecks()
+            // ========================================
+            // Kubernetes Probes
+            // ========================================
+
+            // Liveness Probe: "Is the application running?"
+            // If it fails, Kubernetes RESTARTS the pod
+            .AddCheck("self",
+                () => HealthCheckResult.Healthy("Application is running"),
+                tags: ["live"])
+
+            // Readiness Probe: "Is the application ready to receive traffic?"
+            // If it fails, Kubernetes REMOVES the pod from Service
+            .AddCheck("readiness",
+                () => HealthCheckResult.Healthy("Application is ready"),
+                tags: ["ready"])
+
+            // ========================================
+            // Critical Dependencies (Database)
+            // ========================================
+
+            // PostgreSQL - Main database
+            .Add(new HealthCheckRegistration(
+                name: "postgres",
+                factory: sp => new PostgresSqlHealthCheck(
+                    sp.GetRequiredService<IConfiguration>(),
+                    sp.GetRequiredService<ILogger<PostgresSqlHealthCheck>>(),
+                    sp),
+                failureStatus: HealthStatus.Degraded,  // Degraded allows app to continue
+                tags: ["dependency", "critical", "database"],
+                timeout: TimeSpan.FromSeconds(15)));
+
+        // ========================================
+        // Health Check Publisher (Periodic Checks)
+        // ========================================
+
+        builder.Services.Configure<HealthCheckPublisherOptions>(options =>
+        {
+            options.Delay = TimeSpan.FromSeconds(5);   // Wait 5s after startup
+            options.Period = TimeSpan.FromSeconds(30); // Execute every 30s
+        });
+
+        return builder;
+    }
+
+    private static async Task WriteHealthCheckResponse(HttpContext context, HealthReport report)
+    {
+        context.Response.ContentType = "application/json";
+
+        var response = new
+        {
+            Status = report.Status.ToString(),
+            Timestamp = DateTime.UtcNow.ToString("yyyy-MM-ddTHH:mm:ss.fffZ"),
+            TotalDuration = $"{report.TotalDuration.TotalMilliseconds}ms",
+            Results = report.Entries.Select(e => new
+            {
+                Name = e.Key,
+                Status = e.Value.Status.ToString(),
+                Duration = $"{e.Value.Duration.TotalMilliseconds}ms",
+                Description = e.Value.Description,
+                Error = e.Value.Exception?.Message,
+                Data = e.Value.Data?.Count > 0 ? e.Value.Data : null,
+                Tags = e.Value.Tags?.ToList()
+            }).ToList(),
+            Summary = new
+            {
+                Total = report.Entries.Count,
+                Healthy = report.Entries.Count(e => e.Value.Status == HealthStatus.Healthy),
+                Degraded = report.Entries.Count(e => e.Value.Status == HealthStatus.Degraded),
+                Unhealthy = report.Entries.Count(e => e.Value.Status == HealthStatus.Unhealthy)
+            }
+        };
+
+        await context.Response.WriteAsync(
+            JsonSerializer.Serialize(response, new JsonSerializerOptions
+            {
+                WriteIndented = true,
+                PropertyNamingPolicy = JsonNamingPolicy.CamelCase
+            }));
+    }
+    public static WebApplication MapHealthEndpoints(this WebApplication app)
+    {
+        var healthCheckOptions = new HealthCheckOptions
+        {
+            Predicate = _ => true,  // All dependencies
+            ResponseWriter = WriteHealthCheckResponse,
+            ResultStatusCodes = new Dictionary<HealthStatus, int>
+            {
+                [HealthStatus.Healthy] = StatusCodes.Status200OK,
+                [HealthStatus.Degraded] = StatusCodes.Status200OK,
+                [HealthStatus.Unhealthy] = StatusCodes.Status503ServiceUnavailable
+            }
+        };
+
+        var liveOptions = new HealthCheckOptions
+        {
+            Predicate = r => r.Tags.Contains("live"),
+            ResponseWriter = WriteHealthCheckResponse,
+            ResultStatusCodes = new Dictionary<HealthStatus, int>
+            {
+                [HealthStatus.Healthy] = StatusCodes.Status200OK,
+                [HealthStatus.Degraded] = StatusCodes.Status200OK,
+                [HealthStatus.Unhealthy] = StatusCodes.Status503ServiceUnavailable
+            }
+        };
+
+        var readyOptions = new HealthCheckOptions
+        {
+            Predicate = r => r.Tags.Contains("ready"),
+            ResponseWriter = WriteHealthCheckResponse,
+            ResultStatusCodes = new Dictionary<HealthStatus, int>
+            {
+                [HealthStatus.Healthy] = StatusCodes.Status200OK,
+                [HealthStatus.Degraded] = StatusCodes.Status200OK,
+                [HealthStatus.Unhealthy] = StatusCodes.Status503ServiceUnavailable
+            }
+        };
+
+        // Map endpoints to both /health and /healthz (K8s compatibility)
+        MapDetailedHealthEndpoints(app, "/health",
+            healthCheckOptions, liveOptions, readyOptions);
+        MapDetailedHealthEndpoints(app, "/healthz",
+            healthCheckOptions, liveOptions, readyOptions);
+
+        return app;
+    }
+
+    private static void MapDetailedHealthEndpoints(WebApplication app, string v, HealthCheckOptions healthCheckOptions, HealthCheckOptions liveOptions, HealthCheckOptions readyOptions)
+    {
+        app.MapHealthChecks(v, healthCheckOptions);
+        app.MapHealthChecks(v + "/live", liveOptions);
+        app.MapHealthChecks(v + "/ready", readyOptions);
+    }
+}

Models/Api/ModelBinder.cs

@@ -11,6 +11,7 @@ using Microsoft.EntityFrameworkCore;
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;
 using Microsoft.OpenApi.Models;
+using NejCommon.Utils;
 using Swashbuckle.AspNetCore.SwaggerGen;
 using System.Linq.Expressions;
 using System.Reflection;
@@ -81,6 +82,8 @@ namespace NejCommon.Models
             {
                 throw new ArgumentNullException(nameof(bindingContext));
             }
+            using var activity = Telemetry.Start();
+            
             var cancellationToken = bindingContext.HttpContext.RequestAborted;
             cancellationToken.ThrowIfCancellationRequested();
 

Utils/PostgresSqlHealthCheck.cs

@@ -0,0 +1,92 @@
+
+using System.Text.RegularExpressions;
+using Microsoft.EntityFrameworkCore;
+using Microsoft.Extensions.Diagnostics.HealthChecks;
+using NejAccountingAPI.Models;
+
+namespace NejCommon.Utils;
+
+public class PostgresSqlHealthCheck : IHealthCheck
+{
+    private readonly IConfiguration _configuration;
+    private readonly ILogger<PostgresSqlHealthCheck> _logger;
+    private readonly IServiceProvider _serviceProvider;
+
+    public PostgresSqlHealthCheck(
+        IConfiguration configuration,
+        ILogger<PostgresSqlHealthCheck> logger,
+        IServiceProvider serviceProvider)
+    {
+        _configuration = configuration;
+        _logger = logger;
+        _serviceProvider = serviceProvider;
+    }
+
+    public async Task<HealthCheckResult> CheckHealthAsync(
+        HealthCheckContext context,
+        CancellationToken cancellationToken = default)
+    {
+        try
+        {
+            // Try to get DbContext via DI
+            using var scope = _serviceProvider.CreateScope();
+            var dbContext = scope.ServiceProvider
+                .GetRequiredService<AppDbContext>();
+
+            // Execute simple query to check connectivity
+            var canConnect = await dbContext.Database
+                .CanConnectAsync(cancellationToken);
+
+            if (!canConnect)
+            {
+                return HealthCheckResult.Unhealthy(
+                    "Cannot connect to PostgreSQL database",
+                    data: new Dictionary<string, object>
+                    {
+                        ["timestamp"] = DateTime.UtcNow,
+                        ["database"] = "PostgreSQL"
+                    });
+            }
+
+            // Check for pending migrations
+            var pendingMigrations = await dbContext.Database
+                .GetPendingMigrationsAsync(cancellationToken);
+
+            if (pendingMigrations.Any())
+            {
+                return HealthCheckResult.Degraded(
+                    "Database has pending migrations",
+                    data: new Dictionary<string, object>
+                    {
+                        ["pendingMigrations"] = pendingMigrations.Count(),
+                        ["migrations"] = string.Join(", ", pendingMigrations)
+                    });
+            }
+
+            // Test real query to validate permissions
+            var recordCount = await dbContext.ApiKeys
+                .CountAsync(cancellationToken);
+
+            return HealthCheckResult.Healthy(
+                "PostgreSQL connection is healthy",
+                data: new Dictionary<string, object>
+                {
+                    ["responseTime"] = DateTime.UtcNow
+                });
+        }
+        catch (Exception ex)
+        {
+            _logger.LogError(ex,
+                "PostgreSQL health check failed: {Message}", ex.Message);
+
+            return HealthCheckResult.Unhealthy(
+                "PostgreSQL health check failed",
+                exception: ex,
+                data: new Dictionary<string, object>
+                {
+                    ["error"] = ex.Message,
+                    ["timestamp"] = DateTime.UtcNow
+                });
+        }
+    }
+}

Utils/Telemetry.cs

@@ -0,0 +1,14 @@
+using System.Diagnostics;
+using System.Runtime.CompilerServices;
+
+namespace NejCommon.Utils;
+
+public static class Telemetry
+{
+    public const string SourceName = "Api";
+
+    public static readonly ActivitySource Source = new(SourceName);
+
+    public static Activity? Start([CallerMemberName] string operation = "")
+        => Source.StartActivity(operation);
+}