# 1. ACKNOWLEDGE THE INCIDENT
# Log into incident tracking system
# Create incident ticket with P0 tag

# 2. ASSEMBLE WAR ROOM
# Slack: #incident-response
# Zoom: incidents.zoom.us/warroom
# Google Doc: Incident Log Template

# 3. QUICK HEALTH CHECK
cd /Users/admin/Sites/nself-chat/.backend
nself status

# Check all critical services
docker ps --filter "label=nself.service=critical"

# Check disk space
df -h

# Check memory
free -h

# Check database connectivity
docker exec nself-postgres pg_isready

# Check Hasura
curl -f http://localhost:8080/healthz || echo "HASURA DOWN"

# Check Auth
curl -f http://localhost:4000/healthz || echo "AUTH DOWN"

# 1. IDENTIFY FAILING COMPONENT
# Check service logs
nself logs postgres --tail 100
nself logs hasura --tail 100
nself logs auth --tail 100

# Check Sentry for errors
# https://sentry.io/organizations/[org]/issues/

# Check Grafana dashboards
# http://localhost:3000/d/system-overview

# 2. ISOLATE THE PROBLEM
# Is it a single service or cascading failure?
# Is data at risk?
# Can we fail over to backup?

# 3. ESTIMATE RECOVERY TIME
# Quick fix available? < 30 min
# Requires restore? 1-2 hours
# Requires rebuild? 2-4 hours

# Stop all services writing to DB
docker stop nself-hasura nself-auth

# Assess corruption
docker exec nself-postgres psql -U postgres -c "SELECT 1"

# If corrupted, initiate restore
# See: DISASTER-RECOVERY-PROCEDURES.md

# If successful, restart services
docker start nself-hasura nself-auth

# Verify writes working
docker exec nself-postgres psql -U postgres -d nself_db -c \
  "INSERT INTO health_check (timestamp) VALUES (NOW())"

# Identify crashing service
docker ps -a | grep Restarting

# Get crash logs
docker logs [container-id] --tail 200

# Check resource limits
docker stats --no-stream

# Try safe restart with increased limits
docker update --memory 4g --cpus 2 [container-id]
docker restart [container-id]

# If still failing, rollback to last known good
# See: DISASTER-RECOVERY-PROCEDURES.md

# Check connectivity between services
docker network inspect nself_default

# Test DNS resolution
docker exec nself-hasura ping nself-postgres

# Check firewall rules
sudo iptables -L -n

# Recreate network if needed
docker network rm nself_default
docker network create nself_default
# Restart services to reconnect

# Update status page every 15 minutes
# Template: "We are investigating reports of [issue].
#           Current impact: [description].
#           Next update: [time]"

# Internal updates every 10 minutes in war room
# Use: INCIDENT-STATUS-TEMPLATE.md

# 1. VERIFY ALL SERVICES HEALTHY
./scripts/ops/verify-system-health.sh

# 2. RUN SMOKE TESTS
pnpm test:e2e:smoke

# 3. CHECK DATA INTEGRITY
./scripts/ops/verify-data-integrity.sh

# 4. MONITOR FOR 30 MINUTES
# Watch error rates, response times, resource usage

# 5. DECLARE RESOLUTION
# Update status page
# Send all-clear to stakeholders
# Schedule post-mortem

# Check auth container
docker logs nself-auth --tail 100

# Common fix: JWT secret misconfiguration
docker exec nself-auth env | grep JWT

# Restart auth service
docker restart nself-auth

# Verify login working
curl -X POST http://localhost:4000/v1/auth/signin \
  -H "Content-Type: application/json" \
  -d '{"email":"[email protected]","password":"test123"}'

# Check GraphQL engine
curl http://localhost:8080/healthz

# Check database connection pool
docker exec nself-postgres psql -U postgres -c \
  "SELECT count(*) FROM pg_stat_activity WHERE state = 'active'"

# Check message queue if applicable
docker logs nself-redis --tail 50

# Test message insert directly
docker exec nself-postgres psql -U postgres -d nself_db -c \
  "INSERT INTO nchat_messages (content, user_id, channel_id)
   VALUES ('test', '123', '456') RETURNING id"

# Service health check
nself status

# View all logs
nself logs --follow

# Database backup
./scripts/ops/backup-database.sh

# Restore from backup
./scripts/ops/restore-database.sh [backup-file]

# Check resource usage
docker stats --no-stream

# View recent errors
docker logs nself-hasura --since 10m 2>&1 | grep ERROR