API Telemetry - arilonUK/iotagentmesh GitHub Wiki

Telemetry API

The Telemetry API provides enterprise-grade management of time-series data from IoT devices, including high-volume data ingestion, real-time processing, advanced analytics, machine learning integration, and comprehensive data lifecycle management within the IoT Agent Mesh platform.

Overview

The Telemetry system is architected for enterprise-scale IoT deployments and provides:

  • 🚀 High-Volume Data Ingestion: Process millions of data points per day with optimized batch processing
  • ⚡ Real-Time Data Streaming: WebSocket subscriptions with sub-second latency
  • 🤖 Advanced Analytics: Statistical analysis, forecasting, and anomaly detection
  • 🧠 Machine Learning Integration: Predictive maintenance and intelligent monitoring
  • 📊 Flexible Schema Management: Dynamic data validation and transformation
  • 🏢 Enterprise Features: Multi-region deployment, compliance tools, and audit logging
  • 🔄 Data Lifecycle Management: Automated archival, compression, and retention policies
  • 🛡️ Security & Compliance: GDPR compliance, encryption, and audit trails

Base Endpoints

/rest/v1/telemetry              # Core CRUD operations
/functions/v1/telemetry-*       # Advanced processing functions

Authentication

All endpoints require JWT authentication with appropriate organization access permissions.

Required Headers:

Authorization: Bearer YOUR_JWT_TOKEN
apikey: YOUR_SUPABASE_ANON_KEY
Content-Type: application/json

📥 Data Ingestion

Single Data Point Ingestion

Submit a single telemetry data point from an IoT agent.

POST /rest/v1/telemetry

Request Body

{
  "agent_id": "agent_01H8K2M3N4P5Q6R7S8T9U0V1W2",
  "timestamp": "2024-07-23T16:45:00Z",
  "data": {
    "temperature": 23.5,
    "humidity": 65.2,
    "pressure": 1013.25,
    "battery_level": 87,
    "signal_strength": -65,
    "location": {
      "lat": 40.7128,
      "lon": -74.0060,
      "altitude": 10.5
    },
    "status": "normal",
    "vibration": {
      "x": 0.02,
      "y": 0.01,
      "z": 0.98
    }
  },
  "metadata": {
    "firmware_version": "1.4.2",
    "sensor_status": "calibrated",
    "collection_method": "automatic",
    "quality_score": 0.98,
    "processing_flags": ["validated", "calibrated"]
  }
}

Response

{
  "data": [
    {
      "id": "telem_01H8K3L4M5N6O7P8Q9R0S1T2U3V4",
      "agent_id": "agent_01H8K2M3N4P5Q6R7S8T9U0V1W2",
      "timestamp": "2024-07-23T16:45:00Z",
      "data": {
        "temperature": 23.5,
        "humidity": 65.2,
        "pressure": 1013.25,
        "battery_level": 87,
        "signal_strength": -65,
        "location": {
          "lat": 40.7128,
          "lon": -74.0060,
          "altitude": 10.5
        },
        "status": "normal",
        "vibration": {
          "x": 0.02,
          "y": 0.01,
          "z": 0.98
        }
      },
      "metadata": {
        "firmware_version": "1.4.2",
        "sensor_status": "calibrated",
        "collection_method": "automatic",
        "quality_score": 0.98,
        "processing_flags": ["validated", "calibrated"]
      },
      "processed_at": "2024-07-23T16:45:01.234Z",
      "created_at": "2024-07-23T16:45:01.234Z"
    }
  ],
  "ingestion_stats": {
    "processing_time_ms": 12,
    "validation_passed": true,
    "transformations_applied": 0
  }
}

Code Examples

JavaScript/TypeScript:

const { data: telemetry, error } = await supabase
  .from('telemetry')
  .insert({
    agent_id: 'agent_01H8K2M3N4P5Q6R7S8T9U0V1W2',
    data: {
      temperature: 23.5,
      humidity: 65.2,
      pressure: 1013.25,
      battery_level: 87,
      location: { lat: 40.7128, lon: -74.0060 }
    },
    metadata: {
      firmware_version: '1.4.2',
      quality_score: 0.98
    }
  })
  .select()
  .single()

if (error) throw error
console.log('Telemetry ingested:', telemetry)

Python:

telemetry_data = {
    "agent_id": "agent_01H8K2M3N4P5Q6R7S8T9U0V1W2",
    "data": {
        "temperature": 23.5,
        "humidity": 65.2,
        "pressure": 1013.25,
        "battery_level": 87
    },
    "metadata": {
        "firmware_version": "1.4.2",
        "quality_score": 0.98
    }
}

response = supabase.table('telemetry').insert(telemetry_data).execute()
telemetry = response.data[0]

High-Volume Batch Ingestion

Submit multiple telemetry data points in a single request for maximum throughput.

POST /functions/v1/telemetry-batch

Request Body

{
  "batch_config": {
    "batch_id": "batch_01H8K3L4M5N6O7P8Q9R0S1T2U3V4",
    "validate_schema": true,
    "allow_duplicates": false,
    "compression": "gzip",
    "priority": "normal"
  },
  "telemetry": [
    {
      "agent_id": "agent_01H8K2M3N4P5Q6R7S8T9U0V1W2",
      "timestamp": "2024-07-23T16:45:00Z",
      "data": {
        "temperature": 23.5,
        "humidity": 65.2,
        "pressure": 1013.25
      }
    },
    {
      "agent_id": "agent_01H8K2M3N4P5Q6R7S8T9U0V1W2",
      "timestamp": "2024-07-23T16:46:00Z",
      "data": {
        "temperature": 23.8,
        "humidity": 64.9,
        "pressure": 1013.20
      }
    },
    {
      "agent_id": "agent_02H8K2M3N4P5Q6R7S8T9U0V1W3",
      "timestamp": "2024-07-23T16:45:00Z",
      "data": {
        "connected_devices": 25,
        "network_utilization": 34.2,
        "uptime_seconds": 864000
      }
    }
  ]
}

Response

{
  "batch_id": "batch_01H8K3L4M5N6O7P8Q9R0S1T2U3V4",
  "processing_summary": {
    "total_submitted": 3,
    "successfully_ingested": 3,
    "failed": 0,
    "duplicates_rejected": 0,
    "validation_errors": 0
  },
  "performance_metrics": {
    "processing_time_ms": 45,
    "ingestion_rate_per_second": 67,
    "compression_ratio": 0.73,
    "memory_usage_mb": 2.1
  },
  "results": [
    {
      "id": "telem_01H8K3L4M5N6O7P8Q9R0S1T2U3V4",
      "status": "success",
      "agent_id": "agent_01H8K2M3N4P5Q6R7S8T9U0V1W2",
      "processing_time_ms": 12
    },
    {
      "id": "telem_01H8K3L4M5N6O7P8Q9R0S1T2U3V5",
      "status": "success",
      "agent_id": "agent_01H8K2M3N4P5Q6R7S8T9U0V1W2",
      "processing_time_ms": 15
    },
    {
      "id": "telem_01H8K3L4M5N6O7P8Q9R0S1T2U3V6",
      "status": "success",
      "agent_id": "agent_02H8K2M3N4P5Q6R7S8T9U0V1W3",
      "processing_time_ms": 18
    }
  ],
  "next_batch_recommendations": {
    "optimal_batch_size": 150,
    "suggested_interval_ms": 3000
  }
}

Code Examples

JavaScript/TypeScript:

const response = await supabase.functions.invoke('telemetry-batch', {
  body: {
    batch_config: {
      validate_schema: true,
      allow_duplicates: false,
      priority: 'high'
    },
    telemetry: [
      {
        agent_id: 'agent_01H8K2M3N4P5Q6R7S8T9U0V1W2',
        data: { temperature: 23.5, humidity: 65.2 }
      },
      {
        agent_id: 'agent_01H8K2M3N4P5Q6R7S8T9U0V1W2',
        data: { temperature: 23.8, humidity: 64.9 }
      }
    ]
  }
})

console.log(Batch processed: ${response.data.processing_summary.successfully_ingested} records)

Stream Ingestion for Ultra-High Volume

For extremely high-volume scenarios (10,000+ points/second).

POST /functions/v1/telemetry-stream-ingest

Request Body

{
  "stream_config": {
    "agent_id": "agent_01H8K2M3N4P5Q6R7S8T9U0V1W2",
    "format": "compressed_json",
    "buffer_size": 5000,
    "flush_interval_ms": 2000,
    "compression": "zstd",
    "priority": "realtime"
  },
  "data_stream": [
    {"ts": "2024-07-23T16:45:00.000Z", "temp": 23.5, "hum": 65.2, "bat": 87},
    {"ts": "2024-07-23T16:45:01.000Z", "temp": 23.6, "hum": 65.1, "bat": 87},
    {"ts": "2024-07-23T16:45:02.000Z", "temp": 23.4, "hum": 65.3, "bat": 87},
    {"ts": "2024-07-23T16:45:03.000Z", "temp": 23.7, "hum": 65.0, "bat": 86}
  ]
}

Response

{
  "stream_id": "stream_01H8K3L4M5N6O7P8Q9R0S1T2U3V4",
  "status": "active",
  "performance_metrics": {
    "ingestion_rate_per_second": 2500,
    "buffer_utilization": 0.72,
    "compression_ratio": 0.68,
    "latency_p95_ms": 45
  },
  "cost_optimization": {
    "estimated_monthly_cost": "$25.30",
    "savings_vs_individual": "$45.20"
  },
  "stream_health": {
    "backpressure": "normal",
    "error_rate": 0.001,
    "memory_usage_mb": 128
  }
}

🔍 Advanced Querying

Complex Telemetry Queries

Retrieve telemetry data with advanced filtering, aggregation, and optimization.

GET /rest/v1/telemetry

Advanced Query Examples

// Time-range query with field extraction
const { data } = await supabase
  .from('telemetry')
  .select('timestamp, data->>temperature, data->>humidity, data->>location')
  .eq('agent_id', 'agent_123')
  .gte('timestamp', '2024-07-23T00:00:00Z')
  .lte('timestamp', '2024-07-23T23:59:59Z')
  .order('timestamp', { ascending: false })
  .limit(1000)

// Complex condition querying
const { data: filtered } = await supabase
.from('telemetry')
.select('*')
.and('data->>temperature.gte.20,data->>temperature.lte.30')
.gt('data->>humidity', 60)
.contains('metadata', { quality_score: { gte: 0.9 } })


// Geospatial queries
const { data: locationFiltered } = await supabase
.from('telemetry')
.select('*')
.gte('data->location->>lat', 40.0)
.lte('data->location->>lat', 41.0)
.gte('data->location->>lon', -75.0)
.lte('data->location->>lon', -73.0)


// Multi-agent queries with OR conditions
const { data: multiAgent } = await supabase
.from('telemetry')
.select('*')
.or('data->>temperature.gt.35,data->>battery_level.lt.20,data->>status.eq.error')
.in('agent_id', ['agent_123', 'agent_456', 'agent_789'])

Get Latest Telemetry with Caching

Retrieve the most recent telemetry data with intelligent caching.

GET /functions/v1/telemetry-latest

Query Parameters

Parameter Type Description Example
agent_id string Get latest for specific agent agent_123
agent_ids array Get latest for multiple agents ["agent_123", "agent_456"]
fields array Specific data fields to return ["temperature", "humidity"]
include_metadata boolean Include processing metadata true
cache_ttl integer Cache TTL in seconds 30

Response

{
  "data": [
    {
      "agent_id": "agent_01H8K2M3N4P5Q6R7S8T9U0V1W2",
      "agent_name": "Temperature Sensor 01",
      "latest_telemetry": {
        "id": "telem_01H8K3L4M5N6O7P8Q9R0S1T2U3V4",
        "timestamp": "2024-07-23T16:45:00Z",
        "data": {
          "temperature": 23.5,
          "humidity": 65.2,
          "pressure": 1013.25,
          "battery_level": 87
        },
        "age_seconds": 30,
        "quality_score": 0.98,
        "processing_latency_ms": 12
      },
      "status_indicators": {
        "connectivity": "excellent",
        "data_quality": "high",
        "battery_status": "good"
      }
    }
  ],
  "cache_info": {
    "cache_hit": true,
    "cache_age_seconds": 15,
    "ttl_remaining_seconds": 15
  },
  "response_time_ms": 8
}

📊 Advanced Analytics

Time-Series Aggregation

Perform sophisticated time-based aggregations with multiple functions.

POST /functions/v1/telemetry-aggregate

Request Body

{
  "agent_id": "agent_01H8K2M3N4P5Q6R7S8T9U0V1W2",
  "aggregation_config": {
    "field": "temperature",
    "functions": ["avg", "min", "max", "stddev", "percentile_95"],
    "interval": "15m",
    "timezone": "America/New_York",
    "fill_gaps": true,
    "interpolation_method": "linear"
  },
  "time_range": {
    "start": "2024-07-23T00:00:00Z",
    "end": "2024-07-23T23:59:59Z"
  },
  "advanced_options": {
    "outlier_detection": true,
    "anomaly_flagging": true,
    "quality_weighting": true,
    "confidence_intervals": true
  }
}

Response

{
  "aggregation_id": "agg_01H8K3L4M5N6O7P8Q9R0S1T2U3V4",
  "agent_id": "agent_01H8K2M3N4P5Q6R7S8T9U0V1W2",
  "field": "temperature",
  "interval": "15m",
  "data": [
    {
      "timestamp": "2024-07-23T00:00:00-04:00",
      "avg": 22.3,
      "min": 21.8,
      "max": 22.9,
      "stddev": 0.35,
      "percentile_95": 22.7,
      "sample_count": 15,
      "quality_score": 0.98,
      "confidence_interval": [22.1, 22.5],
      "anomalies_detected": 0,
      "outliers_removed": 1
    },
    {
      "timestamp": "2024-07-23T00:15:00-04:00",
      "avg": 22.1,
      "min": 21.5,
      "max": 22.8,
      "stddev": 0.42,
      "percentile_95": 22.6,
      "sample_count": 15,
      "quality_score": 0.95,
      "confidence_interval": [21.9, 22.3],
      "anomalies_detected": 1,
      "outliers_removed": 0
    }
  ],
  "summary_statistics": {
    "total_samples": 1440,
    "intervals_processed": 96,
    "data_completeness": 0.97,
    "overall_quality": 0.96,
    "anomalies_detected": 3,
    "outliers_removed": 7
  },
  "performance_metrics": {
    "computation_time_ms": 450,
    "cache_utilization": 0.73,
    "memory_usage_mb": 15.2
  }
}

Multi-Field Statistical Analysis

Comprehensive statistical analysis across multiple telemetry fields.

POST /functions/v1/telemetry-statistics

Request Body

{
  "analysis_config": {
    "agent_ids": ["agent_01H8K2M3N4P5Q6R7S8T9U0V1W2"],
    "fields": ["temperature", "humidity", "pressure", "battery_level"],
    "time_range": {
      "start": "2024-07-01T00:00:00Z",
      "end": "2024-07-31T23:59:59Z"
    },
    "analysis_types": [
      "descriptive", "distribution", "correlation", 
      "trend_analysis", "seasonality", "anomaly_detection"
    ]
  },
  "advanced_options": {
    "confidence_level": 0.95,
    "outlier_method": "modified_z_score",
    "seasonality_periods": [24, 168, 720],
    "correlation_methods": ["pearson", "spearman", "kendall"]
  }
}

Response

{
  "analysis_id": "stats_01H8K3L4M5N6O7P8Q9R0S1T2U3V4",
  "agent_id": "agent_01H8K2M3N4P5Q6R7S8T9U0V1W2",
  "analysis_period": "31 days",
  "field_analysis": {
    "temperature": {
      "descriptive_stats": {
        "count": 44640,
        "mean": 23.47,
        "median": 23.5,
        "mode": 23.0,
        "std_dev": 2.34,
        "variance": 5.48,
        "skewness": -0.12,
        "kurtosis": 2.98,
        "range": 11.6,
        "iqr": 3.4,
        "cv": 0.10
      },
      "percentiles": {
        "p1": 18.2, "p5": 19.1, "p10": 20.1, "p25": 21.8,
        "p50": 23.5, "p75": 25.2, "p90": 26.8, "p95": 27.9, "p99": 29.5
      },
      "distribution_analysis": {
        "type": "normal",
        "parameters": {"mu": 23.47, "sigma": 2.34},
        "goodness_of_fit": 0.94,
        "normality_tests": {
          "shapiro_wilk": {"statistic": 0.996, "p_value": 0.15},
          "jarque_bera": {"statistic": 45.2, "p_value": 0.12}
        }
      },
      "trend_analysis": {
        "trend_detected": true,
        "direction": "increasing",
        "slope": 0.025,
        "r_squared": 0.15,
        "significance": 0.001,
        "change_points": [
          {"date": "2024-07-15", "magnitude": 1.2}
        ]
      },
      "seasonality": {
        "daily_pattern": {
          "detected": true,
          "amplitude": 3.2,
          "phase_offset_hours": 14.5,
          "strength": 0.78,
          "peak_hour": 14,
          "trough_hour": 6
        },
        "weekly_pattern": {
          "detected": false,
          "strength": 0.12
        },
        "monthly_pattern": {
          "detected": true,
          "strength": 0.35,
          "phase": "mid_month_peak"
        }
      },
      "anomaly_analysis": {
        "total_anomalies": 23,
        "anomaly_rate": 0.0005,
        "severity_distribution": {
          "low": 15,
          "medium": 6,
          "high": 2,
          "critical": 0
        },
        "temporal_patterns": {
          "morning": 8, "afternoon": 12, "evening": 2, "night": 1
        },
        "clustering": {
          "isolated": 18,
          "clustered": 5,
          "burst_events": 1
        }
      }
    }
  },
  "correlation_matrix": {
    "temperature_humidity": {
      "pearson": -0.72,
      "spearman": -0.68,
      "kendall": -0.51,
      "significance": 0.001,
      "interpretation": "strong negative correlation",
      "confidence_interval": [-0.75, -0.69]
    },
    "temperature_pressure": {
      "pearson": 0.31,
      "spearman": 0.29,
      "kendall": 0.20,
      "significance": 0.05,
      "interpretation": "weak positive correlation"
    }
  },
  "insights_generated": [
    "Temperature shows strong daily seasonality with peak at 2:30 PM",
    "Strong negative correlation between temperature and humidity indicates inverse relationship",
    "Gradual warming trend observed: 0.025°C increase per day",
    "2 high-severity anomalies detected on July 15th require investigation",
    "Monthly pattern suggests mid-month temperature peaks"
  ],
  "recommendations": [
    "Investigate temperature spike events on July 15th",
    "Consider humidity compensation algorithm based on temperature correlation",
    "Implement seasonal adjustment for baseline temperature thresholds",
    "Monitor daily peak temperature trends for equipment health"
  ],
  "computation_metadata": {
    "computed_at": "2024-07-23T17:00:00Z",
    "computation_time_ms": 2340,
    "data_quality_score": 0.96,
    "confidence_level": 0.95
  }
}

🤖 Machine Learning Integration

Advanced Anomaly Detection

Multi-algorithm ensemble anomaly detection with real-time processing.

POST /functions/v1/telemetry-anomaly-detection

Request Body

{
  "detection_config": {
    "agent_id": "agent_01H8K2M3N4P5Q6R7S8T9U0V1W2",
    "model_type": "ensemble",
    "algorithms": [
      {
        "type": "isolation_forest",
        "parameters": {
          "contamination": 0.1,
          "n_estimators": 100,
          "max_samples": "auto",
          "random_state": 42
        },
        "weight": 0.4
      },
      {
        "type": "one_class_svm",
        "parameters": {
          "nu": 0.05,
          "kernel": "rbf",
          "gamma": "scale"
        },
        "weight": 0.3
      },
      {
        "type": "statistical_outlier",
        "parameters": {
          "method": "modified_z_score",
          "threshold": 3.5,
          "window_size": 100
        },
        "weight": 0.3
      }
    ]
  },
  "feature_config": {
    "features": ["temperature", "humidity", "pressure", "battery_level"],
    "preprocessing": {
      "scaling": "standard",
      "outlier_removal": true,
      "feature_selection": "auto"
    }
  },
  "training_config": {
    "training_period": "30d",
    "validation_split": 0.2,
    "cross_validation_folds": 5,
    "auto_retrain": true,
    "retrain_interval": "7d",
    "performance_threshold": 0.85
  },
  "detection_settings": {
    "sensitivity": "medium",
    "real_time_processing": true,
    "batch_size": 100,
    "alert_threshold": 0.8,
    "min_anomaly_duration": "5m",
    "context_window": "1h"
  }
}

Response

{
  "detector_id": "ad_01H8K3L4M5N6O7P8Q9R0S1T2U3V4",
  "agent_id": "agent_01H8K2M3N4P5Q6R7S8T9U0V1W2",
  "model_status": "active",
  "training_results": {
    "training_samples": 43200,
    "validation_samples": 10800,
    "model_performance": {
      "accuracy": 0.94,
      "precision": 0.89,
      "recall": 0.91,
      "f1_score": 0.90,
      "auc_roc": 0.96,
      "false_positive_rate": 0.05
    },
    "cross_validation_scores": [0.92, 0.94, 0.93, 0.95, 0.91],
    "feature_importance": {
      "temperature": 0.35,
      "pressure": 0.28,
      "humidity": 0.22,
      "battery_level": 0.15
    }
  },
  "recent_detections": [
    {
      "detection_id": "det_01H8K3L4M5N6O7P8Q9R0S1T2U3V4",
      "timestamp": "2024-07-23T16:30:00Z",
      "anomaly_score": 0.87,
      "severity": "high",
      "duration": "7m 30s",
      "affected_features": {
        "temperature": {
          "score": 0.92,
          "deviation": "+3.2σ",
          "baseline": 23.5,
          "current": 31.2
        },
        "pressure": {
          "score": 0.78,
          "deviation": "+2.8σ",
          "baseline": 1013.25,
          "current": 1025.8
        }
      },
      "contextual_factors": {
        "time_of_day": "afternoon_peak",
        "recent_patterns": "increasing_trend",
        "environmental_conditions": "high_ambient_temperature"
      },
      "confidence": 0.94,
      "risk_assessment": "equipment_stress",
      "recommended_actions": [
        "Check equipment cooling system",
        "Verify sensor calibration",
        "Inspect for environmental factors",
        "Consider maintenance window"
      ]
    }
  ],
  "real_time_config": {
    "processing_enabled": true,
    "latency_p95_ms": 150,
    "throughput_per_second": 1000,
    "memory_usage_mb": 45,
    "cpu_utilization": 0.15
  },
  "maintenance_schedule": {
    "next_retrain": "2024-07-30T17:00:00Z",
    "model_drift_check": "2024-07-24T17:00:00Z",
    "performance_review": "2024-08-01T00:00:00Z"
  }
}

Predictive Maintenance

ML-powered predictive maintenance with failure probability analysis.

POST /functions/v1/telemetry-predictive-maintenance

Request Body

{
  "maintenance_config": {
    "agent_id": "agent_01H8K2M3N4P5Q6R7S8T9U0V1W2",
    "equipment_type": "industrial_pump",
    "model
⚠️ **GitHub.com Fallback** ⚠️