# Python service instrumentation with OTel
from opentelemetry import trace, metrics
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.export import BatchSpanProcessor
from opentelemetry.sdk.metrics import MeterProvider
from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter
from opentelemetry.exporter.otlp.proto.grpc.metric_exporter import OTLPMetricExporter
from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor
from opentelemetry.instrumentation.httpx import HTTPXClientInstrumentor
from opentelemetry.instrumentation.sqlalchemy import SQLAlchemyInstrumentor
import logging

def setup_observability(service_name: str, service_version: str):
    """Configure OpenTelemetry for the service."""
    
    # Resource attributes - identify this service
    from opentelemetry.sdk.resources import Resource
    resource = Resource.create({
        "service.name": service_name,
        "service.version": service_version,
        "deployment.environment": os.environ.get("ENV", "development"),
        "service.instance.id": socket.gethostname(),
    })
    
    # Tracing setup
    tracer_provider = TracerProvider(resource=resource)
    tracer_provider.add_span_processor(
        BatchSpanProcessor(
            OTLPSpanExporter(
                endpoint="http://otel-collector:4317",
                insecure=True
            )
        )
    )
    trace.set_tracer_provider(tracer_provider)
    
    # Metrics setup
    metric_reader = PeriodicExportingMetricReader(
        OTLPMetricExporter(
            endpoint="http://otel-collector:4317",
            insecure=True
        ),
        export_interval_millis=10000
    )
    meter_provider = MeterProvider(resource=resource, metric_readers=[metric_reader])
    metrics.set_meter_provider(meter_provider)
    
    # Auto-instrument popular libraries
    FastAPIInstrumentor.instrument()
    HTTPXClientInstrumentor.instrument()
    SQLAlchemyInstrumentor.instrument(enable_commenter=True)
    
    # Structured logging with trace context
    logging.basicConfig(
        format='%(asctime)s %(levelname)s %(name)s trace_id=%(otelTraceID)s span_id=%(otelSpanID)s %(message)s'
    )
    
    return trace.get_tracer(service_name)

# Usage in your FastAPI app
from fastapi import FastAPI
from opentelemetry import trace
from opentelemetry.trace import Status, StatusCode

app = FastAPI()
tracer = setup_observability("payment-service", "2.4.1")

# Custom business metrics
meter = metrics.get_meter("payment-service")
payment_counter = meter.create_counter(
    "payments.processed",
    description="Number of payments processed"
)
payment_duration = meter.create_histogram(
    "payments.duration",
    description="Payment processing duration",
    unit="ms"
)

@app.post("/payments")
async def process_payment(payment: PaymentRequest):
    # Automatic trace from FastAPI instrumentation
    # Add custom span for business logic
    with tracer.start_as_current_span("validate-payment") as span:
        span.set_attribute("payment.amount", payment.amount)
        span.set_attribute("payment.currency", payment.currency)
        span.set_attribute("payment.method", payment.method)
        
        try:
            result = await validate_payment(payment)
            
            if not result.valid:
                span.set_status(Status(StatusCode.ERROR, result.reason))
                span.set_attribute("payment.invalid_reason", result.reason)
                raise HTTPException(422, result.reason)
                
        except Exception as e:
            span.record_exception(e)
            span.set_status(Status(StatusCode.ERROR))
            raise
    
    import time
    start = time.time()
    
    with tracer.start_as_current_span("charge-card") as span:
        charge_result = await charge_card(payment)
        
        duration_ms = (time.time() - start) * 1000
        
        # Record metrics
        payment_counter.add(1, {
            "status": "success",
            "method": payment.method,
            "currency": payment.currency
        })
        payment_duration.record(duration_ms, {
            "method": payment.method
        })
    
    return charge_result

# Generate Grafana dashboards with grafonnet (Python)
from grafanalib.core import (
    Dashboard, TimeSeries, Target, GridPos, 
    Threshold, RED, YELLOW, GREEN
)

dashboard = Dashboard(
    title="Payment Service SLOs",
    time="now-1h",
    panels=[
        TimeSeries(
            title="Payment Success Rate",
            targets=[
                Target(
                    expr='sum(rate(payments_processed_total{status="success"}[5m])) / sum(rate(payments_processed_total[5m]))',
                    legendFormat="Success Rate"
                )
            ],
            thresholds=[
                Threshold(value=None, color=RED),
                Threshold(value=0.99, color=YELLOW),
                Threshold(value=0.999, color=GREEN),
            ],
            gridPos=GridPos(h=8, w=12, x=0, y=0)
        ),
        TimeSeries(
            title="P99 Latency",
            targets=[
                Target(
                    expr='histogram_quantile(0.99, sum(rate(payments_duration_bucket[5m])) by (le))',
                    legendFormat="P99"
                ),
                Target(
                    expr='histogram_quantile(0.50, sum(rate(payments_duration_bucket[5m])) by (le))',
                    legendFormat="P50"
                )
            ],
            gridPos=GridPos(h=8, w=12, x=12, y=0)
        )
    ]
).auto_panel_ids()

# AIOps: Anomaly detection with OpenTelemetry data
from sklearn.ensemble import IsolationForest
import numpy as np

class AnomalyDetector:
    def __init__(self, metric_window_hours: int = 24):
        self.model = IsolationForest(contamination=0.01)
        self.window = metric_window_hours
        self.trained = False
    
    def train(self, historical_metrics: np.ndarray):
        """Train on normal traffic patterns."""
        self.model.fit(historical_metrics)
        self.trained = True
    
    def predict(self, current_metrics: np.ndarray) -> list[bool]:
        """Returns True for anomalous data points."""
        if not self.trained:
            raise RuntimeError("Model not trained")
        
        predictions = self.model.predict(current_metrics)
        return [p == -1 for p in predictions]
    
    async def check_and_alert(self, prometheus_client):
        """Fetch recent metrics and check for anomalies."""
        # Fetch error rate, latency, throughput
        features = await self.fetch_features(prometheus_client)
        
        anomalies = self.predict(features)
        
        if any(anomalies):
            # Call LLM to explain the anomaly
            explanation = await explain_anomaly(features, anomalies)
            await send_alert(f"Anomaly detected: {explanation}")