APM 最佳实践：从业人员操作指南

# Auto-instrumentation handles this automatically
@app.route('/api/orders')
def create_order():
    # Add manual span only for critical business logic
    with tracer.start_as_current_span("order.validation") as span:
        span.set_attribute("order.value", order_total)
        if not validate_order(order_data):
            span.set_status(Status(StatusCode.ERROR))
            return 400

Copy to clipboardCopy to clipboard

# Outgoing request - inject context
headers = {}
propagate.inject(headers)
response = requests.post('http://service-b/process', headers=headers)

Copy to clipboardCopy to clipboard

@RestController
public class OrderController {
    
    @PostMapping("/orders")
    public ResponseEntity<Order> createOrder(@RequestBody OrderRequest request) {
        // Auto-instrumentation captures this endpoint automatically
        // Add custom business context
        Span.current().setAttributes(Attributes.of(
            stringKey("order.value"), String.valueOf(request.getTotal()),
            stringKey("user.tier"), request.getUserTier()
        ));
        
        return ResponseEntity.ok(processOrder(request));
    }
}

Copy to clipboardCopy to clipboard

# Good - bounded cardinality
span.set_attribute("user.tier", user.subscription_tier)  # 3-5 values
span.set_attribute("http.status_code", response.status_code)  # ~10 values

# Bad - unbounded cardinality  
span.set_attribute("user.id", user.id)  # Millions of values
span.set_attribute("request.timestamp", now())  # Infinite values

Copy to clipboardCopy to clipboard

slos:
  - name: checkout_availability
    target: 99.9%
    window: 7d
  - name: checkout_latency  
    target: 95%  # 95% of requests under 500ms
    window: 7d

Copy to clipboardCopy to clipboard

order_processing_duration = Histogram(
    "order_processing_seconds",
    "Time to process orders", 
    ["payment_method", "order_size"]
)

with order_processing_duration.labels(
    payment_method=payment.method,
    order_size=get_size_bucket(order.total)
).time():
    process_order(order)

Copy to clipboardCopy to clipboard

// Synthetic check for critical user flow
const syntheticCheck = async () => {
    const span = tracer.startSpan('synthetic.checkout_flow');
    try {
        await loginUser();
        await addItemToCart();
        await completePurchase();
        span.setStatus({code: SpanStatusCode.OK});
    } catch (error) {
        span.recordException(error);
        span.setStatus({code: SpanStatusCode.ERROR});
        throw error;
    } finally {
        span.end();
    }
};

Copy to clipboardCopy to clipboard

# Event-driven systems - propagate context through messages
def publish_order_event(order_data):
    headers = {}
    propagate.inject(headers)
    
    message = {
        'data': order_data,
        'trace_headers': headers  # Preserve trace context
    }
    kafka_producer.send('order-events', message)

Copy to clipboardCopy to clipboard

FROM traces-apm*
| WHERE user.id == "user_12345" 
  AND @timestamp >= "2024-06-06T09:00:00" 
  AND @timestamp <= "2024-06-06T10:00:00"
| EVAL duration_ms = transaction.duration.us / 1000
| KEEP trace.id, duration_ms, transaction.name, service.name, transaction.result
| WHERE duration_ms > 2000
| SORT duration_ms DESC
| LIMIT 10

Copy to clipboardCopy to clipboard

# N+1 query problem detected by APM
def get_user_orders_slow(user_id):
    user = User.query.get(user_id)
    orders = []
    for order_id in user.order_ids:  # Each iteration = 1 DB query
        orders.append(Order.query.get(order_id))
    return orders

# Optimized after APM analysis
def get_user_orders_fast(user_id):
    return Order.query.filter(Order.user_id == user_id).all()  # Single query

Copy to clipboardCopy to clipboard

// Java - Auto-propagation with Spring Cloud
@PostMapping("/orders")
public ResponseEntity<Order> createOrder(@RequestBody OrderRequest request) {
    Span.current().setAttributes(Attributes.of(
        stringKey("order.type"), request.getOrderType(),
        longKey("order.value"), request.getTotalValue()));
    
    // OpenFeign automatically propagates context to downstream services
    return paymentClient.processPayment(request.getPaymentData());}

Copy to clipboardCopy to clipboard

// Go - Manual context extraction and propagation
func processHandler(w http.ResponseWriter, r *http.Request) {
    ctx := otel.GetTextMapPropagator().Extract(r.Context(), 
                                              propagation.HeaderCarrier(r.Header))
    ctx, span := tracer.Start(ctx, "process_payment")
    defer span.End()
    // Continue with trace context maintained}

Copy to clipboardCopy to clipboard

# Synthetic spans with correlation IDs for mainframe calls
with tracer.start_as_current_span("mainframe.account_lookup") as span:
    correlation_id = format(span.get_span_context().trace_id, '032x')
    
    logger.info("CICS call started", extra={
        "correlation_id": correlation_id,
        "trace_id": span.get_span_context().trace_id
    })
    
    result = call_mainframe_service(account_data, correlation_id)
    span.set_attribute("account.status", result.status)

Copy to clipboardCopy to clipboard

-- Find slow requests during complaint timeframe
FROM traces-apm*
| WHERE user.id == "user_12345" AND @timestamp >= "2024-06-06T09:00:00"
| EVAL duration_ms = transaction.duration.us / 1000
| WHERE duration_ms > 2000
| STATS avg_duration = AVG(duration_ms) BY service.name, transaction.name
| SORT avg_duration DESC

-- Correlate errors across service boundaries
FROM traces-apm*
| WHERE trace.id == "44b3c2c06e15d444a770b87daab45c0a"
| EVAL is_error = CASE(transaction.result == "error", 1, 0)
| STATS error_rate = SUM(is_error) / COUNT(*) * 100 BY service.name
| WHERE error_rate > 0

Copy to clipboardCopy to clipboard

# Producer - inject context into message
headers = {}
propagate.inject(headers)
message = {
    'data': order_data,
    'trace_headers': headers  # Preserve trace context
}
await kafka_producer.send('order-events', message)

# Consumer - extract and continue trace
trace_headers = message.get('trace_headers', {})
context = propagate.extract(trace_headers)
with tracer.start_as_current_span("order.process", context=context):
    await process_order(message['data'])

Copy to clipboardCopy to clipboard