HTTP propagation lets trace context flow across service boundaries, making distributed tracing actually work.
Imagine you’ve got a request hitting your API gateway. That gateway then calls service A, which calls service B. Without HTTP propagation, each of those calls would start a new, independent trace. Service B wouldn’t know it’s part of the same request that hit the gateway. OpenTelemetry solves this by injecting a special header into outgoing HTTP requests and extracting it from incoming ones.
Let’s see this in action.
Here’s a simplified Go service that receives an HTTP request, starts a new span for itself, and then calls another service.
package main
import (
"context"
"fmt"
"log"
"net/http"
"os"
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/exporters/stdout/stdouttrace"
"go.opentelemetry.io/otel/propagation"
"go.opentelemetry.io/otel/sdk/resource"
"go.opentelemetry.io/otel/sdk/trace"
semconv "go.opentelemetry.io/otel/semconv/v1.17.0"
)
// initTracer sets up the OpenTelemetry tracer.
func initTracer() (*trace.TracerProvider, error) {
// Define resource attributes for this service.
res, err := resource.New(context.Background(),
resource.WithAttributes(
semconv.ServiceName("service-a"),
semconv.ServiceVersion("1.0.0"),
),
)
if err != nil {
return nil, fmt.Errorf("failed to create resource: %w", err)
}
// Create a stdout exporter to print traces to the console.
exporter, err := stdouttrace.New(stdouttrace.WithPrettyPrint())
if err != nil {
return nil, fmt.Errorf("failed to create stdout exporter: %w", err)
}
// Create a tracer provider with the exporter and resource.
tp := trace.NewTracerProvider(
trace.WithResource(res),
trace.WithBatcher(exporter),
)
// Set the global propagator to W3C Trace Context and Baggage.
otel.SetPropagator(propagation.NewCompositeTextMapPropagator(
propagation.TraceContext{},
propagation.Baggage{},
))
return tp, nil
}
// getTracer returns the global tracer.
func getTracer() *trace.TracerProvider {
tp, err := initTracer()
if err != nil {
log.Fatalf("Failed to initialize tracer: %v", err)
}
return tp
}
// handler is the HTTP handler for incoming requests.
func handler(w http.ResponseWriter, r *http.Request) {
// Extract context from incoming request headers.
// This is where propagation happens on the receiving end.
ctx := otel.GetTextMapPropagator().Extract(r.Context(), propagation.HeaderCarrier(r.Header))
// Get the tracer for this service.
tracer := getTracer().Tracer("service-a-tracer")
// Start a new span, parented by the extracted context.
ctx, span := tracer.Start(ctx, "incoming-request")
defer span.End()
span.AddEvent("Processing request...")
// Simulate calling another service.
err := callServiceB(ctx)
if err != nil {
span.RecordError(err)
http.Error(w, "Error calling service B", http.StatusInternalServerError)
return
}
fmt.Fprintln(w, "Request processed successfully!")
}
// callServiceB simulates calling another service.
func callServiceB(ctx context.Context) error {
tracer := getTracer().Tracer("service-a-tracer")
ctx, span := tracer.Start(ctx, "call-service-b")
defer span.End()
// Create a new HTTP request.
req, err := http.NewRequestWithContext(ctx, "GET", "http://localhost:8081/serviceb", nil)
if err != nil {
return fmt.Errorf("failed to create request: %w", err)
}
// Inject context into the outgoing request headers.
// This is where propagation happens on the sending end.
otel.GetTextMapPropagator().Inject(ctx, propagation.HeaderCarrier(req.Header))
client := &http.Client{}
resp, err := client.Do(req)
if err != nil {
return fmt.Errorf("failed to do request: %w", err)
}
defer resp.Body.Close()
span.AddEvent("Service B called successfully.")
return nil
}
func main() {
// Initialize tracer
tp := getTracer()
defer func() {
if err := tp.Shutdown(context.Background()); err != nil {
log.Fatalf("Error shutting down tracer provider: %v", err)
}
}()
http.HandleFunc("/servicea", handler)
fmt.Println("Service A listening on :8080")
log.Fatal(http.ListenAndServe(":8080", nil))
}
And here’s the service it calls, service-b:
package main
import (
"context"
"fmt"
"log"
"net/http"
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/exporters/stdout/stdouttrace"
"go.opentelemetry.io/otel/propagation"
"go.opentelemetry.io/otel/sdk/resource"
"go.opentelemetry.io/otel/sdk/trace"
semconv "go.opentelemetry.io/otel/semconv/v1.17.0"
)
// initTracer sets up the OpenTelemetry tracer.
func initTracer() (*trace.TracerProvider, error) {
res, err := resource.New(context.Background(),
resource.WithAttributes(
semconv.ServiceName("service-b"),
semconv.ServiceVersion("1.0.0"),
),
)
if err != nil {
return nil, fmt.Errorf("failed to create resource: %w", err)
}
exporter, err := stdouttrace.New(stdouttrace.WithPrettyPrint())
if err != nil {
return nil, fmt.Errorf("failed to create stdout exporter: %w", err)
}
tp := trace.NewTracerProvider(
trace.WithResource(res),
trace.WithBatcher(exporter),
)
otel.SetPropagator(propagation.NewCompositeTextMapPropagator(
propagation.TraceContext{},
propagation.Baggage{},
))
return tp, nil
}
// getTracer returns the global tracer.
func getTracer() *trace.TracerProvider {
tp, err := initTracer()
if err != nil {
log.Fatalf("Failed to initialize tracer: %v", err)
}
return tp
}
// handler is the HTTP handler for incoming requests.
func handler(w http.ResponseWriter, r *http.Request) {
// Extract context from incoming request headers.
ctx := otel.GetTextMapPropagator().Extract(r.Context(), propagation.HeaderCarrier(r.Header))
tracer := getTracer().Tracer("service-b-tracer")
// Start a new span, parented by the extracted context.
ctx, span := tracer.Start(ctx, "incoming-request-service-b")
defer span.End()
span.AddEvent("Processing request in Service B...")
fmt.Fprintln(w, "Service B processed successfully!")
}
func main() {
tp := getTracer()
defer func() {
if err := tp.Shutdown(context.Background()); err != nil {
log.Fatalf("Error shutting down tracer provider: %v", err)
}
}()
http.HandleFunc("/serviceb", handler)
fmt.Println("Service B listening on :8081")
log.Fatal(http.ListenAndServe(":8081", nil))
}
When you run both services and hit http://localhost:8080/servicea with curl, you’ll see output like this in your console:
Service A Output:
{
"traceId": "d8a475b78b88762a3658f89300a05f8a",
"spanId": "08666e32508931c1",
"parentSpanId": "0000000000000000",
"name": "incoming-request",
"kind": 1,
"startTimeUnixNano": 1678886400000000000,
"endTimeUnixNano": 1678886400100000000,
"attributes": [],
"events": [
{
"name": "Processing request...",
"timeUnixNano": 1678886400050000000
}
],
"status": {
"code": 2
},
"traceState": "",
"resource": {
"attributes": [
{
"key": "service.name",
"value": {
"stringValue": "service-a"
}
},
{
"key": "service.version",
"value": {
"stringValue": "1.0.0"
}
}
]
},
"instrumentationLibrary": {
"name": "service-a-tracer",
"version": ""
}
}
{
"traceId": "d8a475b78b88762a3658f89300a05f8a",
"spanId": "02a6189b575f8b1c",
"parentSpanId": "08666e32508931c1",
"name": "call-service-b",
"kind": 2,
"startTimeUnixNano": 1678886400080000000,
"endTimeUnixNano": 1678886400090000000,
"attributes": [],
"events": [
{
"name": "Service B called successfully.",
"timeUnixNano": 1678886400085000000
}
],
"status": {
"code": 2
},
"traceState": "",
"resource": {
"attributes": [
{
"key": "service.name",
"value": {
"stringValue": "service-a"
}
},
{
"key": "service.version",
"value": {
"stringValue": "1.0.0"
}
}
]
},
"instrumentationLibrary": {
"name": "service-a-tracer",
"version": ""
}
}
Service B Output:
{
"traceId": "d8a475b78b88762a3658f89300a05f8a",
"spanId": "04201152f3d49e5b",
"parentSpanId": "02a6189b575f8b1c",
"name": "incoming-request-service-b",
"kind": 1,
"startTimeUnixNano": 1678886400095000000,
"endTimeUnixNano": 1678886400105000000,
"attributes": [],
"events": [
{
"name": "Processing request in Service B...",
"timeUnixNano": 1678886400100000000
}
],
"status": {
"code": 2
},
"traceState": "",
"resource": {
"attributes": [
{
"key": "service.name",
"value": {
"stringValue": "service-b"
}
},
{
"key": "service.version",
"value": {
"stringValue": "1.0.0"
}
}
]
},
"instrumentationLibrary": {
"name": "service-b-tracer",
"version": ""
}
}
Notice how traceId is identical across all spans. Crucially, parentSpanId of the incoming-request-service-b span in Service B matches the spanId of the call-service-b span in Service A. This establishes the parent-child relationship, creating a unified trace.
The key to this is the otel.GetTextMapPropagator() call. This retrieves the configured propagator (in our case, TraceContext and Baggage) and uses its Inject and Extract methods.
Inject takes a context and a carrier (like an http.Header) and writes the trace context information into the carrier as HTTP headers. By default, TraceContext uses traceparent and tracestate headers.
Extract does the reverse: it takes a context and a carrier and reads the trace context information from the headers, returning a new context that includes this information. This new context is then used to start the next span, ensuring it’s correctly linked to the parent.
The propagation.HeaderCarrier(r.Header) is a simple adapter that lets OpenTelemetry treat http.Header like a map of string keys and string values, which is exactly what it needs to read and write HTTP headers.
The propagation.TraceContext{} and propagation.Baggage{} are standard formats. TraceContext is the W3C standard for carrying trace information (trace ID, span ID, sampling decision). Baggage is for carrying arbitrary key-value pairs that should propagate across service calls, often used for user IDs or request-specific metadata. The CompositeTextMapPropagator allows you to combine multiple propagators, so you can support both standards or custom ones simultaneously.
When you inject, the traceparent header might look like this: 00-d8a475b78b88762a3658f89300a05f8a-02a6189b575f8b1c-01. The parts are: version, trace ID, parent span ID, and flags (e.g., sampled). When Service B receives this, its Extract function reads this header and populates its context, allowing its Start call to correctly set the parentSpanId.
The mental model is that trace context is a piece of data that needs to be passed along with requests. HTTP headers are the natural place to do this for HTTP-based communication. OpenTelemetry provides a standardized way to serialize and deserialize this context into and out of those headers.
What most people don’t realize is that the kind field in the span JSON (1 for Server, 2 for Client) is crucial. When Service A calls Service B, it creates a client span (kind: 2). This client span’s ID becomes the parent span ID for the server span (kind: 1) that Service B starts upon receiving the request. This is how the link is formed. If you only ever have server spans, you’re not seeing the full picture of request flow.
The next concept to explore is sampling.