diff --git a/metrics/debug_endpoints_test.go b/metrics/debug_endpoints_test.go new file mode 100644 index 00000000..b4fd4c27 --- /dev/null +++ b/metrics/debug_endpoints_test.go @@ -0,0 +1,171 @@ +package metrics + +import ( + "encoding/json" + "expvar" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "testing" + + "github.com/google/uuid" + "github.com/rs/zerolog" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + _ "golang.org/x/net/trace" // ensure /debug/requests and /debug/events are registered on http.DefaultServeMux + + "github.com/cloudflare/cloudflared/diagnostic" +) + +// cmdlineMarker identifies os.Args in a response body. The base name of the test +// binary is used rather than the full path because the path may contain +// characters that are escaped in JSON responses (e.g. backslashes on Windows). +var cmdlineMarker = filepath.Base(os.Args[0]) + +func testHandler(t *testing.T) *http.ServeMux { + t.Helper() + + log := zerolog.Nop() + return newMetricsHandler(Config{ + DiagnosticHandler: diagnostic.NewDiagnosticHandler( + &log, 0, nil, uuid.Nil, uuid.Nil, nil, map[string]string{}, nil, + ), + }, &log) +} + +// TestDefaultServeMuxLeaksCmdline documents why http.DefaultServeMux must never be +// mounted on the metrics server: handlers registered on it by imported packages +// expose os.Args, which may contain secrets such as tunnel tokens. +func TestDefaultServeMuxLeaksCmdline(t *testing.T) { + t.Parallel() + + for _, path := range []string{"/debug/pprof/cmdline", "/debug/vars"} { + req := httptest.NewRequest(http.MethodGet, path, nil) + w := httptest.NewRecorder() + http.DefaultServeMux.ServeHTTP(w, req) + + require.Equal(t, http.StatusOK, w.Code, "path %s", path) + require.Contains(t, w.Body.String(), cmdlineMarker, "path %s", path) + } +} + +func TestAllowlistedDebugEndpointsAreRegistered(t *testing.T) { + t.Parallel() + + handler := testHandler(t) + for path, pattern := range map[string]string{ + "/debug/pprof/": "/debug/pprof/", + "/debug/pprof/heap": "/debug/pprof/", + "/debug/pprof/goroutine": "/debug/pprof/", + "/debug/pprof/profile": "/debug/pprof/profile", + "/debug/pprof/symbol": "/debug/pprof/symbol", + "/debug/pprof/trace": "/debug/pprof/trace", + "/debug/vars": "GET /debug/vars", + } { + req := httptest.NewRequest(http.MethodGet, path, nil) + _, got := handler.Handler(req) + assert.Equal(t, pattern, got, "path %s", path) + } +} + +func TestAllowlistedDebugEndpointsServe(t *testing.T) { + t.Parallel() + + handler := testHandler(t) + for _, path := range []string{ + "/debug/pprof/", + "/debug/pprof/heap", + "/debug/pprof/goroutine", + "/debug/pprof/allocs", + "/debug/pprof/symbol", + } { + req := httptest.NewRequest(http.MethodGet, path, nil) + w := httptest.NewRecorder() + handler.ServeHTTP(w, req) + + assert.Equal(t, http.StatusOK, w.Code, "path %s", path) + } +} + +func TestNonAllowlistedDebugEndpointsAreNotServed(t *testing.T) { + t.Parallel() + + handler := testHandler(t) + for _, path := range []string{ + "/debug/", + "/debug/pprof/cmdline", + "/debug/requests", + "/debug/events", + // Path variants that http.ServeMux normalizes. + "/debug/vars/", + "//debug/vars", + "/debug/./vars", + "/debug/pprof/../vars", + "/debug/pprof/cmdline/", + "/debug//pprof/cmdline", + } { + for _, method := range []string{http.MethodGet, http.MethodHead, http.MethodPost} { + req := httptest.NewRequest(method, path, nil) + w := httptest.NewRecorder() + handler.ServeHTTP(w, req) + + assert.NotEqual(t, http.StatusOK, w.Code, "%s %s", method, path) + assert.NotContains(t, w.Body.String(), cmdlineMarker, "%s %s", method, path) + } + } +} + +func TestExpvarEndpointServesOnlyAllowlistedVars(t *testing.T) { + t.Parallel() + + // Sanity check: the default expvar set includes the sensitive "cmdline" var. + require.NotNil(t, expvar.Get("cmdline")) + + handler := testHandler(t) + req := httptest.NewRequest(http.MethodGet, "/debug/vars", nil) + w := httptest.NewRecorder() + handler.ServeHTTP(w, req) + + require.Equal(t, http.StatusOK, w.Code) + assert.Equal(t, "application/json; charset=utf-8", w.Header().Get("Content-Type")) + assert.NotContains(t, w.Body.String(), cmdlineMarker) + + var vars map[string]json.RawMessage + require.NoError(t, json.Unmarshal(w.Body.Bytes(), &vars)) + keys := make([]string, 0, len(vars)) + for key := range vars { + keys = append(keys, key) + } + assert.ElementsMatch(t, allowedExpvars, keys) + assert.NotContains(t, vars, "cmdline") + + var memstats map[string]any + require.NoError(t, json.Unmarshal(vars["memstats"], &memstats)) + assert.Contains(t, memstats, "HeapAlloc") +} + +func TestExpvarEndpointRejectsNonGetMethods(t *testing.T) { + t.Parallel() + + handler := testHandler(t) + for _, method := range []string{http.MethodPost, http.MethodPut, http.MethodDelete} { + req := httptest.NewRequest(method, "/debug/vars", nil) + w := httptest.NewRecorder() + handler.ServeHTTP(w, req) + + assert.Equal(t, http.StatusMethodNotAllowed, w.Code, "method %s", method) + assert.NotContains(t, w.Body.String(), cmdlineMarker, "method %s", method) + } +} + +func TestPprofCmdlineEndpointIsNotFound(t *testing.T) { + t.Parallel() + + handler := testHandler(t) + req := httptest.NewRequest(http.MethodGet, "/debug/pprof/cmdline", nil) + w := httptest.NewRecorder() + handler.ServeHTTP(w, req) + + assert.Equal(t, http.StatusNotFound, w.Code) +} diff --git a/metrics/metrics.go b/metrics/metrics.go index abc62504..3dd54109 100644 --- a/metrics/metrics.go +++ b/metrics/metrics.go @@ -2,10 +2,12 @@ package metrics import ( "context" + "encoding/json" + "expvar" "fmt" "net" "net/http" - _ "net/http/pprof" //nolint:gosec // G108: the sensitive /debug/pprof/cmdline endpoint is explicitly blocked in newMetricsHandler + "net/http/pprof" //nolint:gosec // G108: http.DefaultServeMux is not served; only allowlisted handlers are mounted in installDebugEndpoints "runtime" "sync" "time" @@ -14,7 +16,6 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/rs/zerolog" - "golang.org/x/net/trace" "github.com/cloudflare/cloudflared/diagnostic" ) @@ -70,12 +71,7 @@ func newMetricsHandler( log *zerolog.Logger, ) *http.ServeMux { router := http.NewServeMux() - // Block /debug/pprof/cmdline to prevent leaking secret command-line arguments - // (e.g. tunnel tokens) that are exposed via os.Args. - router.HandleFunc("/debug/pprof/cmdline", func(w http.ResponseWriter, r *http.Request) { - http.Error(w, "forbidden", http.StatusForbidden) - }) - router.Handle("/debug/", http.DefaultServeMux) + installDebugEndpoints(router) router.Handle("/metrics", promhttp.Handler()) router.HandleFunc("/healthcheck", func(w http.ResponseWriter, r *http.Request) { _, _ = fmt.Fprintf(w, "OK\n") @@ -104,6 +100,47 @@ func newMetricsHandler( return router } +// allowedExpvars lists the expvar variables served on /debug/vars. The default +// "cmdline" variable is intentionally excluded because it publishes os.Args. +var allowedExpvars = []string{"memstats"} + +// installDebugEndpoints registers an explicit allowlist of debug endpoints. +// +// http.DefaultServeMux must never be mounted on the metrics server: any imported +// package may register handlers on it, some of which expose os.Args and thus +// secrets such as tunnel tokens (e.g. /debug/pprof/cmdline from net/http/pprof +// and the "cmdline" variable on /debug/vars from expvar). +func installDebugEndpoints(router *http.ServeMux) { + // pprof.Index serves the index page and the runtime profiles + // (heap, goroutine, goroutineleak, allocs, block, mutex, threadcreate). + router.HandleFunc("/debug/pprof/", pprof.Index) + router.HandleFunc("/debug/pprof/profile", pprof.Profile) + router.HandleFunc("/debug/pprof/symbol", pprof.Symbol) + router.HandleFunc("/debug/pprof/trace", pprof.Trace) + // Unlike the pprof handlers above, which must accept any method (`go tool pprof` + // POSTs to /debug/pprof/symbol), this mirrors the standard expvar registration + // and only accepts GET (and HEAD). + router.HandleFunc("GET /debug/vars", expvarHandler) +} + +// expvarHandler serves only the allowlisted expvar variables, in the same JSON +// object format as the standard expvar handler. +func expvarHandler(w http.ResponseWriter, _ *http.Request) { + vars := make(map[string]json.RawMessage, len(allowedExpvars)) + for _, name := range allowedExpvars { + if v := expvar.Get(name); v != nil { + vars[name] = json.RawMessage(v.String()) + } + } + body, err := json.MarshalIndent(vars, "", " ") + if err != nil { + http.Error(w, fmt.Sprintf("failed to encode expvars: %v", err), http.StatusInternalServerError) + return + } + w.Header().Set("Content-Type", "application/json; charset=utf-8") + _, _ = w.Write(append(body, '\n')) +} + // CreateMetricsListener will create a new [net.Listener] by using an // known set of ports when the default address is passed with the fallback // of choosing a random port when none is available. @@ -147,8 +184,6 @@ func ServeMetrics( log *zerolog.Logger, ) (err error) { var wg sync.WaitGroup - // Metrics port is privileged, so no need for further access control - trace.AuthRequest = func(*http.Request) (bool, bool) { return true, true } h := newMetricsHandler(config, log) server := &http.Server{ ReadTimeout: 10 * time.Second, diff --git a/metrics/pprof_cmdline_test.go b/metrics/pprof_cmdline_test.go deleted file mode 100644 index 69840303..00000000 --- a/metrics/pprof_cmdline_test.go +++ /dev/null @@ -1,48 +0,0 @@ -package metrics - -import ( - "net/http" - "net/http/httptest" - "testing" - - "github.com/google/uuid" - "github.com/rs/zerolog" - "github.com/stretchr/testify/assert" - - "github.com/cloudflare/cloudflared/diagnostic" -) - -func testHandler(t *testing.T) *http.ServeMux { - t.Helper() - - log := zerolog.Nop() - return newMetricsHandler(Config{ - DiagnosticHandler: diagnostic.NewDiagnosticHandler( - &log, 0, nil, uuid.Nil, uuid.Nil, nil, map[string]string{}, nil, - ), - }, &log) -} - -func TestPprofCmdlineEndpointIsBlocked(t *testing.T) { - t.Parallel() - - handler := testHandler(t) - req := httptest.NewRequest(http.MethodGet, "/debug/pprof/cmdline", nil) - w := httptest.NewRecorder() - handler.ServeHTTP(w, req) - - assert.Equal(t, http.StatusForbidden, w.Code) -} - -func TestOtherPprofEndpointsStillWork(t *testing.T) { - t.Parallel() - - handler := testHandler(t) - - // /debug/pprof/ index should still be served by DefaultServeMux - req := httptest.NewRequest(http.MethodGet, "/debug/pprof/", nil) - w := httptest.NewRecorder() - handler.ServeHTTP(w, req) - - assert.Equal(t, http.StatusOK, w.Code) -}