diff --git a/cmd/cloudflared/tunnel/cmd.go b/cmd/cloudflared/tunnel/cmd.go index 30a4862d..f46d8769 100644 --- a/cmd/cloudflared/tunnel/cmd.go +++ b/cmd/cloudflared/tunnel/cmd.go @@ -420,12 +420,16 @@ func StartServer( return err } connectorID := tunnelConfig.ClientConfig.ConnectorID + precheckReports := diagnostic.NewPrecheckReportStore() // Run connectivity pre-checks for cloudflared. This runs in a separate // goroutine, as we want to keep initializing cloudflared while prechecks // are running. Prechecks are controlled via DNS flag for remote kill-switch capability. if !tunnelConfig.ClientConfig.ConnectionFeaturesSnapshot().SkipPrechecks && !c.Bool(cfdflags.NoPrechecks) { - go runPrechecks(c, log, tunnelConfig.Region, tunnelConfig.ClientConfig.ConnectionFeaturesSnapshot().PostQuantum) + go func() { + report := runPrechecks(c, log, tunnelConfig.Region, tunnelConfig.ClientConfig.ConnectionFeaturesSnapshot().PostQuantum) + precheckReports.Set(report) + }() } // Disable ICMP packet routing for quick tunnels @@ -502,6 +506,7 @@ func StartServer( tracker, cliFlags, sources, + diagnostic.WithPrecheckReportStore(precheckReports), ) metricsConfig := metrics.Config{ ReadyServer: readinessServer, @@ -530,7 +535,7 @@ func StartServer( // runPrechecks executes connectivity pre-checks and logs the results. // Pre-checks are diagnostic only and do not gate tunnel startup. -func runPrechecks(c *cli.Context, log *zerolog.Logger, region string, pqMode features.PostQuantumMode) { +func runPrechecks(c *cli.Context, log *zerolog.Logger, region string, pqMode features.PostQuantumMode) prechecks.Report { ipVersion := allregions.Auto if ipVersionStr := c.String(cfdflags.EdgeIpVersion); ipVersionStr != "" { parsedVersion, err := parseConfigIPVersion(ipVersionStr) @@ -562,6 +567,7 @@ func runPrechecks(c *cli.Context, log *zerolog.Logger, region string, pqMode fea // Also log structured results for log aggregation report.LogEvent(log) + return report } func waitToShutdown(wg *sync.WaitGroup, diff --git a/component-tests/test_prechecks.py b/component-tests/test_prechecks.py index c74036b3..e8937efa 100644 --- a/component-tests/test_prechecks.py +++ b/component-tests/test_prechecks.py @@ -7,7 +7,7 @@ Scope These tests verify the end-to-end behavior of cloudflared pre-checks: - that the human-readable table written to the log output has the correct structure and content, -- that structured JSON log lines are emitted with the expected fields, and +- that the diagnostic endpoint exposes the completed report, and - that running the `diag` subcommand against a live tunnel instance produces a zip archive that contains prechecks.json. @@ -30,11 +30,10 @@ stdout/stderr design The pre-checks table is emitted via cliutil.LogTable, which wraps the content in an ASCII box and logs each line at Info level through zerolog. zerolog writes to stderr, which the test harness merges into stdout (stderr=STDOUT in -Popen). We poll a --logfile for the "precheck complete" sentinel before -leaving the `with` block, ensuring the goroutine has finished. We then call -cfd.terminate(). After the `with` block exits, the process is dead and all -output has been captured by CloudflaredProcess's background reader thread. We -read the accumulated lines from cfd.stdout_lines. +Popen). We poll /diag/precheck before leaving the `with` block, ensuring the +goroutine has finished. We then call cfd.terminate(). After the `with` block +exits, the process is dead and all output has been captured by +CloudflaredProcess's background reader thread. Box format (cliutil.asciiBox with padding=2, title="CONNECTIVITY PRE-CHECKS"): +----...----+ @@ -52,6 +51,8 @@ import subprocess import time import zipfile as zipfilemod +import requests + from constants import METRICS_PORT from util import LOGGER, start_cloudflared, wait_tunnel_ready @@ -106,10 +107,6 @@ ACTION_HTTP2_BLOCKED = "Allow outbound TCP on port 7844." SUMMARY_HEALTHY = "SUMMARY: Environment is healthy. cloudflared will use 'quic' as primary protocol." SUMMARY_CRITICAL = "SUMMARY: Environment has critical failures. cloudflared may not be able to establish a tunnel." -# structured log constants (result.go) - -LOG_MSG_PRECHECK = "precheck" -LOG_MSG_PRECHECK_COMPLETE = "precheck complete" STATUS_PASS_LOG = "pass" UNREACHABLE_EDGE = "192.0.2.1:7844" @@ -120,46 +117,21 @@ PRECHECK_POLL_INTERVAL_SECS = 1 # ---------- helpers ---------- -def _poll_log_file_for_precheck_complete(log_file: str, timeout: float) -> list[dict]: - """ - Poll a JSON log file until a 'precheck complete' line appears or timeout - expires. Returns all precheck-related log lines found. - - cloudflared's --logfile writes one JSON object per line. Polling keeps - the test fast on healthy networks and still tolerates slow CI hosts. - - We re-read from the beginning of the file on every poll because the file - is append-only, small, and tracking a byte offset would add complexity with - no meaningful performance benefit for a ~15 s total window. - """ +def _poll_precheck_endpoint(timeout: float) -> dict: + """Poll /diag/precheck until a completed report is available.""" + url = f"http://localhost:{METRICS_PORT}/diag/precheck" deadline = time.monotonic() + timeout while time.monotonic() < deadline: - lines = _read_precheck_log_lines_from_file(log_file) - if any(l.get("message") == LOG_MSG_PRECHECK_COMPLETE for l in lines): - return lines + try: + response = requests.get(url, timeout=1) + if response.status_code == 200: + return response.json() + assert response.status_code == 204, \ + f"Expected {url} to return 200 or 204, got {response.status_code}" + except requests.exceptions.RequestException: + pass time.sleep(PRECHECK_POLL_INTERVAL_SECS) - return _read_precheck_log_lines_from_file(log_file) - - -def _read_precheck_log_lines_from_file(log_file: str) -> list[dict]: - """Parse all precheck-related JSON log lines from a --logfile path.""" - result = [] - try: - with open(log_file, "r") as f: - for raw_line in f: - raw_line = raw_line.strip() - if not raw_line: - continue - try: - obj = json.loads(raw_line) - except json.JSONDecodeError: - continue - msg = obj.get("message") or obj.get("msg", "") - if msg in (LOG_MSG_PRECHECK, LOG_MSG_PRECHECK_COMPLETE): - result.append(obj) - except FileNotFoundError: - pass - return result + raise AssertionError(f"Timed out waiting for a completed precheck report from {url}") # stdout table parse @@ -235,25 +207,21 @@ def _rows_for(rows: list[TableRow], component: str) -> list[TableRow]: # log assertions -def _assert_precheck_summary_log( - log_lines: list[dict], +def _assert_precheck_report( + report: dict, *, hard_fail: bool, suggested_protocol: str | None = None, ): - """Assert the 'precheck complete' summary log line has the expected fields.""" - summary_lines = [l for l in log_lines if l.get("message") == LOG_MSG_PRECHECK_COMPLETE] - assert len(summary_lines) == 1, \ - f"Expected exactly one '{LOG_MSG_PRECHECK_COMPLETE}' log line; got {summary_lines}" - summary = summary_lines[0] - - assert summary.get("hard_fail") is hard_fail, \ - f"Expected hard_fail={hard_fail} in summary log: {summary}" + """Assert the diagnostic report has the expected summary fields.""" + assert report.get("run_id"), f"Expected a run_id in precheck report: {report}" + assert report.get("hard_fail") is hard_fail, \ + f"Expected hard_fail={hard_fail} in precheck report: {report}" if suggested_protocol is not None: - assert summary.get("suggested_protocol") == suggested_protocol, \ + assert report.get("suggested_protocol") == suggested_protocol, \ (f"Expected suggested_protocol={suggested_protocol!r}; " - f"got {summary.get('suggested_protocol')!r}") + f"got {report.get('suggested_protocol')!r}") # ---------- Tests ---------- @@ -265,12 +233,11 @@ class TestPrechecksHappyPath: - every row's component, target, status, and details - no ERROR/WARNING action lines - the exact summary line - - the structured log summary (hard_fail=false, suggested_protocol=quic) + - the diagnostic report summary (hard_fail=false, suggested_protocol=quic) """ def test_prechecks_pass_on_healthy_connection(self, tmp_path, component_tests_config): - log_file = str(tmp_path / "cloudflared.log") - config = component_tests_config({"logfile": log_file}) + config = component_tests_config({}) with start_cloudflared( tmp_path, @@ -281,10 +248,7 @@ class TestPrechecksHappyPath: capture_output=True, ) as cfd: wait_tunnel_ready(tunnel_url=config.get_url(), require_min_connections=1) - # Poll the log file for the sentinel before signalling the process. - log_lines = _poll_log_file_for_precheck_complete( - log_file, timeout=PRECHECK_POLL_TIMEOUT_SECS - ) + report = _poll_precheck_endpoint(timeout=PRECHECK_POLL_TIMEOUT_SECS) # Signal shutdown. cfd.terminate() @@ -293,7 +257,7 @@ class TestPrechecksHappyPath: stdout = b"".join(cfd.stdout_lines).decode(errors="replace") LOGGER.debug(f"[happy-path] stdout:\n{stdout}") - LOGGER.debug(f"[happy-path] log_lines:\n{log_lines}") + LOGGER.debug(f"[happy-path] precheck report:\n{report}") # Strip zerolog console prefixes so pattern matching works on raw messages. messages = "\n".join(_strip_log_prefix(l) for l in stdout.splitlines()) @@ -351,14 +315,12 @@ class TestPrechecksHappyPath: assert SUMMARY_HEALTHY in messages, \ f"Expected healthy summary;\ngot:\n{stdout}" - # ── structured log ─────────────────────────────────────────────────── - assert len(log_lines) > 0, \ - "Expected at least one structured precheck log line in log file" - for line in log_lines: - if line.get("message") == LOG_MSG_PRECHECK: - assert line.get("status") == STATUS_PASS_LOG, \ - f"Expected status=pass in precheck log line: {line}" - _assert_precheck_summary_log(log_lines, hard_fail=False, suggested_protocol="quic") + # ── diagnostic report ──────────────────────────────────────────────── + assert len(report["results"]) == 7 + for result in report["results"]: + assert result["status"] == STATUS_PASS_LOG, \ + f"Expected status=pass in precheck report: {result}" + _assert_precheck_report(report, hard_fail=False, suggested_protocol="quic") class TestPrechecksHardFail: @@ -372,15 +334,14 @@ class TestPrechecksHardFail: - HTTP/2 row: FAIL with correct details + ERROR action - API row: PASS (api.cloudflare.com:443 is independently reachable) - the exact critical summary line - - the structured log summary (hard_fail=true) + - the diagnostic report summary (hard_fail=true) This test does NOT call wait_tunnel_ready because the tunnel will not connect to the unreachable address. """ def test_prechecks_hard_fail_when_edge_unreachable(self, tmp_path, component_tests_config): - log_file = str(tmp_path / "cloudflared.log") - config = component_tests_config({"logfile": log_file}) + config = component_tests_config({}) with start_cloudflared( tmp_path, @@ -394,15 +355,13 @@ class TestPrechecksHardFail: new_process=True, capture_output=True, ) as cfd: - log_lines = _poll_log_file_for_precheck_complete( - log_file, timeout=PRECHECK_POLL_TIMEOUT_SECS - ) + report = _poll_precheck_endpoint(timeout=PRECHECK_POLL_TIMEOUT_SECS) cfd.terminate() stdout = b"".join(cfd.stdout_lines).decode(errors="replace") LOGGER.debug(f"[hard-fail] stdout:\n{stdout}") - LOGGER.debug(f"[hard-fail] log_lines:\n{log_lines}") + LOGGER.debug(f"[hard-fail] precheck report:\n{report}") # Strip zerolog console prefixes so pattern matching works on raw messages. messages = "\n".join(_strip_log_prefix(l) for l in stdout.splitlines()) @@ -454,7 +413,7 @@ class TestPrechecksHardFail: assert SUMMARY_CRITICAL in messages, \ f"Expected critical summary;\ngot:\n{stdout}" - _assert_precheck_summary_log(log_lines, hard_fail=True, suggested_protocol=None) + _assert_precheck_report(report, hard_fail=True, suggested_protocol=None) class TestPreChecksDiag: diff --git a/diagnostic/consts.go b/diagnostic/consts.go index 57db38c8..637e66d4 100644 --- a/diagnostic/consts.go +++ b/diagnostic/consts.go @@ -18,6 +18,7 @@ const ( cliConfigurationEndpoint = "/diag/configuration" tunnelStateEndpoint = "/diag/tunnel" systemInformationEndpoint = "/diag/system" + precheckEndpoint = "/diag/precheck" memoryDumpEndpoint = "debug/pprof/heap" goroutineDumpEndpoint = "debug/pprof/goroutine" metricsEndpoint = "metrics" diff --git a/diagnostic/handlers.go b/diagnostic/handlers.go index e4d85db4..6ec4d348 100644 --- a/diagnostic/handlers.go +++ b/diagnostic/handlers.go @@ -6,11 +6,14 @@ import ( "net/http" "os" "strconv" + "strings" + "sync" "time" "github.com/google/uuid" "github.com/rs/zerolog" + "github.com/cloudflare/cloudflared/prechecks" "github.com/cloudflare/cloudflared/tunnelstate" ) @@ -23,6 +26,51 @@ type Handler struct { tracker *tunnelstate.ConnTracker cliFlags map[string]string icmpSources []string + precheckReports *PrecheckReportStore +} + +// HandlerOption configures a diagnostic Handler. +type HandlerOption func(*Handler) + +// PrecheckReportStore safely shares a completed precheck report between the +// startup precheck and diagnostic HTTP goroutines. +type PrecheckReportStore struct { + mu sync.RWMutex + report *prechecks.Report +} + +// NewPrecheckReportStore creates an empty precheck report store. +func NewPrecheckReportStore() *PrecheckReportStore { + return &PrecheckReportStore{} +} + +// WithPrecheckReportStore configures the report store used by /diag/precheck. +func WithPrecheckReportStore(store *PrecheckReportStore) HandlerOption { + return func(handler *Handler) { + if store != nil { + handler.precheckReports = store + } + } +} + +// Set publishes a completed startup precheck report. +func (store *PrecheckReportStore) Set(report prechecks.Report) { + report.Results = append([]prechecks.CheckResult(nil), report.Results...) + store.mu.Lock() + store.report = &report + store.mu.Unlock() +} + +func (store *PrecheckReportStore) get() (prechecks.Report, bool) { + store.mu.RLock() + defer store.mu.RUnlock() + if store.report == nil { + return prechecks.Report{}, false + } + + report := *store.report + report.Results = append([]prechecks.CheckResult(nil), report.Results...) + return report, true } func NewDiagnosticHandler( @@ -34,6 +82,7 @@ func NewDiagnosticHandler( tracker *tunnelstate.ConnTracker, cliFlags map[string]string, icmpSources []string, + options ...HandlerOption, ) *Handler { logger := log.With().Logger() if timeout == 0 { @@ -41,7 +90,7 @@ func NewDiagnosticHandler( } cliFlags[configurationKeyUID] = strconv.Itoa(os.Getuid()) - return &Handler{ + handler := &Handler{ log: &logger, timeout: timeout, systemCollector: systemCollector, @@ -50,13 +99,65 @@ func NewDiagnosticHandler( tracker: tracker, cliFlags: cliFlags, icmpSources: icmpSources, + precheckReports: NewPrecheckReportStore(), } + for _, option := range options { + option(handler) + } + return handler } func (handler *Handler) InstallEndpoints(router *http.ServeMux) { router.HandleFunc(cliConfigurationEndpoint, handler.ConfigurationHandler) router.HandleFunc(tunnelStateEndpoint, handler.TunnelStateHandler) router.HandleFunc(systemInformationEndpoint, handler.SystemHandler) + router.HandleFunc(precheckEndpoint, handler.PrecheckHandler) +} + +type PrecheckResult struct { + Component string `json:"component"` + Target string `json:"target"` + Status string `json:"status"` + Details string `json:"details"` + Action string `json:"action,omitempty"` +} + +type PrecheckResponse struct { + RunID uuid.UUID `json:"run_id"` + Results []PrecheckResult `json:"results"` + HardFail bool `json:"hard_fail"` + SuggestedProtocol string `json:"suggested_protocol,omitempty"` +} + +func (handler *Handler) PrecheckHandler(writer http.ResponseWriter, _ *http.Request) { + report, ok := handler.precheckReports.get() + if !ok { + writer.WriteHeader(http.StatusNoContent) + return + } + + response := PrecheckResponse{ + RunID: report.RunID, + Results: make([]PrecheckResult, 0, len(report.Results)), + HardFail: report.HasHardFail(), + } + if report.SuggestedProtocol != nil { + response.SuggestedProtocol = report.SuggestedProtocol.String() + } + for _, result := range report.Results { + response.Results = append(response.Results, PrecheckResult{ + Component: result.Component, + Target: result.Target, + Status: strings.ToLower(result.ProbeStatus.String()), + Details: result.Details, + Action: result.Action, + }) + } + + writer.Header().Set("Content-Type", "application/json") + if err := json.NewEncoder(writer).Encode(response); err != nil { + handler.log.Error().Err(err).Msg("error occurred whilst serializing precheck report") + } } type SystemInformationResponse struct { @@ -133,12 +234,3 @@ func (handler *Handler) ConfigurationHandler(writer http.ResponseWriter, _ *http writer.WriteHeader(http.StatusInternalServerError) } } - -func writeResponse(w http.ResponseWriter, bytes []byte, logger *zerolog.Logger) { - bytesWritten, err := w.Write(bytes) - if err != nil { - logger.Error().Err(err).Msg("error occurred writing response") - } else if bytesWritten != len(bytes) { - logger.Error().Msgf("error incomplete write response %d/%d", bytesWritten, len(bytes)) - } -} diff --git a/diagnostic/handlers_test.go b/diagnostic/handlers_test.go index 2849241c..98fa2098 100644 --- a/diagnostic/handlers_test.go +++ b/diagnostic/handlers_test.go @@ -17,6 +17,7 @@ import ( "github.com/cloudflare/cloudflared/connection" "github.com/cloudflare/cloudflared/diagnostic" + "github.com/cloudflare/cloudflared/prechecks" "github.com/cloudflare/cloudflared/tunnelstate" ) @@ -25,11 +26,6 @@ type SystemCollectorMock struct { err error } -const ( - systemInformationKey = "sikey" - errorKey = "errkey" -) - func newTrackerFromConns(t *testing.T, connections []tunnelstate.IndexedConnectionInfo) *tunnelstate.ConnTracker { t.Helper() @@ -222,3 +218,49 @@ func TestConfigurationHandler(t *testing.T) { }) } } + +func TestPrecheckHandler(t *testing.T) { + t.Parallel() + + log := zerolog.Nop() + store := diagnostic.NewPrecheckReportStore() + handler := diagnostic.NewDiagnosticHandler( + &log, 0, nil, uuid.Nil, uuid.Nil, nil, map[string]string{}, nil, + diagnostic.WithPrecheckReportStore(store), + ) + router := http.NewServeMux() + handler.InstallEndpoints(router) + + pending := httptest.NewRecorder() + router.ServeHTTP(pending, httptest.NewRequest(http.MethodGet, "/diag/precheck", nil)) + assert.Equal(t, http.StatusNoContent, pending.Code) + + protocol := connection.QUIC + report := prechecks.Report{ + RunID: uuid.New(), + Results: []prechecks.CheckResult{ + { + Type: prechecks.ProbeTypeDNS, + Component: "DNS Resolution", + Target: "region1.v2.argotunnel.com", + ProbeStatus: prechecks.Pass, + Details: "DNS resolved successfully", + }, + }, + SuggestedProtocol: &protocol, + } + store.Set(report) + + recorder := httptest.NewRecorder() + router.ServeHTTP(recorder, httptest.NewRequest(http.MethodGet, "/diag/precheck", nil)) + require.Equal(t, http.StatusOK, recorder.Code) + assert.Equal(t, "application/json", recorder.Header().Get("Content-Type")) + + var response diagnostic.PrecheckResponse + require.NoError(t, json.NewDecoder(recorder.Body).Decode(&response)) + assert.Equal(t, report.RunID, response.RunID) + assert.False(t, response.HardFail) + assert.Equal(t, "quic", response.SuggestedProtocol) + require.Len(t, response.Results, 1) + assert.Equal(t, "pass", response.Results[0].Status) +} diff --git a/prechecks/result.go b/prechecks/result.go index c0eaea5d..fcb6cbeb 100644 --- a/prechecks/result.go +++ b/prechecks/result.go @@ -131,6 +131,11 @@ func (r Report) hasHardFail() bool { return quicFail && http2Fail } +// HasHardFail reports whether the environment cannot establish a tunnel. +func (r Report) HasHardFail() bool { + return r.hasHardFail() +} + // hasWarn returns true when the environment is degraded but functional: // - Exactly one transport (QUIC or HTTP/2) failed, OR // - The Management API is unreachable (auto-updates unavailable).