Clarify worker failures in operator UI

This commit is contained in:
Emil
2026-07-23 22:59:51 +03:00
parent c7956c4683
commit 6bac7dad3c
3 changed files with 49 additions and 1 deletions
@@ -15,7 +15,7 @@
<section class="summary"><div class="summary-head"><div><span id="status" class="status status-{{statusClass .Status}}">{{statusLabel .Status}}</span><p id="hint" class="hint">{{statusHint .Status}}</p></div><small>Summary refreshes automatically every two seconds.</small></div><div class="bar" aria-label="Progress"><span id="progress-bar" style="width:{{progressPercent .Completed .Failed .Total}}%"></span></div><p id="progress" class="muted">{{.Completed}} of {{.Total}} tasks complete</p><div class="numbers"><div class="number"><b id="total">{{.Total}}</b><small>total shards</small></div><div class="number"><b id="completed">{{.Completed}}</b><small>complete</small></div><div class="number"><b id="pending">{{.Pending}}</b><small>waiting</small></div><div class="number"><b id="active">{{add .Leased .Running}}</b><small>with workers</small></div><div class="number"><b id="failed">{{.Failed}}</b><small>failed</small></div></div><details><summary>Technical details</summary><p>Job ID: <code>{{.ID}}</code><br>Workload: <code>{{.Workload}}</code><br>Created: {{time .CreatedAt}}</p></details></section>
<section class="notice"><strong>What can be downloaded now?</strong><br><code>partial_result</code> files come from individual shards. They are useful for checking the pipeline, but are not a merged final CSV because the reducer is not implemented yet.</section>
<h2>Shard tasks</h2><p class="muted">If a task fails, its code and message appear here. Refresh the page to update the detailed rows.</p>
<div class="table-wrap"><table><tr><th>Shard</th><th>State</th><th>Attempt</th><th>Worker / lease</th><th>Error</th></tr>{{range .Tasks}}<tr><td>#{{.ChunkIndex}}</td><td><span class="status status-{{statusClass .Status}}">{{statusLabel .Status}}</span></td><td>{{.Attempt}} / {{.MaxAttempts}}</td><td>{{if .LeaseOwner}}<code>{{.LeaseOwner}}</code>{{if .LeaseExpiresAt}}<br><small>until {{time .LeaseExpiresAt}}</small>{{end}}{{else}}<span class="muted"></span>{{end}}</td><td class="error">{{if .ErrorCode}}<code>{{.ErrorCode}}</code><br>{{.ErrorMessage}}{{else}}<span class="muted"></span>{{end}}</td></tr>{{else}}<tr><td colspan="5" class="empty">No tasks have appeared yet.</td></tr>{{end}}</table></div>
<div class="table-wrap"><table><tr><th>Shard</th><th>State</th><th>Attempt</th><th>Worker / lease</th><th>Error</th></tr>{{range .Tasks}}<tr><td>#{{.ChunkIndex}}</td><td><span class="status status-{{statusClass .Status}}">{{statusLabel .Status}}</span></td><td>{{.Attempt}} / {{.MaxAttempts}}</td><td>{{if .LeaseOwner}}<code>{{.LeaseOwner}}</code>{{if .LeaseExpiresAt}}<br><small>until {{time .LeaseExpiresAt}}</small>{{end}}{{else}}<span class="muted"></span>{{end}}</td><td class="error">{{if .ErrorCode}}<strong>{{taskErrorLabel .ErrorCode}}</strong><br><small>{{taskErrorHint .ErrorCode}}</small>{{else}}<span class="muted"></span>{{end}}</td></tr>{{else}}<tr><td colspan="5" class="empty">No tasks have appeared yet.</td></tr>{{end}}</table></div>
<h2>Coordinator artifacts</h2>
<div class="table-wrap"><table><tr><th>Type</th><th>File</th><th>Size</th><th>Integrity check</th><th></th></tr>{{range .Artifacts}}<tr><td>{{if .Diagnostic}}<strong>Partial result</strong><br><small>diagnostic</small>{{else}}{{.Kind}}{{end}}</td><td>{{.Filename}}</td><td>{{bytes .SizeBytes}}</td><td><code>{{.SHA256}}</code></td><td>{{if .Downloadable}}<a class="download" href="/ui/jobs/{{$.ID}}/artifacts/{{.ID}}">Download CSV</a>{{else}}<span class="muted">Unavailable</span>{{end}}</td></tr>{{else}}<tr><td colspan="5" class="empty">No artifacts yet. The worker uploads a CSV after it completes a shard.</td></tr>{{end}}</table></div>
</main>
+39
View File
@@ -23,6 +23,8 @@ var uiTemplates = template.Must(template.New("ui").Funcs(template.FuncMap{
"statusLabel": uiStatusLabel,
"statusHint": uiStatusHint,
"statusClass": uiStatusClass,
"taskErrorLabel": uiTaskErrorLabel,
"taskErrorHint": uiTaskErrorHint,
"workerStatusLabel": uiWorkerStatusLabel,
"workloadLabel": uiWorkloadLabel,
"progressPercent": uiProgressPercent,
@@ -95,6 +97,43 @@ func uiWorkerStatusLabel(status string) string {
}
}
// uiTaskErrorLabel deliberately maps worker implementation errors to an
// operator-facing diagnosis. Raw subprocess commands and local paths belong in
// the worker terminal, not in the web UI.
func uiTaskErrorLabel(errorCode string) string {
switch errorCode {
case "CalledProcessError":
return "Local calculation failed"
case "ValueError":
return "Task input could not be processed"
case "CoordinatorTransientError":
return "Coordinator connection was interrupted"
case "CoordinatorConflictError":
return "Worker lease was no longer valid"
case "FileNotFoundError":
return "Local task file is missing"
default:
return errorCode
}
}
func uiTaskErrorHint(errorCode string) string {
switch errorCode {
case "CalledProcessError":
return "The local SciMesh command stopped before it could upload a result. Check the worker terminal for the original error."
case "ValueError":
return "The coordinator task or its downloaded input did not meet the worker validation rules."
case "CoordinatorTransientError":
return "The worker will retry after the coordinator connection is available again."
case "CoordinatorConflictError":
return "Another worker or a lease timeout changed this task before completion."
case "FileNotFoundError":
return "The worker could not find one of its local task files. Restart it with an absolute --work-dir."
default:
return "Check the worker terminal for the original error details."
}
}
func uiWorkloadLabel(workload string) string {
switch workload {
case "similarity-search", "similarity_search":
@@ -33,3 +33,12 @@ func TestUIProgressPercent(t *testing.T) {
t.Errorf("empty progress = %d, want 0", got)
}
}
func TestUITaskErrorPresentationDoesNotExposeCommand(t *testing.T) {
if got := uiTaskErrorLabel("CalledProcessError"); got != "Local calculation failed" {
t.Errorf("error label = %q", got)
}
if got := uiTaskErrorHint("CalledProcessError"); got == "" {
t.Error("error hint must explain the failure")
}
}