feat(xtask): Add log overview.

This patch adds the `log overview` task that generates a standalone HTML
report representing the logs as a tree where each node is a target, and
each leaf is a log location with occurrences, spans and fields.

Each node displays the sum of errors and warnings for this node.
It helps to quickly spot the problematic targets, and it guides to
exploration of the node without taking the reader's hand by trying to
draw conclusions. We don't want to guide the reader to a mistake: we
just want to guide the reader to draw its own conclusions.

Each line can be highlighted. When a node is folded/closed, it is also
highlighted if at least one of its child is highlighted.

The occurrences of logs are displayed on a timeline.
This commit is contained in:
Ivan Enderlin
2026-02-05 16:39:51 +01:00
parent f1e41222f8
commit e1e86a78c9
5 changed files with 775 additions and 4 deletions
Generated
+1
View File
@@ -7365,6 +7365,7 @@ dependencies = [
"clap",
"fs_extra",
"indexmap",
"lazy_static",
"regex",
"serde",
"serde_json",
+1
View File
@@ -18,6 +18,7 @@ chrono.workspace = true
clap = { workspace = true, features = ["derive"] }
fs_extra = "1"
indexmap.workspace = true
lazy_static = "1.5.0"
regex = "1"
serde = { workspace = true, features = ["derive"] }
serde_json.workspace = true
+23 -4
View File
@@ -1,7 +1,9 @@
mod overview;
mod sync;
use std::path::PathBuf;
use clap::{Args, Subcommand};
mod sync;
use crate::{Result, sh, workspace};
@@ -14,12 +16,26 @@ pub struct LogArgs {
/// Analysis around logs (formatted by `matrix-sdk-ffi`).
#[derive(Subcommand)]
enum LogCommand {
/// Visualise the sync requests and responses with a table and a duration
/// graph.
Sync {
/// Overview the logs as a tree where each node is a target and each leaf is
/// a log location with occurrences, spans and fields.
Overview {
/// The file containing the logs to analyse.
#[clap(long)]
log_file: PathBuf,
/// The output file that will receive the HTML report.
#[clap(long)]
html_output_file: PathBuf,
},
/// Visualise the sync requests and responses with a table and a duration
/// graph.
Sync {
/// The file containing the logs to analyse.
#[clap(long)]
log_file: PathBuf,
/// The output file that will receive the HTML report.
#[clap(long)]
html_output_file: PathBuf,
},
@@ -31,6 +47,9 @@ impl LogArgs {
let _p = sh.push_dir(workspace::root_path()?);
match self.cmd {
LogCommand::Overview { log_file, html_output_file } => {
overview::run(log_file, html_output_file)?
}
LogCommand::Sync { log_file, html_output_file } => {
sync::run(log_file, html_output_file)?
}
+481
View File
@@ -0,0 +1,481 @@
use std::{
collections::{BTreeMap, HashMap},
fmt, fs,
io::{self, BufRead, Write},
ops::Not,
path,
str::FromStr,
};
use chrono::{DateTime, FixedOffset};
use lazy_static::lazy_static;
use regex::{Regex, RegexBuilder};
use crate::Result;
const OUTPUT_TEMPLATE: &str = include_str!("overview.template.html");
lazy_static! {
static ref LINE_PARSER: Regex = {
RegexBuilder::new(
r#"
# Let's start.
^
# Datetime of the log line.
(?<datetime>\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+Z)
# Log level.
\s+(?<level>\S+)
# Target.
\s(?<target>matrix_[\w_]+(::[\w_]+)*):
# The log message. We don't care about it.
(?<message>.*)
# The source file and line.
\|\scrates/
(?<source_file>[^:]+)
:(?<source_line>\d+)
# The spans.
\s\|\sspans:\s
(?<spans>.+)
"#,
)
.ignore_whitespace(true)
.build()
.expect("Failed to build the `line_parser` regex")
};
static ref MESSAGE_PARSER: Regex = {
RegexBuilder::new(
r#"
# Let's start.
^
# Anything (trimmed)…
\s*(?<message>.*?)
# … until optional fields!
(\s(?<fields>[\w\d_]+=.*))?$
"#,
)
.ignore_whitespace(true)
.build()
.expect("Failed to build the `message_parser` regex")
};
static ref FIELDS_PARSER: Regex = {
RegexBuilder::new(
r#"
# Let's start.
^
# A name.
(?<name>[\w\d_]+)
# Equal
=
# A value, which can be anything: it stops when a new field is found.
(?<value>.*?)
# The new field (and everything else).
(?<next_fields>\s[\w\d_]+=.+)?$
"#,
)
.ignore_whitespace(true)
.build()
.expect("Failed to build the `fields_parser` regex")
};
}
pub(super) fn run(log_path: path::PathBuf, output_path: path::PathBuf) -> Result<()> {
let line_parser = &*LINE_PARSER;
let log_file = fs::File::open(&log_path)?;
let mut output_file = io::BufWriter::new(fs::File::create(&output_path)?);
let reader = io::BufReader::new(log_file);
let mut number_of_analysed_lines = 0;
let mut number_of_matched_lines = 0;
let mut tree = TargetTree::default();
for (line_nth, line) in reader.lines().enumerate().map(|(nth, line)| {
(
nth + 1,
line.unwrap_or_else(|error| {
panic!("Failed to read line #{nth}\n{error}");
}),
)
}) {
number_of_analysed_lines += 1;
if let Some(captures) = line_parser.captures(&line) {
number_of_matched_lines += 1;
let date_time = DateTime::parse_from_rfc3339(
captures.name("datetime").expect("Failed to capture `datetime`").as_str(),
)
.expect("Failed to parse `datetime`");
let full_target = captures.name("target").expect("Failed to capture `target`").as_str();
let level = captures
.name("level")
.expect("Failed to capture `level`")
.as_str()
.parse()
.expect("Failed to parse the `level`");
let message = captures.name("message").expect("Failed to capture `message`").as_str();
let source_file =
captures.name("source_file").expect("Failed to capture `source_file`").as_str();
let source_line = captures
.name("source_line")
.expect("Failed to capture `source_line`")
.as_str()
.parse()
.expect("Failed to parse the `source_line`");
let spans = captures.name("spans").expect("Failed to capture `spans`").as_str();
let mut tree_node = &mut tree;
for target in full_target.split("::") {
tree_node =
match tree_node.entry(target.to_owned()).or_insert_with(|| Target::Node {
number_of_errors: 0,
number_of_warnings: 0,
node: TargetTree::default(),
}) {
Target::Leaf(_) => panic!("Expect a `Node`, not a `Leaf`"),
Target::Node { number_of_errors, number_of_warnings, node } => {
// Adjust number of specific log levels all the targets.
match level {
Level::Error => *number_of_errors += 1,
Level::Warn => *number_of_warnings += 1,
_ => (),
}
node
}
};
}
let span =
Span { message: message.to_owned(), log_line: line_nth, raw: spans.to_owned() };
tree_node
// `LogLevels`
.entry("".to_owned())
.or_insert_with(|| Target::Leaf(LogLevels::default()))
.as_leaf_mut()
.expect("Expect a `Leaf`, not a `Node`")
// `LogLocations`
.entry(level)
.or_insert_with(LogLocations::default)
// `SourceFile`
.entry(source_file.to_owned())
.or_insert_with(Spans::default)
// `Spans`
.entry(source_line)
.or_insert_with(BTreeMap::default)
// `Span`!
.insert(date_time, span);
}
}
fn render_target_tree(tree: TargetTree, buffer: &mut io::BufWriter<fs::File>) {
if tree.is_empty() {
return;
}
writeln!(buffer, "<ul>").unwrap();
for (target_name, target) in tree {
match target {
Target::Node { node, number_of_errors, number_of_warnings } => {
render_node(target_name, node, number_of_errors, number_of_warnings, buffer);
}
Target::Leaf(log_levels) => {
render_leaf(log_levels, buffer);
}
}
}
writeln!(buffer, "</ul>").unwrap();
}
fn render_node(
target_name: String,
node: TargetTree,
number_of_errors: usize,
number_of_warnings: usize,
buffer: &mut io::BufWriter<fs::File>,
) {
writeln!(
buffer,
"<li>\n\
<details><summary>{target_name}\
<span class=\"aside\">\
<span class=\"highlight\"><input type=\"checkbox\" title=\"Highlight\" /></span>\
<span class=\"error\" title=\"Number of errors\">{number_of_errors}</span>\
<span class=\"warn\" title=\"Number of warnings\">{number_of_warnings}</span>\
</span>\
</summary>"
)
.unwrap();
render_target_tree(node, buffer);
writeln!(buffer, "</details></li>").unwrap();
}
fn render_leaf(log_levels: LogLevels, buffer: &mut io::BufWriter<fs::File>) {
writeln!(
buffer,
"<li>\n\
<details><summary>(root)\
<span class=\"aside\">\
<span class=\"highlight\"><input type=\"checkbox\" title=\"Highlight\" /></span>\
</span>\
</summary>"
)
.unwrap();
if log_levels.is_empty().not() {
writeln!(buffer, "<ul>").unwrap();
for (level, locations) in log_levels {
render_locations(level, locations, buffer);
}
writeln!(buffer, "</ul>").unwrap();
}
writeln!(buffer, "</details></li>").unwrap();
}
fn render_locations(
level: Level,
locations: LogLocations,
buffer: &mut io::BufWriter<fs::File>,
) {
for (source_file, spans) in locations {
writeln!(
buffer,
"<li>\n\
<details><summary><span class=\"{level_class}\">{level}</span> <code>{source_file}</code>\
<span class=\"aside\">\
<span class=\"highlight\"><input type=\"checkbox\" title=\"Highlight\" /></span>\
</span>\
</summary>\n\
<ul>",
level_class = level.css(),
)
.unwrap();
render_spans(spans, buffer);
writeln!(
buffer,
"</ul>\n\
</details></li>"
)
.unwrap();
}
}
fn render_spans(spans: Spans, buffer: &mut io::BufWriter<fs::File>) {
for (source_line, all_spans) in spans {
writeln!(
buffer,
"<li>\n\
<details><summary>{n} ocurrrence{s} at line <code>{source_line}</code>\
<span class=\"aside\">\
<span class=\"highlight\"><input type=\"checkbox\" title=\"Highlight\" /></span>\
</span>\
</summary>\n\
<ol class=\"spans\">",
n = all_spans.len(),
s = if all_spans.len() > 1 { "s" } else { "" }
)
.unwrap();
for (date_time, span) in all_spans {
render_span(date_time, span, buffer);
}
writeln!(
buffer,
"</ol>\n\
</details></li>"
)
.unwrap();
}
}
fn render_span(
date_time: DateTime<FixedOffset>,
span: Span,
buffer: &mut io::BufWriter<fs::File>,
) {
let Span { message, log_line, .. } = span;
let message_parser = &*MESSAGE_PARSER;
let fields_parser = &*FIELDS_PARSER;
if let Some(captures) = message_parser.captures(&message) {
writeln!(
buffer,
"<li><time datetime=\"{date_time}\">{time}</time> {message} <span title=\"Log line\">[#{log_line}]</span>\
<span class=\"aside\">\
<span class=\"highlight\"><input type=\"checkbox\" title=\"Highlight\" /></span>\
</span>\
",
message = captures.name("message").expect("Failed to parse the `message`").as_str(),
date_time = date_time.format("%+"),
time = date_time.format("%H:%M:%S%.3f"),
)
.unwrap();
if let Some(fields) = captures.name("fields") {
writeln!(buffer, "<ul class=\"fields\"></li>").unwrap();
let mut fields = &message[fields.start()..];
while let Some(captures) = fields_parser.captures(fields) {
let name = captures
.name("name")
.expect("Failed to parse the `name` of the field")
.as_str();
let value = captures
.name("value")
.expect("Failed to parse the `value` of the field")
.as_str();
writeln!(buffer, "<li><span>{name}</span><span>{value}</span></li>").unwrap();
if let Some(next_fields) = captures.name("next_fields") {
fields = &message[next_fields.start()..];
} else {
break;
}
}
writeln!(buffer, "</ul>").unwrap();
}
writeln!(buffer, "</li>").unwrap();
} else {
writeln!(buffer, "<li>{message}</li>").unwrap();
}
}
let output = OUTPUT_TEMPLATE.replace("{log_file}", &log_path.to_string_lossy());
output_file.write_all(output.as_bytes()).expect(
"Failed to write the
output",
);
render_target_tree(tree, &mut output_file);
output_file.write_all("</main></body></html>".as_bytes()).expect("Failed to write the output");
println!(
"\nNumber of analysed log lines: {number_of_analysed_lines}\n\
Number of matched lines: {number_of_matched_lines}\n\
Output file: {output_path:?}\n\
Done!"
);
Ok(())
}
type TargetName = String;
type TargetTree = BTreeMap<TargetName, Target>;
#[derive(Debug)]
enum Target {
Node {
/// Number of errors for this node entirely (including sub-trees).
number_of_errors: usize,
/// Number of warnings for this node entirely (including sub-trees).
number_of_warnings: usize,
/// The Node.
node: TargetTree,
},
Leaf(LogLevels),
}
#[derive(Debug, Eq, PartialEq, Ord, PartialOrd)]
enum Level {
Error,
Warn,
Info,
Debug,
Trace,
}
type LogLevels = BTreeMap<Level, LogLocations>;
type SourceFile = String;
type SourceLine = usize;
type LogLocations = HashMap<SourceFile, Spans>;
type Spans = BTreeMap<SourceLine, BTreeMap<DateTime<FixedOffset>, Span>>;
#[derive(Debug)]
struct Span {
message: String,
log_line: usize,
#[allow(dead_code)]
raw: String,
}
impl Target {
fn as_leaf_mut(&mut self) -> Option<&mut LogLevels> {
match self {
Self::Leaf(log_levels) => Some(log_levels),
Self::Node { .. } => None,
}
}
}
impl Level {
fn css(&self) -> &str {
match self {
Self::Error => "error",
Self::Warn => "warn",
Self::Info => "info",
Self::Debug => "debug",
Self::Trace => "trace",
}
}
}
impl FromStr for Level {
type Err = ();
fn from_str(str: &str) -> Result<Self, Self::Err> {
Ok(match str {
"ERROR" => Self::Error,
"WARN" => Self::Warn,
"INFO" => Self::Info,
"DEBUG" => Self::Debug,
"TRACE" => Self::Trace,
_ => return Err(()),
})
}
}
impl fmt::Display for Level {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(
formatter,
"{}",
match self {
Self::Error => "ERROR",
Self::Warn => "WARNING",
Self::Info => "INFO",
Self::Debug => "DEBUG",
Self::Trace => "TRACE",
}
)
}
}
+269
View File
@@ -0,0 +1,269 @@
<!doctype html>
<html lang="en">
<head>
<meta http-equiv="content-type" content="text/html; charset=utf-8">
<meta http-equiv="content-security-policy" content="default-src 'self'; script-src 'unsafe-inline'; style-src 'unsafe-inline'; img-src data:">
<meta name="viewport" content="width=device-width, minimum-scale=1">
<style>
/* Reset */
*, *::before, *::after {
box-sizing: border-box;
}
* { margin: 0 }
:root {
--space-very-small: .25rem;
--space-small: .5rem;
--space: 1rem;
--space-large: 2rem;
--space-very-large: 4rem;
--border-radius: 3px;
--line-height: 1.5rem;
--color-accent: oklch(59.1% .236 10.25);
--color-text: oklch(.84 0 0);
--color-canvas: oklch(0.258 0.007 285.867);
--color-canvas-lighter: oklch(0.303 0.007 285.966);
--color-canvas-lighter-2: oklch(0.343 0.009 285.935);
--color-canvas-lighter-3: oklch(.6 0 0);
--color-white: oklch(1 0 0);
--color-red: oklch(.503 .172 25);
--color-orange: oklch(.793 .171 70.67);
--color-blue: oklch(0.579 0.191 252.0);
--color-green: oklch(0.571 0.181 145.0);
--color-back: oklch(0 0 0);
--color-yellow: oklch(.968 .211 109.769);
--color-yellow-light: oklch(.568 .211 109.769);
}
.content-grid {
--padding-inline: 1rem;
--content-max-width: 70ch;
--breakout-max-width: 85ch;
--breakout-size: calc((var(--breakout-max-width) - var(--content-max-width)) / 2);
display: grid;
grid-template-columns:
[full-width-start]
minmax(var(--padding-inline), 1fr)
[breakout-start]
minmax(0, var(--breakout-size))
[content-start]
min(
100% - (var(--padding-inline) * 2),
var(--content-max-width)
)
[content-end]
minmax(0, var(--breakout-size))
[breakout-end]
minmax(var(--padding-inline), 1fr)
[full-width-end];
> * {
grid-column: content;
}
> .breakout {
grid-column: breakout;
}
> .full-width {
grid-column: full-width;
}
}
html {
color-scheme: dark;
scroll-padding-top: var(--space-very-large);
}
body {
font-size: .95rem;
/* Thanks modernfontstacks.com */
font-family: Inter, Roboto, 'Helvetica Neue', 'Arial Nova', 'Nimbus Sans', Arial, sans-serif;
font-weight: normal;
font-variant-numeric: proportional-nums slashed-zero;
color: var(--color-text);
background: var(--color-canvas);
scroll-behaviour: smooth;
}
header {
text-align: center;
text-wrap: balanced;
margin-block: var(--space) var(--space-large);
}
code {
/* Thanks modernfontstacks.com */
font-family: ui-monospace, 'Cascadia Code', 'Source Code Pro', Menlo, Consolas, 'DejaVu Sans Mono', monospace;
font-weight: normal;
font-size: .9em;
line-height: var(--line-height);
}
main {
--_aside-column-width: 10rem;
position: relative;
line-height: var(--line-height);
& > ul {
padding-inline-start: calc(var(--_aside-column-width) + var(--space));
background-image: linear-gradient(var(--color-canvas-lighter) 50%, var(--color-canvas) 50%);
background-size: 100% calc(var(--line-height) * 2);
}
ul {
list-style: none;
ul {
padding-inline-start: var(--space);
}
}
ul.fields {
display: grid;
grid-template-columns: max-content 1fr;
gap: var(--space-large);
> li {
display: grid;
grid-column: 1 / -1;
grid-template-columns: subgrid;
}
}
ol.spans {
--_line-width: 2px;
--_color: var(--color-canvas-lighter-3);
list-style-type: "●";
list-style-position: outside;
margin-inline-start: .2rem;
padding-inline-start: var(--space);
border: var(--_color) solid;
border-width: 0 0 0 var(--_line-width);
> li {
margin-inline-start: calc(-1ch - var(--_line-width) / 2);
padding-inline-start: var(--space);
&::marker { color: var(--_color) }
> time {
position: absolute;
transform: translateX(calc(-100% - var(--space) * 2 - .8rem));
font-variant-numeric: slashed-zero tabular-nums;
}
}
}
summary {
cursor: pointer;
height: var(--line-height);
}
code {
line-height: var(--line-height);
}
.aside {
position: absolute;
left: 0;
width: var(--_aside-column-width);
font-variant-numeric: slashed-zero tabular-nums;
> * {
display: inline-block;
&.highlight {
min-width: 1.5rem;
text-align: center;
/* hide the highlight box by default */
input { visibility: hidden }
/* highlight the line when the input is checked */
&:has(input:checked)::before {
position: absolute;
content: "";
left: 0;
height: var(--line-height);
width: 100vw;
border: var(--color-yellow) solid;
border-width: 1px 0;
pointer-events: none;
}
}
&.error, &.warn {
min-width: 3.5rem;
margin-inline: .2rem;
text-align: end;
}
}
/* shows the highlight checkbox on `:hover` */
:is(summary:hover, li:hover) > & > .highlight > input { visibility: visible }
}
/* highlight the parent node if a child line is checked */
details:not([open]):has(.highlight > input:checked)::before {
position: absolute;
content: "";
left: 0;
height: var(--line-height);
width: 100vw;
border: oklch(.568 .211 109.769) solid;
border-width: 0 0 2px 0;
pointer-events: none;
}
.error, .warn, .info, .debug, .trace {
display: inline-block;
font-size: .9em;
vertical-align: center;
line-height: 1.4em;
padding-inline: var(--space-very-small);
border-radius: var(--border-radius);
color: var(--color-white);
}
.error { background: var(--color-red) }
.warn { background: var(--color-orange) }
.info { background: var(--color-blue) }
.debug { background: var(--color-green) }
.trace { background: var(--color-black) }
}
</style>
<title>Overview</title>
</head>
<body class="content-grid">
<header>
<h1>Overview of <code>{log_file}</code></h1>
</header>
<main class="full-width">
<!--
the tree and the end of file is written by Rust for performance reason:
</main>
</body>
</html>
-->