Add external processor support

CHANGELOG.md

@@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Added
 
 - Allow `partition_table_offset` to be specified in the config file. (for #699)
+- Support external log-processors (#705)
 
 ### Changed
 

cargo-espflash/src/main.rs

@@ -323,7 +323,7 @@ fn flash(args: FlashArgs, config: &Config) -> Result<()> {
         build(&args.build_args, &cargo_config, chip).wrap_err("Failed to build project")?;
 
     // Read the ELF data from the build path and load it to the target.
-    let elf_data = fs::read(build_ctx.artifact_path).into_diagnostic()?;
+    let elf_data = fs::read(build_ctx.artifact_path.clone()).into_diagnostic()?;
 
     print_board_info(&mut flasher)?;
 
@@ -368,6 +368,8 @@ fn flash(args: FlashArgs, config: &Config) -> Result<()> {
             args.flash_args.monitor_baud.unwrap_or(default_baud),
             args.flash_args.log_format,
             true,
+            args.flash_args.processors,
+            Some(build_ctx.artifact_path),
         )
     } else {
         Ok(())

espflash/src/bin/espflash.rs

@@ -30,7 +30,7 @@ pub struct Cli {
     subcommand: Commands,
 
     /// Do not check for updates
-    #[clap(short, long, global = true, action)]
+    #[clap(short = 'S', long, global = true, action)]
     skip_update_check: bool,
 }
 
@@ -303,6 +303,8 @@ fn flash(args: FlashArgs, config: &Config) -> Result<()> {
             args.flash_args.monitor_baud.unwrap_or(default_baud),
             args.flash_args.log_format,
             true,
+            args.flash_args.processors,
+            Some(args.image),
         )
     } else {
         Ok(())

espflash/src/cli/mod.rs

@@ -156,6 +156,9 @@ pub struct FlashArgs {
     pub no_skip: bool,
     #[clap(flatten)]
     pub image: ImageArgs,
+    /// External log processors to use (comma separated executables)
+    #[arg(long, requires = "monitor")]
+    pub processors: Option<String>,
 }
 
 /// Operations for partitions tables
@@ -262,6 +265,9 @@ pub struct MonitorArgs {
     /// Logging format.
     #[arg(long, short = 'L', default_value = "serial", requires = "elf")]
     pub log_format: LogFormat,
+    /// External log processors to use (comma separated executables)
+    #[arg(long)]
+    processors: Option<String>,
 }
 
 #[derive(Debug, Args)]
@@ -418,7 +424,7 @@ pub fn serial_monitor(args: MonitorArgs, config: &Config) -> Result<()> {
     let mut flasher = connect(&args.connect_args, config, true, true)?;
     let pid = flasher.get_usb_pid()?;
 
-    let elf = if let Some(elf_path) = args.elf {
+    let elf = if let Some(elf_path) = args.elf.clone() {
         let path = fs::canonicalize(elf_path).into_diagnostic()?;
         let data = fs::read(path).into_diagnostic()?;
 
@@ -447,6 +453,8 @@ pub fn serial_monitor(args: MonitorArgs, config: &Config) -> Result<()> {
         args.connect_args.baud.unwrap_or(default_baud),
         args.log_format,
         !args.non_interactive,
+        args.processors,
+        args.elf,
     )
 }
 

espflash/src/cli/monitor/external_processors.rs

@@ -0,0 +1,162 @@
+#![allow(clippy::needless_doctest_main)]
+//! External processor support
+//!
+//! Via the command line argument `--processors` you can instruct espflash to run external executables to pre-process
+//! the logs received from the target. Multiple processors are supported by separating them via `,`. Processors are executed in the specified order.
+//!
+//! You can use full-qualified paths or run an executable which is already in the search path.
+//!
+//! A processors reads from stdin and output to stdout. Be aware this runs before further processing by espflash.
+//! i.e. addresses are not resolved and when using `defmt` you will see encoded data.
+//!
+//! Additionally be aware that you might receive chunked data which is not always split at valid UTF character boundaries.
+//!
+//! The executable will get the path of the ELF file as the first argument if available.
+//!
+//! Example processor which turns some letters into uppercase
+//! ```rust,no-run
+//! use std::io::{stdin, stdout, Read, Write};
+//!
+//! fn main() {
+//!     let args: Vec<String> = std::env::args().collect();
+//!     println!("ELF file: {:?}", args[1]);
+//!
+//!     let mut buf = [0u8; 1024];
+//!     loop {
+//!         if let Ok(len) = stdin().read(&mut buf) {
+//!             for b in &mut buf[..len] {
+//!                 *b = if b"abdfeo".contains(b) {
+//!                     b.to_ascii_uppercase()
+//!                 } else {
+//!                     *b
+//!                 };
+//!             }
+//!
+//!             stdout().write(&buf[..len]).unwrap();
+//!             stdout().flush().unwrap();
+//!         } else {
+//!             // ignored
+//!         }
+//!     }
+//! }
+//! ```
+
+use std::{
+    fmt::Display,
+    io::{Read, Write},
+    path::PathBuf,
+    process::{Child, ChildStdin, Stdio},
+    sync::mpsc,
+};
+
+use miette::Diagnostic;
+
+#[derive(Debug)]
+pub struct Error {
+    executable: String,
+}
+
+impl Display for Error {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "Failed to launch '{}'", self.executable)
+    }
+}
+
+impl std::error::Error for Error {}
+
+impl Diagnostic for Error {}
+
+struct Processor {
+    rx: mpsc::Receiver<u8>,
+    stdin: ChildStdin,
+    child: Child,
+}
+
+impl Processor {
+    pub fn new(child: Child) -> Self {
+        let mut child = child;
+        let (tx, rx) = mpsc::channel::<u8>();
+
+        let mut stdout = child.stdout.take().unwrap();
+        let stdin = child.stdin.take().unwrap();
+
+        std::thread::spawn(move || {
+            let mut buffer = [0u8; 1024];
+            loop {
+                if let Ok(len) = stdout.read(&mut buffer) {
+                    for b in &buffer[..len] {
+                        if tx.send(*b).is_err() {
+                            break;
+                        }
+                    }
+                }
+            }
+        });
+
+        Self { rx, stdin, child }
+    }
+
+    pub fn try_receive(&mut self) -> Vec<u8> {
+        let mut res = Vec::new();
+        while let Ok(b) = self.rx.try_recv() {
+            res.push(b);
+        }
+        res
+    }
+
+    pub fn send(&mut self, data: Vec<u8>) {
+        let _ignored = self.stdin.write(&data).ok();
+    }
+}
+
+impl Drop for Processor {
+    fn drop(&mut self) {
+        self.child.kill().unwrap();
+    }
+}
+
+pub struct ExternalProcessors {
+    processors: Vec<Processor>,
+}
+
+impl ExternalProcessors {
+    pub fn new(processors: Option<String>, elf: Option<PathBuf>) -> Result<Self, Error> {
+        let mut args = Vec::new();
+
+        if let Some(elf) = elf {
+            args.push(elf.as_os_str().to_str().unwrap().to_string());
+        };
+
+        let mut spawned = Vec::new();
+        if let Some(processors) = processors {
+            for processor in processors.split(",") {
+                let processor = std::process::Command::new(processor)
+                    .args(args.clone())
+                    .stdin(Stdio::piped())
+                    .stdout(Stdio::piped())
+                    .stderr(Stdio::inherit())
+                    .spawn()
+                    .map_err(|_| Error {
+                        executable: processor.to_string(),
+                    })?;
+                spawned.push(Processor::new(processor));
+            }
+        }
+
+        Ok(Self {
+            processors: spawned,
+        })
+    }
+
+    pub fn process(&mut self, read: &[u8]) -> Vec<u8> {
+        let mut buffer = Vec::new();
+        buffer.extend_from_slice(read);
+
+        for processor in &mut self.processors {
+            processor.send(buffer);
+            buffer = processor.try_receive();
+        }
+
+        buffer
+    }
+}

espflash/src/cli/monitor/mod.rs

@@ -12,6 +12,7 @@
 
 use std::{
     io::{stdout, ErrorKind, Read, Write},
+    path::PathBuf,
     time::Duration,
 };
 
@@ -20,6 +21,7 @@ use crossterm::{
     event::{poll, read, Event, KeyCode, KeyEvent, KeyModifiers},
     terminal::{disable_raw_mode, enable_raw_mode},
 };
+use external_processors::ExternalProcessors;
 use log::error;
 use miette::{IntoDiagnostic, Result};
 #[cfg(feature = "serialport")]
@@ -31,6 +33,7 @@ use crate::{
     connection::{reset::reset_after_flash, Port},
 };
 
+pub mod external_processors;
 pub mod parser;
 
 mod line_endings;
@@ -66,13 +69,16 @@ impl Drop for RawModeGuard {
 }
 
 /// Open a serial monitor on the given serial port, using the given input parser.
+#[allow(clippy::too_many_arguments)]
 pub fn monitor(
     mut serial: Port,
     elf: Option<&[u8]>,
     pid: u16,
     baud: u32,
     log_format: LogFormat,
     interactive_mode: bool,
+    processors: Option<String>,
+    elf_file: Option<PathBuf>,
 ) -> miette::Result<()> {
     if interactive_mode {
         println!("Commands:");
@@ -101,6 +107,8 @@ pub fn monitor(
         LogFormat::Serial => Box::new(parser::serial::Serial),
     };
 
+    let mut external_processors = ExternalProcessors::new(processors, elf_file)?;
+
     let mut buff = [0; 1024];
     loop {
         let read_count = match serial.read(&mut buff) {
@@ -110,7 +118,8 @@ pub fn monitor(
             err => err.into_diagnostic(),
         }?;
 
-        parser.feed(&buff[0..read_count], &mut stdout);
+        let processed = external_processors.process(&buff[0..read_count]);
+        parser.feed(&processed, &mut stdout);
 
         // Don't forget to flush the writer!
         stdout.flush().ok();