//! ZHILA material: decode an uploaded file, find its seams, cut sixteen faces. //! //! The import path is intentionally CPU-bound and off the audio thread. The //! archive receives mono f32 samples at the source rate; the engine resamples //! playback through its rate while the output stays at 48 kHz. use std::io::Cursor; use std::sync::Arc; use blake3::Hasher; use symphonia::core::audio::SampleBuffer; use symphonia::core::codecs::DecoderOptions; use symphonia::core::errors::Error as SymphoniaError; use symphonia::core::formats::FormatOptions; use symphonia::core::io::MediaSourceStream; use symphonia::core::meta::MetadataOptions; use symphonia::core::probe::Hint; use thiserror::Error; use zhila_core::{AudioFace, Core}; use zhila_protocol::{CoreSummary, FaceSummary, Seal}; const FACE_COUNT: usize = 16; const PEAK_BINS: usize = 1400; const ONSET_HOPS_PER_SECOND: f32 = 100.0; // 10 ms energy hops #[derive(Debug, Error)] pub enum MaterialError { #[error("could not decode the audio: {0}")] Decode(String), #[error("the file has no audio track")] NoTrack, #[error("the audio is too short to cut")] TooShort, } /// Decode `bytes`, analyse the seams and build a core. `name` is used both as /// the display name and as the format hint (its extension). pub fn import_bytes(name: &str, seal: Seal, bytes: Vec) -> Result { let mut hasher = Hasher::new(); hasher.update(&bytes); let digest = hasher.finalize(); let id = format!("core-{}", &digest.to_hex()[..10]); let (samples, sample_rate) = decode(name, bytes)?; if samples.len() < 1_000 { return Err(MaterialError::TooShort); } let (faces, face_summaries) = analyse(&samples, sample_rate); let peaks = waveform_peaks(&samples); let duration_sec = samples.len() as f32 / sample_rate as f32; let summary = CoreSummary { id, name: display_name(name), seal, sample_rate, duration_sec, faces: face_summaries, peaks, }; Ok(Core { summary, samples: Arc::from(samples), faces, }) } fn display_name(name: &str) -> String { let base = std::path::Path::new(name) .file_stem() .and_then(|stem| stem.to_str()) .unwrap_or("core") .trim(); if base.is_empty() { "core".into() } else { base.chars().take(48).collect() } } fn decode(name: &str, bytes: Vec) -> Result<(Vec, u32), MaterialError> { let source = MediaSourceStream::new(Box::new(Cursor::new(bytes)), Default::default()); let mut hint = Hint::new(); if let Some(extension) = std::path::Path::new(name) .extension() .and_then(|extension| extension.to_str()) { hint.with_extension(extension); } let probed = symphonia::default::get_probe() .format( &hint, source, &FormatOptions::default(), &MetadataOptions::default(), ) .map_err(|error| MaterialError::Decode(error.to_string()))?; let mut format = probed.format; let track = format.default_track().ok_or(MaterialError::NoTrack)?; let track_id = track.id; let codec_params = track.codec_params.clone(); let sample_rate = codec_params .sample_rate .ok_or_else(|| MaterialError::Decode("missing sample rate".into()))?; let mut decoder = symphonia::default::get_codecs() .make(&codec_params, &DecoderOptions::default()) .map_err(|error| MaterialError::Decode(error.to_string()))?; let mut samples: Vec = Vec::with_capacity(48_000 * 8); loop { let packet = match format.next_packet() { Ok(packet) => packet, Err(SymphoniaError::IoError(_)) => break, Err(SymphoniaError::ResetRequired) => break, Err(error) => return Err(MaterialError::Decode(error.to_string())), }; if packet.track_id() != track_id { continue; } match decoder.decode(&packet) { Ok(decoded) => { let spec = *decoded.spec(); let channels = spec.channels.count().max(1); let mut buffer = SampleBuffer::::new(decoded.capacity() as u64, spec); buffer.copy_interleaved_ref(decoded); for frame in buffer.samples().chunks(channels) { let mono = frame.iter().sum::() / channels as f32; samples.push(mono.clamp(-1.0, 1.0)); } } Err(SymphoniaError::DecodeError(_)) => continue, Err(error) => return Err(MaterialError::Decode(error.to_string())), } } if samples.is_empty() { return Err(MaterialError::Decode("no samples decoded".into())); } Ok((samples, sample_rate)) } fn analyse(samples: &[f32], sample_rate: u32) -> (Vec, Vec) { let hop = (sample_rate as f32 / ONSET_HOPS_PER_SECOND).max(64.0) as usize; let energy: Vec = samples .chunks(hop) .map(|chunk| { chunk.iter().map(|sample| sample * sample).sum::() / chunk.len().max(1) as f32 }) .collect(); let mut flux = vec![0.0f32; energy.len()]; for index in 1..energy.len() { flux[index] = (energy[index] - energy[index - 1]).max(0.0); } let mean = flux.iter().sum::() / flux.len().max(1) as f32; let variance = flux .iter() .map(|value| (value - mean) * (value - mean)) .sum::() / flux.len().max(1) as f32; let threshold = mean + variance.sqrt() * 0.55; let min_gap = (sample_rate as f32 * 0.09 / hop as f32).max(1.0) as usize; let mut candidates: Vec<(usize, f32)> = Vec::new(); for index in 1..flux.len().saturating_sub(1) { let value = flux[index]; if value > threshold && value >= flux[index - 1] && value > flux[index + 1] { candidates.push((index, value)); } } candidates.sort_by(|a, b| b.1.total_cmp(&a.1)); let mut onsets: Vec = Vec::new(); for (index, _) in candidates { if onsets.len() >= FACE_COUNT - 1 { break; } if onsets .iter() .all(|existing| existing.abs_diff(index) >= min_gap) { onsets.push(index); } } let mut boundaries: Vec = onsets.into_iter().map(|index| index * hop).collect(); boundaries.push(0); boundaries.push(samples.len()); boundaries.sort_unstable(); boundaries.dedup(); while boundaries.len() < FACE_COUNT + 1 { let Some((split_at, _)) = boundaries .windows(2) .enumerate() .max_by_key(|(_, pair)| pair[1] - pair[0]) else { break; }; let pair = &boundaries[split_at..split_at + 2]; let midpoint = pair[0] + (pair[1] - pair[0]) / 2; if midpoint == pair[0] || midpoint == pair[1] { break; } boundaries.insert(split_at + 1, midpoint); } if boundaries.len() > FACE_COUNT + 1 { boundaries.truncate(FACE_COUNT + 1); let last = boundaries.len() - 1; boundaries[last] = samples.len(); } let mut faces = Vec::with_capacity(FACE_COUNT); let mut summaries = Vec::with_capacity(FACE_COUNT); for index in 0..FACE_COUNT { let start = boundaries[index]; let end = boundaries[index + 1].max(start + 1).min(samples.len()); let segment = &samples[start..end]; let rms = (segment.iter().map(|sample| sample * sample).sum::() / segment.len().max(1) as f32) .sqrt(); faces.push(AudioFace { index: index as u8, start: start as u32, end: end as u32, rms, }); summaries.push(FaceSummary { index: index as u8, rank: 0, start_sec: start as f32 / sample_rate as f32, end_sec: end as f32 / sample_rate as f32, rms, }); } let mut ranked: Vec = (0..summaries.len()).collect(); ranked.sort_by(|a, b| summaries[*b].rms.total_cmp(&summaries[*a].rms)); for (rank, index) in ranked.into_iter().enumerate() { summaries[index].rank = rank as u8; } (faces, summaries) } fn waveform_peaks(samples: &[f32]) -> Vec { let mut peaks = vec![0.0f32; PEAK_BINS]; let chunk = (samples.len() as f32 / PEAK_BINS as f32).ceil().max(1.0) as usize; for (index, frame) in samples.chunks(chunk).take(PEAK_BINS).enumerate() { peaks[index] = frame .iter() .fold(0.0f32, |peak, sample| peak.max(sample.abs())); } let maximum = peaks.iter().fold(0.0f32, |peak, value| peak.max(*value)); if maximum > 1e-6 { for peak in &mut peaks { *peak = (*peak / maximum).min(1.0); } } peaks } #[cfg(test)] mod tests { use super::*; fn click_track(seconds: f32, sample_rate: u32) -> Vec { let frames = (seconds * sample_rate as f32) as usize; let mut samples = vec![0.0f32; frames]; let mut position = 0; while position < frames { for offset in 0..960 { if position + offset < frames { let envelope = 1.0 - offset as f32 / 960.0; samples[position + offset] = (offset as f32 * 0.12).sin() * envelope * 0.7; } } position += (sample_rate / 6) as usize; } let mut bytes = Vec::new(); let spec = hound::WavSpec { channels: 1, sample_rate, bits_per_sample: 16, sample_format: hound::SampleFormat::Int, }; let mut writer = hound::WavWriter::new(Cursor::new(&mut bytes), spec).unwrap(); for sample in samples { writer .write_sample((sample.clamp(-1.0, 1.0) * i16::MAX as f32) as i16) .unwrap(); } writer.finalize().unwrap(); bytes } #[test] fn imports_a_click_track_into_sixteen_faces() { let bytes = click_track(2.0, 48_000); let core = import_bytes("clicks.wav", Seal::Found, bytes).unwrap(); assert_eq!(core.summary.sample_rate, 48_000); assert_eq!(core.faces.len(), FACE_COUNT); assert_eq!(core.summary.faces.len(), FACE_COUNT); assert_eq!(core.summary.peaks.len(), PEAK_BINS); assert!(core.summary.duration_sec > 1.9); assert_eq!(core.summary.name, "clicks"); assert_eq!(core.summary.seal, Seal::Found); assert!(core.faces.iter().all(|face| face.start < face.end)); assert!(core.summary.faces.iter().all(|face| face.rank < 16)); } #[test] fn short_audio_is_rejected() { let bytes = click_track(0.01, 48_000); assert!(matches!( import_bytes("short.wav", Seal::Found, bytes), Err(MaterialError::TooShort) )); } }