317 lines
11 KiB
Rust
317 lines
11 KiB
Rust
//! ZHILA material: decode an uploaded file, find its seams, cut sixteen faces.
|
|
//!
|
|
//! The import path is intentionally CPU-bound and off the audio thread. The
|
|
//! archive receives mono f32 samples at the source rate; the engine resamples
|
|
//! playback through its rate while the output stays at 48 kHz.
|
|
|
|
use std::io::Cursor;
|
|
use std::sync::Arc;
|
|
|
|
use blake3::Hasher;
|
|
use symphonia::core::audio::SampleBuffer;
|
|
use symphonia::core::codecs::DecoderOptions;
|
|
use symphonia::core::errors::Error as SymphoniaError;
|
|
use symphonia::core::formats::FormatOptions;
|
|
use symphonia::core::io::MediaSourceStream;
|
|
use symphonia::core::meta::MetadataOptions;
|
|
use symphonia::core::probe::Hint;
|
|
use thiserror::Error;
|
|
use zhila_core::{AudioFace, Core};
|
|
use zhila_protocol::{CoreSummary, FaceSummary, Seal};
|
|
|
|
const FACE_COUNT: usize = 16;
|
|
const PEAK_BINS: usize = 1400;
|
|
const ONSET_HOPS_PER_SECOND: f32 = 100.0; // 10 ms energy hops
|
|
|
|
#[derive(Debug, Error)]
|
|
pub enum MaterialError {
|
|
#[error("could not decode the audio: {0}")]
|
|
Decode(String),
|
|
#[error("the file has no audio track")]
|
|
NoTrack,
|
|
#[error("the audio is too short to cut")]
|
|
TooShort,
|
|
}
|
|
|
|
/// Decode `bytes`, analyse the seams and build a core. `name` is used both as
|
|
/// the display name and as the format hint (its extension).
|
|
pub fn import_bytes(name: &str, seal: Seal, bytes: Vec<u8>) -> Result<Core, MaterialError> {
|
|
let mut hasher = Hasher::new();
|
|
hasher.update(&bytes);
|
|
let digest = hasher.finalize();
|
|
let id = format!("core-{}", &digest.to_hex()[..10]);
|
|
|
|
let (samples, sample_rate) = decode(name, bytes)?;
|
|
if samples.len() < 1_000 {
|
|
return Err(MaterialError::TooShort);
|
|
}
|
|
let (faces, face_summaries) = analyse(&samples, sample_rate);
|
|
let peaks = waveform_peaks(&samples);
|
|
let duration_sec = samples.len() as f32 / sample_rate as f32;
|
|
let summary = CoreSummary {
|
|
id,
|
|
name: display_name(name),
|
|
seal,
|
|
sample_rate,
|
|
duration_sec,
|
|
faces: face_summaries,
|
|
peaks,
|
|
};
|
|
Ok(Core {
|
|
summary,
|
|
samples: Arc::from(samples),
|
|
faces,
|
|
})
|
|
}
|
|
|
|
fn display_name(name: &str) -> String {
|
|
let base = std::path::Path::new(name)
|
|
.file_stem()
|
|
.and_then(|stem| stem.to_str())
|
|
.unwrap_or("core")
|
|
.trim();
|
|
if base.is_empty() {
|
|
"core".into()
|
|
} else {
|
|
base.chars().take(48).collect()
|
|
}
|
|
}
|
|
|
|
fn decode(name: &str, bytes: Vec<u8>) -> Result<(Vec<f32>, u32), MaterialError> {
|
|
let source = MediaSourceStream::new(Box::new(Cursor::new(bytes)), Default::default());
|
|
let mut hint = Hint::new();
|
|
if let Some(extension) = std::path::Path::new(name)
|
|
.extension()
|
|
.and_then(|extension| extension.to_str())
|
|
{
|
|
hint.with_extension(extension);
|
|
}
|
|
let probed = symphonia::default::get_probe()
|
|
.format(
|
|
&hint,
|
|
source,
|
|
&FormatOptions::default(),
|
|
&MetadataOptions::default(),
|
|
)
|
|
.map_err(|error| MaterialError::Decode(error.to_string()))?;
|
|
let mut format = probed.format;
|
|
let track = format.default_track().ok_or(MaterialError::NoTrack)?;
|
|
let track_id = track.id;
|
|
let codec_params = track.codec_params.clone();
|
|
let sample_rate = codec_params
|
|
.sample_rate
|
|
.ok_or_else(|| MaterialError::Decode("missing sample rate".into()))?;
|
|
let mut decoder = symphonia::default::get_codecs()
|
|
.make(&codec_params, &DecoderOptions::default())
|
|
.map_err(|error| MaterialError::Decode(error.to_string()))?;
|
|
|
|
let mut samples: Vec<f32> = Vec::with_capacity(48_000 * 8);
|
|
loop {
|
|
let packet = match format.next_packet() {
|
|
Ok(packet) => packet,
|
|
Err(SymphoniaError::IoError(_)) => break,
|
|
Err(SymphoniaError::ResetRequired) => break,
|
|
Err(error) => return Err(MaterialError::Decode(error.to_string())),
|
|
};
|
|
if packet.track_id() != track_id {
|
|
continue;
|
|
}
|
|
match decoder.decode(&packet) {
|
|
Ok(decoded) => {
|
|
let spec = *decoded.spec();
|
|
let channels = spec.channels.count().max(1);
|
|
let mut buffer = SampleBuffer::<f32>::new(decoded.capacity() as u64, spec);
|
|
buffer.copy_interleaved_ref(decoded);
|
|
for frame in buffer.samples().chunks(channels) {
|
|
let mono = frame.iter().sum::<f32>() / channels as f32;
|
|
samples.push(mono.clamp(-1.0, 1.0));
|
|
}
|
|
}
|
|
Err(SymphoniaError::DecodeError(_)) => continue,
|
|
Err(error) => return Err(MaterialError::Decode(error.to_string())),
|
|
}
|
|
}
|
|
if samples.is_empty() {
|
|
return Err(MaterialError::Decode("no samples decoded".into()));
|
|
}
|
|
Ok((samples, sample_rate))
|
|
}
|
|
|
|
fn analyse(samples: &[f32], sample_rate: u32) -> (Vec<AudioFace>, Vec<FaceSummary>) {
|
|
let hop = (sample_rate as f32 / ONSET_HOPS_PER_SECOND).max(64.0) as usize;
|
|
let energy: Vec<f32> = samples
|
|
.chunks(hop)
|
|
.map(|chunk| {
|
|
chunk.iter().map(|sample| sample * sample).sum::<f32>() / chunk.len().max(1) as f32
|
|
})
|
|
.collect();
|
|
let mut flux = vec![0.0f32; energy.len()];
|
|
for index in 1..energy.len() {
|
|
flux[index] = (energy[index] - energy[index - 1]).max(0.0);
|
|
}
|
|
let mean = flux.iter().sum::<f32>() / flux.len().max(1) as f32;
|
|
let variance = flux
|
|
.iter()
|
|
.map(|value| (value - mean) * (value - mean))
|
|
.sum::<f32>()
|
|
/ flux.len().max(1) as f32;
|
|
let threshold = mean + variance.sqrt() * 0.55;
|
|
let min_gap = (sample_rate as f32 * 0.09 / hop as f32).max(1.0) as usize;
|
|
|
|
let mut candidates: Vec<(usize, f32)> = Vec::new();
|
|
for index in 1..flux.len().saturating_sub(1) {
|
|
let value = flux[index];
|
|
if value > threshold && value >= flux[index - 1] && value > flux[index + 1] {
|
|
candidates.push((index, value));
|
|
}
|
|
}
|
|
candidates.sort_by(|a, b| b.1.total_cmp(&a.1));
|
|
|
|
let mut onsets: Vec<usize> = Vec::new();
|
|
for (index, _) in candidates {
|
|
if onsets.len() >= FACE_COUNT - 1 {
|
|
break;
|
|
}
|
|
if onsets
|
|
.iter()
|
|
.all(|existing| existing.abs_diff(index) >= min_gap)
|
|
{
|
|
onsets.push(index);
|
|
}
|
|
}
|
|
|
|
let mut boundaries: Vec<usize> = onsets.into_iter().map(|index| index * hop).collect();
|
|
boundaries.push(0);
|
|
boundaries.push(samples.len());
|
|
boundaries.sort_unstable();
|
|
boundaries.dedup();
|
|
while boundaries.len() < FACE_COUNT + 1 {
|
|
let Some((split_at, _)) = boundaries
|
|
.windows(2)
|
|
.enumerate()
|
|
.max_by_key(|(_, pair)| pair[1] - pair[0])
|
|
else {
|
|
break;
|
|
};
|
|
let pair = &boundaries[split_at..split_at + 2];
|
|
let midpoint = pair[0] + (pair[1] - pair[0]) / 2;
|
|
if midpoint == pair[0] || midpoint == pair[1] {
|
|
break;
|
|
}
|
|
boundaries.insert(split_at + 1, midpoint);
|
|
}
|
|
if boundaries.len() > FACE_COUNT + 1 {
|
|
boundaries.truncate(FACE_COUNT + 1);
|
|
let last = boundaries.len() - 1;
|
|
boundaries[last] = samples.len();
|
|
}
|
|
|
|
let mut faces = Vec::with_capacity(FACE_COUNT);
|
|
let mut summaries = Vec::with_capacity(FACE_COUNT);
|
|
for index in 0..FACE_COUNT {
|
|
let start = boundaries[index];
|
|
let end = boundaries[index + 1].max(start + 1).min(samples.len());
|
|
let segment = &samples[start..end];
|
|
let rms = (segment.iter().map(|sample| sample * sample).sum::<f32>()
|
|
/ segment.len().max(1) as f32)
|
|
.sqrt();
|
|
faces.push(AudioFace {
|
|
index: index as u8,
|
|
start: start as u32,
|
|
end: end as u32,
|
|
rms,
|
|
});
|
|
summaries.push(FaceSummary {
|
|
index: index as u8,
|
|
rank: 0,
|
|
start_sec: start as f32 / sample_rate as f32,
|
|
end_sec: end as f32 / sample_rate as f32,
|
|
rms,
|
|
});
|
|
}
|
|
|
|
let mut ranked: Vec<usize> = (0..summaries.len()).collect();
|
|
ranked.sort_by(|a, b| summaries[*b].rms.total_cmp(&summaries[*a].rms));
|
|
for (rank, index) in ranked.into_iter().enumerate() {
|
|
summaries[index].rank = rank as u8;
|
|
}
|
|
|
|
(faces, summaries)
|
|
}
|
|
|
|
fn waveform_peaks(samples: &[f32]) -> Vec<f32> {
|
|
let mut peaks = vec![0.0f32; PEAK_BINS];
|
|
let chunk = (samples.len() as f32 / PEAK_BINS as f32).ceil().max(1.0) as usize;
|
|
for (index, frame) in samples.chunks(chunk).take(PEAK_BINS).enumerate() {
|
|
peaks[index] = frame
|
|
.iter()
|
|
.fold(0.0f32, |peak, sample| peak.max(sample.abs()));
|
|
}
|
|
let maximum = peaks.iter().fold(0.0f32, |peak, value| peak.max(*value));
|
|
if maximum > 1e-6 {
|
|
for peak in &mut peaks {
|
|
*peak = (*peak / maximum).min(1.0);
|
|
}
|
|
}
|
|
peaks
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
|
|
fn click_track(seconds: f32, sample_rate: u32) -> Vec<u8> {
|
|
let frames = (seconds * sample_rate as f32) as usize;
|
|
let mut samples = vec![0.0f32; frames];
|
|
let mut position = 0;
|
|
while position < frames {
|
|
for offset in 0..960 {
|
|
if position + offset < frames {
|
|
let envelope = 1.0 - offset as f32 / 960.0;
|
|
samples[position + offset] = (offset as f32 * 0.12).sin() * envelope * 0.7;
|
|
}
|
|
}
|
|
position += (sample_rate / 6) as usize;
|
|
}
|
|
let mut bytes = Vec::new();
|
|
let spec = hound::WavSpec {
|
|
channels: 1,
|
|
sample_rate,
|
|
bits_per_sample: 16,
|
|
sample_format: hound::SampleFormat::Int,
|
|
};
|
|
let mut writer = hound::WavWriter::new(Cursor::new(&mut bytes), spec).unwrap();
|
|
for sample in samples {
|
|
writer
|
|
.write_sample((sample.clamp(-1.0, 1.0) * i16::MAX as f32) as i16)
|
|
.unwrap();
|
|
}
|
|
writer.finalize().unwrap();
|
|
bytes
|
|
}
|
|
|
|
#[test]
|
|
fn imports_a_click_track_into_sixteen_faces() {
|
|
let bytes = click_track(2.0, 48_000);
|
|
let core = import_bytes("clicks.wav", Seal::Found, bytes).unwrap();
|
|
assert_eq!(core.summary.sample_rate, 48_000);
|
|
assert_eq!(core.faces.len(), FACE_COUNT);
|
|
assert_eq!(core.summary.faces.len(), FACE_COUNT);
|
|
assert_eq!(core.summary.peaks.len(), PEAK_BINS);
|
|
assert!(core.summary.duration_sec > 1.9);
|
|
assert_eq!(core.summary.name, "clicks");
|
|
assert_eq!(core.summary.seal, Seal::Found);
|
|
assert!(core.faces.iter().all(|face| face.start < face.end));
|
|
assert!(core.summary.faces.iter().all(|face| face.rank < 16));
|
|
}
|
|
|
|
#[test]
|
|
fn short_audio_is_rejected() {
|
|
let bytes = click_track(0.01, 48_000);
|
|
assert!(matches!(
|
|
import_bytes("short.wav", Seal::Found, bytes),
|
|
Err(MaterialError::TooShort)
|
|
));
|
|
}
|
|
}
|