P1a: material import, sixteen faces, sample engine and quarry
This commit is contained in:
Emil
@@ -0,0 +1,316 @@
|
||||
//! ZHILA material: decode an uploaded file, find its seams, cut sixteen faces.
|
||||
//!
|
||||
//! The import path is intentionally CPU-bound and off the audio thread. The
|
||||
//! archive receives mono f32 samples at the source rate; the engine resamples
|
||||
//! playback through its rate while the output stays at 48 kHz.
|
||||
|
||||
use std::io::Cursor;
|
||||
use std::sync::Arc;
|
||||
|
||||
use blake3::Hasher;
|
||||
use symphonia::core::audio::SampleBuffer;
|
||||
use symphonia::core::codecs::DecoderOptions;
|
||||
use symphonia::core::errors::Error as SymphoniaError;
|
||||
use symphonia::core::formats::FormatOptions;
|
||||
use symphonia::core::io::MediaSourceStream;
|
||||
use symphonia::core::meta::MetadataOptions;
|
||||
use symphonia::core::probe::Hint;
|
||||
use thiserror::Error;
|
||||
use zhila_core::{AudioFace, Core};
|
||||
use zhila_protocol::{CoreSummary, FaceSummary, Seal};
|
||||
|
||||
const FACE_COUNT: usize = 16;
|
||||
const PEAK_BINS: usize = 1400;
|
||||
const ONSET_HOPS_PER_SECOND: f32 = 100.0; // 10 ms energy hops
|
||||
|
||||
#[derive(Debug, Error)]
|
||||
pub enum MaterialError {
|
||||
#[error("could not decode the audio: {0}")]
|
||||
Decode(String),
|
||||
#[error("the file has no audio track")]
|
||||
NoTrack,
|
||||
#[error("the audio is too short to cut")]
|
||||
TooShort,
|
||||
}
|
||||
|
||||
/// Decode `bytes`, analyse the seams and build a core. `name` is used both as
|
||||
/// the display name and as the format hint (its extension).
|
||||
pub fn import_bytes(name: &str, seal: Seal, bytes: Vec<u8>) -> Result<Core, MaterialError> {
|
||||
let mut hasher = Hasher::new();
|
||||
hasher.update(&bytes);
|
||||
let digest = hasher.finalize();
|
||||
let id = format!("core-{}", &digest.to_hex()[..10]);
|
||||
|
||||
let (samples, sample_rate) = decode(name, bytes)?;
|
||||
if samples.len() < 1_000 {
|
||||
return Err(MaterialError::TooShort);
|
||||
}
|
||||
let (faces, face_summaries) = analyse(&samples, sample_rate);
|
||||
let peaks = waveform_peaks(&samples);
|
||||
let duration_sec = samples.len() as f32 / sample_rate as f32;
|
||||
let summary = CoreSummary {
|
||||
id,
|
||||
name: display_name(name),
|
||||
seal,
|
||||
sample_rate,
|
||||
duration_sec,
|
||||
faces: face_summaries,
|
||||
peaks,
|
||||
};
|
||||
Ok(Core {
|
||||
summary,
|
||||
samples: Arc::from(samples),
|
||||
faces,
|
||||
})
|
||||
}
|
||||
|
||||
fn display_name(name: &str) -> String {
|
||||
let base = std::path::Path::new(name)
|
||||
.file_stem()
|
||||
.and_then(|stem| stem.to_str())
|
||||
.unwrap_or("core")
|
||||
.trim();
|
||||
if base.is_empty() {
|
||||
"core".into()
|
||||
} else {
|
||||
base.chars().take(48).collect()
|
||||
}
|
||||
}
|
||||
|
||||
fn decode(name: &str, bytes: Vec<u8>) -> Result<(Vec<f32>, u32), MaterialError> {
|
||||
let source = MediaSourceStream::new(Box::new(Cursor::new(bytes)), Default::default());
|
||||
let mut hint = Hint::new();
|
||||
if let Some(extension) = std::path::Path::new(name)
|
||||
.extension()
|
||||
.and_then(|extension| extension.to_str())
|
||||
{
|
||||
hint.with_extension(extension);
|
||||
}
|
||||
let probed = symphonia::default::get_probe()
|
||||
.format(
|
||||
&hint,
|
||||
source,
|
||||
&FormatOptions::default(),
|
||||
&MetadataOptions::default(),
|
||||
)
|
||||
.map_err(|error| MaterialError::Decode(error.to_string()))?;
|
||||
let mut format = probed.format;
|
||||
let track = format.default_track().ok_or(MaterialError::NoTrack)?;
|
||||
let track_id = track.id;
|
||||
let codec_params = track.codec_params.clone();
|
||||
let sample_rate = codec_params
|
||||
.sample_rate
|
||||
.ok_or_else(|| MaterialError::Decode("missing sample rate".into()))?;
|
||||
let mut decoder = symphonia::default::get_codecs()
|
||||
.make(&codec_params, &DecoderOptions::default())
|
||||
.map_err(|error| MaterialError::Decode(error.to_string()))?;
|
||||
|
||||
let mut samples: Vec<f32> = Vec::with_capacity(48_000 * 8);
|
||||
loop {
|
||||
let packet = match format.next_packet() {
|
||||
Ok(packet) => packet,
|
||||
Err(SymphoniaError::IoError(_)) => break,
|
||||
Err(SymphoniaError::ResetRequired) => break,
|
||||
Err(error) => return Err(MaterialError::Decode(error.to_string())),
|
||||
};
|
||||
if packet.track_id() != track_id {
|
||||
continue;
|
||||
}
|
||||
match decoder.decode(&packet) {
|
||||
Ok(decoded) => {
|
||||
let spec = *decoded.spec();
|
||||
let channels = spec.channels.count().max(1);
|
||||
let mut buffer = SampleBuffer::<f32>::new(decoded.capacity() as u64, spec);
|
||||
buffer.copy_interleaved_ref(decoded);
|
||||
for frame in buffer.samples().chunks(channels) {
|
||||
let mono = frame.iter().sum::<f32>() / channels as f32;
|
||||
samples.push(mono.clamp(-1.0, 1.0));
|
||||
}
|
||||
}
|
||||
Err(SymphoniaError::DecodeError(_)) => continue,
|
||||
Err(error) => return Err(MaterialError::Decode(error.to_string())),
|
||||
}
|
||||
}
|
||||
if samples.is_empty() {
|
||||
return Err(MaterialError::Decode("no samples decoded".into()));
|
||||
}
|
||||
Ok((samples, sample_rate))
|
||||
}
|
||||
|
||||
fn analyse(samples: &[f32], sample_rate: u32) -> (Vec<AudioFace>, Vec<FaceSummary>) {
|
||||
let hop = (sample_rate as f32 / ONSET_HOPS_PER_SECOND).max(64.0) as usize;
|
||||
let energy: Vec<f32> = samples
|
||||
.chunks(hop)
|
||||
.map(|chunk| {
|
||||
chunk.iter().map(|sample| sample * sample).sum::<f32>() / chunk.len().max(1) as f32
|
||||
})
|
||||
.collect();
|
||||
let mut flux = vec![0.0f32; energy.len()];
|
||||
for index in 1..energy.len() {
|
||||
flux[index] = (energy[index] - energy[index - 1]).max(0.0);
|
||||
}
|
||||
let mean = flux.iter().sum::<f32>() / flux.len().max(1) as f32;
|
||||
let variance = flux
|
||||
.iter()
|
||||
.map(|value| (value - mean) * (value - mean))
|
||||
.sum::<f32>()
|
||||
/ flux.len().max(1) as f32;
|
||||
let threshold = mean + variance.sqrt() * 0.55;
|
||||
let min_gap = (sample_rate as f32 * 0.09 / hop as f32).max(1.0) as usize;
|
||||
|
||||
let mut candidates: Vec<(usize, f32)> = Vec::new();
|
||||
for index in 1..flux.len().saturating_sub(1) {
|
||||
let value = flux[index];
|
||||
if value > threshold && value >= flux[index - 1] && value > flux[index + 1] {
|
||||
candidates.push((index, value));
|
||||
}
|
||||
}
|
||||
candidates.sort_by(|a, b| b.1.total_cmp(&a.1));
|
||||
|
||||
let mut onsets: Vec<usize> = Vec::new();
|
||||
for (index, _) in candidates {
|
||||
if onsets.len() >= FACE_COUNT - 1 {
|
||||
break;
|
||||
}
|
||||
if onsets
|
||||
.iter()
|
||||
.all(|existing| existing.abs_diff(index) >= min_gap)
|
||||
{
|
||||
onsets.push(index);
|
||||
}
|
||||
}
|
||||
|
||||
let mut boundaries: Vec<usize> = onsets.into_iter().map(|index| index * hop).collect();
|
||||
boundaries.push(0);
|
||||
boundaries.push(samples.len());
|
||||
boundaries.sort_unstable();
|
||||
boundaries.dedup();
|
||||
while boundaries.len() < FACE_COUNT + 1 {
|
||||
let Some((split_at, _)) = boundaries
|
||||
.windows(2)
|
||||
.enumerate()
|
||||
.max_by_key(|(_, pair)| pair[1] - pair[0])
|
||||
else {
|
||||
break;
|
||||
};
|
||||
let pair = &boundaries[split_at..split_at + 2];
|
||||
let midpoint = pair[0] + (pair[1] - pair[0]) / 2;
|
||||
if midpoint == pair[0] || midpoint == pair[1] {
|
||||
break;
|
||||
}
|
||||
boundaries.insert(split_at + 1, midpoint);
|
||||
}
|
||||
if boundaries.len() > FACE_COUNT + 1 {
|
||||
boundaries.truncate(FACE_COUNT + 1);
|
||||
let last = boundaries.len() - 1;
|
||||
boundaries[last] = samples.len();
|
||||
}
|
||||
|
||||
let mut faces = Vec::with_capacity(FACE_COUNT);
|
||||
let mut summaries = Vec::with_capacity(FACE_COUNT);
|
||||
for index in 0..FACE_COUNT {
|
||||
let start = boundaries[index];
|
||||
let end = boundaries[index + 1].max(start + 1).min(samples.len());
|
||||
let segment = &samples[start..end];
|
||||
let rms = (segment.iter().map(|sample| sample * sample).sum::<f32>()
|
||||
/ segment.len().max(1) as f32)
|
||||
.sqrt();
|
||||
faces.push(AudioFace {
|
||||
index: index as u8,
|
||||
start: start as u32,
|
||||
end: end as u32,
|
||||
rms,
|
||||
});
|
||||
summaries.push(FaceSummary {
|
||||
index: index as u8,
|
||||
rank: 0,
|
||||
start_sec: start as f32 / sample_rate as f32,
|
||||
end_sec: end as f32 / sample_rate as f32,
|
||||
rms,
|
||||
});
|
||||
}
|
||||
|
||||
let mut ranked: Vec<usize> = (0..summaries.len()).collect();
|
||||
ranked.sort_by(|a, b| summaries[*b].rms.total_cmp(&summaries[*a].rms));
|
||||
for (rank, index) in ranked.into_iter().enumerate() {
|
||||
summaries[index].rank = rank as u8;
|
||||
}
|
||||
|
||||
(faces, summaries)
|
||||
}
|
||||
|
||||
fn waveform_peaks(samples: &[f32]) -> Vec<f32> {
|
||||
let mut peaks = vec![0.0f32; PEAK_BINS];
|
||||
let chunk = (samples.len() as f32 / PEAK_BINS as f32).ceil().max(1.0) as usize;
|
||||
for (index, frame) in samples.chunks(chunk).take(PEAK_BINS).enumerate() {
|
||||
peaks[index] = frame
|
||||
.iter()
|
||||
.fold(0.0f32, |peak, sample| peak.max(sample.abs()));
|
||||
}
|
||||
let maximum = peaks.iter().fold(0.0f32, |peak, value| peak.max(*value));
|
||||
if maximum > 1e-6 {
|
||||
for peak in &mut peaks {
|
||||
*peak = (*peak / maximum).min(1.0);
|
||||
}
|
||||
}
|
||||
peaks
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn click_track(seconds: f32, sample_rate: u32) -> Vec<u8> {
|
||||
let frames = (seconds * sample_rate as f32) as usize;
|
||||
let mut samples = vec![0.0f32; frames];
|
||||
let mut position = 0;
|
||||
while position < frames {
|
||||
for offset in 0..960 {
|
||||
if position + offset < frames {
|
||||
let envelope = 1.0 - offset as f32 / 960.0;
|
||||
samples[position + offset] = (offset as f32 * 0.12).sin() * envelope * 0.7;
|
||||
}
|
||||
}
|
||||
position += (sample_rate / 6) as usize;
|
||||
}
|
||||
let mut bytes = Vec::new();
|
||||
let spec = hound::WavSpec {
|
||||
channels: 1,
|
||||
sample_rate,
|
||||
bits_per_sample: 16,
|
||||
sample_format: hound::SampleFormat::Int,
|
||||
};
|
||||
let mut writer = hound::WavWriter::new(Cursor::new(&mut bytes), spec).unwrap();
|
||||
for sample in samples {
|
||||
writer
|
||||
.write_sample((sample.clamp(-1.0, 1.0) * i16::MAX as f32) as i16)
|
||||
.unwrap();
|
||||
}
|
||||
writer.finalize().unwrap();
|
||||
bytes
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn imports_a_click_track_into_sixteen_faces() {
|
||||
let bytes = click_track(2.0, 48_000);
|
||||
let core = import_bytes("clicks.wav", Seal::Found, bytes).unwrap();
|
||||
assert_eq!(core.summary.sample_rate, 48_000);
|
||||
assert_eq!(core.faces.len(), FACE_COUNT);
|
||||
assert_eq!(core.summary.faces.len(), FACE_COUNT);
|
||||
assert_eq!(core.summary.peaks.len(), PEAK_BINS);
|
||||
assert!(core.summary.duration_sec > 1.9);
|
||||
assert_eq!(core.summary.name, "clicks");
|
||||
assert_eq!(core.summary.seal, Seal::Found);
|
||||
assert!(core.faces.iter().all(|face| face.start < face.end));
|
||||
assert!(core.summary.faces.iter().all(|face| face.rank < 16));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn short_audio_is_rejected() {
|
||||
let bytes = click_track(0.01, 48_000);
|
||||
assert!(matches!(
|
||||
import_bytes("short.wav", Seal::Found, bytes),
|
||||
Err(MaterialError::TooShort)
|
||||
));
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user