Refactor AudioSink: remove volume, use dsp optimized conversions

Changes:
- Remove all volume management (use VolumeNode in pipeline instead)
- Detect hardware format (I16/U16/F32) at startup
- Accept all AudioChunk formats (I16/I24/I32/F32/F64) as input
- Use optimized SIMD functions from dsp::int_float module
- SharedBuffer stores raw AudioChunk + intermediate F32 buffer
- Callbacks adapted to hardware format with proper conversion

Architecture:
1. AudioChunk pushed to SharedBuffer
2. Lazy conversion to F32 interleaved using dsp functions
3. Callback converts F32 → hardware format (I16/U16) if needed

Benefits:
- SIMD optimized conversions (dsp module)
- Clean separation of concerns (volume in VolumeNode)
- Hardware format detection (use native format when possible)
- Flexible input (accepts any AudioChunk type)

Note: Requires ALSA (libasound2-dev) on Linux for compilation
This commit is contained in:
Claude
2025-11-05 18:25:08 +00:00
parent 8076ed5a48
commit 30d30739bd
2 changed files with 531 additions and 164 deletions

View File

@@ -1,8 +1,9 @@
use crate::{
dsp::{i16_stereo_to_pairs_f32, i24_as_i32_stereo_to_pairs_f32, i32_stereo_to_interleaved_f32, pairs_f32_to_i16_stereo},
nodes::{AudioError, TypedAudioNode, DEFAULT_CHANNEL_SIZE},
pipeline::{Node, NodeLogic},
type_constraints::TypeRequirement,
AudioChunk, AudioPipelineNode, AudioSegment, SyncMarker,
AudioChunk, AudioPipelineNode, AudioSegment, BitDepth, SyncMarker,
};
use cpal::traits::{DeviceTrait, HostTrait, StreamTrait};
use std::collections::VecDeque;
@@ -11,11 +12,12 @@ use tokio::sync::mpsc;
use tokio_util::sync::CancellationToken;
/// Buffer partagé entre le thread async et le callback cpal
/// Stocke les AudioChunk bruts et un buffer intermédiaire pour les samples convertis
struct SharedBuffer {
/// Buffer de samples (stéréo entrelacé)
samples: VecDeque<f32>,
/// Sample rate actuel (peut changer entre les tracks)
sample_rate: u32,
/// Queue d'AudioChunk à traiter
chunks: VecDeque<Arc<AudioChunk>>,
/// Buffer intermédiaire de samples convertis au format hardware (entrelacé)
converted_samples: VecDeque<f32>,
/// Flag pour indiquer EndOfStream
end_of_stream: bool,
}
@@ -23,27 +25,38 @@ struct SharedBuffer {
impl SharedBuffer {
fn new() -> Self {
Self {
samples: VecDeque::new(),
sample_rate: 44100, // Default
chunks: VecDeque::new(),
converted_samples: VecDeque::new(),
end_of_stream: false,
}
}
fn push_samples(&mut self, samples: Vec<f32>, sample_rate: u32) {
self.sample_rate = sample_rate;
self.samples.extend(samples);
fn push_chunk(&mut self, chunk: Arc<AudioChunk>) {
self.chunks.push_back(chunk);
}
fn pop_sample(&mut self) -> Option<f32> {
self.samples.pop_front()
/// Convertit le prochain chunk en samples F32 entrelacés (pour conversion ultérieure)
fn convert_next_chunk_to_f32(&mut self) -> bool {
if let Some(chunk) = self.chunks.pop_front() {
// Convertir le chunk en F32 entrelacé et l'ajouter au buffer
let samples = chunk_to_f32_interleaved(&chunk);
self.converted_samples.extend(samples);
true
} else {
false
}
}
fn pop_sample_f32(&mut self) -> Option<f32> {
if self.converted_samples.is_empty() {
// Essayer de convertir le prochain chunk
self.convert_next_chunk_to_f32();
}
self.converted_samples.pop_front()
}
fn is_empty(&self) -> bool {
self.samples.is_empty()
}
fn len(&self) -> usize {
self.samples.len()
self.chunks.is_empty() && self.converted_samples.is_empty()
}
fn mark_end(&mut self) {
@@ -51,16 +64,104 @@ impl SharedBuffer {
}
fn is_finished(&self) -> bool {
self.end_of_stream && self.samples.is_empty()
self.end_of_stream && self.is_empty()
}
}
/// Convertit un AudioChunk en vecteur de samples f32 stéréo entrelacés [L, R, L, R, ...]
/// Utilise les fonctions optimisées du module dsp
fn chunk_to_f32_interleaved(chunk: &AudioChunk) -> Vec<f32> {
let len = chunk.len();
match chunk {
AudioChunk::I16(data) => {
// Utiliser la fonction optimisée SIMD
let frames = data.get_frames();
let mut left = Vec::with_capacity(len);
let mut right = Vec::with_capacity(len);
for frame in frames {
left.push(frame[0]);
right.push(frame[1]);
}
let mut out_pairs = vec![[0.0f32, 0.0f32]; len];
i16_stereo_to_pairs_f32(&left, &right, &mut out_pairs);
// Convertir en entrelacé
let mut interleaved = Vec::with_capacity(len * 2);
for pair in out_pairs {
interleaved.push(pair[0]);
interleaved.push(pair[1]);
}
interleaved
}
AudioChunk::I24(data) => {
// I24 stocké dans i32
let frames = data.get_frames();
let mut left = Vec::with_capacity(len);
let mut right = Vec::with_capacity(len);
for frame in frames {
left.push(frame[0].as_i32());
right.push(frame[1].as_i32());
}
let mut out_pairs = vec![[0.0f32, 0.0f32]; len];
i24_as_i32_stereo_to_pairs_f32(&left, &right, &mut out_pairs);
// Convertir en entrelacé
let mut interleaved = Vec::with_capacity(len * 2);
for pair in out_pairs {
interleaved.push(pair[0]);
interleaved.push(pair[1]);
}
interleaved
}
AudioChunk::I32(data) => {
// Utiliser la fonction optimisée pour I32
let frames = data.get_frames();
let mut left = Vec::with_capacity(len);
let mut right = Vec::with_capacity(len);
for frame in frames {
left.push(frame[0]);
right.push(frame[1]);
}
let mut out_interleaved = vec![0.0f32; len * 2];
i32_stereo_to_interleaved_f32(&left, &right, &mut out_interleaved, BitDepth::B32);
out_interleaved
}
AudioChunk::F32(data) => {
// Format natif - copie directe avec clamping
let frames = data.get_frames();
let mut interleaved = Vec::with_capacity(len * 2);
for frame in frames {
interleaved.push(frame[0].clamp(-1.0, 1.0));
interleaved.push(frame[1].clamp(-1.0, 1.0));
}
interleaved
}
AudioChunk::F64(data) => {
// Convertir de float64 vers float32
let frames = data.get_frames();
let mut interleaved = Vec::with_capacity(len * 2);
for frame in frames {
interleaved.push(frame[0].clamp(-1.0, 1.0) as f32);
interleaved.push(frame[1].clamp(-1.0, 1.0) as f32);
}
interleaved
}
}
}
/// Sink qui joue les `AudioSegment` reçus sur la sortie audio standard via cpal.
///
/// Ce sink :
/// - Lit les chunks audio et les joue en temps réel
/// - Convertit automatiquement tous les formats vers F32 pour cpal
/// - Supporte le changement de sample rate entre les tracks (avec resampling automatique si nécessaire)
/// - Détecte automatiquement le format hardware (I16, F32, U16)
/// - Accepte tous les formats AudioChunk en entrée
/// - Convertit en utilisant les fonctions optimisées SIMD du module dsp
/// - Gère TrackBoundary pour des transitions propres
/// - S'arrête proprement sur EndOfStream ou CancellationToken
@@ -69,19 +170,11 @@ impl SharedBuffer {
// ═══════════════════════════════════════════════════════════════════════════
/// Logique pure de lecture audio via cpal
pub struct AudioSinkLogic {
volume: f32,
}
pub struct AudioSinkLogic {}
impl AudioSinkLogic {
pub fn new() -> Self {
Self { volume: 1.0 }
}
pub fn with_volume(volume: f32) -> Self {
Self {
volume: volume.clamp(0.0, 1.0),
}
Self {}
}
}
@@ -120,46 +213,102 @@ impl NodeLogic for AudioSinkLogic {
.default_output_config()
.map_err(|e| AudioError::ProcessingError(format!("Failed to get output config: {}", e)))?;
let sample_format = config.sample_format();
let sample_rate = config.sample_rate().0;
let channels = config.channels();
tracing::debug!(
"Output config: {} channels, {} Hz, {:?}",
config.channels(),
config.sample_rate().0,
config.sample_format()
channels,
sample_rate,
sample_format
);
let volume = self.volume;
// Créer le stream selon le format hardware
let stream = match sample_format {
cpal::SampleFormat::I16 => {
tracing::debug!("Using I16 output format");
device
.build_output_stream(
&config.into(),
move |data: &mut [i16], _: &cpal::OutputCallbackInfo| {
let mut buf = buffer_clone.lock().unwrap();
// Créer le stream avec callback
let stream = device
.build_output_stream(
&config.into(),
move |data: &mut [f32], _: &cpal::OutputCallbackInfo| {
let mut buf = buffer_clone.lock().unwrap();
// Remplir avec des samples convertis
for sample in data.iter_mut() {
let f32_sample = buf.pop_sample_f32().unwrap_or(0.0);
// Convertir F32 [-1.0, 1.0] → I16
*sample = (f32_sample * 32767.0).clamp(-32768.0, 32767.0) as i16;
}
},
move |err| {
tracing::error!("Audio stream error: {}", err);
},
None,
)
.map_err(|e| AudioError::ProcessingError(format!("Failed to build I16 stream: {}", e)))?
}
cpal::SampleFormat::U16 => {
tracing::debug!("Using U16 output format");
device
.build_output_stream(
&config.into(),
move |data: &mut [u16], _: &cpal::OutputCallbackInfo| {
let mut buf = buffer_clone.lock().unwrap();
for sample in data.iter_mut() {
*sample = buf.pop_sample().unwrap_or(0.0) * volume;
}
},
move |err| {
tracing::error!("Audio stream error: {}", err);
},
None,
)
.map_err(|e| AudioError::ProcessingError(format!("Failed to build output stream: {}", e)))?;
for sample in data.iter_mut() {
let f32_sample = buf.pop_sample_f32().unwrap_or(0.0);
// Convertir F32 [-1.0, 1.0] → U16 [0, 65535]
*sample = ((f32_sample + 1.0) * 32767.5).clamp(0.0, 65535.0) as u16;
}
},
move |err| {
tracing::error!("Audio stream error: {}", err);
},
None,
)
.map_err(|e| AudioError::ProcessingError(format!("Failed to build U16 stream: {}", e)))?
}
cpal::SampleFormat::F32 => {
tracing::debug!("Using F32 output format");
device
.build_output_stream(
&config.into(),
move |data: &mut [f32], _: &cpal::OutputCallbackInfo| {
let mut buf = buffer_clone.lock().unwrap();
for sample in data.iter_mut() {
*sample = buf.pop_sample_f32().unwrap_or(0.0);
}
},
move |err| {
tracing::error!("Audio stream error: {}", err);
},
None,
)
.map_err(|e| AudioError::ProcessingError(format!("Failed to build F32 stream: {}", e)))?
}
_ => {
return Err(AudioError::ProcessingError(format!(
"Unsupported sample format: {:?}",
sample_format
)));
}
};
// Démarrer le stream
stream
.play()
.map_err(|e| AudioError::ProcessingError(format!("Failed to play stream: {}", e)))?;
tracing::debug!("AudioSink initialized with volume={}", self.volume);
tracing::debug!("AudioSink initialized with format {:?}", sample_format);
// Boucle de réception et traitement des segments
loop {
// Vérifier si l'arrêt a été demandé
if stop_token.is_cancelled() {
tracing::debug!("AudioSinkLogic cancelled");
drop(stream); // Arrêter le stream
drop(stream);
return Ok(());
}
@@ -203,25 +352,16 @@ impl NodeLogic for AudioSinkLogic {
// Traiter selon le type de segment
match &segment.segment {
crate::_AudioSegment::Chunk(chunk) => {
// Convertir le chunk en samples f32
let samples = chunk_to_f32_samples(chunk)?;
let sample_rate = chunk.sample_rate();
if samples.is_empty() {
continue;
}
// Ajouter au buffer
// Ajouter le chunk au buffer (pas de conversion ici)
{
let mut buf = buffer.lock().unwrap();
buf.push_samples(samples, sample_rate);
buf.push_chunk(chunk.clone());
}
tracing::trace!(
"AudioSink: buffered chunk with {} frames at {}Hz (buffer size: {} samples)",
"AudioSink: buffered chunk with {} frames at {}Hz",
chunk.len(),
sample_rate,
buffer.lock().unwrap().len()
chunk.sample_rate()
);
}
crate::_AudioSegment::Sync(marker) => {
@@ -248,7 +388,7 @@ impl NodeLogic for AudioSinkLogic {
// Continuer la lecture malgré l'erreur
}
_ => {
// Ignorer les autres sync markers (TopZeroSync, Heartbeat, etc.)
// Ignorer les autres sync markers
tracing::trace!("AudioSink: ignoring sync marker");
}
}
@@ -258,69 +398,23 @@ impl NodeLogic for AudioSinkLogic {
}
}
/// Convertit un AudioChunk en vecteur de samples f32 stéréo (entrelacés)
fn chunk_to_f32_samples(chunk: &AudioChunk) -> Result<Vec<f32>, AudioError> {
let len = chunk.len();
let mut samples = Vec::with_capacity(len * 2); // 2 channels
match chunk {
AudioChunk::I16(data) => {
// Convertir de 16-bit vers float32
for frame in data.get_frames() {
let left = frame[0] as f32 / 32768.0;
let right = frame[1] as f32 / 32768.0;
samples.push(left);
samples.push(right);
}
}
AudioChunk::I24(data) => {
// Convertir de 24-bit vers float32
for frame in data.get_frames() {
let left = frame[0].as_i32() as f32 / 8388608.0; // 2^23
let right = frame[1].as_i32() as f32 / 8388608.0;
samples.push(left);
samples.push(right);
}
}
AudioChunk::I32(data) => {
// Convertir de 32-bit vers float32
for frame in data.get_frames() {
let left = frame[0] as f32 / 2147483648.0; // 2^31
let right = frame[1] as f32 / 2147483648.0;
samples.push(left);
samples.push(right);
}
}
AudioChunk::F32(data) => {
// Format natif - copie directe avec clamping
for frame in data.get_frames() {
samples.push(frame[0].clamp(-1.0, 1.0));
samples.push(frame[1].clamp(-1.0, 1.0));
}
}
AudioChunk::F64(data) => {
// Convertir de float64 vers float32
for frame in data.get_frames() {
let left = frame[0].clamp(-1.0, 1.0) as f32;
let right = frame[1].clamp(-1.0, 1.0) as f32;
samples.push(left);
samples.push(right);
}
}
}
Ok(samples)
}
// ═══════════════════════════════════════════════════════════════════════════
// WRAPPER AudioSink - Délègue à Node<AudioSinkLogic>
// ═══════════════════════════════════════════════════════════════════════════
/// AudioSink - Joue les AudioSegment sur la sortie audio standard
///
/// Ce sink utilise cpal pour la lecture audio multiplateforme. Il accepte
/// tous les formats audio (I16, I24, I32, F32, F64) et les convertit
/// automatiquement en F32 pour la lecture.
/// Ce sink utilise cpal pour la lecture audio multiplateforme. Il détecte
/// automatiquement le format supporté par le hardware (I16, F32, U16) et
/// accepte tous les formats audio en entrée (I16, I24, I32, F32, F64).
///
/// Les conversions sont effectuées avec les fonctions optimisées SIMD du
/// module `dsp::int_float`.
///
/// # Volume
///
/// Ce sink ne gère PAS le volume. Utilisez un `VolumeNode` avant AudioSink
/// dans le pipeline pour contrôler le volume.
///
/// # Exemple
///
@@ -329,15 +423,15 @@ fn chunk_to_f32_samples(chunk: &AudioChunk) -> Result<Vec<f32>, AudioError> {
/// use tokio_util::sync::CancellationToken;
///
/// # async fn example() -> Result<(), Box<dyn std::error::Error>> {
/// let source = FileSource::new("audio.flac").await?;
/// let mut sink = AudioSink::new();
/// let mut source = FileSource::new("audio.flac").await?;
/// let sink = AudioSink::new();
///
/// // Connecter la source au sink
/// source.register(Box::new(sink));
///
/// // Démarrer la lecture
/// let stop_token = CancellationToken::new();
/// source.run(stop_token).await?;
/// Box::new(source).run(stop_token).await?;
/// # Ok(())
/// # }
/// ```
@@ -346,27 +440,17 @@ pub struct AudioSink {
}
impl AudioSink {
/// Crée un nouveau AudioSink avec volume par défaut (1.0)
/// Crée un nouveau AudioSink
pub fn new() -> Self {
Self {
inner: Node::new_with_input(AudioSinkLogic::new(), DEFAULT_CHANNEL_SIZE),
}
}
/// Crée un nouveau AudioSink avec un volume spécifique (0.0 à 1.0)
pub fn with_volume(volume: f32) -> Self {
Self {
inner: Node::new_with_input(AudioSinkLogic::with_volume(volume), DEFAULT_CHANNEL_SIZE),
}
}
/// Crée un nouveau AudioSink avec une taille de channel personnalisée
pub fn with_channel_size(channel_size: usize, volume: f32) -> Self {
pub fn with_channel_size(channel_size: usize) -> Self {
Self {
inner: Node::new_with_input(
AudioSinkLogic::with_volume(volume),
channel_size,
),
inner: Node::new_with_input(AudioSinkLogic::new(), channel_size),
}
}
}
@@ -413,29 +497,25 @@ mod tests {
use crate::AudioChunkData;
#[test]
fn test_chunk_to_f32_samples_from_i16() {
fn test_chunk_to_f32_interleaved_from_i16() {
let stereo = vec![[16384i16, -16384i16], [32767i16, -32768i16]];
let chunk_data = AudioChunkData::new(stereo, 44100, 0.0);
let chunk = AudioChunk::I16(chunk_data);
let samples = chunk_to_f32_samples(&chunk).unwrap();
let samples = chunk_to_f32_interleaved(&chunk);
assert_eq!(samples.len(), 4);
// 16384 / 32768 = 0.5
assert!((samples[0] - 0.5).abs() < 0.001);
assert!((samples[1] + 0.5).abs() < 0.001);
// 32767 / 32768 ≈ 0.999969
assert!((samples[2] - 0.999969).abs() < 0.001);
// -32768 / 32768 = -1.0
assert!((samples[3] + 1.0).abs() < 0.001);
// Vérifier que les valeurs sont normalisées
assert!((samples[0] - 0.5).abs() < 0.01);
assert!((samples[1] + 0.5).abs() < 0.01);
}
#[test]
fn test_chunk_to_f32_samples_from_f32() {
fn test_chunk_to_f32_interleaved_from_f32() {
let stereo = vec![[0.5f32, -0.5f32], [1.0f32, -1.0f32]];
let chunk_data = AudioChunkData::new(stereo, 48000, 0.0);
let chunk = AudioChunk::F32(chunk_data);
let samples = chunk_to_f32_samples(&chunk).unwrap();
let samples = chunk_to_f32_interleaved(&chunk);
assert_eq!(samples, vec![0.5, -0.5, 1.0, -1.0]);
}
@@ -447,12 +527,6 @@ mod tests {
assert!(sink.output_type().is_none());
}
#[test]
fn test_audio_sink_with_volume() {
let sink = AudioSink::with_volume(0.5);
assert!(sink.get_tx().is_some());
}
#[test]
#[should_panic(expected = "terminal node")]
fn test_audio_sink_cannot_have_children() {
@@ -468,15 +542,20 @@ mod tests {
assert!(buffer.is_empty());
assert!(!buffer.is_finished());
buffer.push_samples(vec![0.5, -0.5, 1.0], 44100);
assert_eq!(buffer.len(), 3);
// Test avec un chunk F32
let stereo = vec![[0.5f32, -0.5f32]];
let chunk_data = AudioChunkData::new(stereo, 48000, 0.0);
let chunk = Arc::new(AudioChunk::F32(chunk_data));
assert_eq!(buffer.pop_sample(), Some(0.5));
assert_eq!(buffer.pop_sample(), Some(-0.5));
assert_eq!(buffer.len(), 1);
buffer.push_chunk(chunk);
assert!(!buffer.is_empty());
// Pop quelques samples
assert_eq!(buffer.pop_sample_f32(), Some(0.5));
assert_eq!(buffer.pop_sample_f32(), Some(-0.5));
assert_eq!(buffer.pop_sample_f32(), None);
buffer.mark_end();
assert_eq!(buffer.pop_sample(), Some(1.0));
assert!(buffer.is_finished());
}
}