1use crate::media::{Error, Result};
2use crate::opus::celt::CeltDecoder;
3use crate::opus::celt::tables::OVERLAP;
4use crate::opus::packet::{Bandwidth, Mode, Packet};
5use crate::opus::range::RangeDecoder;
6use crate::opus::silk::{LostFlag, SilkDecoder};
7
8pub const SAMPLE_RATES: [u32; 5] = [8000, 12000, 16000, 24000, 48000];
9
10pub struct StreamDecoder {
11 rate: u32,
12 channels: usize,
13 celt: CeltDecoder,
14 silk: SilkDecoder,
15 stream_channels: usize,
16 bandwidth: Bandwidth,
17 mode: Option<Mode>,
18 prev_mode: Option<Mode>,
19 frame_size: usize,
20 prev_redundancy: bool,
21 last_packet_duration: usize,
22 range_final: u32,
23 gain_q8: i16,
24 pcm_silk: Vec<f32>,
25}
26
27impl StreamDecoder {
28 pub fn new(rate: u32, channels: usize) -> Result<StreamDecoder> {
29 if !SAMPLE_RATES.contains(&rate) {
30 return Err(Error::Invalid(
31 "sample rate must be 8000, 12000, 16000, 24000 or 48000".into(),
32 ));
33 }
34 if !(1..=2).contains(&channels) {
35 return Err(Error::Invalid("channels must be 1 or 2".into()));
36 }
37 Ok(StreamDecoder {
38 rate,
39 channels,
40 celt: CeltDecoder::new(rate, channels),
41 silk: SilkDecoder::new(rate, channels),
42 stream_channels: channels,
43 bandwidth: Bandwidth::Fullband,
44 mode: None,
45 prev_mode: None,
46 frame_size: rate as usize / 400,
47 prev_redundancy: false,
48 last_packet_duration: 0,
49 range_final: 0,
50 gain_q8: 0,
51 pcm_silk: Vec::new(),
52 })
53 }
54
55 pub fn sample_rate(&self) -> u32 {
56 self.rate
57 }
58
59 pub fn channels(&self) -> usize {
60 self.channels
61 }
62
63 pub fn final_range(&self) -> u32 {
64 self.range_final
65 }
66
67 pub fn last_packet_duration(&self) -> usize {
68 self.last_packet_duration
69 }
70
71 pub fn set_gain_q8(&mut self, gain: i16) {
72 self.gain_q8 = gain;
73 }
74
75 pub fn set_phase_inversion_disabled(&mut self, disabled: bool) {
76 self.celt.disable_inv = disabled;
77 }
78
79 pub fn reset(&mut self) {
80 self.celt.reset();
81 self.silk.reset();
82 self.stream_channels = self.channels;
83 self.bandwidth = Bandwidth::Fullband;
84 self.mode = None;
85 self.prev_mode = None;
86 self.frame_size = self.rate as usize / 400;
87 self.prev_redundancy = false;
88 self.last_packet_duration = 0;
89 self.range_final = 0;
90 }
91
92 pub fn decode_interleaved(
93 &mut self,
94 packet: Option<&[u8]>,
95 out: &mut Vec<f32>,
96 ) -> Result<usize> {
97 match packet {
98 Some(p) if !p.is_empty() => self.decode_packet(p, out),
99 _ => {
100 let n = if self.last_packet_duration > 0 {
101 self.last_packet_duration
102 } else {
103 self.rate as usize / 50
104 };
105 self.conceal(n, out)
106 }
107 }
108 }
109
110 pub fn conceal(&mut self, n: usize, out: &mut Vec<f32>) -> Result<usize> {
111 let f2_5 = self.rate as usize / 400;
112 if n == 0 || !n.is_multiple_of(f2_5) {
113 return Err(Error::Invalid(
114 "concealment length must be a multiple of 2.5 ms".into(),
115 ));
116 }
117 let start = out.len();
118 out.resize(start + n * self.channels, 0.0);
119 let mut done = 0;
120 while done < n {
121 let r = self.decode_frame(
122 None,
123 &mut out[start + done * self.channels..],
124 n - done,
125 false,
126 )?;
127 done += r;
128 }
129 self.last_packet_duration = n;
130 Ok(n)
131 }
132
133 pub fn decode_fec(&mut self, packet: &[u8], n: usize, out: &mut Vec<f32>) -> Result<usize> {
134 let p = Packet::parse(packet)?;
135 let pfs = p.toc.frame_samples(self.rate);
136 if n < pfs || p.toc.mode == Mode::CeltOnly || self.mode == Some(Mode::CeltOnly) {
137 return self.conceal(n, out);
138 }
139 let start = out.len();
140 if n > pfs {
141 self.conceal(n - pfs, out)?;
142 }
143 self.mode = Some(p.toc.mode);
144 self.bandwidth = p.toc.bandwidth;
145 self.frame_size = pfs;
146 self.stream_channels = p.toc.channels();
147 let off = out.len();
148 out.resize(off + pfs * self.channels, 0.0);
149 self.decode_frame(Some(p.frames[0]), &mut out[off..], pfs, true)?;
150 self.last_packet_duration = n;
151 debug_assert_eq!(out.len() - start, n * self.channels);
152 Ok(n)
153 }
154
155 fn decode_packet(&mut self, data: &[u8], out: &mut Vec<f32>) -> Result<usize> {
156 let p = Packet::parse(data)?;
157 let fs = p.toc.frame_samples(self.rate);
158 self.mode = Some(p.toc.mode);
159 self.bandwidth = p.toc.bandwidth;
160 self.frame_size = fs;
161 self.stream_channels = p.toc.channels();
162 let start = out.len();
163 out.resize(start + fs * p.frames.len() * self.channels, 0.0);
164 let mut nb = 0;
165 for f in &p.frames {
166 let r =
167 self.decode_frame(Some(f), &mut out[start + nb * self.channels..], fs, false)?;
168 nb += r;
169 }
170 out.truncate(start + nb * self.channels);
171 self.last_packet_duration = nb;
172 Ok(nb)
173 }
174
175 fn decode_frame(
176 &mut self,
177 data: Option<&[u8]>,
178 pcm: &mut [f32],
179 frame_size: usize,
180 fec: bool,
181 ) -> Result<usize> {
182 let ch = self.channels;
183 let fs = self.rate as usize;
184 let f20 = fs / 50;
185 let f10 = f20 >> 1;
186 let f5 = f10 >> 1;
187 let f2_5 = f5 >> 1;
188 let mut frame_size = frame_size.min(fs / 25 * 3);
189 let data = match data {
190 Some(d) if d.len() > 1 => Some(d),
191 _ => {
192 frame_size = frame_size.min(self.frame_size);
193 None
194 }
195 };
196 let (mut audiosize, mode, bandwidth);
197 match data {
198 Some(_) => {
199 audiosize = self.frame_size;
200 mode = self.mode.unwrap_or(Mode::CeltOnly);
201 bandwidth = Some(self.bandwidth);
202 }
203 None => {
204 audiosize = frame_size;
205 bandwidth = None;
206 mode = match self.prev_mode {
207 None => {
208 pcm[..audiosize * ch].fill(0.0);
209 return Ok(audiosize);
210 }
211 Some(m) => m,
212 };
213 if audiosize > f20 {
214 let mut done = 0;
215 while done < audiosize {
216 let r = self.decode_frame(
217 None,
218 &mut pcm[done * ch..],
219 (audiosize - done).min(f20),
220 false,
221 )?;
222 done += r;
223 }
224 return Ok(frame_size);
225 } else if audiosize < f20 {
226 if audiosize > f10 {
227 audiosize = f10;
228 } else if mode != Mode::SilkOnly && audiosize > f5 && audiosize < f10 {
229 audiosize = f5;
230 }
231 }
232 }
233 }
234 let data_len = data.map_or(0, |d| d.len());
235 let mut dec_storage;
236 let mut dec: Option<&mut RangeDecoder> = match data {
237 Some(d) => {
238 dec_storage = RangeDecoder::new(d);
239 Some(&mut dec_storage)
240 }
241 None => None,
242 };
243 let mut transition = false;
244 if data.is_some()
245 && self.prev_mode.is_some()
246 && ((mode == Mode::CeltOnly
247 && self.prev_mode != Some(Mode::CeltOnly)
248 && !self.prev_redundancy)
249 || (mode != Mode::CeltOnly && self.prev_mode == Some(Mode::CeltOnly)))
250 {
251 transition = true;
252 }
253 let mut pcm_transition = Vec::new();
254 if transition && mode == Mode::CeltOnly {
255 pcm_transition.resize(f5 * ch, 0.0);
256 let n = f5.min(audiosize);
257 self.decode_frame(None, &mut pcm_transition, n, false)?;
258 }
259 if audiosize > frame_size {
260 return Err(Error::Invalid("frame larger than output buffer".into()));
261 }
262 frame_size = audiosize;
263
264 let mut pcm_silk = std::mem::take(&mut self.pcm_silk);
265 pcm_silk.clear();
266 if mode != Mode::CeltOnly {
267 pcm_silk.resize(frame_size.max(f10) * ch, 0.0);
268 if self.prev_mode == Some(Mode::CeltOnly) {
269 self.silk.reset();
270 }
271 let payload_ms = (1000 * audiosize / fs).max(10);
272 let internal_rate = if mode == Mode::SilkOnly {
273 match bandwidth {
274 Some(Bandwidth::Narrowband) => 8000,
275 Some(Bandwidth::Mediumband) => 12000,
276 _ => 16000,
277 }
278 } else {
279 16000
280 };
281 if data.is_some() {
282 self.silk.set_stream(self.stream_channels, internal_rate);
283 }
284 let lost = if data.is_none() {
285 LostFlag::Lost
286 } else if fec {
287 LostFlag::Lbrr
288 } else {
289 LostFlag::Normal
290 };
291 let mut decoded = 0;
292 while decoded < frame_size {
293 let first = decoded == 0;
294 let n = match self.silk.decode(
295 dec.as_deref_mut(),
296 lost,
297 first,
298 payload_ms,
299 &mut pcm_silk[decoded * ch..],
300 ) {
301 Ok(n) => n,
302 Err(e) => {
303 if lost != LostFlag::Normal {
304 let n = frame_size - decoded;
305 pcm_silk[decoded * ch..(decoded + n) * ch].fill(0.0);
306 n
307 } else {
308 return Err(e);
309 }
310 }
311 };
312 if n == 0 {
313 break;
314 }
315 decoded += n;
316 }
317 }
318
319 let mut start_band = 0;
320 let mut redundancy = false;
321 let mut celt_to_silk = false;
322 let mut redundancy_bytes = 0usize;
323 let mut len = data_len;
324 if !fec
325 && mode != Mode::CeltOnly
326 && let Some(d) = dec.as_deref_mut()
327 && d.tell() + 17 + 20 * (self.mode == Some(Mode::Hybrid)) as i32 <= 8 * len as i32
328 {
329 redundancy = if mode == Mode::Hybrid {
330 d.bit_logp(12)
331 } else {
332 true
333 };
334 if redundancy {
335 celt_to_silk = d.bit_logp(1);
336 redundancy_bytes = if mode == Mode::Hybrid {
337 d.uint(256) as usize + 2
338 } else {
339 len - ((d.tell() as usize + 7) >> 3)
340 };
341 if redundancy_bytes > len || (len - redundancy_bytes) * 8 < d.tell() as usize {
342 len = 0;
343 redundancy_bytes = 0;
344 redundancy = false;
345 } else {
346 len -= redundancy_bytes;
347 }
348 d.shrink_storage(redundancy_bytes);
349 }
350 }
351 if mode != Mode::CeltOnly {
352 start_band = 17;
353 }
354 if redundancy {
355 transition = false;
356 }
357 if transition && mode != Mode::CeltOnly {
358 pcm_transition.resize(f5 * ch, 0.0);
359 let n = f5.min(audiosize);
360 self.decode_frame(None, &mut pcm_transition, n, false)?;
361 }
362 if let Some(bw) = bandwidth {
363 self.celt.end = bw.celt_end_band();
364 }
365 self.celt.stream_channels = self.stream_channels;
366 let mut redundant_audio = vec![0f32; if redundancy { f5 * ch } else { 0 }];
367 let mut redundant_rng = 0u32;
368 let red_data = data.map(|d| &d[len..len + redundancy_bytes]);
369 if redundancy && celt_to_silk {
370 self.celt.start = 0;
371 self.celt
372 .decode(red_data, f5, &mut redundant_audio, false, None)?;
373 redundant_rng = self.celt.rng;
374 }
375 self.celt.start = start_band;
376
377 if mode != Mode::SilkOnly {
378 let celt_frame = f20.min(frame_size);
379 if self.prev_mode.is_some() && Some(mode) != self.prev_mode && !self.prev_redundancy {
380 self.celt.reset();
381 }
382 let cdata = if fec { None } else { data.map(|d| &d[..len]) };
383 let cdec = if fec { None } else { dec.as_deref_mut() };
384 self.celt.decode(cdata, celt_frame, pcm, false, cdec)?;
385 } else {
386 pcm[..frame_size * ch].fill(0.0);
387 if self.prev_mode == Some(Mode::Hybrid)
388 && !(redundancy && celt_to_silk && self.prev_redundancy)
389 {
390 self.celt.start = 0;
391 let silence = [0xFFu8, 0xFF];
392 self.celt.decode(Some(&silence), f2_5, pcm, true, None)?;
393 }
394 }
395 if mode != Mode::CeltOnly {
396 for (o, s) in pcm[..frame_size * ch].iter_mut().zip(&pcm_silk) {
397 *o += *s;
398 }
399 }
400 let window = &crate::opus::celt::mode().window;
401 if redundancy && !celt_to_silk {
402 self.celt.reset();
403 self.celt.start = 0;
404 self.celt
405 .decode(red_data, f5, &mut redundant_audio, false, None)?;
406 redundant_rng = self.celt.rng;
407 let o = ch * (frame_size - f2_5);
408 let tail: Vec<f32> = pcm[o..o + f2_5 * ch].to_vec();
409 smooth_fade(
410 &tail,
411 &redundant_audio[ch * f2_5..],
412 &mut pcm[o..],
413 f2_5,
414 ch,
415 window,
416 self.rate,
417 );
418 }
419 if redundancy && celt_to_silk {
420 pcm[..f2_5 * ch].copy_from_slice(&redundant_audio[..f2_5 * ch]);
421 let cur: Vec<f32> = pcm[ch * f2_5..ch * 2 * f2_5].to_vec();
422 smooth_fade(
423 &redundant_audio[ch * f2_5..],
424 &cur,
425 &mut pcm[ch * f2_5..],
426 f2_5,
427 ch,
428 window,
429 self.rate,
430 );
431 }
432 if transition {
433 if audiosize >= f5 {
434 pcm[..f2_5 * ch].copy_from_slice(&pcm_transition[..f2_5 * ch]);
435 let cur: Vec<f32> = pcm[ch * f2_5..ch * 2 * f2_5].to_vec();
436 smooth_fade(
437 &pcm_transition[ch * f2_5..],
438 &cur,
439 &mut pcm[ch * f2_5..],
440 f2_5,
441 ch,
442 window,
443 self.rate,
444 );
445 } else {
446 let cur: Vec<f32> = pcm[..ch * f2_5].to_vec();
447 smooth_fade(&pcm_transition, &cur, pcm, f2_5, ch, window, self.rate);
448 }
449 }
450 if self.gain_q8 != 0 {
451 let g = (6.488_141e-4f32 * self.gain_q8 as f32).exp2();
452 for v in pcm[..frame_size * ch].iter_mut() {
453 *v *= g;
454 }
455 }
456 self.range_final = match dec.as_deref() {
457 Some(d) if data_len > 1 => d.rng() ^ redundant_rng,
458 _ => 0,
459 };
460 self.prev_mode = Some(mode);
461 self.prev_redundancy = redundancy && !celt_to_silk;
462 self.pcm_silk = pcm_silk;
463 Ok(audiosize)
464 }
465}
466
467fn smooth_fade(
468 in1: &[f32],
469 in2: &[f32],
470 out: &mut [f32],
471 overlap: usize,
472 ch: usize,
473 window: &[f32; OVERLAP],
474 rate: u32,
475) {
476 let inc = (48000 / rate) as usize;
477 for c in 0..ch {
478 for i in 0..overlap {
479 let w = window[i * inc] * window[i * inc];
480 out[i * ch + c] = w * in2[i * ch + c] + (1.0 - w) * in1[i * ch + c];
481 }
482 }
483}