pito_codecs/media/front/
video.rs

1use std::io::Read;
2use std::path::{Path, PathBuf};
3use std::sync::Arc;
4
5use crate::gpu::{GpuFrame, GpuScaler};
6use crate::mp4::Reader;
7
8use super::demux::Demux;
9use super::ffmpeg::{Child, Ffmpeg, pix_fmt};
10use super::native;
11use super::picture::{Color, even, fit_width, resize};
12use super::probe::{Probe, probe};
13use super::route::{Caps, Choice, Prefs, pick_decode};
14use crate::media::{Error, Frame, PixelFormat, Result, Route, TimeBase};
15
16#[derive(Clone)]
17pub struct Source {
18    pub path: PathBuf,
19    pub probe: Probe,
20    pub route: Route,
21    pub reasons: Vec<String>,
22    pub ffmpeg: Option<Ffmpeg>,
23    pub ffmpeg_codec: Option<String>,
24    pub demux: Option<Demux>,
25}
26
27impl std::fmt::Debug for Video {
28    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
29        write!(f, "Video({} {}x{})", self.route, self.width, self.height)
30    }
31}
32
33pub fn source(path: &Path, prefs: &Prefs) -> Result<Source> {
34    let probe = probe(path, prefs)?;
35    let caps = Caps::detect(prefs);
36    let demux = Demux::open(path).ok();
37    let (mut route, mut reasons, mut ffmpeg_codec) = match probe.video_codec {
38        Some(codec) => {
39            let p = pick_decode(codec, prefs, &caps)?;
40            (p.route, p.reasons, p.ffmpeg_codec)
41        }
42        None => match (&prefs.decode, &caps.ffmpeg) {
43            (Choice::Ffmpeg(u), Some(_)) => (Route::Ffmpeg, Vec::new(), u.codec.clone()),
44            _ => {
45                return Err(Error::Unsupported(format!(
46                    "{} video decodes only through ffmpeg picked in the settings",
47                    probe.video_codec_name
48                )));
49            }
50        },
51    };
52    if route != Route::Ffmpeg && demux.is_none() {
53        match (&caps.ffmpeg, &prefs.decode) {
54            (Some(_), Choice::Ffmpeg(u)) => {
55                reasons.push("our MP4 and MKV readers can't open this file".to_string());
56                route = Route::Ffmpeg;
57                ffmpeg_codec = u.codec.clone();
58            }
59            _ => {
60                return Err(Error::Unsupported(
61                    "our MP4 and MKV readers can't open this file; pick ffmpeg in the settings to read it"
62                        .to_string(),
63                ));
64            }
65        }
66    }
67    Ok(Source {
68        path: path.to_path_buf(),
69        probe,
70        route,
71        reasons,
72        ffmpeg: caps.ffmpeg,
73        ffmpeg_codec,
74        demux,
75    })
76}
77
78impl Source {
79    pub fn ffmpeg(&self) -> Result<&Ffmpeg> {
80        self.ffmpeg
81            .as_ref()
82            .ok_or_else(|| Error::Unsupported("ffmpeg is not picked in the settings".to_string()))
83    }
84
85    pub fn native(&self) -> Error {
86        Error::Unsupported(format!(
87            "the {} route for {} is not in this build",
88            self.route, self.probe.video_codec_name
89        ))
90    }
91
92    pub fn demux(&self) -> Result<&Demux> {
93        self.demux.as_ref().ok_or_else(|| {
94            Error::Unsupported("our MP4 and MKV readers can't open this file".to_string())
95        })
96    }
97
98    pub fn mp4(&self) -> Option<&Arc<Reader>> {
99        self.demux.as_ref().and_then(Demux::mp4)
100    }
101}
102
103enum Inner {
104    Ffmpeg { child: Child, next: i64, got: u64 },
105    Native(Box<native::Frames>, (u32, u32)),
106}
107
108pub struct Video {
109    inner: Inner,
110    pub route: Route,
111    pub reasons: Vec<String>,
112    pub width: u32,
113    pub height: u32,
114    pub format: PixelFormat,
115    pub time_base: TimeBase,
116    left: Option<u64>,
117}
118
119pub fn decode_video(path: &Path, from: f64, height: Option<u32>, prefs: &Prefs) -> Result<Video> {
120    let src = source(path, prefs)?;
121    let p = &src.probe;
122    let size = height
123        .filter(|h| *h > 0 && p.height > 0 && *h != p.height)
124        .map(|h| (fit_width(p.width, p.height, h), h));
125    open(&src, from, None, size, None)
126}
127
128pub fn open(
129    src: &Source,
130    from: f64,
131    frames: Option<u64>,
132    size: Option<(u32, u32)>,
133    fps: Option<(u32, u32)>,
134) -> Result<Video> {
135    let p = &src.probe;
136    match src.route {
137        Route::Cpu | Route::Gpu => {
138            let decoded = native::Frames::open(src.demux()?.clone(), from, src.route)?;
139            let mut reasons = src.reasons.clone();
140            reasons.extend(decoded.reason.clone());
141            return Ok(native_video(
142                decoded,
143                reasons,
144                size.unwrap_or((p.width, p.height)),
145                fps.unwrap_or(p.fps),
146                format_of(p),
147                frames,
148            ));
149        }
150        Route::Ffmpeg => {}
151    }
152    let f = src.ffmpeg()?;
153    let format = format_of(p);
154    let (width, out_height) = size.unwrap_or((p.width, p.height));
155    if width == 0 || out_height == 0 {
156        return Err(Error::Invalid(format!("a picture of {width}x{out_height}")));
157    }
158    let fps = fps.unwrap_or(p.fps);
159    let mut args: Vec<String> = f.decoder_args(src.ffmpeg_codec.as_deref());
160    args.extend([
161        "-noautorotate".into(),
162        "-ss".into(),
163        format!("{:.6}", from.max(0.0)),
164        "-i".into(),
165        src.path.display().to_string(),
166        "-map".into(),
167        "0:v:0".into(),
168        "-fps_mode".into(),
169        "cfr".into(),
170        "-r".into(),
171        format!("{}/{}", fps.0, fps.1),
172    ]);
173    if let Some(n) = frames {
174        args.extend(["-frames:v".into(), n.to_string()]);
175    }
176    if (width, out_height) != (p.width, p.height) {
177        args.extend([
178            "-vf".into(),
179            format!("scale={width}:{out_height}:flags=area"),
180        ]);
181    }
182    args.extend([
183        "-f".into(),
184        "rawvideo".into(),
185        "-pix_fmt".into(),
186        pix_fmt(format).into(),
187        "-".into(),
188    ]);
189    let child = f.spawn(&args, false, true)?;
190    Ok(Video {
191        inner: Inner::Ffmpeg {
192            child,
193            next: frame_at(from.max(0.0), fps),
194            got: 0,
195        },
196        route: Route::Ffmpeg,
197        reasons: src.reasons.clone(),
198        width,
199        height: out_height,
200        format,
201        time_base: TimeBase::new(fps.1, fps.0),
202        left: frames,
203    })
204}
205
206impl Video {
207    pub fn recycle(&mut self, frame: Frame) {
208        if let Inner::Native(frames, _) = &mut self.inner {
209            frames.recycle(frame);
210        }
211    }
212
213    fn read(&mut self) -> Result<Option<Frame>> {
214        if self.left == Some(0) {
215            return Ok(None);
216        }
217        let (child, next, got) = match &mut self.inner {
218            Inner::Ffmpeg { child, next, got } => (child, next, got),
219            Inner::Native(frames, fps) => {
220                let Some(mut frame) = frames.next_frame()? else {
221                    return Ok(None);
222                };
223                frame.pts = frame_at(frames.seconds(frame.pts), *fps);
224                frame.time_base = self.time_base;
225                if (frame.width, frame.height) != (self.width, self.height) {
226                    frame = resize(&frame, self.width, self.height);
227                }
228                if let Some(n) = self.left.as_mut() {
229                    *n -= 1;
230                }
231                return Ok(Some(frame));
232            }
233        };
234        let mut frame = Frame::new(self.format, self.width, self.height, *next, self.time_base);
235        let Some(out) = child.stdout.as_mut() else {
236            return Ok(None);
237        };
238        for (i, plane) in frame.planes.iter_mut().enumerate() {
239            match out.read_exact(&mut plane.data) {
240                Ok(()) => {}
241                Err(e) if e.kind() == std::io::ErrorKind::UnexpectedEof && i == 0 => {
242                    self.left = Some(0);
243                    let err = child.stderr_now();
244                    if !err.is_empty() && *got == 0 {
245                        return Err(Error::Backend(format!("ffmpeg: {err}")));
246                    }
247                    return Ok(None);
248                }
249                Err(e) => return Err(Error::Io(e)),
250            }
251        }
252        *next += 1;
253        *got += 1;
254        if let Some(n) = self.left.as_mut() {
255            *n -= 1;
256        }
257        Ok(Some(frame))
258    }
259}
260
261impl Iterator for Video {
262    type Item = Result<Frame>;
263
264    fn next(&mut self) -> Option<Result<Frame>> {
265        self.read().transpose()
266    }
267}
268
269pub const IN_FLIGHT: usize = 8;
270
271pub enum VideoFrame {
272    Gpu(GpuFrame),
273    Cpu(Frame),
274}
275
276impl VideoFrame {
277    pub fn pts(&self) -> i64 {
278        match self {
279            VideoFrame::Gpu(f) => f.pts(),
280            VideoFrame::Cpu(f) => f.pts,
281        }
282    }
283
284    pub fn size(&self) -> (u32, u32) {
285        match self {
286            VideoFrame::Gpu(f) => f.size(),
287            VideoFrame::Cpu(f) => (f.width, f.height),
288        }
289    }
290
291    pub fn on_gpu(&self) -> bool {
292        matches!(self, VideoFrame::Gpu(_))
293    }
294}
295
296enum GpuRoute {
297    Gpu {
298        scaler: Option<GpuScaler>,
299        frames: Box<native::Frames>,
300        fps: (u32, u32),
301    },
302    Cpu(Video),
303}
304
305pub struct GpuVideo {
306    inner: GpuRoute,
307    pub route: Route,
308    pub reasons: Vec<String>,
309    pub width: u32,
310    pub height: u32,
311    pub format: PixelFormat,
312    pub time_base: TimeBase,
313    pub color: Color,
314}
315
316impl std::fmt::Debug for GpuVideo {
317    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
318        write!(
319            f,
320            "GpuVideo({} {}x{}, {} on the GPU)",
321            self.route,
322            self.width,
323            self.height,
324            if self.on_gpu() { "frames" } else { "no frames" }
325        )
326    }
327}
328
329fn format_of(p: &Probe) -> PixelFormat {
330    if p.bit_depth > 8 {
331        PixelFormat::Yuv420p10
332    } else {
333        PixelFormat::Yuv420p8
334    }
335}
336
337pub(super) fn frame_at(seconds: f64, fps: (u32, u32)) -> i64 {
338    (seconds * fps.0 as f64 / fps.1.max(1) as f64).round() as i64
339}
340
341fn stream_color(src: &Source) -> Color {
342    let found = src
343        .demux
344        .as_ref()
345        .and_then(|d| d.stream(d.video().ok()?).ok()?.color);
346    let (matrix, full_range) = found.unwrap_or((2, false));
347    Color { matrix, full_range }
348}
349
350pub fn decode_video_gpu(
351    path: &Path,
352    from: f64,
353    size: Option<(u32, u32)>,
354    prefs: &Prefs,
355) -> Result<GpuVideo> {
356    let src = source(path, prefs)?;
357    let p = &src.probe;
358    let size = size
359        .filter(|&(w, h)| w > 0 && h > 0)
360        .map(|(w, h)| (even(w), even(h)))
361        .unwrap_or((p.width, p.height));
362    let fps = p.fps;
363    let color = stream_color(&src);
364    let format = format_of(p);
365    if src.route == Route::Gpu {
366        let frames = native::Frames::open(src.demux()?.clone(), from, Route::Gpu)?;
367        let mut reasons = src.reasons.clone();
368        reasons.extend(frames.reason.clone());
369        if frames.route() == Route::Gpu {
370            let ten_bit = format == PixelFormat::Yuv420p10;
371            let scaler = match GpuScaler::with_frames(size.0, size.1, ten_bit, IN_FLIGHT) {
372                Ok(s) => Some(s),
373                Err(e) if size == (p.width, p.height) => {
374                    reasons.push(format!(
375                        "no GPU scaler ({e}); frames are the decoder's own surfaces"
376                    ));
377                    None
378                }
379                Err(e) => {
380                    reasons.push(format!("no GPU scaler ({e}); frames are scaled on the CPU"));
381                    None
382                }
383            };
384            return Ok(GpuVideo {
385                inner: GpuRoute::Gpu {
386                    scaler,
387                    frames: Box::new(frames),
388                    fps,
389                },
390                route: Route::Gpu,
391                reasons,
392                width: size.0,
393                height: size.1,
394                format,
395                time_base: TimeBase::new(fps.1, fps.0),
396                color,
397            });
398        }
399        let video = native_video(frames, reasons, size, fps, format, None);
400        return Ok(GpuVideo::cpu(video, color));
401    }
402    let video = open(&src, from, None, Some(size), Some(fps))?;
403    Ok(GpuVideo::cpu(video, color))
404}
405
406fn native_video(
407    frames: native::Frames,
408    reasons: Vec<String>,
409    size: (u32, u32),
410    fps: (u32, u32),
411    format: PixelFormat,
412    left: Option<u64>,
413) -> Video {
414    Video {
415        route: frames.route(),
416        inner: Inner::Native(Box::new(frames), fps),
417        reasons,
418        width: size.0,
419        height: size.1,
420        format,
421        time_base: TimeBase::new(fps.1, fps.0),
422        left,
423    }
424}
425
426impl GpuVideo {
427    fn cpu(video: Video, color: Color) -> Self {
428        Self {
429            route: video.route,
430            reasons: video.reasons.clone(),
431            width: video.width,
432            height: video.height,
433            format: video.format,
434            time_base: video.time_base,
435            color,
436            inner: GpuRoute::Cpu(video),
437        }
438    }
439
440    pub fn on_gpu(&self) -> bool {
441        matches!(self.inner, GpuRoute::Gpu { .. })
442    }
443
444    pub fn recycle(&mut self, frame: VideoFrame) {
445        let VideoFrame::Cpu(frame) = frame else {
446            return;
447        };
448        match &mut self.inner {
449            GpuRoute::Gpu { frames, .. } => frames.recycle(frame),
450            GpuRoute::Cpu(video) => video.recycle(frame),
451        }
452    }
453
454    fn read(&mut self) -> Result<Option<VideoFrame>> {
455        let size = (self.width, self.height);
456        let time_base = self.time_base;
457        let (frames, scaler, fps) = match &mut self.inner {
458            GpuRoute::Cpu(video) => return Ok(video.read()?.map(VideoFrame::Cpu)),
459            GpuRoute::Gpu {
460                frames,
461                scaler,
462                fps,
463            } => (frames, scaler, *fps),
464        };
465        let Some(mut frame) = frames.next_gpu()? else {
466            return Ok(None);
467        };
468        let pts = frame_at(frames.seconds(frame.pts()), fps);
469        if !frame.on_gpu() {
470            let Some(mut cpu) = frame.into_cpu() else {
471                return Ok(None);
472            };
473            cpu.pts = pts;
474            cpu.time_base = time_base;
475            if (cpu.width, cpu.height) != size {
476                cpu = resize(&cpu, size.0, size.1);
477            }
478            return Ok(Some(VideoFrame::Cpu(cpu)));
479        }
480        frame.retime(pts, time_base);
481        match scaler {
482            Some(s) => Ok(Some(VideoFrame::Gpu(s.scale(&frame)?))),
483            None if frame.size() == size => Ok(Some(VideoFrame::Gpu(frame))),
484            None => {
485                let cpu = frame.download()?;
486                Ok(Some(VideoFrame::Cpu(resize(&cpu, size.0, size.1))))
487            }
488        }
489    }
490}
491
492impl Iterator for GpuVideo {
493    type Item = Result<VideoFrame>;
494
495    fn next(&mut self) -> Option<Result<VideoFrame>> {
496        self.read().transpose()
497    }
498}
pito_codecs/media/front/video.rs

pito_codecs/media/front/
video.rs
