1use super::*;
2use crate::media::isa::dispatch;
3use crate::vp9::spec_tables::SUBPEL_FILTERS;
4
5const WIN: usize = 64 + 7;
6
7#[inline(always)]
8pub(super) fn taps(filter: u8, frac: usize) -> &'static [i16; 8] {
9 let o = ((filter as usize & 3) * 16 + frac) * 8;
10 SUBPEL_FILTERS[o..o + 8].try_into().unwrap()
11}
12
13pub fn mc(
14 r: &Plane,
15 x16: isize,
16 y16: isize,
17 w: usize,
18 h: usize,
19 filter: u8,
20 dst: &mut [u8],
21 ds: usize,
22) {
23 let (x0, y0) = (x16 >> 4, y16 >> 4);
24 let (fx, fy) = ((x16 & 15) as usize, (y16 & 15) as usize);
25 if r.inside(x0 - 3, y0 - 3, w + 7, h + 7) {
26 filter_block(
27 r.data,
28 r.stride,
29 r.at(x0 - 3, y0 - 3),
30 w,
31 h,
32 filter,
33 fx,
34 fy,
35 dst,
36 ds,
37 );
38 } else {
39 mc_edge(r, x0, y0, w, h, filter, fx, fy, dst, ds);
40 }
41}
42
43#[cold]
44fn mc_edge(
45 r: &Plane,
46 x0: isize,
47 y0: isize,
48 w: usize,
49 h: usize,
50 filter: u8,
51 fx: usize,
52 fy: usize,
53 dst: &mut [u8],
54 ds: usize,
55) {
56 let mut win = [0u8; WIN * WIN + 16];
57 let b = r.border as isize;
58 let (lx, ly) = (r.width as isize - 1 + b, r.height as isize - 1 + b);
59 for row in 0..h + 7 {
60 let yy = (y0 - 3 + row as isize).clamp(-b, ly);
61 let base = r.at(0, yy) as isize;
62 let d = &mut win[row * WIN..row * WIN + w + 7];
63 for (c, v) in d.iter_mut().enumerate() {
64 let xx = (x0 - 3 + c as isize).clamp(-b, lx);
65 *v = r.data[(base + xx) as usize];
66 }
67 }
68 filter_block(&win, WIN, 0, w, h, filter, fx, fy, dst, ds);
69}
70
71pub fn filter_block(
72 src: &[u8],
73 ss: usize,
74 so: usize,
75 w: usize,
76 h: usize,
77 filter: u8,
78 fx: usize,
79 fy: usize,
80 dst: &mut [u8],
81 ds: usize,
82) {
83 assert!(matches!(w, 4 | 8 | 16 | 32 | 64) && h > 0 && h <= 64);
84 assert!(src.len() >= so + (h + 6) * ss + w + 7);
85 assert!(dst.len() >= (h - 1) * ds + w);
86 match (fx, fy) {
87 (0, 0) => {
88 for i in 0..h {
89 let s = so + (i + 3) * ss + 3;
90 dst[i * ds..i * ds + w].copy_from_slice(&src[s..s + w]);
91 }
92 }
93 (_, 0) => h_pass(src, ss, so + 3 * ss, w, h, taps(filter, fx), dst, ds),
94 (0, _) => v_pass(src, ss, so + 3, w, h, taps(filter, fy), dst, ds),
95 _ => {
96 let (f0, f1) = (taps(filter, fx), taps(filter, fy));
97 match w {
98 4 | 8 => {
99 let mut tmp = [0u8; 8 * WIN + 16];
100 h_pass(src, ss, so, w, h + 7, f0, &mut tmp, w);
101 v_pass(&tmp, w, 0, w, h, f1, dst, ds);
102 }
103 16 => {
104 let mut tmp = [0u8; 16 * WIN + 16];
105 h_pass(src, ss, so, w, h + 7, f0, &mut tmp, w);
106 v_pass(&tmp, w, 0, w, h, f1, dst, ds);
107 }
108 _ => {
109 let mut tmp = [0u8; 64 * WIN + 16];
110 h_pass(src, ss, so, w, h + 7, f0, &mut tmp, w);
111 v_pass(&tmp, w, 0, w, h, f1, dst, ds);
112 }
113 }
114 }
115 }
116}
117
118pub(super) fn h_pass(
119 src: &[u8],
120 ss: usize,
121 so: usize,
122 w: usize,
123 h: usize,
124 f: &[i16; 8],
125 dst: &mut [u8],
126 ds: usize,
127) {
128 #[cfg(target_arch = "x86_64")]
129 if crate::media::isa::avx2() && src.len() >= so + (h - 1) * ss + w + super::mc_avx2::OVERREAD {
130 unsafe { super::mc_avx2::h_pass(src.as_ptr().add(so), ss, w, h, f, dst.as_mut_ptr(), ds) };
131 return;
132 }
133 h_pass_any(src, ss, so, w, h, f, dst, ds)
134}
135
136pub(super) fn v_pass(
137 src: &[u8],
138 ss: usize,
139 so: usize,
140 w: usize,
141 h: usize,
142 f: &[i16; 8],
143 dst: &mut [u8],
144 ds: usize,
145) {
146 #[cfg(target_arch = "x86_64")]
147 if crate::media::isa::avx2() {
148 unsafe { super::mc_avx2::v_pass(src.as_ptr().add(so), ss, w, h, f, dst.as_mut_ptr(), ds) };
149 return;
150 }
151 v_pass_any(src, ss, so, w, h, f, dst, ds)
152}
153
154#[inline(always)]
155fn narrow(acc: u16) -> u8 {
156 if acc < 0xC000 {
157 (acc >> 7).min(255) as u8
158 } else {
159 0
160 }
161}
162
163dispatch! {
164 fn h_pass_any(src: &[u8], ss: usize, so: usize, w: usize, h: usize, f: &[i16; 8], dst: &mut [u8], ds: usize) {
165 match w {
166 4 => h_rows::<4>(src, ss, so, h, f, dst, ds),
167 8 => h_rows::<8>(src, ss, so, h, f, dst, ds),
168 16 => h_rows::<16>(src, ss, so, h, f, dst, ds),
169 32 => h_rows::<32>(src, ss, so, h, f, dst, ds),
170 _ => h_rows::<64>(src, ss, so, h, f, dst, ds),
171 }
172 }
173}
174
175dispatch! {
176 fn v_pass_any(src: &[u8], ss: usize, so: usize, w: usize, h: usize, f: &[i16; 8], dst: &mut [u8], ds: usize) {
177 match w {
178 4 => v_rows::<4>(src, ss, so, h, f, dst, ds),
179 8 => v_rows::<8>(src, ss, so, h, f, dst, ds),
180 16 => v_rows::<16>(src, ss, so, h, f, dst, ds),
181 32 => v_rows::<32>(src, ss, so, h, f, dst, ds),
182 _ => v_rows::<64>(src, ss, so, h, f, dst, ds),
183 }
184 }
185}
186
187#[inline(always)]
188fn h_rows<const W: usize>(
189 src: &[u8],
190 ss: usize,
191 so: usize,
192 h: usize,
193 f: &[i16; 8],
194 dst: &mut [u8],
195 ds: usize,
196) {
197 let c: [u16; 8] = f.map(|v| v as u16);
198 for i in 0..h {
199 let s: &[u8; 72] = match src[so + i * ss..].first_chunk::<72>() {
200 Some(s) => s,
201 None => {
202 let mut t = [0u8; 72];
203 let rest = &src[so + i * ss..];
204 let n = rest.len().min(W + 7);
205 t[..n].copy_from_slice(&rest[..n]);
206 h_row::<W>(&t, &c, &mut dst[i * ds..i * ds + W]);
207 continue;
208 }
209 };
210 h_row::<W>(s, &c, &mut dst[i * ds..i * ds + W]);
211 }
212}
213
214#[inline(always)]
215fn h_row<const W: usize>(s: &[u8; 72], c: &[u16; 8], o: &mut [u8]) {
216 let mut acc = [64u16; W];
217 for t in 0..8 {
218 let p: &[u8; W] = s[t..t + W].try_into().unwrap();
219 for j in 0..W {
220 acc[j] = acc[j].wrapping_add(c[t].wrapping_mul(p[j] as u16));
221 }
222 }
223 let o: &mut [u8; W] = o.try_into().unwrap();
224 for j in 0..W {
225 o[j] = narrow(acc[j]);
226 }
227}
228
229#[inline(always)]
230fn v_rows<const W: usize>(
231 src: &[u8],
232 ss: usize,
233 so: usize,
234 h: usize,
235 f: &[i16; 8],
236 dst: &mut [u8],
237 ds: usize,
238) {
239 let c: [u16; 8] = f.map(|v| v as u16);
240 for i in 0..h {
241 let mut acc = [64u16; W];
242 for t in 0..8 {
243 let b = so + (i + t) * ss;
244 let p: &[u8; W] = src[b..b + W].try_into().unwrap();
245 for j in 0..W {
246 acc[j] = acc[j].wrapping_add(c[t].wrapping_mul(p[j] as u16));
247 }
248 }
249 let o: &mut [u8; W] = (&mut dst[i * ds..i * ds + W]).try_into().unwrap();
250 for j in 0..W {
251 o[j] = narrow(acc[j]);
252 }
253 }
254}
255
256dispatch! {
257 pub fn average(a: &[u8], b: &[u8], w: usize, h: usize, dst: &mut [u8], ds: usize) {
258 for i in 0..h {
259 let (ra, rb) = (&a[i * w..i * w + w], &b[i * w..i * w + w]);
260 for ((d, &x), &y) in dst[i * ds..i * ds + w].iter_mut().zip(ra).zip(rb) {
261 *d = ((x as u16 + y as u16 + 1) >> 1) as u8;
262 }
263 }
264 }
265}
266
267#[derive(Clone, Copy, Debug)]
268pub struct MvLimits {
269 pub row_min: i32,
270 pub row_max: i32,
271 pub col_min: i32,
272 pub col_max: i32,
273}
274
275impl MvLimits {
276 pub fn new(mi_row: usize, mi_col: usize, bsize: u8, mi_rows: usize, mi_cols: usize) -> Self {
277 let bh = (BLOCK_H[bsize as usize] / 8).max(1) as i32;
278 let bw = (BLOCK_W[bsize as usize] / 8).max(1) as i32;
279 let (r, c) = (mi_row as i32, mi_col as i32);
280 let to_top = -(r * 8 * 16);
281 let to_bottom = (mi_rows as i32 - bh - r) * 8 * 16;
282 let to_left = -(c * 8 * 16);
283 let to_right = (mi_cols as i32 - bw - c) * 8 * 16;
284 let spel_left = (4 + bw * 8) << 4;
285 let spel_top = (4 + bh * 8) << 4;
286 let row_min = to_top - spel_top;
287 let col_min = to_left - spel_left;
288 MvLimits {
289 row_min: (row_min + 1) >> 1,
290 row_max: (to_bottom + spel_top - 16).max(row_min) >> 1,
291 col_min: (col_min + 1) >> 1,
292 col_max: (to_right + spel_left - 16).max(col_min) >> 1,
293 }
294 }
295
296 pub fn clamp(&self, mv: Mv) -> Mv {
297 Mv::new(
298 (mv.row as i32).clamp(self.row_min, self.row_max) as i16,
299 (mv.col as i32).clamp(self.col_min, self.col_max) as i16,
300 )
301 }
302
303 pub fn contains(&self, mv: Mv) -> bool {
304 (self.row_min..=self.row_max).contains(&(mv.row as i32))
305 && (self.col_min..=self.col_max).contains(&(mv.col as i32))
306 }
307}
308
309#[derive(Clone, Copy, Debug)]
310pub struct BlockPos {
311 pub mi_row: usize,
312 pub mi_col: usize,
313 pub bsize: u8,
314 pub mi_rows: usize,
315 pub mi_cols: usize,
316}
317
318fn clamped_mv16(p: &BlockPos, mv: Mv, sx: usize, sy: usize) -> (isize, isize) {
319 let bh = (BLOCK_H[p.bsize as usize] / 8).max(1) as i32;
320 let bw = (BLOCK_W[p.bsize as usize] / 8).max(1) as i32;
321 let (r, c) = (p.mi_row as i32, p.mi_col as i32);
322 let to_top = (-(r * 8 * 16)) >> sy;
323 let to_bottom = ((p.mi_rows as i32 - bh - r) * 8 * 16) >> sy;
324 let to_left = (-(c * 8 * 16)) >> sx;
325 let to_right = ((p.mi_cols as i32 - bw - c) * 8 * 16) >> sx;
326 let spel_left = (4 + ((bw * 8) >> sx)) << 4;
327 let spel_top = (4 + ((bh * 8) >> sy)) << 4;
328 let row = ((2 * mv.row as i32) >> sy).clamp(
329 to_top - spel_top,
330 (to_bottom + spel_top - 16).max(to_top - spel_top),
331 );
332 let col = ((2 * mv.col as i32) >> sx).clamp(
333 to_left - spel_left,
334 (to_right + spel_left - 16).max(to_left - spel_left),
335 );
336 (row as isize, col as isize)
337}
338
339pub fn predict_plane(
340 refs: &[Plane],
341 mvs: &[Mv],
342 p: &BlockPos,
343 plane: usize,
344 filter: u8,
345 dst: &mut [u8],
346 ds: usize,
347) {
348 let (sx, sy) = if plane > 0 { (1, 1) } else { (0, 0) };
349 let w = (BLOCK_W[p.bsize as usize] >> sx).max(4);
350 let h = (BLOCK_H[p.bsize as usize] >> sy).max(4);
351 let x = ((p.mi_col * 8) >> sx) as isize;
352 let y = ((p.mi_row * 8) >> sy) as isize;
353 let (r0, c0) = clamped_mv16(p, mvs[0], sx, sy);
354 if mvs.len() < 2 {
355 mc(
356 &refs[0],
357 (x << 4) + c0,
358 (y << 4) + r0,
359 w,
360 h,
361 filter,
362 dst,
363 ds,
364 );
365 return;
366 }
367 let mut a = [0u8; 64 * 64];
368 let mut b = [0u8; 64 * 64];
369 mc(
370 &refs[0],
371 (x << 4) + c0,
372 (y << 4) + r0,
373 w,
374 h,
375 filter,
376 &mut a,
377 w,
378 );
379 let (r1, c1) = clamped_mv16(p, mvs[1], sx, sy);
380 mc(
381 &refs[1],
382 (x << 4) + c1,
383 (y << 4) + r1,
384 w,
385 h,
386 filter,
387 &mut b,
388 w,
389 );
390 average(&a, &b, w, h, dst, ds);
391}
392
393pub fn predict_block(
394 refs: &[[Plane; 3]],
395 mvs: &[Mv],
396 p: &BlockPos,
397 filter: u8,
398 dst: [&mut [u8]; 3],
399 ds: [usize; 3],
400) {
401 for (plane, d) in dst.into_iter().enumerate() {
402 let r: [Plane; 2] = [refs[0][plane], refs[refs.len() - 1][plane]];
403 predict_plane(&r[..mvs.len().min(2)], mvs, p, plane, filter, d, ds[plane]);
404 }
405}