; /*
; * Provide SSE luma and chroma mc functions for HEVC/VVC decoding
; * Copyright (c) 2013 Pierre-Edouard LEPERE
; * Copyright (c) 2023-2024 Nuo Mi
; * Copyright (c) 2023-2024 Wu Jianhua
; *
; * This file is part of FFmpeg.
; *
; * FFmpeg is free software; you can redistribute it and/or
; * modify it under the terms of the GNU Lesser General Public
; * License as published by the Free Software Foundation; either
; * version 2.1 of the License, or (at your option) any later version.
; *
; * FFmpeg is distributed in the hope that it will be useful,
; * but WITHOUT ANY WARRANTY; without even the implied warranty of
; * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
; * Lesser General Public License for more details.
; *
; * You should have received a copy of the GNU Lesser General Public
; * License along with FFmpeg; if not, write to the Free Software
; * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
; */
%include "libavutil/x86/x86util.asm"

SECTION_RODATA 32
cextern pw_255
cextern pw_512
cextern pw_2048
cextern pw_1023
cextern pw_1024
cextern pw_4096
cextern pw_8192
%define scale_8 pw_512
%define scale_10 pw_2048
%define scale_12 pw_8192
%define max_pixels_8 pw_255
%define max_pixels_10 pw_1023
max_pixels_12:          times 16 dw ((1 << 12)-1)
cextern pb_0

SECTION .text
%macro SIMPLE_LOAD 4    ;width, bitd, tab, r1
%if %1 == 2 || (%2 == 8 && %1 <= 4)
    movd              %4, [%3]                                               ; load data from source
%elif %1 == 4 || (%2 == 8 && %1 <= 8)
    movq              %4, [%3]                                               ; load data from source
%elif notcpuflag(avx)
    movu              %4, [%3]                                               ; load data from source
%elif %1 <= 8 || (%2 == 8 && %1 <= 16)
    movdqu           %4, [%3]
%else
    movu              %4, [%3]
%endif
%endmacro

%macro VPBROADCASTW 2
%if notcpuflag(avx2)
    movd           %1, %2
    pshuflw        %1, %1, 0
    punpcklwd      %1, %1
%else
    vpbroadcastw   %1, %2
%endif
%endmacro

%macro MC_4TAP_FILTER 4 ; bitdepth, filter, a, b,
    VPBROADCASTW   %3, [%2q + 0 * 2]  ; coeff 0, 1
    VPBROADCASTW   %4, [%2q + 1 * 2]  ; coeff 2, 3
%if %1 != 8
    pmovsxbw       %3, xmm%3
    pmovsxbw       %4, xmm%4
%endif
%endmacro

%macro MC_4TAP_HV_FILTER 1
    VPBROADCASTW  m12, [vfq + 0 * 2]  ; vf 0, 1
    VPBROADCASTW  m13, [vfq + 1 * 2]  ; vf 2, 3
    VPBROADCASTW  m14, [hfq + 0 * 2]  ; hf 0, 1
    VPBROADCASTW  m15, [hfq + 1 * 2]  ; hf 2, 3

    pmovsxbw      m12, xm12
    pmovsxbw      m13, xm13
%if %1 != 8
    pmovsxbw      m14, xm14
    pmovsxbw      m15, xm15
%endif
    lea           r3srcq, [srcstrideq*3]
%endmacro

%macro MC_8TAP_SAVE_FILTER 5    ;offset, mm registers
    mova [rsp + %1 + 0*mmsize], %2
    mova [rsp + %1 + 1*mmsize], %3
    mova [rsp + %1 + 2*mmsize], %4
    mova [rsp + %1 + 3*mmsize], %5
%endmacro

%macro MC_8TAP_FILTER 2-3 ;bitdepth, filter, offset
    VPBROADCASTW                      m12, [%2q + 0 * 2]  ; coeff 0, 1
    VPBROADCASTW                      m13, [%2q + 1 * 2]  ; coeff 2, 3
    VPBROADCASTW                      m14, [%2q + 2 * 2]  ; coeff 4, 5
    VPBROADCASTW                      m15, [%2q + 3 * 2]  ; coeff 6, 7
%if %0 == 3
    MC_8TAP_SAVE_FILTER                %3, m12, m13, m14, m15
%endif

%if %1 != 8
    pmovsxbw                          m12, xm12
    pmovsxbw                          m13, xm13
    pmovsxbw                          m14, xm14
    pmovsxbw                          m15, xm15
    %if %0 == 3
    MC_8TAP_SAVE_FILTER     %3 + 4*mmsize, m12, m13, m14, m15
    %endif
%elif %0 == 3
    pmovsxbw                          m8, xm12
    pmovsxbw                          m9, xm13
    pmovsxbw                         m10, xm14
    pmovsxbw                         m11, xm15
    MC_8TAP_SAVE_FILTER     %3 + 4*mmsize, m8, m9, m10, m11
%endif

%endmacro

%macro MC_4TAP_LOAD 4
%if (%1 == 8 && %4 <= 4)
%define %%load movd
%elif (%1 == 8 && %4 <= 8) || (%1 > 8 && %4 <= 4)
%define %%load movq
%else
%define %%load movdqu
%endif

    %%load            m0, [%2q ]
%ifnum %3
    %%load            m1, [%2q+  %3]
    %%load            m2, [%2q+2*%3]
    %%load            m3, [%2q+3*%3]
%else
    %%load            m1, [%2q+  %3q]
    %%load            m2, [%2q+2*%3q]
    %%load            m3, [%2q+r3srcq]
%endif
%if %1 == 8
%if %4 > 8
    SBUTTERFLY        bw, 0, 1, 7
    SBUTTERFLY        bw, 2, 3, 7
%else
    punpcklbw         m0, m1
    punpcklbw         m2, m3
%endif
%else
%if %4 > 4
    SBUTTERFLY        wd, 0, 1, 7
    SBUTTERFLY        wd, 2, 3, 7
%else
    punpcklwd         m0, m1
    punpcklwd         m2, m3
%endif
%endif
%endmacro

%macro MC_8TAP_H_LOAD 4
%assign %%stride (%1+7)/8
%if %1 == 8
%if %3 <= 4
%define %%load movd
%elif %3 == 8
%define %%load movq
%else
%define %%load movu
%endif
%else
%if %3 == 2
%define %%load movd
%elif %3 == 4
%define %%load movq
%else
%define %%load movu
%endif
%endif
    %%load            m0, [%2-3*%%stride]        ;load data from source
    %%load            m1, [%2-2*%%stride]
    %%load            m2, [%2-%%stride  ]
    %%load            m3, [%2           ]
    %%load            m4, [%2+%%stride  ]
    %%load            m5, [%2+2*%%stride]
    %%load            m6, [%2+3*%%stride]
    %%load            m7, [%2+4*%%stride]

%if %1 == 8
%if %3 > 8
    SBUTTERFLY        wd, 0, 1, %4
    SBUTTERFLY        wd, 2, 3, %4
    SBUTTERFLY        wd, 4, 5, %4
    SBUTTERFLY        wd, 6, 7, %4
%else
    punpcklbw         m0, m1
    punpcklbw         m2, m3
    punpcklbw         m4, m5
    punpcklbw         m6, m7
%endif
%else
%if %3 > 4
    SBUTTERFLY        dq, 0, 1, %4
    SBUTTERFLY        dq, 2, 3, %4
    SBUTTERFLY        dq, 4, 5, %4
    SBUTTERFLY        dq, 6, 7, %4
%else
    punpcklwd         m0, m1
    punpcklwd         m2, m3
    punpcklwd         m4, m5
    punpcklwd         m6, m7
%endif
%endif
%endmacro

%macro MC_8TAP_V_LOAD 5
    lea              %5q, [%2]
    sub              %5q, r3srcq
    movu              m0, [%5q            ]      ;load x- 3*srcstride
    movu              m1, [%5q+   %3q     ]      ;load x- 2*srcstride
    movu              m2, [%5q+ 2*%3q     ]      ;load x-srcstride
    movu              m3, [%2       ]      ;load x
    movu              m4, [%2+   %3q]      ;load x+stride
    movu              m5, [%2+ 2*%3q]      ;load x+2*stride
    movu              m6, [%2+r3srcq]      ;load x+3*stride
    movu              m7, [%2+ 4*%3q]      ;load x+4*stride
%if %1 == 8
%if %4 > 8
    SBUTTERFLY        bw, 0, 1, 8
    SBUTTERFLY        bw, 2, 3, 8
    SBUTTERFLY        bw, 4, 5, 8
    SBUTTERFLY        bw, 6, 7, 8
%else
    punpcklbw         m0, m1
    punpcklbw         m2, m3
    punpcklbw         m4, m5
    punpcklbw         m6, m7
%endif
%else
%if %4 > 4
    SBUTTERFLY        wd, 0, 1, 8
    SBUTTERFLY        wd, 2, 3, 8
    SBUTTERFLY        wd, 4, 5, 8
    SBUTTERFLY        wd, 6, 7, 8
%else
    punpcklwd         m0, m1
    punpcklwd         m2, m3
    punpcklwd         m4, m5
    punpcklwd         m6, m7
%endif
%endif
%endmacro

%macro PEL_12STORE2 3
    movd           [%1], %2
%endmacro
%macro PEL_12STORE4 3
    movq           [%1], %2
%endmacro
%macro PEL_12STORE6 3
    movq           [%1], %2
    psrldq            %2, 8
    movd         [%1+8], %2
%endmacro
%macro PEL_12STORE8 3
    movdqu         [%1], %2
%endmacro
%macro PEL_12STORE12 3
    PEL_12STORE8     %1, %2, %3
    movq        [%1+16], %3
%endmacro
%macro PEL_12STORE16 3
%if cpuflag(avx2)
    movu            [%1], %2
%else
    PEL_12STORE8      %1, %2, %3
    movdqu       [%1+16], %3
%endif
%endmacro

%macro PEL_10STORE2 3
    movd           [%1], %2
%endmacro
%macro PEL_10STORE4 3
    movq           [%1], %2
%endmacro
%macro PEL_10STORE6 3
    movq           [%1], %2
    psrldq            %2, 8
    movd         [%1+8], %2
%endmacro
%macro PEL_10STORE8 3
    movdqu         [%1], %2
%endmacro
%macro PEL_10STORE12 3
    PEL_10STORE8     %1, %2, %3
    movq        [%1+16], %3
%endmacro
%macro PEL_10STORE16 3
%if cpuflag(avx2)
    movu            [%1], %2
%else
    PEL_10STORE8      %1, %2, %3
    movdqu       [%1+16], %3
%endif
%endmacro
%macro PEL_10STORE32 3
    PEL_10STORE16     %1, %2, %3
    movu         [%1+32], %3
%endmacro

%macro PEL_8STORE2 3
    pextrw          [%1], %2, 0
%endmacro
%macro PEL_8STORE4 3
    movd            [%1], %2
%endmacro
%macro PEL_8STORE6 3
    movd            [%1], %2
    pextrw        [%1+4], %2, 2
%endmacro
%macro PEL_8STORE8 3
    movq           [%1], %2
%endmacro
%macro PEL_8STORE12 3
    movq            [%1], %2
    psrldq            %2, 8
    movd          [%1+8], %2
%endmacro
%macro PEL_8STORE16 3
%if cpuflag(avx2)
    movdqu        [%1], %2
%else
    movu          [%1], %2
%endif ; avx
%endmacro
%macro PEL_8STORE32 3
    movu          [%1], %2
%endmacro

%macro LOOP_END 3
    add              %1q, dststrideq             ; dst += dststride
    add              %2q, %3q                    ; src += srcstride
    dec          heightd                         ; cmp height
    jnz               .loop                      ; height loop
%endmacro


%macro MC_PIXEL_COMPUTE 2-3 ;width, bitdepth
%if %2 == 8
%if cpuflag(avx2) && %0 ==3
%if %1 > 16
    vextracti128 xm1, m0, 1
    pmovzxbw      m1, xm1
    psllw         m1, 14-%2
%endif
    pmovzxbw      m0, xm0
%else ; not avx
%if %1 > 8
    punpckhbw     m1, m0, m2
    psllw         m1, 14-%2
%endif
    punpcklbw     m0, m2
%endif
%endif ;avx
    psllw         m0, 14-%2
%endmacro

%macro MC_4TAP_COMPUTE 4-8 ; bitdepth, width, filter1, filter2, HV/m0, m2, m1, m3
%if %0 == 8
%define %%reg0 %5
%define %%reg2 %6
%define %%reg1 %7
%define %%reg3 %8
%else
%define %%reg0 m0
%define %%reg2 m2
%define %%reg1 m1
%define %%reg3 m3
%endif
%if %1 == 8
%if cpuflag(avx2) && (%0 == 5)
%if %2 > 16
    vperm2i128    m10, m0, m1, q0301
%endif
    vinserti128    m0, m0, xm1, 1
    mova           m1, m10
%if %2 > 16
    vperm2i128    m10, m2, m3, q0301
%endif
    vinserti128    m2, m2, xm3, 1
    mova           m3, m10
%endif
    pmaddubsw      %%reg0, %3   ;x1*c1+x2*c2
    pmaddubsw      %%reg2, %4   ;x3*c3+x4*c4
    paddw          %%reg0, %%reg2
%if %2 > 8
    pmaddubsw      %%reg1, %3
    pmaddubsw      %%reg3, %4
    paddw          %%reg1, %%reg3
%endif
%else
    pmaddwd        %%reg0, %3
    pmaddwd        %%reg2, %4
    paddd          %%reg0, %%reg2
%if %2 > 4
    pmaddwd        %%reg1, %3
    pmaddwd        %%reg3, %4
    paddd          %%reg1, %%reg3
%if %1 != 8
    psrad          %%reg1, %1-8
%endif
%endif
%if %1 != 8
    psrad          %%reg0, %1-8
%endif
    packssdw       %%reg0, %%reg1
%endif
%endmacro

%macro MC_8TAP_HV_COMPUTE 4     ; width, bitdepth, filter

%if %2 == 8
    pmaddubsw         m0, [%3q+0*mmsize]    ;x1*c1+x2*c2
    pmaddubsw         m2, [%3q+1*mmsize]    ;x3*c3+x4*c4
    pmaddubsw         m4, [%3q+2*mmsize]    ;x5*c5+x6*c6
    pmaddubsw         m6, [%3q+3*mmsize]    ;x7*c7+x8*c8
    paddw             m0, m2
    paddw             m4, m6
    paddw             m0, m4
%else
    pmaddwd           m0, [%3q+4*mmsize]
    pmaddwd           m2, [%3q+5*mmsize]
    pmaddwd           m4, [%3q+6*mmsize]
    pmaddwd           m6, [%3q+7*mmsize]
    paddd             m0, m2
    paddd             m4, m6
    paddd             m0, m4
%if %2 != 8
    psrad             m0, %2-8
%endif
%if %1 > 4
    pmaddwd           m1, [%3q+4*mmsize]
    pmaddwd           m3, [%3q+5*mmsize]
    pmaddwd           m5, [%3q+6*mmsize]
    pmaddwd           m7, [%3q+7*mmsize]
    paddd             m1, m3
    paddd             m5, m7
    paddd             m1, m5
%if %2 != 8
    psrad             m1, %2-8
%endif
%endif
    p%4               m0, m1
%endif
%endmacro


%macro MC_8TAP_COMPUTE 2-3     ; width, bitdepth
%if %2 == 8
%if cpuflag(avx2) && (%0 == 3)

    vperm2i128 m10, m0,  m1, q0301
    vinserti128 m0, m0, xm1, 1
    SWAP 1, 10

    vperm2i128 m10, m2,  m3, q0301
    vinserti128 m2, m2, xm3, 1
    SWAP 3, 10


    vperm2i128 m10, m4,  m5, q0301
    vinserti128 m4, m4, xm5, 1
    SWAP 5, 10

    vperm2i128 m10, m6,  m7, q0301
    vinserti128 m6, m6, xm7, 1
    SWAP 7, 10
%endif

    pmaddubsw         m0, m12   ;x1*c1+x2*c2
    pmaddubsw         m2, m13   ;x3*c3+x4*c4
    pmaddubsw         m4, m14   ;x5*c5+x6*c6
    pmaddubsw         m6, m15   ;x7*c7+x8*c8
    paddw             m0, m2
    paddw             m4, m6
    paddw             m0, m4
%if %1 > 8
    pmaddubsw         m1, m12
    pmaddubsw         m3, m13
    pmaddubsw         m5, m14
    pmaddubsw         m7, m15
    paddw             m1, m3
    paddw             m5, m7
    paddw             m1, m5
%endif
%else
    pmaddwd           m0, m12
    pmaddwd           m2, m13
    pmaddwd           m4, m14
    pmaddwd           m6, m15
    paddd             m0, m2
    paddd             m4, m6
    paddd             m0, m4
%if %2 != 8
    psrad             m0, %2-8
%endif
%if %1 > 4
    pmaddwd           m1, m12
    pmaddwd           m3, m13
    pmaddwd           m5, m14
    pmaddwd           m7, m15
    paddd             m1, m3
    paddd             m5, m7
    paddd             m1, m5
%if %2 != 8
    psrad             m1, %2-8
%endif
%endif
%endif
%endmacro
%macro UNI_COMPUTE 5
    pmulhrsw          %3, %5
%if %1 > 8 || (%2 > 8 && %1 > 4)
    pmulhrsw          %4, %5
%endif
%if %2 == 8
    packuswb          %3, %4
%else
    CLIPW             %3, [pb_0], [max_pixels_%2]
%if (%1 > 8 && notcpuflag(avx)) || %1 > 16
    CLIPW             %4, [pb_0], [max_pixels_%2]
%endif
%endif
%endmacro


; ******************************
; void %1_put_pixels(int16_t *dst, ptrdiff_t dststride, const uint8_t *_src, ptrdiff_t srcstride,
;                         int height, const int8_t *hf, const int8_t *vf, int width)
; ******************************

%macro PUT_PIXELS 3
    MC_PIXELS       %1, %2, %3
    MC_UNI_PIXELS   %1, %2, %3
%endmacro

%macro MC_PIXELS 3
cglobal %1_put_pixels%2_%3, 5, 5, 3, dst, dststride, src, srcstride, height
    pxor              m2, m2
.loop:
    SIMPLE_LOAD       %2, %3, srcq, m0
    MC_PIXEL_COMPUTE  %2, %3, 1
    PEL_10STORE%2     dstq, m0, m1
    LOOP_END         dst, src, srcstride
    RET
%endmacro

%macro MC_UNI_PIXELS 3
cglobal %1_put_uni_pixels%2_%3, 5, 5, 2, dst, dststride, src, srcstride, height
.loop:
    SIMPLE_LOAD       %2, %3, srcq, m0
    PEL_%3STORE%2   dstq, m0, m1
    add             dstq, dststrideq             ; dst += dststride
    add             srcq, srcstrideq             ; src += srcstride
    dec          heightd                         ; cmp height
    jnz               .loop                      ; height loop
    RET
%endmacro

%macro PUT_4TAP 3
%if cpuflag(avx2)
%define XMM_REGS  11
%else
%define XMM_REGS  8
%endif

; ******************************
; void %1_put_4tap_hX(int16_t *dst, ptrdiff_t dststride,
;      const uint8_t *_src, ptrdiff_t _srcstride, int height, int8_t *hf, int8_t *vf, int width);
; ******************************
cglobal %1_put_4tap_h%2_%3, 6, 6, XMM_REGS, dst, dststride, src, srcstride, height, hf
%assign %%stride ((%3 + 7)/8)
    MC_4TAP_FILTER       %3, hf, m4, m5
.loop:
    MC_4TAP_LOAD         %3, srcq-%%stride, %%stride, %2
    MC_4TAP_COMPUTE      %3, %2, m4, m5, 1
    PEL_10STORE%2      dstq, m0, m1
    LOOP_END            dst, src, srcstride
    RET

; ******************************
; void %1_put_uni_4tap_hX(uint8_t *dst, ptrdiff_t dststride,
;      const uint8_t *_src, ptrdiff_t _srcstride, int height, int8_t *hf, int8_t *vf, int width);
; ******************************
cglobal %1_put_uni_4tap_h%2_%3, 6, 7, XMM_REGS, dst, dststride, src, srcstride, height, hf
%assign %%stride ((%3 + 7)/8)
    movdqa            m6, [scale_%3]
    MC_4TAP_FILTER    %3, hf, m4, m5
.loop:
    MC_4TAP_LOAD      %3, srcq-%%stride, %%stride, %2
    MC_4TAP_COMPUTE   %3, %2, m4, m5
    UNI_COMPUTE       %2, %3, m0, m1, m6
    PEL_%3STORE%2   dstq, m0, m1
    add             dstq, dststrideq             ; dst += dststride
    add             srcq, srcstrideq             ; src += srcstride
    dec          heightd                         ; cmp height
    jnz               .loop                      ; height loop
    RET

; ******************************
; void %1_put_4tap_v(int16_t *dst, ptrdiff_t dststride,
;      const uint8_t *_src, ptrdiff_t _srcstride, int height, int8_t *hf, int8_t *vf, int width)
; ******************************
cglobal %1_put_4tap_v%2_%3, 7, 7, XMM_REGS, dst, dststride, src, srcstride, height, r3src, vf
    sub             srcq, srcstrideq
    MC_4TAP_FILTER    %3, vf, m4, m5
    lea           r3srcq, [srcstrideq*3]
.loop:
    MC_4TAP_LOAD      %3, srcq, srcstride, %2
    MC_4TAP_COMPUTE   %3, %2, m4, m5, 1
    PEL_10STORE%2     dstq, m0, m1
    LOOP_END          dst, src, srcstride
    RET

; ******************************
; void %1_put_uni_4tap_vX(uint8_t *dst, ptrdiff_t dststride,
;      const uint8_t *_src, ptrdiff_t _srcstride, int height, int8_t *hf, int8_t *vf, int width);
; ******************************
cglobal %1_put_uni_4tap_v%2_%3, 7, 7, XMM_REGS, dst, dststride, src, srcstride, height, r3src, vf
    movdqa            m6, [scale_%3]
    sub             srcq, srcstrideq
    MC_4TAP_FILTER       %3, vf, m4, m5
    lea           r3srcq, [srcstrideq*3]
.loop:
    MC_4TAP_LOAD      %3, srcq, srcstride, %2
    MC_4TAP_COMPUTE   %3, %2, m4, m5
    UNI_COMPUTE       %2, %3, m0, m1, m6
    PEL_%3STORE%2   dstq, m0, m1
    add             dstq, dststrideq             ; dst += dststride
    add             srcq, srcstrideq             ; src += srcstride
    dec          heightd                         ; cmp height
    jnz               .loop                      ; height loop
    RET
%endmacro

%macro PUT_4TAP_HV 3
; ******************************
; void put_4tap_hv(int16_t *dst, ptrdiff_t dststride,
;      const uint8_t *_src, ptrdiff_t _srcstride, int height, int8_t *hf, int8_t *vf, int width)
; ******************************
cglobal %1_put_4tap_hv%2_%3, 7, 8, 16 , dst, dststride, src, srcstride, height, hf, vf, r3src
%assign %%stride ((%3 + 7)/8)
    sub                 srcq, srcstrideq
    MC_4TAP_HV_FILTER    %3
    MC_4TAP_LOAD         %3, srcq-%%stride, %%stride, %2
    MC_4TAP_COMPUTE      %3, %2, m14, m15
%if (%2 > 8 && (%3 == 8))
    SWAP              m8, m1
%endif
    SWAP              m4, m0
    add             srcq, srcstrideq
    MC_4TAP_LOAD         %3, srcq-%%stride, %%stride, %2
    MC_4TAP_COMPUTE      %3, %2, m14, m15
%if (%2 > 8 && (%3 == 8))
    SWAP              m9, m1
%endif
    SWAP              m5, m0
    add             srcq, srcstrideq
    MC_4TAP_LOAD         %3, srcq-%%stride, %%stride, %2
    MC_4TAP_COMPUTE      %3, %2, m14, m15
%if (%2 > 8 && (%3 == 8))
    SWAP             m10, m1
%endif
    SWAP              m6, m0
    add             srcq, srcstrideq
.loop:
    MC_4TAP_LOAD         %3, srcq-%%stride, %%stride, %2
    MC_4TAP_COMPUTE      %3, %2, m14, m15
%if (%2 > 8 && (%3 == 8))
    SWAP             m11, m1
%endif
    SWAP              m7, m0
    punpcklwd         m0, m4, m5
    punpcklwd         m2, m6, m7
%if %2 > 4
    punpckhwd         m1, m4, m5
    punpckhwd         m3, m6, m7
%endif
    MC_4TAP_COMPUTE      14, %2, m12, m13
%if (%2 > 8 && (%3 == 8))
    punpcklwd         m4, m8, m9
    punpcklwd         m2, m10, m11
    punpckhwd         m8, m8, m9
    punpckhwd         m3, m10, m11
    MC_4TAP_COMPUTE      14, %2, m12, m13, m4, m2, m8, m3
%if cpuflag(avx2)
    vinserti128       m2, m0, xm4, 1
    vperm2i128        m3, m0, m4, q0301
    PEL_10STORE%2     dstq, m2, m3
%else
    PEL_10STORE%2     dstq, m0, m4
%endif
%else
    PEL_10STORE%2     dstq, m0, m1
%endif
    movdqa            m4, m5
    movdqa            m5, m6
    movdqa            m6, m7
%if (%2 > 8 && (%3 == 8))
    mova              m8, m9
    mova              m9, m10
    mova             m10, m11
%endif
    LOOP_END         dst, src, srcstride
    RET

cglobal %1_put_uni_4tap_hv%2_%3, 7, 8, 16 , dst, dststride, src, srcstride, height, hf, vf, r3src
%assign %%stride ((%3 + 7)/8)
    sub                srcq, srcstrideq
    MC_4TAP_HV_FILTER    %3
    MC_4TAP_LOAD         %3, srcq-%%stride, %%stride, %2
    MC_4TAP_COMPUTE      %3, %2, m14, m15
%if (%2 > 8 && (%3 == 8))
    SWAP                 m8, m1
%endif
    SWAP                 m4, m0
    add                srcq, srcstrideq
    MC_4TAP_LOAD         %3, srcq-%%stride, %%stride, %2
    MC_4TAP_COMPUTE      %3, %2, m14, m15
%if (%2 > 8 && (%3 == 8))
    SWAP                 m9, m1
%endif
    SWAP                 m5, m0
    add                srcq, srcstrideq
    MC_4TAP_LOAD         %3, srcq-%%stride, %%stride, %2
    MC_4TAP_COMPUTE      %3, %2, m14, m15
%if (%2 > 8 && (%3 == 8))
    SWAP                m10, m1
%endif
    SWAP                 m6, m0
    add                srcq, srcstrideq
.loop:
    MC_4TAP_LOAD         %3, srcq-%%stride, %%stride, %2
    MC_4TAP_COMPUTE      %3, %2, m14, m15
%if (%2 > 8 && (%3 == 8))
    SWAP                m11, m1
%endif
    mova                 m7, m0
    punpcklwd            m0, m4, m5
    punpcklwd            m2, m6, m7
%if %2 > 4
    punpckhwd            m1, m4, m5
    punpckhwd            m3, m6, m7
%endif
    MC_4TAP_COMPUTE      14, %2, m12, m13
%if (%2 > 8 && (%3 == 8))
    punpcklwd            m4, m8, m9
    punpcklwd            m2, m10, m11
    punpckhwd            m8, m8, m9
    punpckhwd            m3, m10, m11
    MC_4TAP_COMPUTE      14, %2, m12, m13, m4, m2, m8, m3
    UNI_COMPUTE          %2, %3, m0, m4, [scale_%3]
%else
    UNI_COMPUTE          %2, %3, m0, m1, [scale_%3]
%endif
    PEL_%3STORE%2      dstq, m0, m1
    mova                 m4, m5
    mova                 m5, m6
    mova                 m6, m7
%if (%2 > 8 && (%3 == 8))
    mova                 m8, m9
    mova                 m9, m10
    mova                m10, m11
%endif
    add                dstq, dststrideq             ; dst += dststride
    add                srcq, srcstrideq             ; src += srcstride
    dec             heightd                         ; cmp height
    jnz               .loop                         ; height loop
    RET
%endmacro

; ******************************
; void put_8tap_hX_X_X(int16_t *dst, ptrdiff_t dststride, const uint8_t *_src, ptrdiff_t srcstride,
;                       int height, const int8_t *hf, const int8_t *vf, int width)
; ******************************

%macro PUT_8TAP 3
cglobal %1_put_8tap_h%2_%3, 6, 6, 16, dst, dststride, src, srcstride, height, hf
    MC_8TAP_FILTER          %3, hf
.loop:
    MC_8TAP_H_LOAD          %3, srcq, %2, 10
    MC_8TAP_COMPUTE         %2, %3, 1
%if %3 > 8
    packssdw                m0, m1
%endif
    PEL_10STORE%2         dstq, m0, m1
    LOOP_END               dst, src, srcstride
    RET

; ******************************
; void put_uni_8tap_hX_X_X(int16_t *dst, ptrdiff_t dststride, const uint8_t *_src, ptrdiff_t srcstride,
;                       int height, const int8_t *hf, const int8_t *vf, int width)
; ******************************
cglobal %1_put_uni_8tap_h%2_%3, 6, 7, 16 , dst, dststride, src, srcstride, height, hf
    mova                 m9, [scale_%3]
    MC_8TAP_FILTER       %3, hf
.loop:
    MC_8TAP_H_LOAD       %3, srcq, %2, 10
    MC_8TAP_COMPUTE      %2, %3
%if %3 > 8
    packssdw             m0, m1
%endif
    UNI_COMPUTE          %2, %3, m0, m1, m9
    PEL_%3STORE%2      dstq, m0, m1
    add                dstq, dststrideq             ; dst += dststride
    add                srcq, srcstrideq             ; src += srcstride
    dec             heightd                         ; cmp height
    jnz               .loop                         ; height loop
    RET


; ******************************
; void put_8tap_vX_X_X(int16_t *dst, ptrdiff_t dststride, const uint8_t *_src, ptrdiff_t srcstride,
;                      int height, const int8_t *hf, const int8_t *vf, int width)
; ******************************
cglobal %1_put_8tap_v%2_%3, 7, 8, 16, dst, dststride, src, srcstride, height, r3src, vf
    MC_8TAP_FILTER        %3, vf
    lea               r3srcq, [srcstrideq*3]
.loop:
    MC_8TAP_V_LOAD        %3, srcq, srcstride, %2, r7
    MC_8TAP_COMPUTE       %2, %3, 1
%if %3 > 8
    packssdw              m0, m1
%endif
    PEL_10STORE%2       dstq, m0, m1
    LOOP_END             dst, src, srcstride
    RET

; ******************************
; void put_uni_8tap_vX_X_X(int16_t *dst, ptrdiff_t dststride, const uint8_t *_src, ptrdiff_t srcstride,
;                       int height, const int8_t *hf, const int8_t *vf, int width)
; ******************************
cglobal %1_put_uni_8tap_v%2_%3, 7, 9, 16, dst, dststride, src, srcstride, height, r3src, vf
    MC_8TAP_FILTER    %3, vf
    movdqa            m9, [scale_%3]
    lea           r3srcq, [srcstrideq*3]
.loop:
    MC_8TAP_V_LOAD    %3, srcq, srcstride, %2, r8
    MC_8TAP_COMPUTE   %2, %3
%if %3 > 8
    packssdw          m0, m1
%endif
    UNI_COMPUTE       %2, %3, m0, m1, m9
    PEL_%3STORE%2   dstq, m0, m1
    add             dstq, dststrideq             ; dst += dststride
    add             srcq, srcstrideq             ; src += srcstride
    dec          heightd                         ; cmp height
    jnz               .loop                      ; height loop
    RET

%endmacro


; ******************************
; void put_8tap_hvX_X(int16_t *dst, ptrdiff_t dststride, const uint8_t *_src, ptrdiff_t srcstride,
;                     int height, const int8_t *hf, const int8_t *vf, int width)
; ******************************
%macro PUT_8TAP_HV 3
cglobal %1_put_8tap_hv%2_%3, 7, 8, 16, 0 - mmsize*16, dst, dststride, src, srcstride, height, hf, vf, r3src
    MC_8TAP_FILTER           %3, hf, 0
    lea                     hfq, [rsp]
    MC_8TAP_FILTER           %3, vf, 8*mmsize
    lea                     vfq, [rsp + 8*mmsize]

    lea                  r3srcq, [srcstrideq*3]
    sub                    srcq, r3srcq

    MC_8TAP_H_LOAD           %3, srcq, %2, 15
    MC_8TAP_HV_COMPUTE       %2, %3, hf, ackssdw
    SWAP                     m8, m0
    add                    srcq, srcstrideq
    MC_8TAP_H_LOAD           %3, srcq, %2, 15
    MC_8TAP_HV_COMPUTE       %2, %3, hf, ackssdw
    SWAP                     m9, m0
    add                    srcq, srcstrideq
    MC_8TAP_H_LOAD           %3, srcq, %2, 15
    MC_8TAP_HV_COMPUTE       %2, %3, hf, ackssdw
    SWAP                    m10, m0
    add                    srcq, srcstrideq
    MC_8TAP_H_LOAD           %3, srcq, %2, 15
    MC_8TAP_HV_COMPUTE       %2, %3, hf, ackssdw
    SWAP                    m11, m0
    add                    srcq, srcstrideq
    MC_8TAP_H_LOAD           %3, srcq, %2, 15
    MC_8TAP_HV_COMPUTE       %2, %3, hf, ackssdw
    SWAP                    m12, m0
    add                    srcq, srcstrideq
    MC_8TAP_H_LOAD           %3, srcq, %2, 15
    MC_8TAP_HV_COMPUTE       %2, %3, hf, ackssdw
    SWAP                    m13, m0
    add                    srcq, srcstrideq
    MC_8TAP_H_LOAD           %3, srcq, %2, 15
    MC_8TAP_HV_COMPUTE       %2, %3, hf, ackssdw
    SWAP                    m14, m0
    add                    srcq, srcstrideq
.loop:
    MC_8TAP_H_LOAD           %3, srcq, %2, 15
    MC_8TAP_HV_COMPUTE       %2, %3, hf, ackssdw
    SWAP                    m15, m0
    punpcklwd                m0, m8, m9
    punpcklwd                m2, m10, m11
    punpcklwd                m4, m12, m13
    punpcklwd                m6, m14, m15
%if %2 > 4
    punpckhwd                m1, m8, m9
    punpckhwd                m3, m10, m11
    punpckhwd                m5, m12, m13
    punpckhwd                m7, m14, m15
%endif
%if %2 <= 4
    movq                     m8, m9
    movq                     m9, m10
    movq                    m10, m11
    movq                    m11, m12
    movq                    m12, m13
    movq                    m13, m14
    movq                    m14, m15
%else
    movdqa                   m8, m9
    movdqa                   m9, m10
    movdqa                  m10, m11
    movdqa                  m11, m12
    movdqa                  m12, m13
    movdqa                  m13, m14
    movdqa                  m14, m15
%endif
    MC_8TAP_HV_COMPUTE       %2, 14, vf, ackssdw
    PEL_10STORE%2          dstq, m0, m1

    LOOP_END                dst, src, srcstride
    RET


cglobal %1_put_uni_8tap_hv%2_%3, 7, 9, 16, 0 - 16*mmsize, dst, dststride, src, srcstride, height, hf, vf, r3src
    MC_8TAP_FILTER           %3, hf, 0
    lea                     hfq, [rsp]
    MC_8TAP_FILTER           %3, vf, 8*mmsize
    lea                     vfq, [rsp + 8*mmsize]
    lea           r3srcq, [srcstrideq*3]
    sub             srcq, r3srcq
    MC_8TAP_H_LOAD       %3, srcq, %2, 15
    MC_8TAP_HV_COMPUTE   %2, %3, hf, ackssdw
    SWAP              m8, m0
    add             srcq, srcstrideq
    MC_8TAP_H_LOAD       %3, srcq, %2, 15
    MC_8TAP_HV_COMPUTE   %2, %3, hf, ackssdw
    SWAP              m9, m0
    add             srcq, srcstrideq
    MC_8TAP_H_LOAD       %3, srcq, %2, 15
    MC_8TAP_HV_COMPUTE   %2, %3, hf, ackssdw
    SWAP             m10, m0
    add             srcq, srcstrideq
    MC_8TAP_H_LOAD       %3, srcq, %2, 15
    MC_8TAP_HV_COMPUTE   %2, %3, hf, ackssdw
    SWAP             m11, m0
    add             srcq, srcstrideq
    MC_8TAP_H_LOAD       %3, srcq, %2, 15
    MC_8TAP_HV_COMPUTE   %2, %3, hf, ackssdw
    SWAP             m12, m0
    add             srcq, srcstrideq
    MC_8TAP_H_LOAD       %3, srcq, %2, 15
    MC_8TAP_HV_COMPUTE   %2, %3, hf, ackssdw
    SWAP             m13, m0
    add             srcq, srcstrideq
    MC_8TAP_H_LOAD       %3, srcq, %2, 15
    MC_8TAP_HV_COMPUTE   %2, %3, hf, ackssdw
    SWAP             m14, m0
    add             srcq, srcstrideq
.loop:
    MC_8TAP_H_LOAD       %3, srcq, %2, 15
    MC_8TAP_HV_COMPUTE   %2, %3, hf, ackssdw
    SWAP             m15, m0
    punpcklwd         m0, m8, m9
    punpcklwd         m2, m10, m11
    punpcklwd         m4, m12, m13
    punpcklwd         m6, m14, m15
%if %2 > 4
    punpckhwd         m1, m8, m9
    punpckhwd         m3, m10, m11
    punpckhwd         m5, m12, m13
    punpckhwd         m7, m14, m15
%endif
    MC_8TAP_HV_COMPUTE   %2, 14, vf, ackusdw
    UNI_COMPUTE       %2, %3, m0, m1, [scale_%3]
    PEL_%3STORE%2   dstq, m0, m1

%if %2 <= 4
    movq              m8, m9
    movq              m9, m10
    movq             m10, m11
    movq             m11, m12
    movq             m12, m13
    movq             m13, m14
    movq             m14, m15
%else
    mova            m8, m9
    mova            m9, m10
    mova           m10, m11
    mova           m11, m12
    mova           m12, m13
    mova           m13, m14
    mova           m14, m15
%endif
    add             dstq, dststrideq             ; dst += dststride
    add             srcq, srcstrideq             ; src += srcstride
    dec          heightd                         ; cmp height
    jnz               .loop                      ; height loop
    RET

%endmacro

%macro H2656PUT_PIXELS 2
    PUT_PIXELS h2656, %1, %2
%endmacro

%macro H2656PUT_4TAP 2
    PUT_4TAP h2656, %1, %2
%endmacro

%macro H2656PUT_4TAP_HV 2
    PUT_4TAP_HV h2656, %1, %2
%endmacro

%macro H2656PUT_8TAP 2
    PUT_8TAP h2656, %1, %2
%endmacro

%macro H2656PUT_8TAP_HV 2
    PUT_8TAP_HV h2656, %1, %2
%endmacro

%if ARCH_X86_64

INIT_XMM sse4
H2656PUT_PIXELS  2, 8
H2656PUT_PIXELS  4, 8
H2656PUT_PIXELS  6, 8
H2656PUT_PIXELS  8, 8
H2656PUT_PIXELS 12, 8
H2656PUT_PIXELS 16, 8

H2656PUT_PIXELS 2, 10
H2656PUT_PIXELS 4, 10
H2656PUT_PIXELS 6, 10
H2656PUT_PIXELS 8, 10

H2656PUT_PIXELS 2, 12
H2656PUT_PIXELS 4, 12
H2656PUT_PIXELS 6, 12
H2656PUT_PIXELS 8, 12

H2656PUT_4TAP 2,  8
H2656PUT_4TAP 4,  8
H2656PUT_4TAP 6,  8
H2656PUT_4TAP 8,  8

H2656PUT_4TAP 12,  8
H2656PUT_4TAP 16, 8

H2656PUT_4TAP 2, 10
H2656PUT_4TAP 4, 10
H2656PUT_4TAP 6, 10
H2656PUT_4TAP 8, 10

H2656PUT_4TAP 2, 12
H2656PUT_4TAP 4, 12
H2656PUT_4TAP 6, 12
H2656PUT_4TAP 8, 12

H2656PUT_4TAP_HV 2,  8
H2656PUT_4TAP_HV 4,  8
H2656PUT_4TAP_HV 6,  8
H2656PUT_4TAP_HV 8,  8
H2656PUT_4TAP_HV 16, 8

H2656PUT_4TAP_HV 2, 10
H2656PUT_4TAP_HV 4, 10
H2656PUT_4TAP_HV 6, 10
H2656PUT_4TAP_HV 8, 10

H2656PUT_4TAP_HV 2, 12
H2656PUT_4TAP_HV 4, 12
H2656PUT_4TAP_HV 6, 12
H2656PUT_4TAP_HV 8, 12

H2656PUT_8TAP  4,  8
H2656PUT_8TAP  8,  8
H2656PUT_8TAP 12, 8
H2656PUT_8TAP 16, 8

H2656PUT_8TAP 4, 10
H2656PUT_8TAP 8, 10

H2656PUT_8TAP 4, 12
H2656PUT_8TAP 8, 12

H2656PUT_8TAP_HV 4, 8
H2656PUT_8TAP_HV 8, 8

H2656PUT_8TAP_HV 4, 10
H2656PUT_8TAP_HV 8, 10

H2656PUT_8TAP_HV 4, 12
H2656PUT_8TAP_HV 8, 12

%if HAVE_AVX2_EXTERNAL
INIT_YMM avx2

H2656PUT_PIXELS  32, 8
H2656PUT_PIXELS  16, 10
H2656PUT_PIXELS  16, 12

H2656PUT_8TAP 32,  8
H2656PUT_8TAP 16, 10
H2656PUT_8TAP 16, 12

H2656PUT_8TAP_HV 16, 10
H2656PUT_8TAP_HV 16, 12

H2656PUT_4TAP 32,  8
H2656PUT_4TAP 16, 10
H2656PUT_4TAP 16, 12

H2656PUT_4TAP_HV 32, 8
H2656PUT_4TAP_HV 16, 10
H2656PUT_4TAP_HV 16, 12

%endif

%endif
