ref: 6db968e9ad1a604fdaa35578df959a8098721a86
parent: 1b9be1a616f38f21f61f2c6b0d51e8763f0c785e
author: Ralph Giles <[email protected]>
date: Tue Jun 13 11:50:07 EDT 2017
Rename SSE 4.1 files to match their targets. Distinguish source files for the SSE 4.1 instruction set extension consistently by their filename. This makes it easier to check the correct flags are being set at build time. Signed-off-by: Jonathan Lennox <[email protected]>
--- a/celt/x86/celt_lpc_sse.c
+++ /dev/null
@@ -1,89 +1,0 @@
-/* Copyright (c) 2014, Cisco Systems, INC
- Written by XiangMingZhu WeiZhou MinPeng YanWang
-
- Redistribution and use in source and binary forms, with or without
- modification, are permitted provided that the following conditions
- are met:
-
- - Redistributions of source code must retain the above copyright
- notice, this list of conditions and the following disclaimer.
-
- - Redistributions in binary form must reproduce the above copyright
- notice, this list of conditions and the following disclaimer in the
- documentation and/or other materials provided with the distribution.
-
- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER
- OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
- EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
- PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
- PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
- LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
- NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-*/
-
-#ifdef HAVE_CONFIG_H
-#include "config.h"
-#endif
-
-#include <xmmintrin.h>
-#include <emmintrin.h>
-#include <smmintrin.h>
-#include "celt_lpc.h"
-#include "stack_alloc.h"
-#include "mathops.h"
-#include "pitch.h"
-#include "x86cpu.h"
-
-#if defined(FIXED_POINT)
-
-void celt_fir_sse4_1(const opus_val16 *x,
- const opus_val16 *num,
- opus_val16 *y,
- int N,
- int ord,
- int arch)
-{- int i,j;
- VARDECL(opus_val16, rnum);
-
- __m128i vecNoA;
- opus_int32 noA ;
- SAVE_STACK;
-
- ALLOC(rnum, ord, opus_val16);
- for(i=0;i<ord;i++)
- rnum[i] = num[ord-i-1];
- noA = EXTEND32(1) << SIG_SHIFT >> 1;
- vecNoA = _mm_set_epi32(noA, noA, noA, noA);
-
- for (i=0;i<N-3;i+=4)
- {- opus_val32 sums[4] = {0};- __m128i vecSum, vecX;
-
- xcorr_kernel(rnum, x+i-ord, sums, ord, arch);
-
- vecSum = _mm_loadu_si128((__m128i *)sums);
- vecSum = _mm_add_epi32(vecSum, vecNoA);
- vecSum = _mm_srai_epi32(vecSum, SIG_SHIFT);
- vecX = OP_CVTEPI16_EPI32_M64(x + i);
- vecSum = _mm_add_epi32(vecSum, vecX);
- vecSum = _mm_packs_epi32(vecSum, vecSum);
- _mm_storel_epi64((__m128i *)(y + i), vecSum);
- }
- for (;i<N;i++)
- {- opus_val32 sum = 0;
- for (j=0;j<ord;j++)
- sum = MAC16_16(sum, rnum[j], x[i+j-ord]);
- y[i] = SATURATE16(ADD32(EXTEND32(x[i]), PSHR32(sum, SIG_SHIFT)));
- }
-
- RESTORE_STACK;
-}
-
-#endif
--- /dev/null
+++ b/celt/x86/celt_lpc_sse4_1.c
@@ -1,0 +1,89 @@
+/* Copyright (c) 2014, Cisco Systems, INC
+ Written by XiangMingZhu WeiZhou MinPeng YanWang
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+
+ - Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+
+ - Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER
+ OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+ LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+ NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*/
+
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include <xmmintrin.h>
+#include <emmintrin.h>
+#include <smmintrin.h>
+#include "celt_lpc.h"
+#include "stack_alloc.h"
+#include "mathops.h"
+#include "pitch.h"
+#include "x86cpu.h"
+
+#if defined(FIXED_POINT)
+
+void celt_fir_sse4_1(const opus_val16 *x,
+ const opus_val16 *num,
+ opus_val16 *y,
+ int N,
+ int ord,
+ int arch)
+{+ int i,j;
+ VARDECL(opus_val16, rnum);
+
+ __m128i vecNoA;
+ opus_int32 noA ;
+ SAVE_STACK;
+
+ ALLOC(rnum, ord, opus_val16);
+ for(i=0;i<ord;i++)
+ rnum[i] = num[ord-i-1];
+ noA = EXTEND32(1) << SIG_SHIFT >> 1;
+ vecNoA = _mm_set_epi32(noA, noA, noA, noA);
+
+ for (i=0;i<N-3;i+=4)
+ {+ opus_val32 sums[4] = {0};+ __m128i vecSum, vecX;
+
+ xcorr_kernel(rnum, x+i-ord, sums, ord, arch);
+
+ vecSum = _mm_loadu_si128((__m128i *)sums);
+ vecSum = _mm_add_epi32(vecSum, vecNoA);
+ vecSum = _mm_srai_epi32(vecSum, SIG_SHIFT);
+ vecX = OP_CVTEPI16_EPI32_M64(x + i);
+ vecSum = _mm_add_epi32(vecSum, vecX);
+ vecSum = _mm_packs_epi32(vecSum, vecSum);
+ _mm_storel_epi64((__m128i *)(y + i), vecSum);
+ }
+ for (;i<N;i++)
+ {+ opus_val32 sum = 0;
+ for (j=0;j<ord;j++)
+ sum = MAC16_16(sum, rnum[j], x[i+j-ord]);
+ y[i] = SATURATE16(ADD32(EXTEND32(x[i]), PSHR32(sum, SIG_SHIFT)));
+ }
+
+ RESTORE_STACK;
+}
+
+#endif
--- a/celt_sources.mk
+++ b/celt_sources.mk
@@ -27,7 +27,7 @@
celt/x86/vq_sse2.c
CELT_SOURCES_SSE4_1 = \
-celt/x86/celt_lpc_sse.c \
+celt/x86/celt_lpc_sse4_1.c \
celt/x86/pitch_sse4_1.c
CELT_SOURCES_ARM = \
--- a/silk/fixed/x86/burg_modified_FIX_sse.c
+++ /dev/null
@@ -1,377 +1,0 @@
-/* Copyright (c) 2014, Cisco Systems, INC
- Written by XiangMingZhu WeiZhou MinPeng YanWang
-
- Redistribution and use in source and binary forms, with or without
- modification, are permitted provided that the following conditions
- are met:
-
- - Redistributions of source code must retain the above copyright
- notice, this list of conditions and the following disclaimer.
-
- - Redistributions in binary form must reproduce the above copyright
- notice, this list of conditions and the following disclaimer in the
- documentation and/or other materials provided with the distribution.
-
- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER
- OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
- EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
- PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
- PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
- LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
- NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-*/
-
-#ifdef HAVE_CONFIG_H
-#include "config.h"
-#endif
-
-#include <xmmintrin.h>
-#include <emmintrin.h>
-#include <smmintrin.h>
-
-#include "SigProc_FIX.h"
-#include "define.h"
-#include "tuning_parameters.h"
-#include "pitch.h"
-#include "celt/x86/x86cpu.h"
-
-#define MAX_FRAME_SIZE 384 /* subfr_length * nb_subfr = ( 0.005 * 16000 + 16 ) * 4 = 384 */
-
-#define QA 25
-#define N_BITS_HEAD_ROOM 2
-#define MIN_RSHIFTS -16
-#define MAX_RSHIFTS (32 - QA)
-
-/* Compute reflection coefficients from input signal */
-void silk_burg_modified_sse4_1(
- opus_int32 *res_nrg, /* O Residual energy */
- opus_int *res_nrg_Q, /* O Residual energy Q value */
- opus_int32 A_Q16[], /* O Prediction coefficients (length order) */
- const opus_int16 x[], /* I Input signal, length: nb_subfr * ( D + subfr_length ) */
- const opus_int32 minInvGain_Q30, /* I Inverse of max prediction gain */
- const opus_int subfr_length, /* I Input signal subframe length (incl. D preceding samples) */
- const opus_int nb_subfr, /* I Number of subframes stacked in x */
- const opus_int D, /* I Order */
- int arch /* I Run-time architecture */
-)
-{- opus_int k, n, s, lz, rshifts, rshifts_extra, reached_max_gain;
- opus_int32 C0, num, nrg, rc_Q31, invGain_Q30, Atmp_QA, Atmp1, tmp1, tmp2, x1, x2;
- const opus_int16 *x_ptr;
- opus_int32 C_first_row[ SILK_MAX_ORDER_LPC ];
- opus_int32 C_last_row[ SILK_MAX_ORDER_LPC ];
- opus_int32 Af_QA[ SILK_MAX_ORDER_LPC ];
- opus_int32 CAf[ SILK_MAX_ORDER_LPC + 1 ];
- opus_int32 CAb[ SILK_MAX_ORDER_LPC + 1 ];
- opus_int32 xcorr[ SILK_MAX_ORDER_LPC ];
-
- __m128i FIRST_3210, LAST_3210, ATMP_3210, TMP1_3210, TMP2_3210, T1_3210, T2_3210, PTR_3210, SUBFR_3210, X1_3210, X2_3210;
- __m128i CONST1 = _mm_set1_epi32(1);
-
- silk_assert( subfr_length * nb_subfr <= MAX_FRAME_SIZE );
-
- /* Compute autocorrelations, added over subframes */
- silk_sum_sqr_shift( &C0, &rshifts, x, nb_subfr * subfr_length );
- if( rshifts > MAX_RSHIFTS ) {- C0 = silk_LSHIFT32( C0, rshifts - MAX_RSHIFTS );
- silk_assert( C0 > 0 );
- rshifts = MAX_RSHIFTS;
- } else {- lz = silk_CLZ32( C0 ) - 1;
- rshifts_extra = N_BITS_HEAD_ROOM - lz;
- if( rshifts_extra > 0 ) {- rshifts_extra = silk_min( rshifts_extra, MAX_RSHIFTS - rshifts );
- C0 = silk_RSHIFT32( C0, rshifts_extra );
- } else {- rshifts_extra = silk_max( rshifts_extra, MIN_RSHIFTS - rshifts );
- C0 = silk_LSHIFT32( C0, -rshifts_extra );
- }
- rshifts += rshifts_extra;
- }
- CAb[ 0 ] = CAf[ 0 ] = C0 + silk_SMMUL( SILK_FIX_CONST( FIND_LPC_COND_FAC, 32 ), C0 ) + 1; /* Q(-rshifts) */
- silk_memset( C_first_row, 0, SILK_MAX_ORDER_LPC * sizeof( opus_int32 ) );
- if( rshifts > 0 ) {- for( s = 0; s < nb_subfr; s++ ) {- x_ptr = x + s * subfr_length;
- for( n = 1; n < D + 1; n++ ) {- C_first_row[ n - 1 ] += (opus_int32)silk_RSHIFT64(
- silk_inner_prod16_aligned_64( x_ptr, x_ptr + n, subfr_length - n, arch ), rshifts );
- }
- }
- } else {- for( s = 0; s < nb_subfr; s++ ) {- int i;
- opus_int32 d;
- x_ptr = x + s * subfr_length;
- celt_pitch_xcorr(x_ptr, x_ptr + 1, xcorr, subfr_length - D, D, arch );
- for( n = 1; n < D + 1; n++ ) {- for ( i = n + subfr_length - D, d = 0; i < subfr_length; i++ )
- d = MAC16_16( d, x_ptr[ i ], x_ptr[ i - n ] );
- xcorr[ n - 1 ] += d;
- }
- for( n = 1; n < D + 1; n++ ) {- C_first_row[ n - 1 ] += silk_LSHIFT32( xcorr[ n - 1 ], -rshifts );
- }
- }
- }
- silk_memcpy( C_last_row, C_first_row, SILK_MAX_ORDER_LPC * sizeof( opus_int32 ) );
-
- /* Initialize */
- CAb[ 0 ] = CAf[ 0 ] = C0 + silk_SMMUL( SILK_FIX_CONST( FIND_LPC_COND_FAC, 32 ), C0 ) + 1; /* Q(-rshifts) */
-
- invGain_Q30 = (opus_int32)1 << 30;
- reached_max_gain = 0;
- for( n = 0; n < D; n++ ) {- /* Update first row of correlation matrix (without first element) */
- /* Update last row of correlation matrix (without last element, stored in reversed order) */
- /* Update C * Af */
- /* Update C * flipud(Af) (stored in reversed order) */
- if( rshifts > -2 ) {- for( s = 0; s < nb_subfr; s++ ) {- x_ptr = x + s * subfr_length;
- x1 = -silk_LSHIFT32( (opus_int32)x_ptr[ n ], 16 - rshifts ); /* Q(16-rshifts) */
- x2 = -silk_LSHIFT32( (opus_int32)x_ptr[ subfr_length - n - 1 ], 16 - rshifts ); /* Q(16-rshifts) */
- tmp1 = silk_LSHIFT32( (opus_int32)x_ptr[ n ], QA - 16 ); /* Q(QA-16) */
- tmp2 = silk_LSHIFT32( (opus_int32)x_ptr[ subfr_length - n - 1 ], QA - 16 ); /* Q(QA-16) */
- for( k = 0; k < n; k++ ) {- C_first_row[ k ] = silk_SMLAWB( C_first_row[ k ], x1, x_ptr[ n - k - 1 ] ); /* Q( -rshifts ) */
- C_last_row[ k ] = silk_SMLAWB( C_last_row[ k ], x2, x_ptr[ subfr_length - n + k ] ); /* Q( -rshifts ) */
- Atmp_QA = Af_QA[ k ];
- tmp1 = silk_SMLAWB( tmp1, Atmp_QA, x_ptr[ n - k - 1 ] ); /* Q(QA-16) */
- tmp2 = silk_SMLAWB( tmp2, Atmp_QA, x_ptr[ subfr_length - n + k ] ); /* Q(QA-16) */
- }
- tmp1 = silk_LSHIFT32( -tmp1, 32 - QA - rshifts ); /* Q(16-rshifts) */
- tmp2 = silk_LSHIFT32( -tmp2, 32 - QA - rshifts ); /* Q(16-rshifts) */
- for( k = 0; k <= n; k++ ) {- CAf[ k ] = silk_SMLAWB( CAf[ k ], tmp1, x_ptr[ n - k ] ); /* Q( -rshift ) */
- CAb[ k ] = silk_SMLAWB( CAb[ k ], tmp2, x_ptr[ subfr_length - n + k - 1 ] ); /* Q( -rshift ) */
- }
- }
- } else {- for( s = 0; s < nb_subfr; s++ ) {- x_ptr = x + s * subfr_length;
- x1 = -silk_LSHIFT32( (opus_int32)x_ptr[ n ], -rshifts ); /* Q( -rshifts ) */
- x2 = -silk_LSHIFT32( (opus_int32)x_ptr[ subfr_length - n - 1 ], -rshifts ); /* Q( -rshifts ) */
- tmp1 = silk_LSHIFT32( (opus_int32)x_ptr[ n ], 17 ); /* Q17 */
- tmp2 = silk_LSHIFT32( (opus_int32)x_ptr[ subfr_length - n - 1 ], 17 ); /* Q17 */
-
- X1_3210 = _mm_set1_epi32( x1 );
- X2_3210 = _mm_set1_epi32( x2 );
- TMP1_3210 = _mm_setzero_si128();
- TMP2_3210 = _mm_setzero_si128();
- for( k = 0; k < n - 3; k += 4 ) {- PTR_3210 = OP_CVTEPI16_EPI32_M64( &x_ptr[ n - k - 1 - 3 ] );
- SUBFR_3210 = OP_CVTEPI16_EPI32_M64( &x_ptr[ subfr_length - n + k ] );
- FIRST_3210 = _mm_loadu_si128( (__m128i *)&C_first_row[ k ] );
- PTR_3210 = _mm_shuffle_epi32( PTR_3210, _MM_SHUFFLE( 0, 1, 2, 3 ) );
- LAST_3210 = _mm_loadu_si128( (__m128i *)&C_last_row[ k ] );
- ATMP_3210 = _mm_loadu_si128( (__m128i *)&Af_QA[ k ] );
-
- T1_3210 = _mm_mullo_epi32( PTR_3210, X1_3210 );
- T2_3210 = _mm_mullo_epi32( SUBFR_3210, X2_3210 );
-
- ATMP_3210 = _mm_srai_epi32( ATMP_3210, 7 );
- ATMP_3210 = _mm_add_epi32( ATMP_3210, CONST1 );
- ATMP_3210 = _mm_srai_epi32( ATMP_3210, 1 );
-
- FIRST_3210 = _mm_add_epi32( FIRST_3210, T1_3210 );
- LAST_3210 = _mm_add_epi32( LAST_3210, T2_3210 );
-
- PTR_3210 = _mm_mullo_epi32( ATMP_3210, PTR_3210 );
- SUBFR_3210 = _mm_mullo_epi32( ATMP_3210, SUBFR_3210 );
-
- _mm_storeu_si128( (__m128i *)&C_first_row[ k ], FIRST_3210 );
- _mm_storeu_si128( (__m128i *)&C_last_row[ k ], LAST_3210 );
-
- TMP1_3210 = _mm_add_epi32( TMP1_3210, PTR_3210 );
- TMP2_3210 = _mm_add_epi32( TMP2_3210, SUBFR_3210 );
- }
-
- TMP1_3210 = _mm_add_epi32( TMP1_3210, _mm_unpackhi_epi64(TMP1_3210, TMP1_3210 ) );
- TMP2_3210 = _mm_add_epi32( TMP2_3210, _mm_unpackhi_epi64(TMP2_3210, TMP2_3210 ) );
- TMP1_3210 = _mm_add_epi32( TMP1_3210, _mm_shufflelo_epi16(TMP1_3210, 0x0E ) );
- TMP2_3210 = _mm_add_epi32( TMP2_3210, _mm_shufflelo_epi16(TMP2_3210, 0x0E ) );
-
- tmp1 += _mm_cvtsi128_si32( TMP1_3210 );
- tmp2 += _mm_cvtsi128_si32( TMP2_3210 );
-
- for( ; k < n; k++ ) {- C_first_row[ k ] = silk_MLA( C_first_row[ k ], x1, x_ptr[ n - k - 1 ] ); /* Q( -rshifts ) */
- C_last_row[ k ] = silk_MLA( C_last_row[ k ], x2, x_ptr[ subfr_length - n + k ] ); /* Q( -rshifts ) */
- Atmp1 = silk_RSHIFT_ROUND( Af_QA[ k ], QA - 17 ); /* Q17 */
- tmp1 = silk_MLA( tmp1, x_ptr[ n - k - 1 ], Atmp1 ); /* Q17 */
- tmp2 = silk_MLA( tmp2, x_ptr[ subfr_length - n + k ], Atmp1 ); /* Q17 */
- }
-
- tmp1 = -tmp1; /* Q17 */
- tmp2 = -tmp2; /* Q17 */
-
- {- __m128i xmm_tmp1, xmm_tmp2;
- __m128i xmm_x_ptr_n_k_x2x0, xmm_x_ptr_n_k_x3x1;
- __m128i xmm_x_ptr_sub_x2x0, xmm_x_ptr_sub_x3x1;
-
- xmm_tmp1 = _mm_set1_epi32( tmp1 );
- xmm_tmp2 = _mm_set1_epi32( tmp2 );
-
- for( k = 0; k <= n - 3; k += 4 ) {- xmm_x_ptr_n_k_x2x0 = OP_CVTEPI16_EPI32_M64( &x_ptr[ n - k - 3 ] );
- xmm_x_ptr_sub_x2x0 = OP_CVTEPI16_EPI32_M64( &x_ptr[ subfr_length - n + k - 1 ] );
-
- xmm_x_ptr_n_k_x2x0 = _mm_shuffle_epi32( xmm_x_ptr_n_k_x2x0, _MM_SHUFFLE( 0, 1, 2, 3 ) );
-
- xmm_x_ptr_n_k_x2x0 = _mm_slli_epi32( xmm_x_ptr_n_k_x2x0, -rshifts - 1 );
- xmm_x_ptr_sub_x2x0 = _mm_slli_epi32( xmm_x_ptr_sub_x2x0, -rshifts - 1 );
-
- /* equal shift right 4 bytes, xmm_x_ptr_n_k_x3x1 = _mm_srli_si128(xmm_x_ptr_n_k_x2x0, 4)*/
- xmm_x_ptr_n_k_x3x1 = _mm_shuffle_epi32( xmm_x_ptr_n_k_x2x0, _MM_SHUFFLE( 0, 3, 2, 1 ) );
- xmm_x_ptr_sub_x3x1 = _mm_shuffle_epi32( xmm_x_ptr_sub_x2x0, _MM_SHUFFLE( 0, 3, 2, 1 ) );
-
- xmm_x_ptr_n_k_x2x0 = _mm_mul_epi32( xmm_x_ptr_n_k_x2x0, xmm_tmp1 );
- xmm_x_ptr_n_k_x3x1 = _mm_mul_epi32( xmm_x_ptr_n_k_x3x1, xmm_tmp1 );
- xmm_x_ptr_sub_x2x0 = _mm_mul_epi32( xmm_x_ptr_sub_x2x0, xmm_tmp2 );
- xmm_x_ptr_sub_x3x1 = _mm_mul_epi32( xmm_x_ptr_sub_x3x1, xmm_tmp2 );
-
- xmm_x_ptr_n_k_x2x0 = _mm_srli_epi64( xmm_x_ptr_n_k_x2x0, 16 );
- xmm_x_ptr_n_k_x3x1 = _mm_slli_epi64( xmm_x_ptr_n_k_x3x1, 16 );
- xmm_x_ptr_sub_x2x0 = _mm_srli_epi64( xmm_x_ptr_sub_x2x0, 16 );
- xmm_x_ptr_sub_x3x1 = _mm_slli_epi64( xmm_x_ptr_sub_x3x1, 16 );
-
- xmm_x_ptr_n_k_x2x0 = _mm_blend_epi16( xmm_x_ptr_n_k_x2x0, xmm_x_ptr_n_k_x3x1, 0xCC );
- xmm_x_ptr_sub_x2x0 = _mm_blend_epi16( xmm_x_ptr_sub_x2x0, xmm_x_ptr_sub_x3x1, 0xCC );
-
- X1_3210 = _mm_loadu_si128( (__m128i *)&CAf[ k ] );
- PTR_3210 = _mm_loadu_si128( (__m128i *)&CAb[ k ] );
-
- X1_3210 = _mm_add_epi32( X1_3210, xmm_x_ptr_n_k_x2x0 );
- PTR_3210 = _mm_add_epi32( PTR_3210, xmm_x_ptr_sub_x2x0 );
-
- _mm_storeu_si128( (__m128i *)&CAf[ k ], X1_3210 );
- _mm_storeu_si128( (__m128i *)&CAb[ k ], PTR_3210 );
- }
-
- for( ; k <= n; k++ ) {- CAf[ k ] = silk_SMLAWW( CAf[ k ], tmp1,
- silk_LSHIFT32( (opus_int32)x_ptr[ n - k ], -rshifts - 1 ) ); /* Q( -rshift ) */
- CAb[ k ] = silk_SMLAWW( CAb[ k ], tmp2,
- silk_LSHIFT32( (opus_int32)x_ptr[ subfr_length - n + k - 1 ], -rshifts - 1 ) ); /* Q( -rshift ) */
- }
- }
- }
- }
-
- /* Calculate nominator and denominator for the next order reflection (parcor) coefficient */
- tmp1 = C_first_row[ n ]; /* Q( -rshifts ) */
- tmp2 = C_last_row[ n ]; /* Q( -rshifts ) */
- num = 0; /* Q( -rshifts ) */
- nrg = silk_ADD32( CAb[ 0 ], CAf[ 0 ] ); /* Q( 1-rshifts ) */
- for( k = 0; k < n; k++ ) {- Atmp_QA = Af_QA[ k ];
- lz = silk_CLZ32( silk_abs( Atmp_QA ) ) - 1;
- lz = silk_min( 32 - QA, lz );
- Atmp1 = silk_LSHIFT32( Atmp_QA, lz ); /* Q( QA + lz ) */
-
- tmp1 = silk_ADD_LSHIFT32( tmp1, silk_SMMUL( C_last_row[ n - k - 1 ], Atmp1 ), 32 - QA - lz ); /* Q( -rshifts ) */
- tmp2 = silk_ADD_LSHIFT32( tmp2, silk_SMMUL( C_first_row[ n - k - 1 ], Atmp1 ), 32 - QA - lz ); /* Q( -rshifts ) */
- num = silk_ADD_LSHIFT32( num, silk_SMMUL( CAb[ n - k ], Atmp1 ), 32 - QA - lz ); /* Q( -rshifts ) */
- nrg = silk_ADD_LSHIFT32( nrg, silk_SMMUL( silk_ADD32( CAb[ k + 1 ], CAf[ k + 1 ] ),
- Atmp1 ), 32 - QA - lz ); /* Q( 1-rshifts ) */
- }
- CAf[ n + 1 ] = tmp1; /* Q( -rshifts ) */
- CAb[ n + 1 ] = tmp2; /* Q( -rshifts ) */
- num = silk_ADD32( num, tmp2 ); /* Q( -rshifts ) */
- num = silk_LSHIFT32( -num, 1 ); /* Q( 1-rshifts ) */
-
- /* Calculate the next order reflection (parcor) coefficient */
- if( silk_abs( num ) < nrg ) {- rc_Q31 = silk_DIV32_varQ( num, nrg, 31 );
- } else {- rc_Q31 = ( num > 0 ) ? silk_int32_MAX : silk_int32_MIN;
- }
-
- /* Update inverse prediction gain */
- tmp1 = ( (opus_int32)1 << 30 ) - silk_SMMUL( rc_Q31, rc_Q31 );
- tmp1 = silk_LSHIFT( silk_SMMUL( invGain_Q30, tmp1 ), 2 );
- if( tmp1 <= minInvGain_Q30 ) {- /* Max prediction gain exceeded; set reflection coefficient such that max prediction gain is exactly hit */
- tmp2 = ( (opus_int32)1 << 30 ) - silk_DIV32_varQ( minInvGain_Q30, invGain_Q30, 30 ); /* Q30 */
- rc_Q31 = silk_SQRT_APPROX( tmp2 ); /* Q15 */
- if( rc_Q31 > 0 ) {- /* Newton-Raphson iteration */
- rc_Q31 = silk_RSHIFT32( rc_Q31 + silk_DIV32( tmp2, rc_Q31 ), 1 ); /* Q15 */
- rc_Q31 = silk_LSHIFT32( rc_Q31, 16 ); /* Q31 */
- if( num < 0 ) {- /* Ensure adjusted reflection coefficients has the original sign */
- rc_Q31 = -rc_Q31;
- }
- }
- invGain_Q30 = minInvGain_Q30;
- reached_max_gain = 1;
- } else {- invGain_Q30 = tmp1;
- }
-
- /* Update the AR coefficients */
- for( k = 0; k < (n + 1) >> 1; k++ ) {- tmp1 = Af_QA[ k ]; /* QA */
- tmp2 = Af_QA[ n - k - 1 ]; /* QA */
- Af_QA[ k ] = silk_ADD_LSHIFT32( tmp1, silk_SMMUL( tmp2, rc_Q31 ), 1 ); /* QA */
- Af_QA[ n - k - 1 ] = silk_ADD_LSHIFT32( tmp2, silk_SMMUL( tmp1, rc_Q31 ), 1 ); /* QA */
- }
- Af_QA[ n ] = silk_RSHIFT32( rc_Q31, 31 - QA ); /* QA */
-
- if( reached_max_gain ) {- /* Reached max prediction gain; set remaining coefficients to zero and exit loop */
- for( k = n + 1; k < D; k++ ) {- Af_QA[ k ] = 0;
- }
- break;
- }
-
- /* Update C * Af and C * Ab */
- for( k = 0; k <= n + 1; k++ ) {- tmp1 = CAf[ k ]; /* Q( -rshifts ) */
- tmp2 = CAb[ n - k + 1 ]; /* Q( -rshifts ) */
- CAf[ k ] = silk_ADD_LSHIFT32( tmp1, silk_SMMUL( tmp2, rc_Q31 ), 1 ); /* Q( -rshifts ) */
- CAb[ n - k + 1 ] = silk_ADD_LSHIFT32( tmp2, silk_SMMUL( tmp1, rc_Q31 ), 1 ); /* Q( -rshifts ) */
- }
- }
-
- if( reached_max_gain ) {- for( k = 0; k < D; k++ ) {- /* Scale coefficients */
- A_Q16[ k ] = -silk_RSHIFT_ROUND( Af_QA[ k ], QA - 16 );
- }
- /* Subtract energy of preceding samples from C0 */
- if( rshifts > 0 ) {- for( s = 0; s < nb_subfr; s++ ) {- x_ptr = x + s * subfr_length;
- C0 -= (opus_int32)silk_RSHIFT64( silk_inner_prod16_aligned_64( x_ptr, x_ptr, D, arch ), rshifts );
- }
- } else {- for( s = 0; s < nb_subfr; s++ ) {- x_ptr = x + s * subfr_length;
- C0 -= silk_LSHIFT32( silk_inner_prod_aligned( x_ptr, x_ptr, D, arch ), -rshifts );
- }
- }
- /* Approximate residual energy */
- *res_nrg = silk_LSHIFT( silk_SMMUL( invGain_Q30, C0 ), 2 );
- *res_nrg_Q = -rshifts;
- } else {- /* Return residual energy */
- nrg = CAf[ 0 ]; /* Q( -rshifts ) */
- tmp1 = (opus_int32)1 << 16; /* Q16 */
- for( k = 0; k < D; k++ ) {- Atmp1 = silk_RSHIFT_ROUND( Af_QA[ k ], QA - 16 ); /* Q16 */
- nrg = silk_SMLAWW( nrg, CAf[ k + 1 ], Atmp1 ); /* Q( -rshifts ) */
- tmp1 = silk_SMLAWW( tmp1, Atmp1, Atmp1 ); /* Q16 */
- A_Q16[ k ] = -Atmp1;
- }
- *res_nrg = silk_SMLAWW( nrg, silk_SMMUL( SILK_FIX_CONST( FIND_LPC_COND_FAC, 32 ), C0 ), -tmp1 );/* Q( -rshifts ) */
- *res_nrg_Q = -rshifts;
- }
-}
--- /dev/null
+++ b/silk/fixed/x86/burg_modified_FIX_sse4_1.c
@@ -1,0 +1,377 @@
+/* Copyright (c) 2014, Cisco Systems, INC
+ Written by XiangMingZhu WeiZhou MinPeng YanWang
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+
+ - Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+
+ - Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER
+ OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+ LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+ NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*/
+
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include <xmmintrin.h>
+#include <emmintrin.h>
+#include <smmintrin.h>
+
+#include "SigProc_FIX.h"
+#include "define.h"
+#include "tuning_parameters.h"
+#include "pitch.h"
+#include "celt/x86/x86cpu.h"
+
+#define MAX_FRAME_SIZE 384 /* subfr_length * nb_subfr = ( 0.005 * 16000 + 16 ) * 4 = 384 */
+
+#define QA 25
+#define N_BITS_HEAD_ROOM 2
+#define MIN_RSHIFTS -16
+#define MAX_RSHIFTS (32 - QA)
+
+/* Compute reflection coefficients from input signal */
+void silk_burg_modified_sse4_1(
+ opus_int32 *res_nrg, /* O Residual energy */
+ opus_int *res_nrg_Q, /* O Residual energy Q value */
+ opus_int32 A_Q16[], /* O Prediction coefficients (length order) */
+ const opus_int16 x[], /* I Input signal, length: nb_subfr * ( D + subfr_length ) */
+ const opus_int32 minInvGain_Q30, /* I Inverse of max prediction gain */
+ const opus_int subfr_length, /* I Input signal subframe length (incl. D preceding samples) */
+ const opus_int nb_subfr, /* I Number of subframes stacked in x */
+ const opus_int D, /* I Order */
+ int arch /* I Run-time architecture */
+)
+{+ opus_int k, n, s, lz, rshifts, rshifts_extra, reached_max_gain;
+ opus_int32 C0, num, nrg, rc_Q31, invGain_Q30, Atmp_QA, Atmp1, tmp1, tmp2, x1, x2;
+ const opus_int16 *x_ptr;
+ opus_int32 C_first_row[ SILK_MAX_ORDER_LPC ];
+ opus_int32 C_last_row[ SILK_MAX_ORDER_LPC ];
+ opus_int32 Af_QA[ SILK_MAX_ORDER_LPC ];
+ opus_int32 CAf[ SILK_MAX_ORDER_LPC + 1 ];
+ opus_int32 CAb[ SILK_MAX_ORDER_LPC + 1 ];
+ opus_int32 xcorr[ SILK_MAX_ORDER_LPC ];
+
+ __m128i FIRST_3210, LAST_3210, ATMP_3210, TMP1_3210, TMP2_3210, T1_3210, T2_3210, PTR_3210, SUBFR_3210, X1_3210, X2_3210;
+ __m128i CONST1 = _mm_set1_epi32(1);
+
+ silk_assert( subfr_length * nb_subfr <= MAX_FRAME_SIZE );
+
+ /* Compute autocorrelations, added over subframes */
+ silk_sum_sqr_shift( &C0, &rshifts, x, nb_subfr * subfr_length );
+ if( rshifts > MAX_RSHIFTS ) {+ C0 = silk_LSHIFT32( C0, rshifts - MAX_RSHIFTS );
+ silk_assert( C0 > 0 );
+ rshifts = MAX_RSHIFTS;
+ } else {+ lz = silk_CLZ32( C0 ) - 1;
+ rshifts_extra = N_BITS_HEAD_ROOM - lz;
+ if( rshifts_extra > 0 ) {+ rshifts_extra = silk_min( rshifts_extra, MAX_RSHIFTS - rshifts );
+ C0 = silk_RSHIFT32( C0, rshifts_extra );
+ } else {+ rshifts_extra = silk_max( rshifts_extra, MIN_RSHIFTS - rshifts );
+ C0 = silk_LSHIFT32( C0, -rshifts_extra );
+ }
+ rshifts += rshifts_extra;
+ }
+ CAb[ 0 ] = CAf[ 0 ] = C0 + silk_SMMUL( SILK_FIX_CONST( FIND_LPC_COND_FAC, 32 ), C0 ) + 1; /* Q(-rshifts) */
+ silk_memset( C_first_row, 0, SILK_MAX_ORDER_LPC * sizeof( opus_int32 ) );
+ if( rshifts > 0 ) {+ for( s = 0; s < nb_subfr; s++ ) {+ x_ptr = x + s * subfr_length;
+ for( n = 1; n < D + 1; n++ ) {+ C_first_row[ n - 1 ] += (opus_int32)silk_RSHIFT64(
+ silk_inner_prod16_aligned_64( x_ptr, x_ptr + n, subfr_length - n, arch ), rshifts );
+ }
+ }
+ } else {+ for( s = 0; s < nb_subfr; s++ ) {+ int i;
+ opus_int32 d;
+ x_ptr = x + s * subfr_length;
+ celt_pitch_xcorr(x_ptr, x_ptr + 1, xcorr, subfr_length - D, D, arch );
+ for( n = 1; n < D + 1; n++ ) {+ for ( i = n + subfr_length - D, d = 0; i < subfr_length; i++ )
+ d = MAC16_16( d, x_ptr[ i ], x_ptr[ i - n ] );
+ xcorr[ n - 1 ] += d;
+ }
+ for( n = 1; n < D + 1; n++ ) {+ C_first_row[ n - 1 ] += silk_LSHIFT32( xcorr[ n - 1 ], -rshifts );
+ }
+ }
+ }
+ silk_memcpy( C_last_row, C_first_row, SILK_MAX_ORDER_LPC * sizeof( opus_int32 ) );
+
+ /* Initialize */
+ CAb[ 0 ] = CAf[ 0 ] = C0 + silk_SMMUL( SILK_FIX_CONST( FIND_LPC_COND_FAC, 32 ), C0 ) + 1; /* Q(-rshifts) */
+
+ invGain_Q30 = (opus_int32)1 << 30;
+ reached_max_gain = 0;
+ for( n = 0; n < D; n++ ) {+ /* Update first row of correlation matrix (without first element) */
+ /* Update last row of correlation matrix (without last element, stored in reversed order) */
+ /* Update C * Af */
+ /* Update C * flipud(Af) (stored in reversed order) */
+ if( rshifts > -2 ) {+ for( s = 0; s < nb_subfr; s++ ) {+ x_ptr = x + s * subfr_length;
+ x1 = -silk_LSHIFT32( (opus_int32)x_ptr[ n ], 16 - rshifts ); /* Q(16-rshifts) */
+ x2 = -silk_LSHIFT32( (opus_int32)x_ptr[ subfr_length - n - 1 ], 16 - rshifts ); /* Q(16-rshifts) */
+ tmp1 = silk_LSHIFT32( (opus_int32)x_ptr[ n ], QA - 16 ); /* Q(QA-16) */
+ tmp2 = silk_LSHIFT32( (opus_int32)x_ptr[ subfr_length - n - 1 ], QA - 16 ); /* Q(QA-16) */
+ for( k = 0; k < n; k++ ) {+ C_first_row[ k ] = silk_SMLAWB( C_first_row[ k ], x1, x_ptr[ n - k - 1 ] ); /* Q( -rshifts ) */
+ C_last_row[ k ] = silk_SMLAWB( C_last_row[ k ], x2, x_ptr[ subfr_length - n + k ] ); /* Q( -rshifts ) */
+ Atmp_QA = Af_QA[ k ];
+ tmp1 = silk_SMLAWB( tmp1, Atmp_QA, x_ptr[ n - k - 1 ] ); /* Q(QA-16) */
+ tmp2 = silk_SMLAWB( tmp2, Atmp_QA, x_ptr[ subfr_length - n + k ] ); /* Q(QA-16) */
+ }
+ tmp1 = silk_LSHIFT32( -tmp1, 32 - QA - rshifts ); /* Q(16-rshifts) */
+ tmp2 = silk_LSHIFT32( -tmp2, 32 - QA - rshifts ); /* Q(16-rshifts) */
+ for( k = 0; k <= n; k++ ) {+ CAf[ k ] = silk_SMLAWB( CAf[ k ], tmp1, x_ptr[ n - k ] ); /* Q( -rshift ) */
+ CAb[ k ] = silk_SMLAWB( CAb[ k ], tmp2, x_ptr[ subfr_length - n + k - 1 ] ); /* Q( -rshift ) */
+ }
+ }
+ } else {+ for( s = 0; s < nb_subfr; s++ ) {+ x_ptr = x + s * subfr_length;
+ x1 = -silk_LSHIFT32( (opus_int32)x_ptr[ n ], -rshifts ); /* Q( -rshifts ) */
+ x2 = -silk_LSHIFT32( (opus_int32)x_ptr[ subfr_length - n - 1 ], -rshifts ); /* Q( -rshifts ) */
+ tmp1 = silk_LSHIFT32( (opus_int32)x_ptr[ n ], 17 ); /* Q17 */
+ tmp2 = silk_LSHIFT32( (opus_int32)x_ptr[ subfr_length - n - 1 ], 17 ); /* Q17 */
+
+ X1_3210 = _mm_set1_epi32( x1 );
+ X2_3210 = _mm_set1_epi32( x2 );
+ TMP1_3210 = _mm_setzero_si128();
+ TMP2_3210 = _mm_setzero_si128();
+ for( k = 0; k < n - 3; k += 4 ) {+ PTR_3210 = OP_CVTEPI16_EPI32_M64( &x_ptr[ n - k - 1 - 3 ] );
+ SUBFR_3210 = OP_CVTEPI16_EPI32_M64( &x_ptr[ subfr_length - n + k ] );
+ FIRST_3210 = _mm_loadu_si128( (__m128i *)&C_first_row[ k ] );
+ PTR_3210 = _mm_shuffle_epi32( PTR_3210, _MM_SHUFFLE( 0, 1, 2, 3 ) );
+ LAST_3210 = _mm_loadu_si128( (__m128i *)&C_last_row[ k ] );
+ ATMP_3210 = _mm_loadu_si128( (__m128i *)&Af_QA[ k ] );
+
+ T1_3210 = _mm_mullo_epi32( PTR_3210, X1_3210 );
+ T2_3210 = _mm_mullo_epi32( SUBFR_3210, X2_3210 );
+
+ ATMP_3210 = _mm_srai_epi32( ATMP_3210, 7 );
+ ATMP_3210 = _mm_add_epi32( ATMP_3210, CONST1 );
+ ATMP_3210 = _mm_srai_epi32( ATMP_3210, 1 );
+
+ FIRST_3210 = _mm_add_epi32( FIRST_3210, T1_3210 );
+ LAST_3210 = _mm_add_epi32( LAST_3210, T2_3210 );
+
+ PTR_3210 = _mm_mullo_epi32( ATMP_3210, PTR_3210 );
+ SUBFR_3210 = _mm_mullo_epi32( ATMP_3210, SUBFR_3210 );
+
+ _mm_storeu_si128( (__m128i *)&C_first_row[ k ], FIRST_3210 );
+ _mm_storeu_si128( (__m128i *)&C_last_row[ k ], LAST_3210 );
+
+ TMP1_3210 = _mm_add_epi32( TMP1_3210, PTR_3210 );
+ TMP2_3210 = _mm_add_epi32( TMP2_3210, SUBFR_3210 );
+ }
+
+ TMP1_3210 = _mm_add_epi32( TMP1_3210, _mm_unpackhi_epi64(TMP1_3210, TMP1_3210 ) );
+ TMP2_3210 = _mm_add_epi32( TMP2_3210, _mm_unpackhi_epi64(TMP2_3210, TMP2_3210 ) );
+ TMP1_3210 = _mm_add_epi32( TMP1_3210, _mm_shufflelo_epi16(TMP1_3210, 0x0E ) );
+ TMP2_3210 = _mm_add_epi32( TMP2_3210, _mm_shufflelo_epi16(TMP2_3210, 0x0E ) );
+
+ tmp1 += _mm_cvtsi128_si32( TMP1_3210 );
+ tmp2 += _mm_cvtsi128_si32( TMP2_3210 );
+
+ for( ; k < n; k++ ) {+ C_first_row[ k ] = silk_MLA( C_first_row[ k ], x1, x_ptr[ n - k - 1 ] ); /* Q( -rshifts ) */
+ C_last_row[ k ] = silk_MLA( C_last_row[ k ], x2, x_ptr[ subfr_length - n + k ] ); /* Q( -rshifts ) */
+ Atmp1 = silk_RSHIFT_ROUND( Af_QA[ k ], QA - 17 ); /* Q17 */
+ tmp1 = silk_MLA( tmp1, x_ptr[ n - k - 1 ], Atmp1 ); /* Q17 */
+ tmp2 = silk_MLA( tmp2, x_ptr[ subfr_length - n + k ], Atmp1 ); /* Q17 */
+ }
+
+ tmp1 = -tmp1; /* Q17 */
+ tmp2 = -tmp2; /* Q17 */
+
+ {+ __m128i xmm_tmp1, xmm_tmp2;
+ __m128i xmm_x_ptr_n_k_x2x0, xmm_x_ptr_n_k_x3x1;
+ __m128i xmm_x_ptr_sub_x2x0, xmm_x_ptr_sub_x3x1;
+
+ xmm_tmp1 = _mm_set1_epi32( tmp1 );
+ xmm_tmp2 = _mm_set1_epi32( tmp2 );
+
+ for( k = 0; k <= n - 3; k += 4 ) {+ xmm_x_ptr_n_k_x2x0 = OP_CVTEPI16_EPI32_M64( &x_ptr[ n - k - 3 ] );
+ xmm_x_ptr_sub_x2x0 = OP_CVTEPI16_EPI32_M64( &x_ptr[ subfr_length - n + k - 1 ] );
+
+ xmm_x_ptr_n_k_x2x0 = _mm_shuffle_epi32( xmm_x_ptr_n_k_x2x0, _MM_SHUFFLE( 0, 1, 2, 3 ) );
+
+ xmm_x_ptr_n_k_x2x0 = _mm_slli_epi32( xmm_x_ptr_n_k_x2x0, -rshifts - 1 );
+ xmm_x_ptr_sub_x2x0 = _mm_slli_epi32( xmm_x_ptr_sub_x2x0, -rshifts - 1 );
+
+ /* equal shift right 4 bytes, xmm_x_ptr_n_k_x3x1 = _mm_srli_si128(xmm_x_ptr_n_k_x2x0, 4)*/
+ xmm_x_ptr_n_k_x3x1 = _mm_shuffle_epi32( xmm_x_ptr_n_k_x2x0, _MM_SHUFFLE( 0, 3, 2, 1 ) );
+ xmm_x_ptr_sub_x3x1 = _mm_shuffle_epi32( xmm_x_ptr_sub_x2x0, _MM_SHUFFLE( 0, 3, 2, 1 ) );
+
+ xmm_x_ptr_n_k_x2x0 = _mm_mul_epi32( xmm_x_ptr_n_k_x2x0, xmm_tmp1 );
+ xmm_x_ptr_n_k_x3x1 = _mm_mul_epi32( xmm_x_ptr_n_k_x3x1, xmm_tmp1 );
+ xmm_x_ptr_sub_x2x0 = _mm_mul_epi32( xmm_x_ptr_sub_x2x0, xmm_tmp2 );
+ xmm_x_ptr_sub_x3x1 = _mm_mul_epi32( xmm_x_ptr_sub_x3x1, xmm_tmp2 );
+
+ xmm_x_ptr_n_k_x2x0 = _mm_srli_epi64( xmm_x_ptr_n_k_x2x0, 16 );
+ xmm_x_ptr_n_k_x3x1 = _mm_slli_epi64( xmm_x_ptr_n_k_x3x1, 16 );
+ xmm_x_ptr_sub_x2x0 = _mm_srli_epi64( xmm_x_ptr_sub_x2x0, 16 );
+ xmm_x_ptr_sub_x3x1 = _mm_slli_epi64( xmm_x_ptr_sub_x3x1, 16 );
+
+ xmm_x_ptr_n_k_x2x0 = _mm_blend_epi16( xmm_x_ptr_n_k_x2x0, xmm_x_ptr_n_k_x3x1, 0xCC );
+ xmm_x_ptr_sub_x2x0 = _mm_blend_epi16( xmm_x_ptr_sub_x2x0, xmm_x_ptr_sub_x3x1, 0xCC );
+
+ X1_3210 = _mm_loadu_si128( (__m128i *)&CAf[ k ] );
+ PTR_3210 = _mm_loadu_si128( (__m128i *)&CAb[ k ] );
+
+ X1_3210 = _mm_add_epi32( X1_3210, xmm_x_ptr_n_k_x2x0 );
+ PTR_3210 = _mm_add_epi32( PTR_3210, xmm_x_ptr_sub_x2x0 );
+
+ _mm_storeu_si128( (__m128i *)&CAf[ k ], X1_3210 );
+ _mm_storeu_si128( (__m128i *)&CAb[ k ], PTR_3210 );
+ }
+
+ for( ; k <= n; k++ ) {+ CAf[ k ] = silk_SMLAWW( CAf[ k ], tmp1,
+ silk_LSHIFT32( (opus_int32)x_ptr[ n - k ], -rshifts - 1 ) ); /* Q( -rshift ) */
+ CAb[ k ] = silk_SMLAWW( CAb[ k ], tmp2,
+ silk_LSHIFT32( (opus_int32)x_ptr[ subfr_length - n + k - 1 ], -rshifts - 1 ) ); /* Q( -rshift ) */
+ }
+ }
+ }
+ }
+
+ /* Calculate nominator and denominator for the next order reflection (parcor) coefficient */
+ tmp1 = C_first_row[ n ]; /* Q( -rshifts ) */
+ tmp2 = C_last_row[ n ]; /* Q( -rshifts ) */
+ num = 0; /* Q( -rshifts ) */
+ nrg = silk_ADD32( CAb[ 0 ], CAf[ 0 ] ); /* Q( 1-rshifts ) */
+ for( k = 0; k < n; k++ ) {+ Atmp_QA = Af_QA[ k ];
+ lz = silk_CLZ32( silk_abs( Atmp_QA ) ) - 1;
+ lz = silk_min( 32 - QA, lz );
+ Atmp1 = silk_LSHIFT32( Atmp_QA, lz ); /* Q( QA + lz ) */
+
+ tmp1 = silk_ADD_LSHIFT32( tmp1, silk_SMMUL( C_last_row[ n - k - 1 ], Atmp1 ), 32 - QA - lz ); /* Q( -rshifts ) */
+ tmp2 = silk_ADD_LSHIFT32( tmp2, silk_SMMUL( C_first_row[ n - k - 1 ], Atmp1 ), 32 - QA - lz ); /* Q( -rshifts ) */
+ num = silk_ADD_LSHIFT32( num, silk_SMMUL( CAb[ n - k ], Atmp1 ), 32 - QA - lz ); /* Q( -rshifts ) */
+ nrg = silk_ADD_LSHIFT32( nrg, silk_SMMUL( silk_ADD32( CAb[ k + 1 ], CAf[ k + 1 ] ),
+ Atmp1 ), 32 - QA - lz ); /* Q( 1-rshifts ) */
+ }
+ CAf[ n + 1 ] = tmp1; /* Q( -rshifts ) */
+ CAb[ n + 1 ] = tmp2; /* Q( -rshifts ) */
+ num = silk_ADD32( num, tmp2 ); /* Q( -rshifts ) */
+ num = silk_LSHIFT32( -num, 1 ); /* Q( 1-rshifts ) */
+
+ /* Calculate the next order reflection (parcor) coefficient */
+ if( silk_abs( num ) < nrg ) {+ rc_Q31 = silk_DIV32_varQ( num, nrg, 31 );
+ } else {+ rc_Q31 = ( num > 0 ) ? silk_int32_MAX : silk_int32_MIN;
+ }
+
+ /* Update inverse prediction gain */
+ tmp1 = ( (opus_int32)1 << 30 ) - silk_SMMUL( rc_Q31, rc_Q31 );
+ tmp1 = silk_LSHIFT( silk_SMMUL( invGain_Q30, tmp1 ), 2 );
+ if( tmp1 <= minInvGain_Q30 ) {+ /* Max prediction gain exceeded; set reflection coefficient such that max prediction gain is exactly hit */
+ tmp2 = ( (opus_int32)1 << 30 ) - silk_DIV32_varQ( minInvGain_Q30, invGain_Q30, 30 ); /* Q30 */
+ rc_Q31 = silk_SQRT_APPROX( tmp2 ); /* Q15 */
+ if( rc_Q31 > 0 ) {+ /* Newton-Raphson iteration */
+ rc_Q31 = silk_RSHIFT32( rc_Q31 + silk_DIV32( tmp2, rc_Q31 ), 1 ); /* Q15 */
+ rc_Q31 = silk_LSHIFT32( rc_Q31, 16 ); /* Q31 */
+ if( num < 0 ) {+ /* Ensure adjusted reflection coefficients has the original sign */
+ rc_Q31 = -rc_Q31;
+ }
+ }
+ invGain_Q30 = minInvGain_Q30;
+ reached_max_gain = 1;
+ } else {+ invGain_Q30 = tmp1;
+ }
+
+ /* Update the AR coefficients */
+ for( k = 0; k < (n + 1) >> 1; k++ ) {+ tmp1 = Af_QA[ k ]; /* QA */
+ tmp2 = Af_QA[ n - k - 1 ]; /* QA */
+ Af_QA[ k ] = silk_ADD_LSHIFT32( tmp1, silk_SMMUL( tmp2, rc_Q31 ), 1 ); /* QA */
+ Af_QA[ n - k - 1 ] = silk_ADD_LSHIFT32( tmp2, silk_SMMUL( tmp1, rc_Q31 ), 1 ); /* QA */
+ }
+ Af_QA[ n ] = silk_RSHIFT32( rc_Q31, 31 - QA ); /* QA */
+
+ if( reached_max_gain ) {+ /* Reached max prediction gain; set remaining coefficients to zero and exit loop */
+ for( k = n + 1; k < D; k++ ) {+ Af_QA[ k ] = 0;
+ }
+ break;
+ }
+
+ /* Update C * Af and C * Ab */
+ for( k = 0; k <= n + 1; k++ ) {+ tmp1 = CAf[ k ]; /* Q( -rshifts ) */
+ tmp2 = CAb[ n - k + 1 ]; /* Q( -rshifts ) */
+ CAf[ k ] = silk_ADD_LSHIFT32( tmp1, silk_SMMUL( tmp2, rc_Q31 ), 1 ); /* Q( -rshifts ) */
+ CAb[ n - k + 1 ] = silk_ADD_LSHIFT32( tmp2, silk_SMMUL( tmp1, rc_Q31 ), 1 ); /* Q( -rshifts ) */
+ }
+ }
+
+ if( reached_max_gain ) {+ for( k = 0; k < D; k++ ) {+ /* Scale coefficients */
+ A_Q16[ k ] = -silk_RSHIFT_ROUND( Af_QA[ k ], QA - 16 );
+ }
+ /* Subtract energy of preceding samples from C0 */
+ if( rshifts > 0 ) {+ for( s = 0; s < nb_subfr; s++ ) {+ x_ptr = x + s * subfr_length;
+ C0 -= (opus_int32)silk_RSHIFT64( silk_inner_prod16_aligned_64( x_ptr, x_ptr, D, arch ), rshifts );
+ }
+ } else {+ for( s = 0; s < nb_subfr; s++ ) {+ x_ptr = x + s * subfr_length;
+ C0 -= silk_LSHIFT32( silk_inner_prod_aligned( x_ptr, x_ptr, D, arch ), -rshifts );
+ }
+ }
+ /* Approximate residual energy */
+ *res_nrg = silk_LSHIFT( silk_SMMUL( invGain_Q30, C0 ), 2 );
+ *res_nrg_Q = -rshifts;
+ } else {+ /* Return residual energy */
+ nrg = CAf[ 0 ]; /* Q( -rshifts ) */
+ tmp1 = (opus_int32)1 << 16; /* Q16 */
+ for( k = 0; k < D; k++ ) {+ Atmp1 = silk_RSHIFT_ROUND( Af_QA[ k ], QA - 16 ); /* Q16 */
+ nrg = silk_SMLAWW( nrg, CAf[ k + 1 ], Atmp1 ); /* Q( -rshifts ) */
+ tmp1 = silk_SMLAWW( tmp1, Atmp1, Atmp1 ); /* Q16 */
+ A_Q16[ k ] = -Atmp1;
+ }
+ *res_nrg = silk_SMLAWW( nrg, silk_SMMUL( SILK_FIX_CONST( FIND_LPC_COND_FAC, 32 ), C0 ), -tmp1 );/* Q( -rshifts ) */
+ *res_nrg_Q = -rshifts;
+ }
+}
--- a/silk/fixed/x86/vector_ops_FIX_sse.c
+++ /dev/null
@@ -1,88 +1,0 @@
-/* Copyright (c) 2014, Cisco Systems, INC
- Written by XiangMingZhu WeiZhou MinPeng YanWang
-
- Redistribution and use in source and binary forms, with or without
- modification, are permitted provided that the following conditions
- are met:
-
- - Redistributions of source code must retain the above copyright
- notice, this list of conditions and the following disclaimer.
-
- - Redistributions in binary form must reproduce the above copyright
- notice, this list of conditions and the following disclaimer in the
- documentation and/or other materials provided with the distribution.
-
- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER
- OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
- EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
- PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
- PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
- LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
- NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-*/
-
-#ifdef HAVE_CONFIG_H
-#include "config.h"
-#endif
-
-#include <xmmintrin.h>
-#include <emmintrin.h>
-#include <smmintrin.h>
-#include "main.h"
-
-#include "SigProc_FIX.h"
-#include "pitch.h"
-
-opus_int64 silk_inner_prod16_aligned_64_sse4_1(
- const opus_int16 *inVec1, /* I input vector 1 */
- const opus_int16 *inVec2, /* I input vector 2 */
- const opus_int len /* I vector lengths */
-)
-{- opus_int i, dataSize8;
- opus_int64 sum;
-
- __m128i xmm_tempa;
- __m128i inVec1_76543210, acc1;
- __m128i inVec2_76543210, acc2;
-
- sum = 0;
- dataSize8 = len & ~7;
-
- acc1 = _mm_setzero_si128();
- acc2 = _mm_setzero_si128();
-
- for( i = 0; i < dataSize8; i += 8 ) {- inVec1_76543210 = _mm_loadu_si128( (__m128i *)(&inVec1[i + 0] ) );
- inVec2_76543210 = _mm_loadu_si128( (__m128i *)(&inVec2[i + 0] ) );
-
- /* only when all 4 operands are -32768 (0x8000), this results in wrap around */
- inVec1_76543210 = _mm_madd_epi16( inVec1_76543210, inVec2_76543210 );
-
- xmm_tempa = _mm_cvtepi32_epi64( inVec1_76543210 );
- /* equal shift right 8 bytes */
- inVec1_76543210 = _mm_shuffle_epi32( inVec1_76543210, _MM_SHUFFLE( 0, 0, 3, 2 ) );
- inVec1_76543210 = _mm_cvtepi32_epi64( inVec1_76543210 );
-
- acc1 = _mm_add_epi64( acc1, xmm_tempa );
- acc2 = _mm_add_epi64( acc2, inVec1_76543210 );
- }
-
- acc1 = _mm_add_epi64( acc1, acc2 );
-
- /* equal shift right 8 bytes */
- acc2 = _mm_shuffle_epi32( acc1, _MM_SHUFFLE( 0, 0, 3, 2 ) );
- acc1 = _mm_add_epi64( acc1, acc2 );
-
- _mm_storel_epi64( (__m128i *)&sum, acc1 );
-
- for( ; i < len; i++ ) {- sum = silk_SMLABB( sum, inVec1[ i ], inVec2[ i ] );
- }
-
- return sum;
-}
--- /dev/null
+++ b/silk/fixed/x86/vector_ops_FIX_sse4_1.c
@@ -1,0 +1,88 @@
+/* Copyright (c) 2014, Cisco Systems, INC
+ Written by XiangMingZhu WeiZhou MinPeng YanWang
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+
+ - Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+
+ - Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER
+ OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+ LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+ NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*/
+
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include <xmmintrin.h>
+#include <emmintrin.h>
+#include <smmintrin.h>
+#include "main.h"
+
+#include "SigProc_FIX.h"
+#include "pitch.h"
+
+opus_int64 silk_inner_prod16_aligned_64_sse4_1(
+ const opus_int16 *inVec1, /* I input vector 1 */
+ const opus_int16 *inVec2, /* I input vector 2 */
+ const opus_int len /* I vector lengths */
+)
+{+ opus_int i, dataSize8;
+ opus_int64 sum;
+
+ __m128i xmm_tempa;
+ __m128i inVec1_76543210, acc1;
+ __m128i inVec2_76543210, acc2;
+
+ sum = 0;
+ dataSize8 = len & ~7;
+
+ acc1 = _mm_setzero_si128();
+ acc2 = _mm_setzero_si128();
+
+ for( i = 0; i < dataSize8; i += 8 ) {+ inVec1_76543210 = _mm_loadu_si128( (__m128i *)(&inVec1[i + 0] ) );
+ inVec2_76543210 = _mm_loadu_si128( (__m128i *)(&inVec2[i + 0] ) );
+
+ /* only when all 4 operands are -32768 (0x8000), this results in wrap around */
+ inVec1_76543210 = _mm_madd_epi16( inVec1_76543210, inVec2_76543210 );
+
+ xmm_tempa = _mm_cvtepi32_epi64( inVec1_76543210 );
+ /* equal shift right 8 bytes */
+ inVec1_76543210 = _mm_shuffle_epi32( inVec1_76543210, _MM_SHUFFLE( 0, 0, 3, 2 ) );
+ inVec1_76543210 = _mm_cvtepi32_epi64( inVec1_76543210 );
+
+ acc1 = _mm_add_epi64( acc1, xmm_tempa );
+ acc2 = _mm_add_epi64( acc2, inVec1_76543210 );
+ }
+
+ acc1 = _mm_add_epi64( acc1, acc2 );
+
+ /* equal shift right 8 bytes */
+ acc2 = _mm_shuffle_epi32( acc1, _MM_SHUFFLE( 0, 0, 3, 2 ) );
+ acc1 = _mm_add_epi64( acc1, acc2 );
+
+ _mm_storel_epi64( (__m128i *)&sum, acc1 );
+
+ for( ; i < len; i++ ) {+ sum = silk_SMLABB( sum, inVec1[ i ], inVec2[ i ] );
+ }
+
+ return sum;
+}
--- a/silk/x86/NSQ_del_dec_sse.c
+++ /dev/null
@@ -1,859 +1,0 @@
-/* Copyright (c) 2014, Cisco Systems, INC
- Written by XiangMingZhu WeiZhou MinPeng YanWang
-
- Redistribution and use in source and binary forms, with or without
- modification, are permitted provided that the following conditions
- are met:
-
- - Redistributions of source code must retain the above copyright
- notice, this list of conditions and the following disclaimer.
-
- - Redistributions in binary form must reproduce the above copyright
- notice, this list of conditions and the following disclaimer in the
- documentation and/or other materials provided with the distribution.
-
- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER
- OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
- EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
- PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
- PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
- LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
- NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-*/
-
-#ifdef HAVE_CONFIG_H
-#include "config.h"
-#endif
-
-#include <xmmintrin.h>
-#include <emmintrin.h>
-#include <smmintrin.h>
-#include "main.h"
-#include "celt/x86/x86cpu.h"
-
-#include "stack_alloc.h"
-
-typedef struct {- opus_int32 sLPC_Q14[ MAX_SUB_FRAME_LENGTH + NSQ_LPC_BUF_LENGTH ];
- opus_int32 RandState[ DECISION_DELAY ];
- opus_int32 Q_Q10[ DECISION_DELAY ];
- opus_int32 Xq_Q14[ DECISION_DELAY ];
- opus_int32 Pred_Q15[ DECISION_DELAY ];
- opus_int32 Shape_Q14[ DECISION_DELAY ];
- opus_int32 sAR2_Q14[ MAX_SHAPE_LPC_ORDER ];
- opus_int32 LF_AR_Q14;
- opus_int32 Seed;
- opus_int32 SeedInit;
- opus_int32 RD_Q10;
-} NSQ_del_dec_struct;
-
-typedef struct {- opus_int32 Q_Q10;
- opus_int32 RD_Q10;
- opus_int32 xq_Q14;
- opus_int32 LF_AR_Q14;
- opus_int32 sLTP_shp_Q14;
- opus_int32 LPC_exc_Q14;
-} NSQ_sample_struct;
-
-typedef NSQ_sample_struct NSQ_sample_pair[ 2 ];
-
-static OPUS_INLINE void silk_nsq_del_dec_scale_states_sse4_1(
- const silk_encoder_state *psEncC, /* I Encoder State */
- silk_nsq_state *NSQ, /* I/O NSQ state */
- NSQ_del_dec_struct psDelDec[], /* I/O Delayed decision states */
- const opus_int32 x_Q3[], /* I Input in Q3 */
- opus_int32 x_sc_Q10[], /* O Input scaled with 1/Gain in Q10 */
- const opus_int16 sLTP[], /* I Re-whitened LTP state in Q0 */
- opus_int32 sLTP_Q15[], /* O LTP state matching scaled input */
- opus_int subfr, /* I Subframe number */
- opus_int nStatesDelayedDecision, /* I Number of del dec states */
- const opus_int LTP_scale_Q14, /* I LTP state scaling */
- const opus_int32 Gains_Q16[ MAX_NB_SUBFR ], /* I */
- const opus_int pitchL[ MAX_NB_SUBFR ], /* I Pitch lag */
- const opus_int signal_type, /* I Signal type */
- const opus_int decisionDelay /* I Decision delay */
-);
-
-/******************************************/
-/* Noise shape quantizer for one subframe */
-/******************************************/
-static OPUS_INLINE void silk_noise_shape_quantizer_del_dec_sse4_1(
- silk_nsq_state *NSQ, /* I/O NSQ state */
- NSQ_del_dec_struct psDelDec[], /* I/O Delayed decision states */
- opus_int signalType, /* I Signal type */
- const opus_int32 x_Q10[], /* I */
- opus_int8 pulses[], /* O */
- opus_int16 xq[], /* O */
- opus_int32 sLTP_Q15[], /* I/O LTP filter state */
- opus_int32 delayedGain_Q10[], /* I/O Gain delay buffer */
- const opus_int16 a_Q12[], /* I Short term prediction coefs */
- const opus_int16 b_Q14[], /* I Long term prediction coefs */
- const opus_int16 AR_shp_Q13[], /* I Noise shaping coefs */
- opus_int lag, /* I Pitch lag */
- opus_int32 HarmShapeFIRPacked_Q14, /* I */
- opus_int Tilt_Q14, /* I Spectral tilt */
- opus_int32 LF_shp_Q14, /* I */
- opus_int32 Gain_Q16, /* I */
- opus_int Lambda_Q10, /* I */
- opus_int offset_Q10, /* I */
- opus_int length, /* I Input length */
- opus_int subfr, /* I Subframe number */
- opus_int shapingLPCOrder, /* I Shaping LPC filter order */
- opus_int predictLPCOrder, /* I Prediction filter order */
- opus_int warping_Q16, /* I */
- opus_int nStatesDelayedDecision, /* I Number of states in decision tree */
- opus_int *smpl_buf_idx, /* I/O Index to newest samples in buffers */
- opus_int decisionDelay /* I */
-);
-
-void silk_NSQ_del_dec_sse4_1(
- const silk_encoder_state *psEncC, /* I Encoder State */
- silk_nsq_state *NSQ, /* I/O NSQ state */
- SideInfoIndices *psIndices, /* I/O Quantization Indices */
- const opus_int32 x_Q3[], /* I Prefiltered input signal */
- opus_int8 pulses[], /* O Quantized pulse signal */
- const opus_int16 PredCoef_Q12[ 2 * MAX_LPC_ORDER ], /* I Short term prediction coefs */
- const opus_int16 LTPCoef_Q14[ LTP_ORDER * MAX_NB_SUBFR ], /* I Long term prediction coefs */
- const opus_int16 AR2_Q13[ MAX_NB_SUBFR * MAX_SHAPE_LPC_ORDER ], /* I Noise shaping coefs */
- const opus_int HarmShapeGain_Q14[ MAX_NB_SUBFR ], /* I Long term shaping coefs */
- const opus_int Tilt_Q14[ MAX_NB_SUBFR ], /* I Spectral tilt */
- const opus_int32 LF_shp_Q14[ MAX_NB_SUBFR ], /* I Low frequency shaping coefs */
- const opus_int32 Gains_Q16[ MAX_NB_SUBFR ], /* I Quantization step sizes */
- const opus_int pitchL[ MAX_NB_SUBFR ], /* I Pitch lags */
- const opus_int Lambda_Q10, /* I Rate/distortion tradeoff */
- const opus_int LTP_scale_Q14 /* I LTP state scaling */
-)
-{- opus_int i, k, lag, start_idx, LSF_interpolation_flag, Winner_ind, subfr;
- opus_int last_smple_idx, smpl_buf_idx, decisionDelay;
- const opus_int16 *A_Q12, *B_Q14, *AR_shp_Q13;
- opus_int16 *pxq;
- VARDECL( opus_int32, sLTP_Q15 );
- VARDECL( opus_int16, sLTP );
- opus_int32 HarmShapeFIRPacked_Q14;
- opus_int offset_Q10;
- opus_int32 RDmin_Q10, Gain_Q10;
- VARDECL( opus_int32, x_sc_Q10 );
- VARDECL( opus_int32, delayedGain_Q10 );
- VARDECL( NSQ_del_dec_struct, psDelDec );
- NSQ_del_dec_struct *psDD;
- SAVE_STACK;
-
- /* Set unvoiced lag to the previous one, overwrite later for voiced */
- lag = NSQ->lagPrev;
-
- silk_assert( NSQ->prev_gain_Q16 != 0 );
-
- /* Initialize delayed decision states */
- ALLOC( psDelDec, psEncC->nStatesDelayedDecision, NSQ_del_dec_struct );
- silk_memset( psDelDec, 0, psEncC->nStatesDelayedDecision * sizeof( NSQ_del_dec_struct ) );
- for( k = 0; k < psEncC->nStatesDelayedDecision; k++ ) {- psDD = &psDelDec[ k ];
- psDD->Seed = ( k + psIndices->Seed ) & 3;
- psDD->SeedInit = psDD->Seed;
- psDD->RD_Q10 = 0;
- psDD->LF_AR_Q14 = NSQ->sLF_AR_shp_Q14;
- psDD->Shape_Q14[ 0 ] = NSQ->sLTP_shp_Q14[ psEncC->ltp_mem_length - 1 ];
- silk_memcpy( psDD->sLPC_Q14, NSQ->sLPC_Q14, NSQ_LPC_BUF_LENGTH * sizeof( opus_int32 ) );
- silk_memcpy( psDD->sAR2_Q14, NSQ->sAR2_Q14, sizeof( NSQ->sAR2_Q14 ) );
- }
-
- offset_Q10 = silk_Quantization_Offsets_Q10[ psIndices->signalType >> 1 ][ psIndices->quantOffsetType ];
- smpl_buf_idx = 0; /* index of oldest samples */
-
- decisionDelay = silk_min_int( DECISION_DELAY, psEncC->subfr_length );
-
- /* For voiced frames limit the decision delay to lower than the pitch lag */
- if( psIndices->signalType == TYPE_VOICED ) {- for( k = 0; k < psEncC->nb_subfr; k++ ) {- decisionDelay = silk_min_int( decisionDelay, pitchL[ k ] - LTP_ORDER / 2 - 1 );
- }
- } else {- if( lag > 0 ) {- decisionDelay = silk_min_int( decisionDelay, lag - LTP_ORDER / 2 - 1 );
- }
- }
-
- if( psIndices->NLSFInterpCoef_Q2 == 4 ) {- LSF_interpolation_flag = 0;
- } else {- LSF_interpolation_flag = 1;
- }
-
- ALLOC( sLTP_Q15,
- psEncC->ltp_mem_length + psEncC->frame_length, opus_int32 );
- ALLOC( sLTP, psEncC->ltp_mem_length + psEncC->frame_length, opus_int16 );
- ALLOC( x_sc_Q10, psEncC->subfr_length, opus_int32 );
- ALLOC( delayedGain_Q10, DECISION_DELAY, opus_int32 );
- /* Set up pointers to start of sub frame */
- pxq = &NSQ->xq[ psEncC->ltp_mem_length ];
- NSQ->sLTP_shp_buf_idx = psEncC->ltp_mem_length;
- NSQ->sLTP_buf_idx = psEncC->ltp_mem_length;
- subfr = 0;
- for( k = 0; k < psEncC->nb_subfr; k++ ) {- A_Q12 = &PredCoef_Q12[ ( ( k >> 1 ) | ( 1 - LSF_interpolation_flag ) ) * MAX_LPC_ORDER ];
- B_Q14 = <PCoef_Q14[ k * LTP_ORDER ];
- AR_shp_Q13 = &AR2_Q13[ k * MAX_SHAPE_LPC_ORDER ];
-
- /* Noise shape parameters */
- silk_assert( HarmShapeGain_Q14[ k ] >= 0 );
- HarmShapeFIRPacked_Q14 = silk_RSHIFT( HarmShapeGain_Q14[ k ], 2 );
- HarmShapeFIRPacked_Q14 |= silk_LSHIFT( (opus_int32)silk_RSHIFT( HarmShapeGain_Q14[ k ], 1 ), 16 );
-
- NSQ->rewhite_flag = 0;
- if( psIndices->signalType == TYPE_VOICED ) {- /* Voiced */
- lag = pitchL[ k ];
-
- /* Re-whitening */
- if( ( k & ( 3 - silk_LSHIFT( LSF_interpolation_flag, 1 ) ) ) == 0 ) {- if( k == 2 ) {- /* RESET DELAYED DECISIONS */
- /* Find winner */
- RDmin_Q10 = psDelDec[ 0 ].RD_Q10;
- Winner_ind = 0;
- for( i = 1; i < psEncC->nStatesDelayedDecision; i++ ) {- if( psDelDec[ i ].RD_Q10 < RDmin_Q10 ) {- RDmin_Q10 = psDelDec[ i ].RD_Q10;
- Winner_ind = i;
- }
- }
- for( i = 0; i < psEncC->nStatesDelayedDecision; i++ ) {- if( i != Winner_ind ) {- psDelDec[ i ].RD_Q10 += ( silk_int32_MAX >> 4 );
- silk_assert( psDelDec[ i ].RD_Q10 >= 0 );
- }
- }
-
- /* Copy final part of signals from winner state to output and long-term filter states */
- psDD = &psDelDec[ Winner_ind ];
- last_smple_idx = smpl_buf_idx + decisionDelay;
- for( i = 0; i < decisionDelay; i++ ) {- last_smple_idx = ( last_smple_idx - 1 ) % DECISION_DELAY;
- if( last_smple_idx < 0 ) last_smple_idx += DECISION_DELAY;
- pulses[ i - decisionDelay ] = (opus_int8)silk_RSHIFT_ROUND( psDD->Q_Q10[ last_smple_idx ], 10 );
- pxq[ i - decisionDelay ] = (opus_int16)silk_SAT16( silk_RSHIFT_ROUND(
- silk_SMULWW( psDD->Xq_Q14[ last_smple_idx ], Gains_Q16[ 1 ] ), 14 ) );
- NSQ->sLTP_shp_Q14[ NSQ->sLTP_shp_buf_idx - decisionDelay + i ] = psDD->Shape_Q14[ last_smple_idx ];
- }
-
- subfr = 0;
- }
-
- /* Rewhiten with new A coefs */
- start_idx = psEncC->ltp_mem_length - lag - psEncC->predictLPCOrder - LTP_ORDER / 2;
- silk_assert( start_idx > 0 );
-
- silk_LPC_analysis_filter( &sLTP[ start_idx ], &NSQ->xq[ start_idx + k * psEncC->subfr_length ],
- A_Q12, psEncC->ltp_mem_length - start_idx, psEncC->predictLPCOrder, psEncC->arch );
-
- NSQ->sLTP_buf_idx = psEncC->ltp_mem_length;
- NSQ->rewhite_flag = 1;
- }
- }
-
- silk_nsq_del_dec_scale_states_sse4_1( psEncC, NSQ, psDelDec, x_Q3, x_sc_Q10, sLTP, sLTP_Q15, k,
- psEncC->nStatesDelayedDecision, LTP_scale_Q14, Gains_Q16, pitchL, psIndices->signalType, decisionDelay );
-
- silk_noise_shape_quantizer_del_dec_sse4_1( NSQ, psDelDec, psIndices->signalType, x_sc_Q10, pulses, pxq, sLTP_Q15,
- delayedGain_Q10, A_Q12, B_Q14, AR_shp_Q13, lag, HarmShapeFIRPacked_Q14, Tilt_Q14[ k ], LF_shp_Q14[ k ],
- Gains_Q16[ k ], Lambda_Q10, offset_Q10, psEncC->subfr_length, subfr++, psEncC->shapingLPCOrder,
- psEncC->predictLPCOrder, psEncC->warping_Q16, psEncC->nStatesDelayedDecision, &smpl_buf_idx, decisionDelay );
-
- x_Q3 += psEncC->subfr_length;
- pulses += psEncC->subfr_length;
- pxq += psEncC->subfr_length;
- }
-
- /* Find winner */
- RDmin_Q10 = psDelDec[ 0 ].RD_Q10;
- Winner_ind = 0;
- for( k = 1; k < psEncC->nStatesDelayedDecision; k++ ) {- if( psDelDec[ k ].RD_Q10 < RDmin_Q10 ) {- RDmin_Q10 = psDelDec[ k ].RD_Q10;
- Winner_ind = k;
- }
- }
-
- /* Copy final part of signals from winner state to output and long-term filter states */
- psDD = &psDelDec[ Winner_ind ];
- psIndices->Seed = psDD->SeedInit;
- last_smple_idx = smpl_buf_idx + decisionDelay;
- Gain_Q10 = silk_RSHIFT32( Gains_Q16[ psEncC->nb_subfr - 1 ], 6 );
- for( i = 0; i < decisionDelay; i++ ) {- last_smple_idx = ( last_smple_idx - 1 ) % DECISION_DELAY;
- if( last_smple_idx < 0 ) last_smple_idx += DECISION_DELAY;
- pulses[ i - decisionDelay ] = (opus_int8)silk_RSHIFT_ROUND( psDD->Q_Q10[ last_smple_idx ], 10 );
- pxq[ i - decisionDelay ] = (opus_int16)silk_SAT16( silk_RSHIFT_ROUND(
- silk_SMULWW( psDD->Xq_Q14[ last_smple_idx ], Gain_Q10 ), 8 ) );
- NSQ->sLTP_shp_Q14[ NSQ->sLTP_shp_buf_idx - decisionDelay + i ] = psDD->Shape_Q14[ last_smple_idx ];
- }
- silk_memcpy( NSQ->sLPC_Q14, &psDD->sLPC_Q14[ psEncC->subfr_length ], NSQ_LPC_BUF_LENGTH * sizeof( opus_int32 ) );
- silk_memcpy( NSQ->sAR2_Q14, psDD->sAR2_Q14, sizeof( psDD->sAR2_Q14 ) );
-
- /* Update states */
- NSQ->sLF_AR_shp_Q14 = psDD->LF_AR_Q14;
- NSQ->lagPrev = pitchL[ psEncC->nb_subfr - 1 ];
-
- /* Save quantized speech signal */
- silk_memmove( NSQ->xq, &NSQ->xq[ psEncC->frame_length ], psEncC->ltp_mem_length * sizeof( opus_int16 ) );
- silk_memmove( NSQ->sLTP_shp_Q14, &NSQ->sLTP_shp_Q14[ psEncC->frame_length ], psEncC->ltp_mem_length * sizeof( opus_int32 ) );
- RESTORE_STACK;
-}
-
-/******************************************/
-/* Noise shape quantizer for one subframe */
-/******************************************/
-static OPUS_INLINE void silk_noise_shape_quantizer_del_dec_sse4_1(
- silk_nsq_state *NSQ, /* I/O NSQ state */
- NSQ_del_dec_struct psDelDec[], /* I/O Delayed decision states */
- opus_int signalType, /* I Signal type */
- const opus_int32 x_Q10[], /* I */
- opus_int8 pulses[], /* O */
- opus_int16 xq[], /* O */
- opus_int32 sLTP_Q15[], /* I/O LTP filter state */
- opus_int32 delayedGain_Q10[], /* I/O Gain delay buffer */
- const opus_int16 a_Q12[], /* I Short term prediction coefs */
- const opus_int16 b_Q14[], /* I Long term prediction coefs */
- const opus_int16 AR_shp_Q13[], /* I Noise shaping coefs */
- opus_int lag, /* I Pitch lag */
- opus_int32 HarmShapeFIRPacked_Q14, /* I */
- opus_int Tilt_Q14, /* I Spectral tilt */
- opus_int32 LF_shp_Q14, /* I */
- opus_int32 Gain_Q16, /* I */
- opus_int Lambda_Q10, /* I */
- opus_int offset_Q10, /* I */
- opus_int length, /* I Input length */
- opus_int subfr, /* I Subframe number */
- opus_int shapingLPCOrder, /* I Shaping LPC filter order */
- opus_int predictLPCOrder, /* I Prediction filter order */
- opus_int warping_Q16, /* I */
- opus_int nStatesDelayedDecision, /* I Number of states in decision tree */
- opus_int *smpl_buf_idx, /* I/O Index to newest samples in buffers */
- opus_int decisionDelay /* I */
-)
-{- opus_int i, j, k, Winner_ind, RDmin_ind, RDmax_ind, last_smple_idx;
- opus_int32 Winner_rand_state;
- opus_int32 LTP_pred_Q14, LPC_pred_Q14, n_AR_Q14, n_LTP_Q14;
- opus_int32 n_LF_Q14, r_Q10, rr_Q10, rd1_Q10, rd2_Q10, RDmin_Q10, RDmax_Q10;
- opus_int32 q1_Q0, q1_Q10, q2_Q10, exc_Q14, LPC_exc_Q14, xq_Q14, Gain_Q10;
- opus_int32 tmp1, tmp2, sLF_AR_shp_Q14;
- opus_int32 *pred_lag_ptr, *shp_lag_ptr, *psLPC_Q14;
- VARDECL( NSQ_sample_pair, psSampleState );
- NSQ_del_dec_struct *psDD;
- NSQ_sample_struct *psSS;
-
- __m128i a_Q12_0123, a_Q12_4567, a_Q12_89AB, a_Q12_CDEF;
- __m128i b_Q12_0123, b_sr_Q12_0123;
- SAVE_STACK;
-
- silk_assert( nStatesDelayedDecision > 0 );
- ALLOC( psSampleState, nStatesDelayedDecision, NSQ_sample_pair );
-
- shp_lag_ptr = &NSQ->sLTP_shp_Q14[ NSQ->sLTP_shp_buf_idx - lag + HARM_SHAPE_FIR_TAPS / 2 ];
- pred_lag_ptr = &sLTP_Q15[ NSQ->sLTP_buf_idx - lag + LTP_ORDER / 2 ];
- Gain_Q10 = silk_RSHIFT( Gain_Q16, 6 );
-
- a_Q12_0123 = OP_CVTEPI16_EPI32_M64( a_Q12 );
- a_Q12_4567 = OP_CVTEPI16_EPI32_M64( a_Q12 + 4 );
-
- if( opus_likely( predictLPCOrder == 16 ) ) {- a_Q12_89AB = OP_CVTEPI16_EPI32_M64( a_Q12 + 8 );
- a_Q12_CDEF = OP_CVTEPI16_EPI32_M64( a_Q12 + 12 );
- }
-
- if( signalType == TYPE_VOICED ){- b_Q12_0123 = OP_CVTEPI16_EPI32_M64( b_Q14 );
- b_sr_Q12_0123 = _mm_shuffle_epi32( b_Q12_0123, _MM_SHUFFLE( 0, 3, 2, 1 ) ); /* equal shift right 4 bytes */
- }
- for( i = 0; i < length; i++ ) {- /* Perform common calculations used in all states */
-
- /* Long-term prediction */
- if( signalType == TYPE_VOICED ) {- /* Unrolled loop */
- /* Avoids introducing a bias because silk_SMLAWB() always rounds to -inf */
- LTP_pred_Q14 = 2;
- {- __m128i tmpa, tmpb, pred_lag_ptr_tmp;
- pred_lag_ptr_tmp = _mm_loadu_si128( (__m128i *)(&pred_lag_ptr[ -3 ] ) );
- pred_lag_ptr_tmp = _mm_shuffle_epi32( pred_lag_ptr_tmp, 0x1B );
- tmpa = _mm_mul_epi32( pred_lag_ptr_tmp, b_Q12_0123 );
- tmpa = _mm_srli_si128( tmpa, 2 );
-
- pred_lag_ptr_tmp = _mm_shuffle_epi32( pred_lag_ptr_tmp, _MM_SHUFFLE( 0, 3, 2, 1 ) );/* equal shift right 4 bytes */
- pred_lag_ptr_tmp = _mm_mul_epi32( pred_lag_ptr_tmp, b_sr_Q12_0123 );
- pred_lag_ptr_tmp = _mm_srli_si128( pred_lag_ptr_tmp, 2 );
- pred_lag_ptr_tmp = _mm_add_epi32( pred_lag_ptr_tmp, tmpa );
-
- tmpb = _mm_shuffle_epi32( pred_lag_ptr_tmp, _MM_SHUFFLE( 0, 0, 3, 2 ) );/* equal shift right 8 bytes */
- pred_lag_ptr_tmp = _mm_add_epi32( pred_lag_ptr_tmp, tmpb );
- LTP_pred_Q14 += _mm_cvtsi128_si32( pred_lag_ptr_tmp );
-
- LTP_pred_Q14 = silk_SMLAWB( LTP_pred_Q14, pred_lag_ptr[ -4 ], b_Q14[ 4 ] );
- LTP_pred_Q14 = silk_LSHIFT( LTP_pred_Q14, 1 ); /* Q13 -> Q14 */
- pred_lag_ptr++;
- }
- } else {- LTP_pred_Q14 = 0;
- }
-
- /* Long-term shaping */
- if( lag > 0 ) {- /* Symmetric, packed FIR coefficients */
- n_LTP_Q14 = silk_SMULWB( silk_ADD32( shp_lag_ptr[ 0 ], shp_lag_ptr[ -2 ] ), HarmShapeFIRPacked_Q14 );
- n_LTP_Q14 = silk_SMLAWT( n_LTP_Q14, shp_lag_ptr[ -1 ], HarmShapeFIRPacked_Q14 );
- n_LTP_Q14 = silk_SUB_LSHIFT32( LTP_pred_Q14, n_LTP_Q14, 2 ); /* Q12 -> Q14 */
- shp_lag_ptr++;
- } else {- n_LTP_Q14 = 0;
- }
- {- __m128i tmpa, tmpb, psLPC_Q14_tmp, a_Q12_tmp;
-
- for( k = 0; k < nStatesDelayedDecision; k++ ) {- /* Delayed decision state */
- psDD = &psDelDec[ k ];
-
- /* Sample state */
- psSS = psSampleState[ k ];
-
- /* Generate dither */
- psDD->Seed = silk_RAND( psDD->Seed );
-
- /* Pointer used in short term prediction and shaping */
- psLPC_Q14 = &psDD->sLPC_Q14[ NSQ_LPC_BUF_LENGTH - 1 + i ];
- /* Short-term prediction */
- silk_assert( predictLPCOrder == 10 || predictLPCOrder == 16 );
- /* Avoids introducing a bias because silk_SMLAWB() always rounds to -inf */
- LPC_pred_Q14 = silk_RSHIFT( predictLPCOrder, 1 );
-
- tmpb = _mm_setzero_si128();
-
- /* step 1 */
- psLPC_Q14_tmp = _mm_loadu_si128( (__m128i *)(&psLPC_Q14[ -3 ] ) ); /* -3, -2 , -1, 0 */
- psLPC_Q14_tmp = _mm_shuffle_epi32( psLPC_Q14_tmp, 0x1B ); /* 0, -1, -2, -3 */
- tmpa = _mm_mul_epi32( psLPC_Q14_tmp, a_Q12_0123 ); /* 0, -1, -2, -3 * 0123 -> 0*0, 2*-2 */
-
- tmpa = _mm_srli_epi64( tmpa, 16 );
- tmpb = _mm_add_epi32( tmpb, tmpa );
-
- psLPC_Q14_tmp = _mm_shuffle_epi32( psLPC_Q14_tmp, _MM_SHUFFLE( 0, 3, 2, 1 ) ); /* equal shift right 4 bytes */
- a_Q12_tmp = _mm_shuffle_epi32( a_Q12_0123, _MM_SHUFFLE(0, 3, 2, 1 ) ); /* equal shift right 4 bytes */
- psLPC_Q14_tmp = _mm_mul_epi32( psLPC_Q14_tmp, a_Q12_tmp ); /* 1*-1, 3*-3 */
- psLPC_Q14_tmp = _mm_srli_epi64( psLPC_Q14_tmp, 16 );
- tmpb = _mm_add_epi32( tmpb, psLPC_Q14_tmp );
-
- /* step 2 */
- psLPC_Q14_tmp = _mm_loadu_si128( (__m128i *)(&psLPC_Q14[ -7 ] ) );
- psLPC_Q14_tmp = _mm_shuffle_epi32( psLPC_Q14_tmp, 0x1B );
- tmpa = _mm_mul_epi32( psLPC_Q14_tmp, a_Q12_4567 );
- tmpa = _mm_srli_epi64( tmpa, 16 );
- tmpb = _mm_add_epi32( tmpb, tmpa );
-
- psLPC_Q14_tmp = _mm_shuffle_epi32( psLPC_Q14_tmp, _MM_SHUFFLE( 0, 3, 2, 1 ) ); /* equal shift right 4 bytes */
- a_Q12_tmp = _mm_shuffle_epi32( a_Q12_4567, _MM_SHUFFLE(0, 3, 2, 1 ) ); /* equal shift right 4 bytes */
- psLPC_Q14_tmp = _mm_mul_epi32( psLPC_Q14_tmp, a_Q12_tmp );
- psLPC_Q14_tmp = _mm_srli_epi64( psLPC_Q14_tmp, 16 );
- tmpb = _mm_add_epi32( tmpb, psLPC_Q14_tmp );
-
- if ( opus_likely( predictLPCOrder == 16 ) )
- {- /* step 3 */
- psLPC_Q14_tmp = _mm_loadu_si128( (__m128i *)(&psLPC_Q14[ -11 ] ) );
- psLPC_Q14_tmp = _mm_shuffle_epi32( psLPC_Q14_tmp, 0x1B );
- tmpa = _mm_mul_epi32( psLPC_Q14_tmp, a_Q12_89AB );
- tmpa = _mm_srli_epi64( tmpa, 16 );
- tmpb = _mm_add_epi32( tmpb, tmpa );
-
- psLPC_Q14_tmp = _mm_shuffle_epi32( psLPC_Q14_tmp, _MM_SHUFFLE( 0, 3, 2, 1 ) ); /* equal shift right 4 bytes */
- a_Q12_tmp = _mm_shuffle_epi32( a_Q12_89AB, _MM_SHUFFLE(0, 3, 2, 1 ) );/* equal shift right 4 bytes */
- psLPC_Q14_tmp = _mm_mul_epi32( psLPC_Q14_tmp, a_Q12_tmp );
- psLPC_Q14_tmp = _mm_srli_epi64( psLPC_Q14_tmp, 16 );
- tmpb = _mm_add_epi32( tmpb, psLPC_Q14_tmp );
-
- /* setp 4 */
- psLPC_Q14_tmp = _mm_loadu_si128( (__m128i *)(&psLPC_Q14[ -15 ] ) );
- psLPC_Q14_tmp = _mm_shuffle_epi32( psLPC_Q14_tmp, 0x1B );
- tmpa = _mm_mul_epi32( psLPC_Q14_tmp, a_Q12_CDEF );
- tmpa = _mm_srli_epi64( tmpa, 16 );
- tmpb = _mm_add_epi32( tmpb, tmpa );
-
- psLPC_Q14_tmp = _mm_shuffle_epi32( psLPC_Q14_tmp, _MM_SHUFFLE( 0, 3, 2, 1 ) ); /* equal shift right 4 bytes */
- a_Q12_tmp = _mm_shuffle_epi32( a_Q12_CDEF, _MM_SHUFFLE(0, 3, 2, 1 ) ); /* equal shift right 4 bytes */
- psLPC_Q14_tmp = _mm_mul_epi32( psLPC_Q14_tmp, a_Q12_tmp );
- psLPC_Q14_tmp = _mm_srli_epi64( psLPC_Q14_tmp, 16 );
- tmpb = _mm_add_epi32( tmpb, psLPC_Q14_tmp );
-
- /* add at last */
- /* equal shift right 8 bytes*/
- tmpa = _mm_shuffle_epi32( tmpb, _MM_SHUFFLE( 0, 0, 3, 2 ) );
- tmpb = _mm_add_epi32( tmpb, tmpa );
- LPC_pred_Q14 += _mm_cvtsi128_si32( tmpb );
- }
- else
- {- /* add at last */
- tmpa = _mm_shuffle_epi32( tmpb, _MM_SHUFFLE( 0, 0, 3, 2 ) ); /* equal shift right 8 bytes*/
- tmpb = _mm_add_epi32( tmpb, tmpa );
- LPC_pred_Q14 += _mm_cvtsi128_si32( tmpb );
-
- LPC_pred_Q14 = silk_SMLAWB( LPC_pred_Q14, psLPC_Q14[ -8 ], a_Q12[ 8 ] );
- LPC_pred_Q14 = silk_SMLAWB( LPC_pred_Q14, psLPC_Q14[ -9 ], a_Q12[ 9 ] );
- }
-
- LPC_pred_Q14 = silk_LSHIFT( LPC_pred_Q14, 4 ); /* Q10 -> Q14 */
-
- /* Noise shape feedback */
- silk_assert( ( shapingLPCOrder & 1 ) == 0 ); /* check that order is even */
- /* Output of lowpass section */
- tmp2 = silk_SMLAWB( psLPC_Q14[ 0 ], psDD->sAR2_Q14[ 0 ], warping_Q16 );
- /* Output of allpass section */
- tmp1 = silk_SMLAWB( psDD->sAR2_Q14[ 0 ], psDD->sAR2_Q14[ 1 ] - tmp2, warping_Q16 );
- psDD->sAR2_Q14[ 0 ] = tmp2;
- n_AR_Q14 = silk_RSHIFT( shapingLPCOrder, 1 );
- n_AR_Q14 = silk_SMLAWB( n_AR_Q14, tmp2, AR_shp_Q13[ 0 ] );
- /* Loop over allpass sections */
- for( j = 2; j < shapingLPCOrder; j += 2 ) {- /* Output of allpass section */
- tmp2 = silk_SMLAWB( psDD->sAR2_Q14[ j - 1 ], psDD->sAR2_Q14[ j + 0 ] - tmp1, warping_Q16 );
- psDD->sAR2_Q14[ j - 1 ] = tmp1;
- n_AR_Q14 = silk_SMLAWB( n_AR_Q14, tmp1, AR_shp_Q13[ j - 1 ] );
- /* Output of allpass section */
- tmp1 = silk_SMLAWB( psDD->sAR2_Q14[ j + 0 ], psDD->sAR2_Q14[ j + 1 ] - tmp2, warping_Q16 );
- psDD->sAR2_Q14[ j + 0 ] = tmp2;
- n_AR_Q14 = silk_SMLAWB( n_AR_Q14, tmp2, AR_shp_Q13[ j ] );
- }
- psDD->sAR2_Q14[ shapingLPCOrder - 1 ] = tmp1;
- n_AR_Q14 = silk_SMLAWB( n_AR_Q14, tmp1, AR_shp_Q13[ shapingLPCOrder - 1 ] );
-
- n_AR_Q14 = silk_LSHIFT( n_AR_Q14, 1 ); /* Q11 -> Q12 */
- n_AR_Q14 = silk_SMLAWB( n_AR_Q14, psDD->LF_AR_Q14, Tilt_Q14 ); /* Q12 */
- n_AR_Q14 = silk_LSHIFT( n_AR_Q14, 2 ); /* Q12 -> Q14 */
-
- n_LF_Q14 = silk_SMULWB( psDD->Shape_Q14[ *smpl_buf_idx ], LF_shp_Q14 ); /* Q12 */
- n_LF_Q14 = silk_SMLAWT( n_LF_Q14, psDD->LF_AR_Q14, LF_shp_Q14 ); /* Q12 */
- n_LF_Q14 = silk_LSHIFT( n_LF_Q14, 2 ); /* Q12 -> Q14 */
-
- /* Input minus prediction plus noise feedback */
- /* r = x[ i ] - LTP_pred - LPC_pred + n_AR + n_Tilt + n_LF + n_LTP */
- tmp1 = silk_ADD32( n_AR_Q14, n_LF_Q14 ); /* Q14 */
- tmp2 = silk_ADD32( n_LTP_Q14, LPC_pred_Q14 ); /* Q13 */
- tmp1 = silk_SUB32( tmp2, tmp1 ); /* Q13 */
- tmp1 = silk_RSHIFT_ROUND( tmp1, 4 ); /* Q10 */
-
- r_Q10 = silk_SUB32( x_Q10[ i ], tmp1 ); /* residual error Q10 */
-
- /* Flip sign depending on dither */
- if ( psDD->Seed < 0 ) {- r_Q10 = -r_Q10;
- }
- r_Q10 = silk_LIMIT_32( r_Q10, -(31 << 10), 30 << 10 );
-
- /* Find two quantization level candidates and measure their rate-distortion */
- q1_Q10 = silk_SUB32( r_Q10, offset_Q10 );
- q1_Q0 = silk_RSHIFT( q1_Q10, 10 );
- if( q1_Q0 > 0 ) {- q1_Q10 = silk_SUB32( silk_LSHIFT( q1_Q0, 10 ), QUANT_LEVEL_ADJUST_Q10 );
- q1_Q10 = silk_ADD32( q1_Q10, offset_Q10 );
- q2_Q10 = silk_ADD32( q1_Q10, 1024 );
- rd1_Q10 = silk_SMULBB( q1_Q10, Lambda_Q10 );
- rd2_Q10 = silk_SMULBB( q2_Q10, Lambda_Q10 );
- } else if( q1_Q0 == 0 ) {- q1_Q10 = offset_Q10;
- q2_Q10 = silk_ADD32( q1_Q10, 1024 - QUANT_LEVEL_ADJUST_Q10 );
- rd1_Q10 = silk_SMULBB( q1_Q10, Lambda_Q10 );
- rd2_Q10 = silk_SMULBB( q2_Q10, Lambda_Q10 );
- } else if( q1_Q0 == -1 ) {- q2_Q10 = offset_Q10;
- q1_Q10 = silk_SUB32( q2_Q10, 1024 - QUANT_LEVEL_ADJUST_Q10 );
- rd1_Q10 = silk_SMULBB( -q1_Q10, Lambda_Q10 );
- rd2_Q10 = silk_SMULBB( q2_Q10, Lambda_Q10 );
- } else { /* q1_Q0 < -1 */- q1_Q10 = silk_ADD32( silk_LSHIFT( q1_Q0, 10 ), QUANT_LEVEL_ADJUST_Q10 );
- q1_Q10 = silk_ADD32( q1_Q10, offset_Q10 );
- q2_Q10 = silk_ADD32( q1_Q10, 1024 );
- rd1_Q10 = silk_SMULBB( -q1_Q10, Lambda_Q10 );
- rd2_Q10 = silk_SMULBB( -q2_Q10, Lambda_Q10 );
- }
- rr_Q10 = silk_SUB32( r_Q10, q1_Q10 );
- rd1_Q10 = silk_RSHIFT( silk_SMLABB( rd1_Q10, rr_Q10, rr_Q10 ), 10 );
- rr_Q10 = silk_SUB32( r_Q10, q2_Q10 );
- rd2_Q10 = silk_RSHIFT( silk_SMLABB( rd2_Q10, rr_Q10, rr_Q10 ), 10 );
-
- if( rd1_Q10 < rd2_Q10 ) {- psSS[ 0 ].RD_Q10 = silk_ADD32( psDD->RD_Q10, rd1_Q10 );
- psSS[ 1 ].RD_Q10 = silk_ADD32( psDD->RD_Q10, rd2_Q10 );
- psSS[ 0 ].Q_Q10 = q1_Q10;
- psSS[ 1 ].Q_Q10 = q2_Q10;
- } else {- psSS[ 0 ].RD_Q10 = silk_ADD32( psDD->RD_Q10, rd2_Q10 );
- psSS[ 1 ].RD_Q10 = silk_ADD32( psDD->RD_Q10, rd1_Q10 );
- psSS[ 0 ].Q_Q10 = q2_Q10;
- psSS[ 1 ].Q_Q10 = q1_Q10;
- }
-
- /* Update states for best quantization */
-
- /* Quantized excitation */
- exc_Q14 = silk_LSHIFT32( psSS[ 0 ].Q_Q10, 4 );
- if ( psDD->Seed < 0 ) {- exc_Q14 = -exc_Q14;
- }
-
- /* Add predictions */
- LPC_exc_Q14 = silk_ADD32( exc_Q14, LTP_pred_Q14 );
- xq_Q14 = silk_ADD32( LPC_exc_Q14, LPC_pred_Q14 );
-
- /* Update states */
- sLF_AR_shp_Q14 = silk_SUB32( xq_Q14, n_AR_Q14 );
- psSS[ 0 ].sLTP_shp_Q14 = silk_SUB32( sLF_AR_shp_Q14, n_LF_Q14 );
- psSS[ 0 ].LF_AR_Q14 = sLF_AR_shp_Q14;
- psSS[ 0 ].LPC_exc_Q14 = LPC_exc_Q14;
- psSS[ 0 ].xq_Q14 = xq_Q14;
-
- /* Update states for second best quantization */
-
- /* Quantized excitation */
- exc_Q14 = silk_LSHIFT32( psSS[ 1 ].Q_Q10, 4 );
- if ( psDD->Seed < 0 ) {- exc_Q14 = -exc_Q14;
- }
-
-
- /* Add predictions */
- LPC_exc_Q14 = silk_ADD32( exc_Q14, LTP_pred_Q14 );
- xq_Q14 = silk_ADD32( LPC_exc_Q14, LPC_pred_Q14 );
-
- /* Update states */
- sLF_AR_shp_Q14 = silk_SUB32( xq_Q14, n_AR_Q14 );
- psSS[ 1 ].sLTP_shp_Q14 = silk_SUB32( sLF_AR_shp_Q14, n_LF_Q14 );
- psSS[ 1 ].LF_AR_Q14 = sLF_AR_shp_Q14;
- psSS[ 1 ].LPC_exc_Q14 = LPC_exc_Q14;
- psSS[ 1 ].xq_Q14 = xq_Q14;
- }
- }
- *smpl_buf_idx = ( *smpl_buf_idx - 1 ) % DECISION_DELAY;
- if( *smpl_buf_idx < 0 ) *smpl_buf_idx += DECISION_DELAY;
- last_smple_idx = ( *smpl_buf_idx + decisionDelay ) % DECISION_DELAY;
-
- /* Find winner */
- RDmin_Q10 = psSampleState[ 0 ][ 0 ].RD_Q10;
- Winner_ind = 0;
- for( k = 1; k < nStatesDelayedDecision; k++ ) {- if( psSampleState[ k ][ 0 ].RD_Q10 < RDmin_Q10 ) {- RDmin_Q10 = psSampleState[ k ][ 0 ].RD_Q10;
- Winner_ind = k;
- }
- }
-
- /* Increase RD values of expired states */
- Winner_rand_state = psDelDec[ Winner_ind ].RandState[ last_smple_idx ];
- for( k = 0; k < nStatesDelayedDecision; k++ ) {- if( psDelDec[ k ].RandState[ last_smple_idx ] != Winner_rand_state ) {- psSampleState[ k ][ 0 ].RD_Q10 = silk_ADD32( psSampleState[ k ][ 0 ].RD_Q10, silk_int32_MAX >> 4 );
- psSampleState[ k ][ 1 ].RD_Q10 = silk_ADD32( psSampleState[ k ][ 1 ].RD_Q10, silk_int32_MAX >> 4 );
- silk_assert( psSampleState[ k ][ 0 ].RD_Q10 >= 0 );
- }
- }
-
- /* Find worst in first set and best in second set */
- RDmax_Q10 = psSampleState[ 0 ][ 0 ].RD_Q10;
- RDmin_Q10 = psSampleState[ 0 ][ 1 ].RD_Q10;
- RDmax_ind = 0;
- RDmin_ind = 0;
- for( k = 1; k < nStatesDelayedDecision; k++ ) {- /* find worst in first set */
- if( psSampleState[ k ][ 0 ].RD_Q10 > RDmax_Q10 ) {- RDmax_Q10 = psSampleState[ k ][ 0 ].RD_Q10;
- RDmax_ind = k;
- }
- /* find best in second set */
- if( psSampleState[ k ][ 1 ].RD_Q10 < RDmin_Q10 ) {- RDmin_Q10 = psSampleState[ k ][ 1 ].RD_Q10;
- RDmin_ind = k;
- }
- }
-
- /* Replace a state if best from second set outperforms worst in first set */
- if( RDmin_Q10 < RDmax_Q10 ) {- silk_memcpy( ( (opus_int32 *)&psDelDec[ RDmax_ind ] ) + i,
- ( (opus_int32 *)&psDelDec[ RDmin_ind ] ) + i, sizeof( NSQ_del_dec_struct ) - i * sizeof( opus_int32) );
- silk_memcpy( &psSampleState[ RDmax_ind ][ 0 ], &psSampleState[ RDmin_ind ][ 1 ], sizeof( NSQ_sample_struct ) );
- }
-
- /* Write samples from winner to output and long-term filter states */
- psDD = &psDelDec[ Winner_ind ];
- if( subfr > 0 || i >= decisionDelay ) {- pulses[ i - decisionDelay ] = (opus_int8)silk_RSHIFT_ROUND( psDD->Q_Q10[ last_smple_idx ], 10 );
- xq[ i - decisionDelay ] = (opus_int16)silk_SAT16( silk_RSHIFT_ROUND(
- silk_SMULWW( psDD->Xq_Q14[ last_smple_idx ], delayedGain_Q10[ last_smple_idx ] ), 8 ) );
- NSQ->sLTP_shp_Q14[ NSQ->sLTP_shp_buf_idx - decisionDelay ] = psDD->Shape_Q14[ last_smple_idx ];
- sLTP_Q15[ NSQ->sLTP_buf_idx - decisionDelay ] = psDD->Pred_Q15[ last_smple_idx ];
- }
- NSQ->sLTP_shp_buf_idx++;
- NSQ->sLTP_buf_idx++;
-
- /* Update states */
- for( k = 0; k < nStatesDelayedDecision; k++ ) {- psDD = &psDelDec[ k ];
- psSS = &psSampleState[ k ][ 0 ];
- psDD->LF_AR_Q14 = psSS->LF_AR_Q14;
- psDD->sLPC_Q14[ NSQ_LPC_BUF_LENGTH + i ] = psSS->xq_Q14;
- psDD->Xq_Q14[ *smpl_buf_idx ] = psSS->xq_Q14;
- psDD->Q_Q10[ *smpl_buf_idx ] = psSS->Q_Q10;
- psDD->Pred_Q15[ *smpl_buf_idx ] = silk_LSHIFT32( psSS->LPC_exc_Q14, 1 );
- psDD->Shape_Q14[ *smpl_buf_idx ] = psSS->sLTP_shp_Q14;
- psDD->Seed = silk_ADD32_ovflw( psDD->Seed, silk_RSHIFT_ROUND( psSS->Q_Q10, 10 ) );
- psDD->RandState[ *smpl_buf_idx ] = psDD->Seed;
- psDD->RD_Q10 = psSS->RD_Q10;
- }
- delayedGain_Q10[ *smpl_buf_idx ] = Gain_Q10;
- }
- /* Update LPC states */
- for( k = 0; k < nStatesDelayedDecision; k++ ) {- psDD = &psDelDec[ k ];
- silk_memcpy( psDD->sLPC_Q14, &psDD->sLPC_Q14[ length ], NSQ_LPC_BUF_LENGTH * sizeof( opus_int32 ) );
- }
- RESTORE_STACK;
-}
-
-static OPUS_INLINE void silk_nsq_del_dec_scale_states_sse4_1(
- const silk_encoder_state *psEncC, /* I Encoder State */
- silk_nsq_state *NSQ, /* I/O NSQ state */
- NSQ_del_dec_struct psDelDec[], /* I/O Delayed decision states */
- const opus_int32 x_Q3[], /* I Input in Q3 */
- opus_int32 x_sc_Q10[], /* O Input scaled with 1/Gain in Q10 */
- const opus_int16 sLTP[], /* I Re-whitened LTP state in Q0 */
- opus_int32 sLTP_Q15[], /* O LTP state matching scaled input */
- opus_int subfr, /* I Subframe number */
- opus_int nStatesDelayedDecision, /* I Number of del dec states */
- const opus_int LTP_scale_Q14, /* I LTP state scaling */
- const opus_int32 Gains_Q16[ MAX_NB_SUBFR ], /* I */
- const opus_int pitchL[ MAX_NB_SUBFR ], /* I Pitch lag */
- const opus_int signal_type, /* I Signal type */
- const opus_int decisionDelay /* I Decision delay */
-)
-{- opus_int i, k, lag;
- opus_int32 gain_adj_Q16, inv_gain_Q31, inv_gain_Q23;
- NSQ_del_dec_struct *psDD;
- __m128i xmm_inv_gain_Q23, xmm_x_Q3_x2x0, xmm_x_Q3_x3x1;
-
- lag = pitchL[ subfr ];
- inv_gain_Q31 = silk_INVERSE32_varQ( silk_max( Gains_Q16[ subfr ], 1 ), 47 );
-
- silk_assert( inv_gain_Q31 != 0 );
-
- /* Calculate gain adjustment factor */
- if( Gains_Q16[ subfr ] != NSQ->prev_gain_Q16 ) {- gain_adj_Q16 = silk_DIV32_varQ( NSQ->prev_gain_Q16, Gains_Q16[ subfr ], 16 );
- } else {- gain_adj_Q16 = (opus_int32)1 << 16;
- }
-
- /* Scale input */
- inv_gain_Q23 = silk_RSHIFT_ROUND( inv_gain_Q31, 8 );
-
- /* prepare inv_gain_Q23 in packed 4 32-bits */
- xmm_inv_gain_Q23 = _mm_set1_epi32(inv_gain_Q23);
-
- for( i = 0; i < psEncC->subfr_length - 3; i += 4 ) {- xmm_x_Q3_x2x0 = _mm_loadu_si128( (__m128i *)(&(x_Q3[ i ] ) ) );
- /* equal shift right 4 bytes*/
- xmm_x_Q3_x3x1 = _mm_shuffle_epi32( xmm_x_Q3_x2x0, _MM_SHUFFLE( 0, 3, 2, 1 ) );
-
- xmm_x_Q3_x2x0 = _mm_mul_epi32( xmm_x_Q3_x2x0, xmm_inv_gain_Q23 );
- xmm_x_Q3_x3x1 = _mm_mul_epi32( xmm_x_Q3_x3x1, xmm_inv_gain_Q23 );
-
- xmm_x_Q3_x2x0 = _mm_srli_epi64( xmm_x_Q3_x2x0, 16 );
- xmm_x_Q3_x3x1 = _mm_slli_epi64( xmm_x_Q3_x3x1, 16 );
-
- xmm_x_Q3_x2x0 = _mm_blend_epi16( xmm_x_Q3_x2x0, xmm_x_Q3_x3x1, 0xCC );
-
- _mm_storeu_si128( (__m128i *)(&(x_sc_Q10[ i ])), xmm_x_Q3_x2x0 );
- }
-
- for( ; i < psEncC->subfr_length; i++ ) {- x_sc_Q10[ i ] = silk_SMULWW( x_Q3[ i ], inv_gain_Q23 );
- }
-
- /* Save inverse gain */
- NSQ->prev_gain_Q16 = Gains_Q16[ subfr ];
-
- /* After rewhitening the LTP state is un-scaled, so scale with inv_gain_Q16 */
- if( NSQ->rewhite_flag ) {- if( subfr == 0 ) {- /* Do LTP downscaling */
- inv_gain_Q31 = silk_LSHIFT( silk_SMULWB( inv_gain_Q31, LTP_scale_Q14 ), 2 );
- }
- for( i = NSQ->sLTP_buf_idx - lag - LTP_ORDER / 2; i < NSQ->sLTP_buf_idx; i++ ) {- silk_assert( i < MAX_FRAME_LENGTH );
- sLTP_Q15[ i ] = silk_SMULWB( inv_gain_Q31, sLTP[ i ] );
- }
- }
-
- /* Adjust for changing gain */
- if( gain_adj_Q16 != (opus_int32)1 << 16 ) {- /* Scale long-term shaping state */
- {- __m128i xmm_gain_adj_Q16, xmm_sLTP_shp_Q14_x2x0, xmm_sLTP_shp_Q14_x3x1;
-
- /* prepare gain_adj_Q16 in packed 4 32-bits */
- xmm_gain_adj_Q16 = _mm_set1_epi32( gain_adj_Q16 );
-
- for( i = NSQ->sLTP_shp_buf_idx - psEncC->ltp_mem_length; i < NSQ->sLTP_shp_buf_idx - 3; i += 4 )
- {- xmm_sLTP_shp_Q14_x2x0 = _mm_loadu_si128( (__m128i *)(&(NSQ->sLTP_shp_Q14[ i ] ) ) );
- /* equal shift right 4 bytes*/
- xmm_sLTP_shp_Q14_x3x1 = _mm_shuffle_epi32( xmm_sLTP_shp_Q14_x2x0, _MM_SHUFFLE( 0, 3, 2, 1 ) );
-
- xmm_sLTP_shp_Q14_x2x0 = _mm_mul_epi32( xmm_sLTP_shp_Q14_x2x0, xmm_gain_adj_Q16 );
- xmm_sLTP_shp_Q14_x3x1 = _mm_mul_epi32( xmm_sLTP_shp_Q14_x3x1, xmm_gain_adj_Q16 );
-
- xmm_sLTP_shp_Q14_x2x0 = _mm_srli_epi64( xmm_sLTP_shp_Q14_x2x0, 16 );
- xmm_sLTP_shp_Q14_x3x1 = _mm_slli_epi64( xmm_sLTP_shp_Q14_x3x1, 16 );
-
- xmm_sLTP_shp_Q14_x2x0 = _mm_blend_epi16( xmm_sLTP_shp_Q14_x2x0, xmm_sLTP_shp_Q14_x3x1, 0xCC );
-
- _mm_storeu_si128( (__m128i *)(&(NSQ->sLTP_shp_Q14[ i ] ) ), xmm_sLTP_shp_Q14_x2x0 );
- }
-
- for( ; i < NSQ->sLTP_shp_buf_idx; i++ ) {- NSQ->sLTP_shp_Q14[ i ] = silk_SMULWW( gain_adj_Q16, NSQ->sLTP_shp_Q14[ i ] );
- }
-
- /* Scale long-term prediction state */
- if( signal_type == TYPE_VOICED && NSQ->rewhite_flag == 0 ) {- for( i = NSQ->sLTP_buf_idx - lag - LTP_ORDER / 2; i < NSQ->sLTP_buf_idx - decisionDelay; i++ ) {- sLTP_Q15[ i ] = silk_SMULWW( gain_adj_Q16, sLTP_Q15[ i ] );
- }
- }
-
- for( k = 0; k < nStatesDelayedDecision; k++ ) {- psDD = &psDelDec[ k ];
-
- /* Scale scalar states */
- psDD->LF_AR_Q14 = silk_SMULWW( gain_adj_Q16, psDD->LF_AR_Q14 );
-
- /* Scale short-term prediction and shaping states */
- for( i = 0; i < NSQ_LPC_BUF_LENGTH; i++ ) {- psDD->sLPC_Q14[ i ] = silk_SMULWW( gain_adj_Q16, psDD->sLPC_Q14[ i ] );
- }
- for( i = 0; i < MAX_SHAPE_LPC_ORDER; i++ ) {- psDD->sAR2_Q14[ i ] = silk_SMULWW( gain_adj_Q16, psDD->sAR2_Q14[ i ] );
- }
- for( i = 0; i < DECISION_DELAY; i++ ) {- psDD->Pred_Q15[ i ] = silk_SMULWW( gain_adj_Q16, psDD->Pred_Q15[ i ] );
- psDD->Shape_Q14[ i ] = silk_SMULWW( gain_adj_Q16, psDD->Shape_Q14[ i ] );
- }
- }
- }
- }
-}
--- /dev/null
+++ b/silk/x86/NSQ_del_dec_sse4_1.c
@@ -1,0 +1,859 @@
+/* Copyright (c) 2014, Cisco Systems, INC
+ Written by XiangMingZhu WeiZhou MinPeng YanWang
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+
+ - Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+
+ - Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER
+ OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+ LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+ NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*/
+
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include <xmmintrin.h>
+#include <emmintrin.h>
+#include <smmintrin.h>
+#include "main.h"
+#include "celt/x86/x86cpu.h"
+
+#include "stack_alloc.h"
+
+typedef struct {+ opus_int32 sLPC_Q14[ MAX_SUB_FRAME_LENGTH + NSQ_LPC_BUF_LENGTH ];
+ opus_int32 RandState[ DECISION_DELAY ];
+ opus_int32 Q_Q10[ DECISION_DELAY ];
+ opus_int32 Xq_Q14[ DECISION_DELAY ];
+ opus_int32 Pred_Q15[ DECISION_DELAY ];
+ opus_int32 Shape_Q14[ DECISION_DELAY ];
+ opus_int32 sAR2_Q14[ MAX_SHAPE_LPC_ORDER ];
+ opus_int32 LF_AR_Q14;
+ opus_int32 Seed;
+ opus_int32 SeedInit;
+ opus_int32 RD_Q10;
+} NSQ_del_dec_struct;
+
+typedef struct {+ opus_int32 Q_Q10;
+ opus_int32 RD_Q10;
+ opus_int32 xq_Q14;
+ opus_int32 LF_AR_Q14;
+ opus_int32 sLTP_shp_Q14;
+ opus_int32 LPC_exc_Q14;
+} NSQ_sample_struct;
+
+typedef NSQ_sample_struct NSQ_sample_pair[ 2 ];
+
+static OPUS_INLINE void silk_nsq_del_dec_scale_states_sse4_1(
+ const silk_encoder_state *psEncC, /* I Encoder State */
+ silk_nsq_state *NSQ, /* I/O NSQ state */
+ NSQ_del_dec_struct psDelDec[], /* I/O Delayed decision states */
+ const opus_int32 x_Q3[], /* I Input in Q3 */
+ opus_int32 x_sc_Q10[], /* O Input scaled with 1/Gain in Q10 */
+ const opus_int16 sLTP[], /* I Re-whitened LTP state in Q0 */
+ opus_int32 sLTP_Q15[], /* O LTP state matching scaled input */
+ opus_int subfr, /* I Subframe number */
+ opus_int nStatesDelayedDecision, /* I Number of del dec states */
+ const opus_int LTP_scale_Q14, /* I LTP state scaling */
+ const opus_int32 Gains_Q16[ MAX_NB_SUBFR ], /* I */
+ const opus_int pitchL[ MAX_NB_SUBFR ], /* I Pitch lag */
+ const opus_int signal_type, /* I Signal type */
+ const opus_int decisionDelay /* I Decision delay */
+);
+
+/******************************************/
+/* Noise shape quantizer for one subframe */
+/******************************************/
+static OPUS_INLINE void silk_noise_shape_quantizer_del_dec_sse4_1(
+ silk_nsq_state *NSQ, /* I/O NSQ state */
+ NSQ_del_dec_struct psDelDec[], /* I/O Delayed decision states */
+ opus_int signalType, /* I Signal type */
+ const opus_int32 x_Q10[], /* I */
+ opus_int8 pulses[], /* O */
+ opus_int16 xq[], /* O */
+ opus_int32 sLTP_Q15[], /* I/O LTP filter state */
+ opus_int32 delayedGain_Q10[], /* I/O Gain delay buffer */
+ const opus_int16 a_Q12[], /* I Short term prediction coefs */
+ const opus_int16 b_Q14[], /* I Long term prediction coefs */
+ const opus_int16 AR_shp_Q13[], /* I Noise shaping coefs */
+ opus_int lag, /* I Pitch lag */
+ opus_int32 HarmShapeFIRPacked_Q14, /* I */
+ opus_int Tilt_Q14, /* I Spectral tilt */
+ opus_int32 LF_shp_Q14, /* I */
+ opus_int32 Gain_Q16, /* I */
+ opus_int Lambda_Q10, /* I */
+ opus_int offset_Q10, /* I */
+ opus_int length, /* I Input length */
+ opus_int subfr, /* I Subframe number */
+ opus_int shapingLPCOrder, /* I Shaping LPC filter order */
+ opus_int predictLPCOrder, /* I Prediction filter order */
+ opus_int warping_Q16, /* I */
+ opus_int nStatesDelayedDecision, /* I Number of states in decision tree */
+ opus_int *smpl_buf_idx, /* I/O Index to newest samples in buffers */
+ opus_int decisionDelay /* I */
+);
+
+void silk_NSQ_del_dec_sse4_1(
+ const silk_encoder_state *psEncC, /* I Encoder State */
+ silk_nsq_state *NSQ, /* I/O NSQ state */
+ SideInfoIndices *psIndices, /* I/O Quantization Indices */
+ const opus_int32 x_Q3[], /* I Prefiltered input signal */
+ opus_int8 pulses[], /* O Quantized pulse signal */
+ const opus_int16 PredCoef_Q12[ 2 * MAX_LPC_ORDER ], /* I Short term prediction coefs */
+ const opus_int16 LTPCoef_Q14[ LTP_ORDER * MAX_NB_SUBFR ], /* I Long term prediction coefs */
+ const opus_int16 AR2_Q13[ MAX_NB_SUBFR * MAX_SHAPE_LPC_ORDER ], /* I Noise shaping coefs */
+ const opus_int HarmShapeGain_Q14[ MAX_NB_SUBFR ], /* I Long term shaping coefs */
+ const opus_int Tilt_Q14[ MAX_NB_SUBFR ], /* I Spectral tilt */
+ const opus_int32 LF_shp_Q14[ MAX_NB_SUBFR ], /* I Low frequency shaping coefs */
+ const opus_int32 Gains_Q16[ MAX_NB_SUBFR ], /* I Quantization step sizes */
+ const opus_int pitchL[ MAX_NB_SUBFR ], /* I Pitch lags */
+ const opus_int Lambda_Q10, /* I Rate/distortion tradeoff */
+ const opus_int LTP_scale_Q14 /* I LTP state scaling */
+)
+{+ opus_int i, k, lag, start_idx, LSF_interpolation_flag, Winner_ind, subfr;
+ opus_int last_smple_idx, smpl_buf_idx, decisionDelay;
+ const opus_int16 *A_Q12, *B_Q14, *AR_shp_Q13;
+ opus_int16 *pxq;
+ VARDECL( opus_int32, sLTP_Q15 );
+ VARDECL( opus_int16, sLTP );
+ opus_int32 HarmShapeFIRPacked_Q14;
+ opus_int offset_Q10;
+ opus_int32 RDmin_Q10, Gain_Q10;
+ VARDECL( opus_int32, x_sc_Q10 );
+ VARDECL( opus_int32, delayedGain_Q10 );
+ VARDECL( NSQ_del_dec_struct, psDelDec );
+ NSQ_del_dec_struct *psDD;
+ SAVE_STACK;
+
+ /* Set unvoiced lag to the previous one, overwrite later for voiced */
+ lag = NSQ->lagPrev;
+
+ silk_assert( NSQ->prev_gain_Q16 != 0 );
+
+ /* Initialize delayed decision states */
+ ALLOC( psDelDec, psEncC->nStatesDelayedDecision, NSQ_del_dec_struct );
+ silk_memset( psDelDec, 0, psEncC->nStatesDelayedDecision * sizeof( NSQ_del_dec_struct ) );
+ for( k = 0; k < psEncC->nStatesDelayedDecision; k++ ) {+ psDD = &psDelDec[ k ];
+ psDD->Seed = ( k + psIndices->Seed ) & 3;
+ psDD->SeedInit = psDD->Seed;
+ psDD->RD_Q10 = 0;
+ psDD->LF_AR_Q14 = NSQ->sLF_AR_shp_Q14;
+ psDD->Shape_Q14[ 0 ] = NSQ->sLTP_shp_Q14[ psEncC->ltp_mem_length - 1 ];
+ silk_memcpy( psDD->sLPC_Q14, NSQ->sLPC_Q14, NSQ_LPC_BUF_LENGTH * sizeof( opus_int32 ) );
+ silk_memcpy( psDD->sAR2_Q14, NSQ->sAR2_Q14, sizeof( NSQ->sAR2_Q14 ) );
+ }
+
+ offset_Q10 = silk_Quantization_Offsets_Q10[ psIndices->signalType >> 1 ][ psIndices->quantOffsetType ];
+ smpl_buf_idx = 0; /* index of oldest samples */
+
+ decisionDelay = silk_min_int( DECISION_DELAY, psEncC->subfr_length );
+
+ /* For voiced frames limit the decision delay to lower than the pitch lag */
+ if( psIndices->signalType == TYPE_VOICED ) {+ for( k = 0; k < psEncC->nb_subfr; k++ ) {+ decisionDelay = silk_min_int( decisionDelay, pitchL[ k ] - LTP_ORDER / 2 - 1 );
+ }
+ } else {+ if( lag > 0 ) {+ decisionDelay = silk_min_int( decisionDelay, lag - LTP_ORDER / 2 - 1 );
+ }
+ }
+
+ if( psIndices->NLSFInterpCoef_Q2 == 4 ) {+ LSF_interpolation_flag = 0;
+ } else {+ LSF_interpolation_flag = 1;
+ }
+
+ ALLOC( sLTP_Q15,
+ psEncC->ltp_mem_length + psEncC->frame_length, opus_int32 );
+ ALLOC( sLTP, psEncC->ltp_mem_length + psEncC->frame_length, opus_int16 );
+ ALLOC( x_sc_Q10, psEncC->subfr_length, opus_int32 );
+ ALLOC( delayedGain_Q10, DECISION_DELAY, opus_int32 );
+ /* Set up pointers to start of sub frame */
+ pxq = &NSQ->xq[ psEncC->ltp_mem_length ];
+ NSQ->sLTP_shp_buf_idx = psEncC->ltp_mem_length;
+ NSQ->sLTP_buf_idx = psEncC->ltp_mem_length;
+ subfr = 0;
+ for( k = 0; k < psEncC->nb_subfr; k++ ) {+ A_Q12 = &PredCoef_Q12[ ( ( k >> 1 ) | ( 1 - LSF_interpolation_flag ) ) * MAX_LPC_ORDER ];
+ B_Q14 = <PCoef_Q14[ k * LTP_ORDER ];
+ AR_shp_Q13 = &AR2_Q13[ k * MAX_SHAPE_LPC_ORDER ];
+
+ /* Noise shape parameters */
+ silk_assert( HarmShapeGain_Q14[ k ] >= 0 );
+ HarmShapeFIRPacked_Q14 = silk_RSHIFT( HarmShapeGain_Q14[ k ], 2 );
+ HarmShapeFIRPacked_Q14 |= silk_LSHIFT( (opus_int32)silk_RSHIFT( HarmShapeGain_Q14[ k ], 1 ), 16 );
+
+ NSQ->rewhite_flag = 0;
+ if( psIndices->signalType == TYPE_VOICED ) {+ /* Voiced */
+ lag = pitchL[ k ];
+
+ /* Re-whitening */
+ if( ( k & ( 3 - silk_LSHIFT( LSF_interpolation_flag, 1 ) ) ) == 0 ) {+ if( k == 2 ) {+ /* RESET DELAYED DECISIONS */
+ /* Find winner */
+ RDmin_Q10 = psDelDec[ 0 ].RD_Q10;
+ Winner_ind = 0;
+ for( i = 1; i < psEncC->nStatesDelayedDecision; i++ ) {+ if( psDelDec[ i ].RD_Q10 < RDmin_Q10 ) {+ RDmin_Q10 = psDelDec[ i ].RD_Q10;
+ Winner_ind = i;
+ }
+ }
+ for( i = 0; i < psEncC->nStatesDelayedDecision; i++ ) {+ if( i != Winner_ind ) {+ psDelDec[ i ].RD_Q10 += ( silk_int32_MAX >> 4 );
+ silk_assert( psDelDec[ i ].RD_Q10 >= 0 );
+ }
+ }
+
+ /* Copy final part of signals from winner state to output and long-term filter states */
+ psDD = &psDelDec[ Winner_ind ];
+ last_smple_idx = smpl_buf_idx + decisionDelay;
+ for( i = 0; i < decisionDelay; i++ ) {+ last_smple_idx = ( last_smple_idx - 1 ) % DECISION_DELAY;
+ if( last_smple_idx < 0 ) last_smple_idx += DECISION_DELAY;
+ pulses[ i - decisionDelay ] = (opus_int8)silk_RSHIFT_ROUND( psDD->Q_Q10[ last_smple_idx ], 10 );
+ pxq[ i - decisionDelay ] = (opus_int16)silk_SAT16( silk_RSHIFT_ROUND(
+ silk_SMULWW( psDD->Xq_Q14[ last_smple_idx ], Gains_Q16[ 1 ] ), 14 ) );
+ NSQ->sLTP_shp_Q14[ NSQ->sLTP_shp_buf_idx - decisionDelay + i ] = psDD->Shape_Q14[ last_smple_idx ];
+ }
+
+ subfr = 0;
+ }
+
+ /* Rewhiten with new A coefs */
+ start_idx = psEncC->ltp_mem_length - lag - psEncC->predictLPCOrder - LTP_ORDER / 2;
+ silk_assert( start_idx > 0 );
+
+ silk_LPC_analysis_filter( &sLTP[ start_idx ], &NSQ->xq[ start_idx + k * psEncC->subfr_length ],
+ A_Q12, psEncC->ltp_mem_length - start_idx, psEncC->predictLPCOrder, psEncC->arch );
+
+ NSQ->sLTP_buf_idx = psEncC->ltp_mem_length;
+ NSQ->rewhite_flag = 1;
+ }
+ }
+
+ silk_nsq_del_dec_scale_states_sse4_1( psEncC, NSQ, psDelDec, x_Q3, x_sc_Q10, sLTP, sLTP_Q15, k,
+ psEncC->nStatesDelayedDecision, LTP_scale_Q14, Gains_Q16, pitchL, psIndices->signalType, decisionDelay );
+
+ silk_noise_shape_quantizer_del_dec_sse4_1( NSQ, psDelDec, psIndices->signalType, x_sc_Q10, pulses, pxq, sLTP_Q15,
+ delayedGain_Q10, A_Q12, B_Q14, AR_shp_Q13, lag, HarmShapeFIRPacked_Q14, Tilt_Q14[ k ], LF_shp_Q14[ k ],
+ Gains_Q16[ k ], Lambda_Q10, offset_Q10, psEncC->subfr_length, subfr++, psEncC->shapingLPCOrder,
+ psEncC->predictLPCOrder, psEncC->warping_Q16, psEncC->nStatesDelayedDecision, &smpl_buf_idx, decisionDelay );
+
+ x_Q3 += psEncC->subfr_length;
+ pulses += psEncC->subfr_length;
+ pxq += psEncC->subfr_length;
+ }
+
+ /* Find winner */
+ RDmin_Q10 = psDelDec[ 0 ].RD_Q10;
+ Winner_ind = 0;
+ for( k = 1; k < psEncC->nStatesDelayedDecision; k++ ) {+ if( psDelDec[ k ].RD_Q10 < RDmin_Q10 ) {+ RDmin_Q10 = psDelDec[ k ].RD_Q10;
+ Winner_ind = k;
+ }
+ }
+
+ /* Copy final part of signals from winner state to output and long-term filter states */
+ psDD = &psDelDec[ Winner_ind ];
+ psIndices->Seed = psDD->SeedInit;
+ last_smple_idx = smpl_buf_idx + decisionDelay;
+ Gain_Q10 = silk_RSHIFT32( Gains_Q16[ psEncC->nb_subfr - 1 ], 6 );
+ for( i = 0; i < decisionDelay; i++ ) {+ last_smple_idx = ( last_smple_idx - 1 ) % DECISION_DELAY;
+ if( last_smple_idx < 0 ) last_smple_idx += DECISION_DELAY;
+ pulses[ i - decisionDelay ] = (opus_int8)silk_RSHIFT_ROUND( psDD->Q_Q10[ last_smple_idx ], 10 );
+ pxq[ i - decisionDelay ] = (opus_int16)silk_SAT16( silk_RSHIFT_ROUND(
+ silk_SMULWW( psDD->Xq_Q14[ last_smple_idx ], Gain_Q10 ), 8 ) );
+ NSQ->sLTP_shp_Q14[ NSQ->sLTP_shp_buf_idx - decisionDelay + i ] = psDD->Shape_Q14[ last_smple_idx ];
+ }
+ silk_memcpy( NSQ->sLPC_Q14, &psDD->sLPC_Q14[ psEncC->subfr_length ], NSQ_LPC_BUF_LENGTH * sizeof( opus_int32 ) );
+ silk_memcpy( NSQ->sAR2_Q14, psDD->sAR2_Q14, sizeof( psDD->sAR2_Q14 ) );
+
+ /* Update states */
+ NSQ->sLF_AR_shp_Q14 = psDD->LF_AR_Q14;
+ NSQ->lagPrev = pitchL[ psEncC->nb_subfr - 1 ];
+
+ /* Save quantized speech signal */
+ silk_memmove( NSQ->xq, &NSQ->xq[ psEncC->frame_length ], psEncC->ltp_mem_length * sizeof( opus_int16 ) );
+ silk_memmove( NSQ->sLTP_shp_Q14, &NSQ->sLTP_shp_Q14[ psEncC->frame_length ], psEncC->ltp_mem_length * sizeof( opus_int32 ) );
+ RESTORE_STACK;
+}
+
+/******************************************/
+/* Noise shape quantizer for one subframe */
+/******************************************/
+static OPUS_INLINE void silk_noise_shape_quantizer_del_dec_sse4_1(
+ silk_nsq_state *NSQ, /* I/O NSQ state */
+ NSQ_del_dec_struct psDelDec[], /* I/O Delayed decision states */
+ opus_int signalType, /* I Signal type */
+ const opus_int32 x_Q10[], /* I */
+ opus_int8 pulses[], /* O */
+ opus_int16 xq[], /* O */
+ opus_int32 sLTP_Q15[], /* I/O LTP filter state */
+ opus_int32 delayedGain_Q10[], /* I/O Gain delay buffer */
+ const opus_int16 a_Q12[], /* I Short term prediction coefs */
+ const opus_int16 b_Q14[], /* I Long term prediction coefs */
+ const opus_int16 AR_shp_Q13[], /* I Noise shaping coefs */
+ opus_int lag, /* I Pitch lag */
+ opus_int32 HarmShapeFIRPacked_Q14, /* I */
+ opus_int Tilt_Q14, /* I Spectral tilt */
+ opus_int32 LF_shp_Q14, /* I */
+ opus_int32 Gain_Q16, /* I */
+ opus_int Lambda_Q10, /* I */
+ opus_int offset_Q10, /* I */
+ opus_int length, /* I Input length */
+ opus_int subfr, /* I Subframe number */
+ opus_int shapingLPCOrder, /* I Shaping LPC filter order */
+ opus_int predictLPCOrder, /* I Prediction filter order */
+ opus_int warping_Q16, /* I */
+ opus_int nStatesDelayedDecision, /* I Number of states in decision tree */
+ opus_int *smpl_buf_idx, /* I/O Index to newest samples in buffers */
+ opus_int decisionDelay /* I */
+)
+{+ opus_int i, j, k, Winner_ind, RDmin_ind, RDmax_ind, last_smple_idx;
+ opus_int32 Winner_rand_state;
+ opus_int32 LTP_pred_Q14, LPC_pred_Q14, n_AR_Q14, n_LTP_Q14;
+ opus_int32 n_LF_Q14, r_Q10, rr_Q10, rd1_Q10, rd2_Q10, RDmin_Q10, RDmax_Q10;
+ opus_int32 q1_Q0, q1_Q10, q2_Q10, exc_Q14, LPC_exc_Q14, xq_Q14, Gain_Q10;
+ opus_int32 tmp1, tmp2, sLF_AR_shp_Q14;
+ opus_int32 *pred_lag_ptr, *shp_lag_ptr, *psLPC_Q14;
+ VARDECL( NSQ_sample_pair, psSampleState );
+ NSQ_del_dec_struct *psDD;
+ NSQ_sample_struct *psSS;
+
+ __m128i a_Q12_0123, a_Q12_4567, a_Q12_89AB, a_Q12_CDEF;
+ __m128i b_Q12_0123, b_sr_Q12_0123;
+ SAVE_STACK;
+
+ silk_assert( nStatesDelayedDecision > 0 );
+ ALLOC( psSampleState, nStatesDelayedDecision, NSQ_sample_pair );
+
+ shp_lag_ptr = &NSQ->sLTP_shp_Q14[ NSQ->sLTP_shp_buf_idx - lag + HARM_SHAPE_FIR_TAPS / 2 ];
+ pred_lag_ptr = &sLTP_Q15[ NSQ->sLTP_buf_idx - lag + LTP_ORDER / 2 ];
+ Gain_Q10 = silk_RSHIFT( Gain_Q16, 6 );
+
+ a_Q12_0123 = OP_CVTEPI16_EPI32_M64( a_Q12 );
+ a_Q12_4567 = OP_CVTEPI16_EPI32_M64( a_Q12 + 4 );
+
+ if( opus_likely( predictLPCOrder == 16 ) ) {+ a_Q12_89AB = OP_CVTEPI16_EPI32_M64( a_Q12 + 8 );
+ a_Q12_CDEF = OP_CVTEPI16_EPI32_M64( a_Q12 + 12 );
+ }
+
+ if( signalType == TYPE_VOICED ){+ b_Q12_0123 = OP_CVTEPI16_EPI32_M64( b_Q14 );
+ b_sr_Q12_0123 = _mm_shuffle_epi32( b_Q12_0123, _MM_SHUFFLE( 0, 3, 2, 1 ) ); /* equal shift right 4 bytes */
+ }
+ for( i = 0; i < length; i++ ) {+ /* Perform common calculations used in all states */
+
+ /* Long-term prediction */
+ if( signalType == TYPE_VOICED ) {+ /* Unrolled loop */
+ /* Avoids introducing a bias because silk_SMLAWB() always rounds to -inf */
+ LTP_pred_Q14 = 2;
+ {+ __m128i tmpa, tmpb, pred_lag_ptr_tmp;
+ pred_lag_ptr_tmp = _mm_loadu_si128( (__m128i *)(&pred_lag_ptr[ -3 ] ) );
+ pred_lag_ptr_tmp = _mm_shuffle_epi32( pred_lag_ptr_tmp, 0x1B );
+ tmpa = _mm_mul_epi32( pred_lag_ptr_tmp, b_Q12_0123 );
+ tmpa = _mm_srli_si128( tmpa, 2 );
+
+ pred_lag_ptr_tmp = _mm_shuffle_epi32( pred_lag_ptr_tmp, _MM_SHUFFLE( 0, 3, 2, 1 ) );/* equal shift right 4 bytes */
+ pred_lag_ptr_tmp = _mm_mul_epi32( pred_lag_ptr_tmp, b_sr_Q12_0123 );
+ pred_lag_ptr_tmp = _mm_srli_si128( pred_lag_ptr_tmp, 2 );
+ pred_lag_ptr_tmp = _mm_add_epi32( pred_lag_ptr_tmp, tmpa );
+
+ tmpb = _mm_shuffle_epi32( pred_lag_ptr_tmp, _MM_SHUFFLE( 0, 0, 3, 2 ) );/* equal shift right 8 bytes */
+ pred_lag_ptr_tmp = _mm_add_epi32( pred_lag_ptr_tmp, tmpb );
+ LTP_pred_Q14 += _mm_cvtsi128_si32( pred_lag_ptr_tmp );
+
+ LTP_pred_Q14 = silk_SMLAWB( LTP_pred_Q14, pred_lag_ptr[ -4 ], b_Q14[ 4 ] );
+ LTP_pred_Q14 = silk_LSHIFT( LTP_pred_Q14, 1 ); /* Q13 -> Q14 */
+ pred_lag_ptr++;
+ }
+ } else {+ LTP_pred_Q14 = 0;
+ }
+
+ /* Long-term shaping */
+ if( lag > 0 ) {+ /* Symmetric, packed FIR coefficients */
+ n_LTP_Q14 = silk_SMULWB( silk_ADD32( shp_lag_ptr[ 0 ], shp_lag_ptr[ -2 ] ), HarmShapeFIRPacked_Q14 );
+ n_LTP_Q14 = silk_SMLAWT( n_LTP_Q14, shp_lag_ptr[ -1 ], HarmShapeFIRPacked_Q14 );
+ n_LTP_Q14 = silk_SUB_LSHIFT32( LTP_pred_Q14, n_LTP_Q14, 2 ); /* Q12 -> Q14 */
+ shp_lag_ptr++;
+ } else {+ n_LTP_Q14 = 0;
+ }
+ {+ __m128i tmpa, tmpb, psLPC_Q14_tmp, a_Q12_tmp;
+
+ for( k = 0; k < nStatesDelayedDecision; k++ ) {+ /* Delayed decision state */
+ psDD = &psDelDec[ k ];
+
+ /* Sample state */
+ psSS = psSampleState[ k ];
+
+ /* Generate dither */
+ psDD->Seed = silk_RAND( psDD->Seed );
+
+ /* Pointer used in short term prediction and shaping */
+ psLPC_Q14 = &psDD->sLPC_Q14[ NSQ_LPC_BUF_LENGTH - 1 + i ];
+ /* Short-term prediction */
+ silk_assert( predictLPCOrder == 10 || predictLPCOrder == 16 );
+ /* Avoids introducing a bias because silk_SMLAWB() always rounds to -inf */
+ LPC_pred_Q14 = silk_RSHIFT( predictLPCOrder, 1 );
+
+ tmpb = _mm_setzero_si128();
+
+ /* step 1 */
+ psLPC_Q14_tmp = _mm_loadu_si128( (__m128i *)(&psLPC_Q14[ -3 ] ) ); /* -3, -2 , -1, 0 */
+ psLPC_Q14_tmp = _mm_shuffle_epi32( psLPC_Q14_tmp, 0x1B ); /* 0, -1, -2, -3 */
+ tmpa = _mm_mul_epi32( psLPC_Q14_tmp, a_Q12_0123 ); /* 0, -1, -2, -3 * 0123 -> 0*0, 2*-2 */
+
+ tmpa = _mm_srli_epi64( tmpa, 16 );
+ tmpb = _mm_add_epi32( tmpb, tmpa );
+
+ psLPC_Q14_tmp = _mm_shuffle_epi32( psLPC_Q14_tmp, _MM_SHUFFLE( 0, 3, 2, 1 ) ); /* equal shift right 4 bytes */
+ a_Q12_tmp = _mm_shuffle_epi32( a_Q12_0123, _MM_SHUFFLE(0, 3, 2, 1 ) ); /* equal shift right 4 bytes */
+ psLPC_Q14_tmp = _mm_mul_epi32( psLPC_Q14_tmp, a_Q12_tmp ); /* 1*-1, 3*-3 */
+ psLPC_Q14_tmp = _mm_srli_epi64( psLPC_Q14_tmp, 16 );
+ tmpb = _mm_add_epi32( tmpb, psLPC_Q14_tmp );
+
+ /* step 2 */
+ psLPC_Q14_tmp = _mm_loadu_si128( (__m128i *)(&psLPC_Q14[ -7 ] ) );
+ psLPC_Q14_tmp = _mm_shuffle_epi32( psLPC_Q14_tmp, 0x1B );
+ tmpa = _mm_mul_epi32( psLPC_Q14_tmp, a_Q12_4567 );
+ tmpa = _mm_srli_epi64( tmpa, 16 );
+ tmpb = _mm_add_epi32( tmpb, tmpa );
+
+ psLPC_Q14_tmp = _mm_shuffle_epi32( psLPC_Q14_tmp, _MM_SHUFFLE( 0, 3, 2, 1 ) ); /* equal shift right 4 bytes */
+ a_Q12_tmp = _mm_shuffle_epi32( a_Q12_4567, _MM_SHUFFLE(0, 3, 2, 1 ) ); /* equal shift right 4 bytes */
+ psLPC_Q14_tmp = _mm_mul_epi32( psLPC_Q14_tmp, a_Q12_tmp );
+ psLPC_Q14_tmp = _mm_srli_epi64( psLPC_Q14_tmp, 16 );
+ tmpb = _mm_add_epi32( tmpb, psLPC_Q14_tmp );
+
+ if ( opus_likely( predictLPCOrder == 16 ) )
+ {+ /* step 3 */
+ psLPC_Q14_tmp = _mm_loadu_si128( (__m128i *)(&psLPC_Q14[ -11 ] ) );
+ psLPC_Q14_tmp = _mm_shuffle_epi32( psLPC_Q14_tmp, 0x1B );
+ tmpa = _mm_mul_epi32( psLPC_Q14_tmp, a_Q12_89AB );
+ tmpa = _mm_srli_epi64( tmpa, 16 );
+ tmpb = _mm_add_epi32( tmpb, tmpa );
+
+ psLPC_Q14_tmp = _mm_shuffle_epi32( psLPC_Q14_tmp, _MM_SHUFFLE( 0, 3, 2, 1 ) ); /* equal shift right 4 bytes */
+ a_Q12_tmp = _mm_shuffle_epi32( a_Q12_89AB, _MM_SHUFFLE(0, 3, 2, 1 ) );/* equal shift right 4 bytes */
+ psLPC_Q14_tmp = _mm_mul_epi32( psLPC_Q14_tmp, a_Q12_tmp );
+ psLPC_Q14_tmp = _mm_srli_epi64( psLPC_Q14_tmp, 16 );
+ tmpb = _mm_add_epi32( tmpb, psLPC_Q14_tmp );
+
+ /* setp 4 */
+ psLPC_Q14_tmp = _mm_loadu_si128( (__m128i *)(&psLPC_Q14[ -15 ] ) );
+ psLPC_Q14_tmp = _mm_shuffle_epi32( psLPC_Q14_tmp, 0x1B );
+ tmpa = _mm_mul_epi32( psLPC_Q14_tmp, a_Q12_CDEF );
+ tmpa = _mm_srli_epi64( tmpa, 16 );
+ tmpb = _mm_add_epi32( tmpb, tmpa );
+
+ psLPC_Q14_tmp = _mm_shuffle_epi32( psLPC_Q14_tmp, _MM_SHUFFLE( 0, 3, 2, 1 ) ); /* equal shift right 4 bytes */
+ a_Q12_tmp = _mm_shuffle_epi32( a_Q12_CDEF, _MM_SHUFFLE(0, 3, 2, 1 ) ); /* equal shift right 4 bytes */
+ psLPC_Q14_tmp = _mm_mul_epi32( psLPC_Q14_tmp, a_Q12_tmp );
+ psLPC_Q14_tmp = _mm_srli_epi64( psLPC_Q14_tmp, 16 );
+ tmpb = _mm_add_epi32( tmpb, psLPC_Q14_tmp );
+
+ /* add at last */
+ /* equal shift right 8 bytes*/
+ tmpa = _mm_shuffle_epi32( tmpb, _MM_SHUFFLE( 0, 0, 3, 2 ) );
+ tmpb = _mm_add_epi32( tmpb, tmpa );
+ LPC_pred_Q14 += _mm_cvtsi128_si32( tmpb );
+ }
+ else
+ {+ /* add at last */
+ tmpa = _mm_shuffle_epi32( tmpb, _MM_SHUFFLE( 0, 0, 3, 2 ) ); /* equal shift right 8 bytes*/
+ tmpb = _mm_add_epi32( tmpb, tmpa );
+ LPC_pred_Q14 += _mm_cvtsi128_si32( tmpb );
+
+ LPC_pred_Q14 = silk_SMLAWB( LPC_pred_Q14, psLPC_Q14[ -8 ], a_Q12[ 8 ] );
+ LPC_pred_Q14 = silk_SMLAWB( LPC_pred_Q14, psLPC_Q14[ -9 ], a_Q12[ 9 ] );
+ }
+
+ LPC_pred_Q14 = silk_LSHIFT( LPC_pred_Q14, 4 ); /* Q10 -> Q14 */
+
+ /* Noise shape feedback */
+ silk_assert( ( shapingLPCOrder & 1 ) == 0 ); /* check that order is even */
+ /* Output of lowpass section */
+ tmp2 = silk_SMLAWB( psLPC_Q14[ 0 ], psDD->sAR2_Q14[ 0 ], warping_Q16 );
+ /* Output of allpass section */
+ tmp1 = silk_SMLAWB( psDD->sAR2_Q14[ 0 ], psDD->sAR2_Q14[ 1 ] - tmp2, warping_Q16 );
+ psDD->sAR2_Q14[ 0 ] = tmp2;
+ n_AR_Q14 = silk_RSHIFT( shapingLPCOrder, 1 );
+ n_AR_Q14 = silk_SMLAWB( n_AR_Q14, tmp2, AR_shp_Q13[ 0 ] );
+ /* Loop over allpass sections */
+ for( j = 2; j < shapingLPCOrder; j += 2 ) {+ /* Output of allpass section */
+ tmp2 = silk_SMLAWB( psDD->sAR2_Q14[ j - 1 ], psDD->sAR2_Q14[ j + 0 ] - tmp1, warping_Q16 );
+ psDD->sAR2_Q14[ j - 1 ] = tmp1;
+ n_AR_Q14 = silk_SMLAWB( n_AR_Q14, tmp1, AR_shp_Q13[ j - 1 ] );
+ /* Output of allpass section */
+ tmp1 = silk_SMLAWB( psDD->sAR2_Q14[ j + 0 ], psDD->sAR2_Q14[ j + 1 ] - tmp2, warping_Q16 );
+ psDD->sAR2_Q14[ j + 0 ] = tmp2;
+ n_AR_Q14 = silk_SMLAWB( n_AR_Q14, tmp2, AR_shp_Q13[ j ] );
+ }
+ psDD->sAR2_Q14[ shapingLPCOrder - 1 ] = tmp1;
+ n_AR_Q14 = silk_SMLAWB( n_AR_Q14, tmp1, AR_shp_Q13[ shapingLPCOrder - 1 ] );
+
+ n_AR_Q14 = silk_LSHIFT( n_AR_Q14, 1 ); /* Q11 -> Q12 */
+ n_AR_Q14 = silk_SMLAWB( n_AR_Q14, psDD->LF_AR_Q14, Tilt_Q14 ); /* Q12 */
+ n_AR_Q14 = silk_LSHIFT( n_AR_Q14, 2 ); /* Q12 -> Q14 */
+
+ n_LF_Q14 = silk_SMULWB( psDD->Shape_Q14[ *smpl_buf_idx ], LF_shp_Q14 ); /* Q12 */
+ n_LF_Q14 = silk_SMLAWT( n_LF_Q14, psDD->LF_AR_Q14, LF_shp_Q14 ); /* Q12 */
+ n_LF_Q14 = silk_LSHIFT( n_LF_Q14, 2 ); /* Q12 -> Q14 */
+
+ /* Input minus prediction plus noise feedback */
+ /* r = x[ i ] - LTP_pred - LPC_pred + n_AR + n_Tilt + n_LF + n_LTP */
+ tmp1 = silk_ADD32( n_AR_Q14, n_LF_Q14 ); /* Q14 */
+ tmp2 = silk_ADD32( n_LTP_Q14, LPC_pred_Q14 ); /* Q13 */
+ tmp1 = silk_SUB32( tmp2, tmp1 ); /* Q13 */
+ tmp1 = silk_RSHIFT_ROUND( tmp1, 4 ); /* Q10 */
+
+ r_Q10 = silk_SUB32( x_Q10[ i ], tmp1 ); /* residual error Q10 */
+
+ /* Flip sign depending on dither */
+ if ( psDD->Seed < 0 ) {+ r_Q10 = -r_Q10;
+ }
+ r_Q10 = silk_LIMIT_32( r_Q10, -(31 << 10), 30 << 10 );
+
+ /* Find two quantization level candidates and measure their rate-distortion */
+ q1_Q10 = silk_SUB32( r_Q10, offset_Q10 );
+ q1_Q0 = silk_RSHIFT( q1_Q10, 10 );
+ if( q1_Q0 > 0 ) {+ q1_Q10 = silk_SUB32( silk_LSHIFT( q1_Q0, 10 ), QUANT_LEVEL_ADJUST_Q10 );
+ q1_Q10 = silk_ADD32( q1_Q10, offset_Q10 );
+ q2_Q10 = silk_ADD32( q1_Q10, 1024 );
+ rd1_Q10 = silk_SMULBB( q1_Q10, Lambda_Q10 );
+ rd2_Q10 = silk_SMULBB( q2_Q10, Lambda_Q10 );
+ } else if( q1_Q0 == 0 ) {+ q1_Q10 = offset_Q10;
+ q2_Q10 = silk_ADD32( q1_Q10, 1024 - QUANT_LEVEL_ADJUST_Q10 );
+ rd1_Q10 = silk_SMULBB( q1_Q10, Lambda_Q10 );
+ rd2_Q10 = silk_SMULBB( q2_Q10, Lambda_Q10 );
+ } else if( q1_Q0 == -1 ) {+ q2_Q10 = offset_Q10;
+ q1_Q10 = silk_SUB32( q2_Q10, 1024 - QUANT_LEVEL_ADJUST_Q10 );
+ rd1_Q10 = silk_SMULBB( -q1_Q10, Lambda_Q10 );
+ rd2_Q10 = silk_SMULBB( q2_Q10, Lambda_Q10 );
+ } else { /* q1_Q0 < -1 */+ q1_Q10 = silk_ADD32( silk_LSHIFT( q1_Q0, 10 ), QUANT_LEVEL_ADJUST_Q10 );
+ q1_Q10 = silk_ADD32( q1_Q10, offset_Q10 );
+ q2_Q10 = silk_ADD32( q1_Q10, 1024 );
+ rd1_Q10 = silk_SMULBB( -q1_Q10, Lambda_Q10 );
+ rd2_Q10 = silk_SMULBB( -q2_Q10, Lambda_Q10 );
+ }
+ rr_Q10 = silk_SUB32( r_Q10, q1_Q10 );
+ rd1_Q10 = silk_RSHIFT( silk_SMLABB( rd1_Q10, rr_Q10, rr_Q10 ), 10 );
+ rr_Q10 = silk_SUB32( r_Q10, q2_Q10 );
+ rd2_Q10 = silk_RSHIFT( silk_SMLABB( rd2_Q10, rr_Q10, rr_Q10 ), 10 );
+
+ if( rd1_Q10 < rd2_Q10 ) {+ psSS[ 0 ].RD_Q10 = silk_ADD32( psDD->RD_Q10, rd1_Q10 );
+ psSS[ 1 ].RD_Q10 = silk_ADD32( psDD->RD_Q10, rd2_Q10 );
+ psSS[ 0 ].Q_Q10 = q1_Q10;
+ psSS[ 1 ].Q_Q10 = q2_Q10;
+ } else {+ psSS[ 0 ].RD_Q10 = silk_ADD32( psDD->RD_Q10, rd2_Q10 );
+ psSS[ 1 ].RD_Q10 = silk_ADD32( psDD->RD_Q10, rd1_Q10 );
+ psSS[ 0 ].Q_Q10 = q2_Q10;
+ psSS[ 1 ].Q_Q10 = q1_Q10;
+ }
+
+ /* Update states for best quantization */
+
+ /* Quantized excitation */
+ exc_Q14 = silk_LSHIFT32( psSS[ 0 ].Q_Q10, 4 );
+ if ( psDD->Seed < 0 ) {+ exc_Q14 = -exc_Q14;
+ }
+
+ /* Add predictions */
+ LPC_exc_Q14 = silk_ADD32( exc_Q14, LTP_pred_Q14 );
+ xq_Q14 = silk_ADD32( LPC_exc_Q14, LPC_pred_Q14 );
+
+ /* Update states */
+ sLF_AR_shp_Q14 = silk_SUB32( xq_Q14, n_AR_Q14 );
+ psSS[ 0 ].sLTP_shp_Q14 = silk_SUB32( sLF_AR_shp_Q14, n_LF_Q14 );
+ psSS[ 0 ].LF_AR_Q14 = sLF_AR_shp_Q14;
+ psSS[ 0 ].LPC_exc_Q14 = LPC_exc_Q14;
+ psSS[ 0 ].xq_Q14 = xq_Q14;
+
+ /* Update states for second best quantization */
+
+ /* Quantized excitation */
+ exc_Q14 = silk_LSHIFT32( psSS[ 1 ].Q_Q10, 4 );
+ if ( psDD->Seed < 0 ) {+ exc_Q14 = -exc_Q14;
+ }
+
+
+ /* Add predictions */
+ LPC_exc_Q14 = silk_ADD32( exc_Q14, LTP_pred_Q14 );
+ xq_Q14 = silk_ADD32( LPC_exc_Q14, LPC_pred_Q14 );
+
+ /* Update states */
+ sLF_AR_shp_Q14 = silk_SUB32( xq_Q14, n_AR_Q14 );
+ psSS[ 1 ].sLTP_shp_Q14 = silk_SUB32( sLF_AR_shp_Q14, n_LF_Q14 );
+ psSS[ 1 ].LF_AR_Q14 = sLF_AR_shp_Q14;
+ psSS[ 1 ].LPC_exc_Q14 = LPC_exc_Q14;
+ psSS[ 1 ].xq_Q14 = xq_Q14;
+ }
+ }
+ *smpl_buf_idx = ( *smpl_buf_idx - 1 ) % DECISION_DELAY;
+ if( *smpl_buf_idx < 0 ) *smpl_buf_idx += DECISION_DELAY;
+ last_smple_idx = ( *smpl_buf_idx + decisionDelay ) % DECISION_DELAY;
+
+ /* Find winner */
+ RDmin_Q10 = psSampleState[ 0 ][ 0 ].RD_Q10;
+ Winner_ind = 0;
+ for( k = 1; k < nStatesDelayedDecision; k++ ) {+ if( psSampleState[ k ][ 0 ].RD_Q10 < RDmin_Q10 ) {+ RDmin_Q10 = psSampleState[ k ][ 0 ].RD_Q10;
+ Winner_ind = k;
+ }
+ }
+
+ /* Increase RD values of expired states */
+ Winner_rand_state = psDelDec[ Winner_ind ].RandState[ last_smple_idx ];
+ for( k = 0; k < nStatesDelayedDecision; k++ ) {+ if( psDelDec[ k ].RandState[ last_smple_idx ] != Winner_rand_state ) {+ psSampleState[ k ][ 0 ].RD_Q10 = silk_ADD32( psSampleState[ k ][ 0 ].RD_Q10, silk_int32_MAX >> 4 );
+ psSampleState[ k ][ 1 ].RD_Q10 = silk_ADD32( psSampleState[ k ][ 1 ].RD_Q10, silk_int32_MAX >> 4 );
+ silk_assert( psSampleState[ k ][ 0 ].RD_Q10 >= 0 );
+ }
+ }
+
+ /* Find worst in first set and best in second set */
+ RDmax_Q10 = psSampleState[ 0 ][ 0 ].RD_Q10;
+ RDmin_Q10 = psSampleState[ 0 ][ 1 ].RD_Q10;
+ RDmax_ind = 0;
+ RDmin_ind = 0;
+ for( k = 1; k < nStatesDelayedDecision; k++ ) {+ /* find worst in first set */
+ if( psSampleState[ k ][ 0 ].RD_Q10 > RDmax_Q10 ) {+ RDmax_Q10 = psSampleState[ k ][ 0 ].RD_Q10;
+ RDmax_ind = k;
+ }
+ /* find best in second set */
+ if( psSampleState[ k ][ 1 ].RD_Q10 < RDmin_Q10 ) {+ RDmin_Q10 = psSampleState[ k ][ 1 ].RD_Q10;
+ RDmin_ind = k;
+ }
+ }
+
+ /* Replace a state if best from second set outperforms worst in first set */
+ if( RDmin_Q10 < RDmax_Q10 ) {+ silk_memcpy( ( (opus_int32 *)&psDelDec[ RDmax_ind ] ) + i,
+ ( (opus_int32 *)&psDelDec[ RDmin_ind ] ) + i, sizeof( NSQ_del_dec_struct ) - i * sizeof( opus_int32) );
+ silk_memcpy( &psSampleState[ RDmax_ind ][ 0 ], &psSampleState[ RDmin_ind ][ 1 ], sizeof( NSQ_sample_struct ) );
+ }
+
+ /* Write samples from winner to output and long-term filter states */
+ psDD = &psDelDec[ Winner_ind ];
+ if( subfr > 0 || i >= decisionDelay ) {+ pulses[ i - decisionDelay ] = (opus_int8)silk_RSHIFT_ROUND( psDD->Q_Q10[ last_smple_idx ], 10 );
+ xq[ i - decisionDelay ] = (opus_int16)silk_SAT16( silk_RSHIFT_ROUND(
+ silk_SMULWW( psDD->Xq_Q14[ last_smple_idx ], delayedGain_Q10[ last_smple_idx ] ), 8 ) );
+ NSQ->sLTP_shp_Q14[ NSQ->sLTP_shp_buf_idx - decisionDelay ] = psDD->Shape_Q14[ last_smple_idx ];
+ sLTP_Q15[ NSQ->sLTP_buf_idx - decisionDelay ] = psDD->Pred_Q15[ last_smple_idx ];
+ }
+ NSQ->sLTP_shp_buf_idx++;
+ NSQ->sLTP_buf_idx++;
+
+ /* Update states */
+ for( k = 0; k < nStatesDelayedDecision; k++ ) {+ psDD = &psDelDec[ k ];
+ psSS = &psSampleState[ k ][ 0 ];
+ psDD->LF_AR_Q14 = psSS->LF_AR_Q14;
+ psDD->sLPC_Q14[ NSQ_LPC_BUF_LENGTH + i ] = psSS->xq_Q14;
+ psDD->Xq_Q14[ *smpl_buf_idx ] = psSS->xq_Q14;
+ psDD->Q_Q10[ *smpl_buf_idx ] = psSS->Q_Q10;
+ psDD->Pred_Q15[ *smpl_buf_idx ] = silk_LSHIFT32( psSS->LPC_exc_Q14, 1 );
+ psDD->Shape_Q14[ *smpl_buf_idx ] = psSS->sLTP_shp_Q14;
+ psDD->Seed = silk_ADD32_ovflw( psDD->Seed, silk_RSHIFT_ROUND( psSS->Q_Q10, 10 ) );
+ psDD->RandState[ *smpl_buf_idx ] = psDD->Seed;
+ psDD->RD_Q10 = psSS->RD_Q10;
+ }
+ delayedGain_Q10[ *smpl_buf_idx ] = Gain_Q10;
+ }
+ /* Update LPC states */
+ for( k = 0; k < nStatesDelayedDecision; k++ ) {+ psDD = &psDelDec[ k ];
+ silk_memcpy( psDD->sLPC_Q14, &psDD->sLPC_Q14[ length ], NSQ_LPC_BUF_LENGTH * sizeof( opus_int32 ) );
+ }
+ RESTORE_STACK;
+}
+
+static OPUS_INLINE void silk_nsq_del_dec_scale_states_sse4_1(
+ const silk_encoder_state *psEncC, /* I Encoder State */
+ silk_nsq_state *NSQ, /* I/O NSQ state */
+ NSQ_del_dec_struct psDelDec[], /* I/O Delayed decision states */
+ const opus_int32 x_Q3[], /* I Input in Q3 */
+ opus_int32 x_sc_Q10[], /* O Input scaled with 1/Gain in Q10 */
+ const opus_int16 sLTP[], /* I Re-whitened LTP state in Q0 */
+ opus_int32 sLTP_Q15[], /* O LTP state matching scaled input */
+ opus_int subfr, /* I Subframe number */
+ opus_int nStatesDelayedDecision, /* I Number of del dec states */
+ const opus_int LTP_scale_Q14, /* I LTP state scaling */
+ const opus_int32 Gains_Q16[ MAX_NB_SUBFR ], /* I */
+ const opus_int pitchL[ MAX_NB_SUBFR ], /* I Pitch lag */
+ const opus_int signal_type, /* I Signal type */
+ const opus_int decisionDelay /* I Decision delay */
+)
+{+ opus_int i, k, lag;
+ opus_int32 gain_adj_Q16, inv_gain_Q31, inv_gain_Q23;
+ NSQ_del_dec_struct *psDD;
+ __m128i xmm_inv_gain_Q23, xmm_x_Q3_x2x0, xmm_x_Q3_x3x1;
+
+ lag = pitchL[ subfr ];
+ inv_gain_Q31 = silk_INVERSE32_varQ( silk_max( Gains_Q16[ subfr ], 1 ), 47 );
+
+ silk_assert( inv_gain_Q31 != 0 );
+
+ /* Calculate gain adjustment factor */
+ if( Gains_Q16[ subfr ] != NSQ->prev_gain_Q16 ) {+ gain_adj_Q16 = silk_DIV32_varQ( NSQ->prev_gain_Q16, Gains_Q16[ subfr ], 16 );
+ } else {+ gain_adj_Q16 = (opus_int32)1 << 16;
+ }
+
+ /* Scale input */
+ inv_gain_Q23 = silk_RSHIFT_ROUND( inv_gain_Q31, 8 );
+
+ /* prepare inv_gain_Q23 in packed 4 32-bits */
+ xmm_inv_gain_Q23 = _mm_set1_epi32(inv_gain_Q23);
+
+ for( i = 0; i < psEncC->subfr_length - 3; i += 4 ) {+ xmm_x_Q3_x2x0 = _mm_loadu_si128( (__m128i *)(&(x_Q3[ i ] ) ) );
+ /* equal shift right 4 bytes*/
+ xmm_x_Q3_x3x1 = _mm_shuffle_epi32( xmm_x_Q3_x2x0, _MM_SHUFFLE( 0, 3, 2, 1 ) );
+
+ xmm_x_Q3_x2x0 = _mm_mul_epi32( xmm_x_Q3_x2x0, xmm_inv_gain_Q23 );
+ xmm_x_Q3_x3x1 = _mm_mul_epi32( xmm_x_Q3_x3x1, xmm_inv_gain_Q23 );
+
+ xmm_x_Q3_x2x0 = _mm_srli_epi64( xmm_x_Q3_x2x0, 16 );
+ xmm_x_Q3_x3x1 = _mm_slli_epi64( xmm_x_Q3_x3x1, 16 );
+
+ xmm_x_Q3_x2x0 = _mm_blend_epi16( xmm_x_Q3_x2x0, xmm_x_Q3_x3x1, 0xCC );
+
+ _mm_storeu_si128( (__m128i *)(&(x_sc_Q10[ i ])), xmm_x_Q3_x2x0 );
+ }
+
+ for( ; i < psEncC->subfr_length; i++ ) {+ x_sc_Q10[ i ] = silk_SMULWW( x_Q3[ i ], inv_gain_Q23 );
+ }
+
+ /* Save inverse gain */
+ NSQ->prev_gain_Q16 = Gains_Q16[ subfr ];
+
+ /* After rewhitening the LTP state is un-scaled, so scale with inv_gain_Q16 */
+ if( NSQ->rewhite_flag ) {+ if( subfr == 0 ) {+ /* Do LTP downscaling */
+ inv_gain_Q31 = silk_LSHIFT( silk_SMULWB( inv_gain_Q31, LTP_scale_Q14 ), 2 );
+ }
+ for( i = NSQ->sLTP_buf_idx - lag - LTP_ORDER / 2; i < NSQ->sLTP_buf_idx; i++ ) {+ silk_assert( i < MAX_FRAME_LENGTH );
+ sLTP_Q15[ i ] = silk_SMULWB( inv_gain_Q31, sLTP[ i ] );
+ }
+ }
+
+ /* Adjust for changing gain */
+ if( gain_adj_Q16 != (opus_int32)1 << 16 ) {+ /* Scale long-term shaping state */
+ {+ __m128i xmm_gain_adj_Q16, xmm_sLTP_shp_Q14_x2x0, xmm_sLTP_shp_Q14_x3x1;
+
+ /* prepare gain_adj_Q16 in packed 4 32-bits */
+ xmm_gain_adj_Q16 = _mm_set1_epi32( gain_adj_Q16 );
+
+ for( i = NSQ->sLTP_shp_buf_idx - psEncC->ltp_mem_length; i < NSQ->sLTP_shp_buf_idx - 3; i += 4 )
+ {+ xmm_sLTP_shp_Q14_x2x0 = _mm_loadu_si128( (__m128i *)(&(NSQ->sLTP_shp_Q14[ i ] ) ) );
+ /* equal shift right 4 bytes*/
+ xmm_sLTP_shp_Q14_x3x1 = _mm_shuffle_epi32( xmm_sLTP_shp_Q14_x2x0, _MM_SHUFFLE( 0, 3, 2, 1 ) );
+
+ xmm_sLTP_shp_Q14_x2x0 = _mm_mul_epi32( xmm_sLTP_shp_Q14_x2x0, xmm_gain_adj_Q16 );
+ xmm_sLTP_shp_Q14_x3x1 = _mm_mul_epi32( xmm_sLTP_shp_Q14_x3x1, xmm_gain_adj_Q16 );
+
+ xmm_sLTP_shp_Q14_x2x0 = _mm_srli_epi64( xmm_sLTP_shp_Q14_x2x0, 16 );
+ xmm_sLTP_shp_Q14_x3x1 = _mm_slli_epi64( xmm_sLTP_shp_Q14_x3x1, 16 );
+
+ xmm_sLTP_shp_Q14_x2x0 = _mm_blend_epi16( xmm_sLTP_shp_Q14_x2x0, xmm_sLTP_shp_Q14_x3x1, 0xCC );
+
+ _mm_storeu_si128( (__m128i *)(&(NSQ->sLTP_shp_Q14[ i ] ) ), xmm_sLTP_shp_Q14_x2x0 );
+ }
+
+ for( ; i < NSQ->sLTP_shp_buf_idx; i++ ) {+ NSQ->sLTP_shp_Q14[ i ] = silk_SMULWW( gain_adj_Q16, NSQ->sLTP_shp_Q14[ i ] );
+ }
+
+ /* Scale long-term prediction state */
+ if( signal_type == TYPE_VOICED && NSQ->rewhite_flag == 0 ) {+ for( i = NSQ->sLTP_buf_idx - lag - LTP_ORDER / 2; i < NSQ->sLTP_buf_idx - decisionDelay; i++ ) {+ sLTP_Q15[ i ] = silk_SMULWW( gain_adj_Q16, sLTP_Q15[ i ] );
+ }
+ }
+
+ for( k = 0; k < nStatesDelayedDecision; k++ ) {+ psDD = &psDelDec[ k ];
+
+ /* Scale scalar states */
+ psDD->LF_AR_Q14 = silk_SMULWW( gain_adj_Q16, psDD->LF_AR_Q14 );
+
+ /* Scale short-term prediction and shaping states */
+ for( i = 0; i < NSQ_LPC_BUF_LENGTH; i++ ) {+ psDD->sLPC_Q14[ i ] = silk_SMULWW( gain_adj_Q16, psDD->sLPC_Q14[ i ] );
+ }
+ for( i = 0; i < MAX_SHAPE_LPC_ORDER; i++ ) {+ psDD->sAR2_Q14[ i ] = silk_SMULWW( gain_adj_Q16, psDD->sAR2_Q14[ i ] );
+ }
+ for( i = 0; i < DECISION_DELAY; i++ ) {+ psDD->Pred_Q15[ i ] = silk_SMULWW( gain_adj_Q16, psDD->Pred_Q15[ i ] );
+ psDD->Shape_Q14[ i ] = silk_SMULWW( gain_adj_Q16, psDD->Shape_Q14[ i ] );
+ }
+ }
+ }
+ }
+}
--- a/silk/x86/NSQ_sse.c
+++ /dev/null
@@ -1,719 +1,0 @@
-/* Copyright (c) 2014, Cisco Systems, INC
- Written by XiangMingZhu WeiZhou MinPeng YanWang
-
- Redistribution and use in source and binary forms, with or without
- modification, are permitted provided that the following conditions
- are met:
-
- - Redistributions of source code must retain the above copyright
- notice, this list of conditions and the following disclaimer.
-
- - Redistributions in binary form must reproduce the above copyright
- notice, this list of conditions and the following disclaimer in the
- documentation and/or other materials provided with the distribution.
-
- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER
- OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
- EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
- PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
- PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
- LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
- NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-*/
-
-#ifdef HAVE_CONFIG_H
-#include "config.h"
-#endif
-
-#include <xmmintrin.h>
-#include <emmintrin.h>
-#include <smmintrin.h>
-#include "main.h"
-#include "celt/x86/x86cpu.h"
-#include "stack_alloc.h"
-
-static OPUS_INLINE void silk_nsq_scale_states_sse4_1(
- const silk_encoder_state *psEncC, /* I Encoder State */
- silk_nsq_state *NSQ, /* I/O NSQ state */
- const opus_int32 x_Q3[], /* I input in Q3 */
- opus_int32 x_sc_Q10[], /* O input scaled with 1/Gain */
- const opus_int16 sLTP[], /* I re-whitened LTP state in Q0 */
- opus_int32 sLTP_Q15[], /* O LTP state matching scaled input */
- opus_int subfr, /* I subframe number */
- const opus_int LTP_scale_Q14, /* I */
- const opus_int32 Gains_Q16[ MAX_NB_SUBFR ], /* I */
- const opus_int pitchL[ MAX_NB_SUBFR ], /* I Pitch lag */
- const opus_int signal_type /* I Signal type */
-);
-
-static OPUS_INLINE void silk_noise_shape_quantizer_10_16_sse4_1(
- silk_nsq_state *NSQ, /* I/O NSQ state */
- opus_int signalType, /* I Signal type */
- const opus_int32 x_sc_Q10[], /* I */
- opus_int8 pulses[], /* O */
- opus_int16 xq[], /* O */
- opus_int32 sLTP_Q15[], /* I/O LTP state */
- const opus_int16 a_Q12[], /* I Short term prediction coefs */
- const opus_int16 b_Q14[], /* I Long term prediction coefs */
- const opus_int16 AR_shp_Q13[], /* I Noise shaping AR coefs */
- opus_int lag, /* I Pitch lag */
- opus_int32 HarmShapeFIRPacked_Q14, /* I */
- opus_int Tilt_Q14, /* I Spectral tilt */
- opus_int32 LF_shp_Q14, /* I */
- opus_int32 Gain_Q16, /* I */
- opus_int offset_Q10, /* I */
- opus_int length, /* I Input length */
- opus_int32 table[][4] /* I */
-);
-
-void silk_NSQ_sse4_1(
- const silk_encoder_state *psEncC, /* I Encoder State */
- silk_nsq_state *NSQ, /* I/O NSQ state */
- SideInfoIndices *psIndices, /* I/O Quantization Indices */
- const opus_int32 x_Q3[], /* I Prefiltered input signal */
- opus_int8 pulses[], /* O Quantized pulse signal */
- const opus_int16 PredCoef_Q12[ 2 * MAX_LPC_ORDER ], /* I Short term prediction coefs */
- const opus_int16 LTPCoef_Q14[ LTP_ORDER * MAX_NB_SUBFR ], /* I Long term prediction coefs */
- const opus_int16 AR2_Q13[ MAX_NB_SUBFR * MAX_SHAPE_LPC_ORDER ], /* I Noise shaping coefs */
- const opus_int HarmShapeGain_Q14[ MAX_NB_SUBFR ], /* I Long term shaping coefs */
- const opus_int Tilt_Q14[ MAX_NB_SUBFR ], /* I Spectral tilt */
- const opus_int32 LF_shp_Q14[ MAX_NB_SUBFR ], /* I Low frequency shaping coefs */
- const opus_int32 Gains_Q16[ MAX_NB_SUBFR ], /* I Quantization step sizes */
- const opus_int pitchL[ MAX_NB_SUBFR ], /* I Pitch lags */
- const opus_int Lambda_Q10, /* I Rate/distortion tradeoff */
- const opus_int LTP_scale_Q14 /* I LTP state scaling */
-)
-{- opus_int k, lag, start_idx, LSF_interpolation_flag;
- const opus_int16 *A_Q12, *B_Q14, *AR_shp_Q13;
- opus_int16 *pxq;
- VARDECL( opus_int32, sLTP_Q15 );
- VARDECL( opus_int16, sLTP );
- opus_int32 HarmShapeFIRPacked_Q14;
- opus_int offset_Q10;
- VARDECL( opus_int32, x_sc_Q10 );
-
- opus_int32 table[ 64 ][ 4 ];
- opus_int32 tmp1;
- opus_int32 q1_Q10, q2_Q10, rd1_Q20, rd2_Q20;
-
- SAVE_STACK;
-
- NSQ->rand_seed = psIndices->Seed;
-
- /* Set unvoiced lag to the previous one, overwrite later for voiced */
- lag = NSQ->lagPrev;
-
- silk_assert( NSQ->prev_gain_Q16 != 0 );
-
- offset_Q10 = silk_Quantization_Offsets_Q10[ psIndices->signalType >> 1 ][ psIndices->quantOffsetType ];
-
- /* 0 */
- q1_Q10 = offset_Q10;
- q2_Q10 = offset_Q10 + ( 1024 - QUANT_LEVEL_ADJUST_Q10 );
- rd1_Q20 = q1_Q10 * Lambda_Q10;
- rd2_Q20 = q2_Q10 * Lambda_Q10;
-
- table[ 32 ][ 0 ] = q1_Q10;
- table[ 32 ][ 1 ] = q2_Q10;
- table[ 32 ][ 2 ] = 2 * (q1_Q10 - q2_Q10);
- table[ 32 ][ 3 ] = (rd1_Q20 - rd2_Q20) + (q1_Q10 * q1_Q10 - q2_Q10 * q2_Q10);
-
- /* -1 */
- q1_Q10 = offset_Q10 - ( 1024 - QUANT_LEVEL_ADJUST_Q10 );
- q2_Q10 = offset_Q10;
- rd1_Q20 = - q1_Q10 * Lambda_Q10;
- rd2_Q20 = q2_Q10 * Lambda_Q10;
-
- table[ 31 ][ 0 ] = q1_Q10;
- table[ 31 ][ 1 ] = q2_Q10;
- table[ 31 ][ 2 ] = 2 * (q1_Q10 - q2_Q10);
- table[ 31 ][ 3 ] = (rd1_Q20 - rd2_Q20) + (q1_Q10 * q1_Q10 - q2_Q10 * q2_Q10);
-
- /* > 0 */
- for (k = 1; k <= 31; k++)
- {- tmp1 = offset_Q10 + silk_LSHIFT( k, 10 );
-
- q1_Q10 = tmp1 - QUANT_LEVEL_ADJUST_Q10;
- q2_Q10 = tmp1 - QUANT_LEVEL_ADJUST_Q10 + 1024;
- rd1_Q20 = q1_Q10 * Lambda_Q10;
- rd2_Q20 = q2_Q10 * Lambda_Q10;
-
- table[ 32 + k ][ 0 ] = q1_Q10;
- table[ 32 + k ][ 1 ] = q2_Q10;
- table[ 32 + k ][ 2 ] = 2 * (q1_Q10 - q2_Q10);
- table[ 32 + k ][ 3 ] = (rd1_Q20 - rd2_Q20) + (q1_Q10 * q1_Q10 - q2_Q10 * q2_Q10);
- }
-
- /* < -1 */
- for (k = -32; k <= -2; k++)
- {- tmp1 = offset_Q10 + silk_LSHIFT( k, 10 );
-
- q1_Q10 = tmp1 + QUANT_LEVEL_ADJUST_Q10;
- q2_Q10 = tmp1 + QUANT_LEVEL_ADJUST_Q10 + 1024;
- rd1_Q20 = - q1_Q10 * Lambda_Q10;
- rd2_Q20 = - q2_Q10 * Lambda_Q10;
-
- table[ 32 + k ][ 0 ] = q1_Q10;
- table[ 32 + k ][ 1 ] = q2_Q10;
- table[ 32 + k ][ 2 ] = 2 * (q1_Q10 - q2_Q10);
- table[ 32 + k ][ 3 ] = (rd1_Q20 - rd2_Q20) + (q1_Q10 * q1_Q10 - q2_Q10 * q2_Q10);
- }
-
- if( psIndices->NLSFInterpCoef_Q2 == 4 ) {- LSF_interpolation_flag = 0;
- } else {- LSF_interpolation_flag = 1;
- }
-
- ALLOC( sLTP_Q15,
- psEncC->ltp_mem_length + psEncC->frame_length, opus_int32 );
- ALLOC( sLTP, psEncC->ltp_mem_length + psEncC->frame_length, opus_int16 );
- ALLOC( x_sc_Q10, psEncC->subfr_length, opus_int32 );
- /* Set up pointers to start of sub frame */
- NSQ->sLTP_shp_buf_idx = psEncC->ltp_mem_length;
- NSQ->sLTP_buf_idx = psEncC->ltp_mem_length;
- pxq = &NSQ->xq[ psEncC->ltp_mem_length ];
- for( k = 0; k < psEncC->nb_subfr; k++ ) {- A_Q12 = &PredCoef_Q12[ (( k >> 1 ) | ( 1 - LSF_interpolation_flag )) * MAX_LPC_ORDER ];
- B_Q14 = <PCoef_Q14[ k * LTP_ORDER ];
- AR_shp_Q13 = &AR2_Q13[ k * MAX_SHAPE_LPC_ORDER ];
-
- /* Noise shape parameters */
- silk_assert( HarmShapeGain_Q14[ k ] >= 0 );
- HarmShapeFIRPacked_Q14 = silk_RSHIFT( HarmShapeGain_Q14[ k ], 2 );
- HarmShapeFIRPacked_Q14 |= silk_LSHIFT( (opus_int32)silk_RSHIFT( HarmShapeGain_Q14[ k ], 1 ), 16 );
-
- NSQ->rewhite_flag = 0;
- if( psIndices->signalType == TYPE_VOICED ) {- /* Voiced */
- lag = pitchL[ k ];
-
- /* Re-whitening */
- if( ( k & ( 3 - silk_LSHIFT( LSF_interpolation_flag, 1 ) ) ) == 0 ) {- /* Rewhiten with new A coefs */
- start_idx = psEncC->ltp_mem_length - lag - psEncC->predictLPCOrder - LTP_ORDER / 2;
- silk_assert( start_idx > 0 );
-
- silk_LPC_analysis_filter( &sLTP[ start_idx ], &NSQ->xq[ start_idx + k * psEncC->subfr_length ],
- A_Q12, psEncC->ltp_mem_length - start_idx, psEncC->predictLPCOrder, psEncC->arch );
-
- NSQ->rewhite_flag = 1;
- NSQ->sLTP_buf_idx = psEncC->ltp_mem_length;
- }
- }
-
- silk_nsq_scale_states_sse4_1( psEncC, NSQ, x_Q3, x_sc_Q10, sLTP, sLTP_Q15, k, LTP_scale_Q14, Gains_Q16, pitchL, psIndices->signalType );
-
- if ( opus_likely( ( 10 == psEncC->shapingLPCOrder ) && ( 16 == psEncC->predictLPCOrder) ) )
- {- silk_noise_shape_quantizer_10_16_sse4_1( NSQ, psIndices->signalType, x_sc_Q10, pulses, pxq, sLTP_Q15, A_Q12, B_Q14,
- AR_shp_Q13, lag, HarmShapeFIRPacked_Q14, Tilt_Q14[ k ], LF_shp_Q14[ k ], Gains_Q16[ k ],
- offset_Q10, psEncC->subfr_length, &(table[32]) );
- }
- else
- {- silk_noise_shape_quantizer( NSQ, psIndices->signalType, x_sc_Q10, pulses, pxq, sLTP_Q15, A_Q12, B_Q14,
- AR_shp_Q13, lag, HarmShapeFIRPacked_Q14, Tilt_Q14[ k ], LF_shp_Q14[ k ], Gains_Q16[ k ], Lambda_Q10,
- offset_Q10, psEncC->subfr_length, psEncC->shapingLPCOrder, psEncC->predictLPCOrder, psEncC->arch );
- }
-
- x_Q3 += psEncC->subfr_length;
- pulses += psEncC->subfr_length;
- pxq += psEncC->subfr_length;
- }
-
- /* Update lagPrev for next frame */
- NSQ->lagPrev = pitchL[ psEncC->nb_subfr - 1 ];
-
- /* Save quantized speech and noise shaping signals */
- silk_memmove( NSQ->xq, &NSQ->xq[ psEncC->frame_length ], psEncC->ltp_mem_length * sizeof( opus_int16 ) );
- silk_memmove( NSQ->sLTP_shp_Q14, &NSQ->sLTP_shp_Q14[ psEncC->frame_length ], psEncC->ltp_mem_length * sizeof( opus_int32 ) );
- RESTORE_STACK;
-}
-
-/***********************************/
-/* silk_noise_shape_quantizer_10_16 */
-/***********************************/
-static OPUS_INLINE void silk_noise_shape_quantizer_10_16_sse4_1(
- silk_nsq_state *NSQ, /* I/O NSQ state */
- opus_int signalType, /* I Signal type */
- const opus_int32 x_sc_Q10[], /* I */
- opus_int8 pulses[], /* O */
- opus_int16 xq[], /* O */
- opus_int32 sLTP_Q15[], /* I/O LTP state */
- const opus_int16 a_Q12[], /* I Short term prediction coefs */
- const opus_int16 b_Q14[], /* I Long term prediction coefs */
- const opus_int16 AR_shp_Q13[], /* I Noise shaping AR coefs */
- opus_int lag, /* I Pitch lag */
- opus_int32 HarmShapeFIRPacked_Q14, /* I */
- opus_int Tilt_Q14, /* I Spectral tilt */
- opus_int32 LF_shp_Q14, /* I */
- opus_int32 Gain_Q16, /* I */
- opus_int offset_Q10, /* I */
- opus_int length, /* I Input length */
- opus_int32 table[][4] /* I */
-)
-{- opus_int i;
- opus_int32 LTP_pred_Q13, LPC_pred_Q10, n_AR_Q12, n_LTP_Q13;
- opus_int32 n_LF_Q12, r_Q10, q1_Q0, q1_Q10, q2_Q10;
- opus_int32 exc_Q14, LPC_exc_Q14, xq_Q14, Gain_Q10;
- opus_int32 tmp1, tmp2, sLF_AR_shp_Q14;
- opus_int32 *psLPC_Q14, *shp_lag_ptr, *pred_lag_ptr;
-
- __m128i xmm_tempa, xmm_tempb;
-
- __m128i xmm_one;
-
- __m128i psLPC_Q14_hi_01234567, psLPC_Q14_hi_89ABCDEF;
- __m128i psLPC_Q14_lo_01234567, psLPC_Q14_lo_89ABCDEF;
- __m128i a_Q12_01234567, a_Q12_89ABCDEF;
-
- __m128i sAR2_Q14_hi_76543210, sAR2_Q14_lo_76543210;
- __m128i AR_shp_Q13_76543210;
-
- shp_lag_ptr = &NSQ->sLTP_shp_Q14[ NSQ->sLTP_shp_buf_idx - lag + HARM_SHAPE_FIR_TAPS / 2 ];
- pred_lag_ptr = &sLTP_Q15[ NSQ->sLTP_buf_idx - lag + LTP_ORDER / 2 ];
- Gain_Q10 = silk_RSHIFT( Gain_Q16, 6 );
-
- /* Set up short term AR state */
- psLPC_Q14 = &NSQ->sLPC_Q14[ NSQ_LPC_BUF_LENGTH - 1 ];
-
- sLF_AR_shp_Q14 = NSQ->sLF_AR_shp_Q14;
- xq_Q14 = psLPC_Q14[ 0 ];
- LTP_pred_Q13 = 0;
-
- /* load a_Q12 */
- xmm_one = _mm_set_epi8( 1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14 );
-
- /* load a_Q12[0] - a_Q12[7] */
- a_Q12_01234567 = _mm_loadu_si128( (__m128i *)(&a_Q12[ 0 ] ) );
- /* load a_Q12[ 8 ] - a_Q12[ 15 ] */
- a_Q12_89ABCDEF = _mm_loadu_si128( (__m128i *)(&a_Q12[ 8 ] ) );
-
- a_Q12_01234567 = _mm_shuffle_epi8( a_Q12_01234567, xmm_one );
- a_Q12_89ABCDEF = _mm_shuffle_epi8( a_Q12_89ABCDEF, xmm_one );
-
- /* load AR_shp_Q13 */
- AR_shp_Q13_76543210 = _mm_loadu_si128( (__m128i *)(&AR_shp_Q13[0] ) );
-
- /* load psLPC_Q14 */
- xmm_one = _mm_set_epi8(15, 14, 11, 10, 7, 6, 3, 2, 13, 12, 9, 8, 5, 4, 1, 0 );
-
- xmm_tempa = _mm_loadu_si128( (__m128i *)(&psLPC_Q14[-16]) );
- xmm_tempb = _mm_loadu_si128( (__m128i *)(&psLPC_Q14[-12]) );
-
- xmm_tempa = _mm_shuffle_epi8( xmm_tempa, xmm_one );
- xmm_tempb = _mm_shuffle_epi8( xmm_tempb, xmm_one );
-
- psLPC_Q14_hi_89ABCDEF = _mm_unpackhi_epi64( xmm_tempa, xmm_tempb );
- psLPC_Q14_lo_89ABCDEF = _mm_unpacklo_epi64( xmm_tempa, xmm_tempb );
-
- xmm_tempa = _mm_loadu_si128( (__m128i *)(&psLPC_Q14[ -8 ]) );
- xmm_tempb = _mm_loadu_si128( (__m128i *)(&psLPC_Q14[ -4 ]) );
-
- xmm_tempa = _mm_shuffle_epi8( xmm_tempa, xmm_one );
- xmm_tempb = _mm_shuffle_epi8( xmm_tempb, xmm_one );
-
- psLPC_Q14_hi_01234567 = _mm_unpackhi_epi64( xmm_tempa, xmm_tempb );
- psLPC_Q14_lo_01234567 = _mm_unpacklo_epi64( xmm_tempa, xmm_tempb );
-
- /* load sAR2_Q14 */
- xmm_tempa = _mm_loadu_si128( (__m128i *)(&(NSQ->sAR2_Q14[ 0 ]) ) );
- xmm_tempb = _mm_loadu_si128( (__m128i *)(&(NSQ->sAR2_Q14[ 4 ]) ) );
-
- xmm_tempa = _mm_shuffle_epi8( xmm_tempa, xmm_one );
- xmm_tempb = _mm_shuffle_epi8( xmm_tempb, xmm_one );
-
- sAR2_Q14_hi_76543210 = _mm_unpackhi_epi64( xmm_tempa, xmm_tempb );
- sAR2_Q14_lo_76543210 = _mm_unpacklo_epi64( xmm_tempa, xmm_tempb );
-
- /* prepare 1 in 8 * 16bit */
- xmm_one = _mm_set1_epi16(1);
-
- for( i = 0; i < length; i++ )
- {- /* Short-term prediction */
- __m128i xmm_hi_07, xmm_hi_8F, xmm_lo_07, xmm_lo_8F;
-
- /* Avoids introducing a bias because silk_SMLAWB() always rounds to -inf */
- LPC_pred_Q10 = 8; /* silk_RSHIFT( predictLPCOrder, 1 ); */
-
- /* shift psLPC_Q14 */
- psLPC_Q14_hi_89ABCDEF = _mm_alignr_epi8( psLPC_Q14_hi_01234567, psLPC_Q14_hi_89ABCDEF, 2 );
- psLPC_Q14_lo_89ABCDEF = _mm_alignr_epi8( psLPC_Q14_lo_01234567, psLPC_Q14_lo_89ABCDEF, 2 );
-
- psLPC_Q14_hi_01234567 = _mm_srli_si128( psLPC_Q14_hi_01234567, 2 );
- psLPC_Q14_lo_01234567 = _mm_srli_si128( psLPC_Q14_lo_01234567, 2 );
-
- psLPC_Q14_hi_01234567 = _mm_insert_epi16( psLPC_Q14_hi_01234567, (xq_Q14 >> 16), 7 );
- psLPC_Q14_lo_01234567 = _mm_insert_epi16( psLPC_Q14_lo_01234567, (xq_Q14), 7 );
-
- /* high part, use pmaddwd, results in 4 32-bit */
- xmm_hi_07 = _mm_madd_epi16( psLPC_Q14_hi_01234567, a_Q12_01234567 );
- xmm_hi_8F = _mm_madd_epi16( psLPC_Q14_hi_89ABCDEF, a_Q12_89ABCDEF );
-
- /* low part, use pmulhw, results in 8 16-bit, note we need simulate unsigned * signed, _mm_srai_epi16(psLPC_Q14_lo_01234567, 15) */
- xmm_tempa = _mm_cmpgt_epi16( _mm_setzero_si128(), psLPC_Q14_lo_01234567 );
- xmm_tempb = _mm_cmpgt_epi16( _mm_setzero_si128(), psLPC_Q14_lo_89ABCDEF );
-
- xmm_tempa = _mm_and_si128( xmm_tempa, a_Q12_01234567 );
- xmm_tempb = _mm_and_si128( xmm_tempb, a_Q12_89ABCDEF );
-
- xmm_lo_07 = _mm_mulhi_epi16( psLPC_Q14_lo_01234567, a_Q12_01234567 );
- xmm_lo_8F = _mm_mulhi_epi16( psLPC_Q14_lo_89ABCDEF, a_Q12_89ABCDEF );
-
- xmm_lo_07 = _mm_add_epi16( xmm_lo_07, xmm_tempa );
- xmm_lo_8F = _mm_add_epi16( xmm_lo_8F, xmm_tempb );
-
- xmm_lo_07 = _mm_madd_epi16( xmm_lo_07, xmm_one );
- xmm_lo_8F = _mm_madd_epi16( xmm_lo_8F, xmm_one );
-
- /* accumulate */
- xmm_hi_07 = _mm_add_epi32( xmm_hi_07, xmm_hi_8F );
- xmm_lo_07 = _mm_add_epi32( xmm_lo_07, xmm_lo_8F );
-
- xmm_hi_07 = _mm_add_epi32( xmm_hi_07, xmm_lo_07 );
-
- xmm_hi_07 = _mm_add_epi32( xmm_hi_07, _mm_unpackhi_epi64(xmm_hi_07, xmm_hi_07 ) );
- xmm_hi_07 = _mm_add_epi32( xmm_hi_07, _mm_shufflelo_epi16(xmm_hi_07, 0x0E ) );
-
- LPC_pred_Q10 += _mm_cvtsi128_si32( xmm_hi_07 );
-
- /* Long-term prediction */
- if ( opus_likely( signalType == TYPE_VOICED ) ) {- /* Unrolled loop */
- /* Avoids introducing a bias because silk_SMLAWB() always rounds to -inf */
- LTP_pred_Q13 = 2;
- {- __m128i b_Q14_3210, b_Q14_0123, pred_lag_ptr_0123;
-
- b_Q14_3210 = OP_CVTEPI16_EPI32_M64( b_Q14 );
- b_Q14_0123 = _mm_shuffle_epi32( b_Q14_3210, 0x1B );
-
- /* loaded: [0] [-1] [-2] [-3] */
- pred_lag_ptr_0123 = _mm_loadu_si128( (__m128i *)(&pred_lag_ptr[ -3 ] ) );
- /* shuffle to [-3] [-2] [-1] [0] and to new xmm */
- xmm_tempa = _mm_shuffle_epi32( pred_lag_ptr_0123, 0x1B );
- /*64-bit multiply, a[2] * b[-2], a[0] * b[0] */
- xmm_tempa = _mm_mul_epi32( xmm_tempa, b_Q14_3210 );
- /* right shift 2 bytes (16 bits), zero extended */
- xmm_tempa = _mm_srli_si128( xmm_tempa, 2 );
-
- /* a[1] * b[-1], a[3] * b[-3] */
- pred_lag_ptr_0123 = _mm_mul_epi32( pred_lag_ptr_0123, b_Q14_0123 );
- pred_lag_ptr_0123 = _mm_srli_si128( pred_lag_ptr_0123, 2 );
-
- pred_lag_ptr_0123 = _mm_add_epi32( pred_lag_ptr_0123, xmm_tempa );
- /* equal shift right 8 bytes*/
- xmm_tempa = _mm_shuffle_epi32( pred_lag_ptr_0123, _MM_SHUFFLE( 0, 0, 3, 2 ) );
- xmm_tempa = _mm_add_epi32( xmm_tempa, pred_lag_ptr_0123 );
-
- LTP_pred_Q13 += _mm_cvtsi128_si32( xmm_tempa );
-
- LTP_pred_Q13 = silk_SMLAWB( LTP_pred_Q13, pred_lag_ptr[ -4 ], b_Q14[ 4 ] );
- pred_lag_ptr++;
- }
- }
-
- /* Noise shape feedback */
- NSQ->sAR2_Q14[ 9 ] = NSQ->sAR2_Q14[ 8 ];
- NSQ->sAR2_Q14[ 8 ] = _mm_cvtsi128_si32( _mm_srli_si128(_mm_unpackhi_epi16( sAR2_Q14_lo_76543210, sAR2_Q14_hi_76543210 ), 12 ) );
-
- sAR2_Q14_hi_76543210 = _mm_slli_si128( sAR2_Q14_hi_76543210, 2 );
- sAR2_Q14_lo_76543210 = _mm_slli_si128( sAR2_Q14_lo_76543210, 2 );
-
- sAR2_Q14_hi_76543210 = _mm_insert_epi16( sAR2_Q14_hi_76543210, (xq_Q14 >> 16), 0 );
- sAR2_Q14_lo_76543210 = _mm_insert_epi16( sAR2_Q14_lo_76543210, (xq_Q14), 0 );
-
- /* high part, use pmaddwd, results in 4 32-bit */
- xmm_hi_07 = _mm_madd_epi16( sAR2_Q14_hi_76543210, AR_shp_Q13_76543210 );
-
- /* low part, use pmulhw, results in 8 16-bit, note we need simulate unsigned * signed,_mm_srai_epi16(sAR2_Q14_lo_76543210, 15) */
- xmm_tempa = _mm_cmpgt_epi16( _mm_setzero_si128(), sAR2_Q14_lo_76543210 );
- xmm_tempa = _mm_and_si128( xmm_tempa, AR_shp_Q13_76543210 );
-
- xmm_lo_07 = _mm_mulhi_epi16( sAR2_Q14_lo_76543210, AR_shp_Q13_76543210 );
- xmm_lo_07 = _mm_add_epi16( xmm_lo_07, xmm_tempa );
-
- xmm_lo_07 = _mm_madd_epi16( xmm_lo_07, xmm_one );
-
- /* accumulate */
- xmm_hi_07 = _mm_add_epi32( xmm_hi_07, xmm_lo_07 );
-
- xmm_hi_07 = _mm_add_epi32( xmm_hi_07, _mm_unpackhi_epi64(xmm_hi_07, xmm_hi_07 ) );
- xmm_hi_07 = _mm_add_epi32( xmm_hi_07, _mm_shufflelo_epi16(xmm_hi_07, 0x0E ) );
-
- n_AR_Q12 = 5 + _mm_cvtsi128_si32( xmm_hi_07 );
-
- n_AR_Q12 = silk_SMLAWB( n_AR_Q12, NSQ->sAR2_Q14[ 8 ], AR_shp_Q13[ 8 ] );
- n_AR_Q12 = silk_SMLAWB( n_AR_Q12, NSQ->sAR2_Q14[ 9 ], AR_shp_Q13[ 9 ] );
-
- n_AR_Q12 = silk_LSHIFT32( n_AR_Q12, 1 ); /* Q11 -> Q12 */
- n_AR_Q12 = silk_SMLAWB( n_AR_Q12, sLF_AR_shp_Q14, Tilt_Q14 );
-
- n_LF_Q12 = silk_SMULWB( NSQ->sLTP_shp_Q14[ NSQ->sLTP_shp_buf_idx - 1 ], LF_shp_Q14 );
- n_LF_Q12 = silk_SMLAWT( n_LF_Q12, sLF_AR_shp_Q14, LF_shp_Q14 );
-
- silk_assert( lag > 0 || signalType != TYPE_VOICED );
-
- /* Combine prediction and noise shaping signals */
- tmp1 = silk_SUB32( silk_LSHIFT32( LPC_pred_Q10, 2 ), n_AR_Q12 ); /* Q12 */
- tmp1 = silk_SUB32( tmp1, n_LF_Q12 ); /* Q12 */
- if( lag > 0 ) {- /* Symmetric, packed FIR coefficients */
- n_LTP_Q13 = silk_SMULWB( silk_ADD32( shp_lag_ptr[ 0 ], shp_lag_ptr[ -2 ] ), HarmShapeFIRPacked_Q14 );
- n_LTP_Q13 = silk_SMLAWT( n_LTP_Q13, shp_lag_ptr[ -1 ], HarmShapeFIRPacked_Q14 );
- n_LTP_Q13 = silk_LSHIFT( n_LTP_Q13, 1 );
- shp_lag_ptr++;
-
- tmp2 = silk_SUB32( LTP_pred_Q13, n_LTP_Q13 ); /* Q13 */
- tmp1 = silk_ADD_LSHIFT32( tmp2, tmp1, 1 ); /* Q13 */
- tmp1 = silk_RSHIFT_ROUND( tmp1, 3 ); /* Q10 */
- } else {- tmp1 = silk_RSHIFT_ROUND( tmp1, 2 ); /* Q10 */
- }
-
- r_Q10 = silk_SUB32( x_sc_Q10[ i ], tmp1 ); /* residual error Q10 */
-
- /* Generate dither */
- NSQ->rand_seed = silk_RAND( NSQ->rand_seed );
-
- /* Flip sign depending on dither */
- tmp2 = -r_Q10;
- if ( NSQ->rand_seed < 0 ) r_Q10 = tmp2;
-
- r_Q10 = silk_LIMIT_32( r_Q10, -(31 << 10), 30 << 10 );
-
- /* Find two quantization level candidates and measure their rate-distortion */
- q1_Q10 = silk_SUB32( r_Q10, offset_Q10 );
- q1_Q0 = silk_RSHIFT( q1_Q10, 10 );
-
- q1_Q10 = table[q1_Q0][0];
- q2_Q10 = table[q1_Q0][1];
-
- if (r_Q10 * table[q1_Q0][2] - table[q1_Q0][3] < 0)
- {- q1_Q10 = q2_Q10;
- }
-
- pulses[ i ] = (opus_int8)silk_RSHIFT_ROUND( q1_Q10, 10 );
-
- /* Excitation */
- exc_Q14 = silk_LSHIFT( q1_Q10, 4 );
-
- tmp2 = -exc_Q14;
- if ( NSQ->rand_seed < 0 ) exc_Q14 = tmp2;
-
- /* Add predictions */
- LPC_exc_Q14 = silk_ADD_LSHIFT32( exc_Q14, LTP_pred_Q13, 1 );
- xq_Q14 = silk_ADD_LSHIFT32( LPC_exc_Q14, LPC_pred_Q10, 4 );
-
- /* Update states */
- psLPC_Q14++;
- *psLPC_Q14 = xq_Q14;
- sLF_AR_shp_Q14 = silk_SUB_LSHIFT32( xq_Q14, n_AR_Q12, 2 );
-
- NSQ->sLTP_shp_Q14[ NSQ->sLTP_shp_buf_idx ] = silk_SUB_LSHIFT32( sLF_AR_shp_Q14, n_LF_Q12, 2 );
- sLTP_Q15[ NSQ->sLTP_buf_idx ] = silk_LSHIFT( LPC_exc_Q14, 1 );
- NSQ->sLTP_shp_buf_idx++;
- NSQ->sLTP_buf_idx++;
-
- /* Make dither dependent on quantized signal */
- NSQ->rand_seed = silk_ADD32_ovflw( NSQ->rand_seed, pulses[ i ] );
- }
-
- NSQ->sLF_AR_shp_Q14 = sLF_AR_shp_Q14;
-
- /* Scale XQ back to normal level before saving */
- psLPC_Q14 = &NSQ->sLPC_Q14[ NSQ_LPC_BUF_LENGTH ];
-
- /* write back sAR2_Q14 */
- xmm_tempa = _mm_unpackhi_epi16( sAR2_Q14_lo_76543210, sAR2_Q14_hi_76543210 );
- xmm_tempb = _mm_unpacklo_epi16( sAR2_Q14_lo_76543210, sAR2_Q14_hi_76543210 );
- _mm_storeu_si128( (__m128i *)(&NSQ->sAR2_Q14[ 4 ]), xmm_tempa );
- _mm_storeu_si128( (__m128i *)(&NSQ->sAR2_Q14[ 0 ]), xmm_tempb );
-
- /* xq[ i ] = (opus_int16)silk_SAT16( silk_RSHIFT_ROUND( silk_SMULWW( psLPC_Q14[ i ], Gain_Q10 ), 8 ) ); */
- {- __m128i xmm_Gain_Q10;
- __m128i xmm_xq_Q14_3210, xmm_xq_Q14_x3x1, xmm_xq_Q14_7654, xmm_xq_Q14_x7x5;
-
- /* prepare (1 << 7) in packed 4 32-bits */
- xmm_tempa = _mm_set1_epi32( (1 << 7) );
-
- /* prepare Gain_Q10 in packed 4 32-bits */
- xmm_Gain_Q10 = _mm_set1_epi32( Gain_Q10 );
-
- /* process xq */
- for (i = 0; i < length - 7; i += 8)
- {- xmm_xq_Q14_3210 = _mm_loadu_si128( (__m128i *)(&(psLPC_Q14[ i + 0 ] ) ) );
- xmm_xq_Q14_7654 = _mm_loadu_si128( (__m128i *)(&(psLPC_Q14[ i + 4 ] ) ) );
-
- /* equal shift right 4 bytes*/
- xmm_xq_Q14_x3x1 = _mm_shuffle_epi32( xmm_xq_Q14_3210, _MM_SHUFFLE( 0, 3, 2, 1 ) );
- /* equal shift right 4 bytes*/
- xmm_xq_Q14_x7x5 = _mm_shuffle_epi32( xmm_xq_Q14_7654, _MM_SHUFFLE( 0, 3, 2, 1 ) );
-
- xmm_xq_Q14_3210 = _mm_mul_epi32( xmm_xq_Q14_3210, xmm_Gain_Q10 );
- xmm_xq_Q14_x3x1 = _mm_mul_epi32( xmm_xq_Q14_x3x1, xmm_Gain_Q10 );
- xmm_xq_Q14_7654 = _mm_mul_epi32( xmm_xq_Q14_7654, xmm_Gain_Q10 );
- xmm_xq_Q14_x7x5 = _mm_mul_epi32( xmm_xq_Q14_x7x5, xmm_Gain_Q10 );
-
- xmm_xq_Q14_3210 = _mm_srli_epi64( xmm_xq_Q14_3210, 16 );
- xmm_xq_Q14_x3x1 = _mm_slli_epi64( xmm_xq_Q14_x3x1, 16 );
- xmm_xq_Q14_7654 = _mm_srli_epi64( xmm_xq_Q14_7654, 16 );
- xmm_xq_Q14_x7x5 = _mm_slli_epi64( xmm_xq_Q14_x7x5, 16 );
-
- xmm_xq_Q14_3210 = _mm_blend_epi16( xmm_xq_Q14_3210, xmm_xq_Q14_x3x1, 0xCC );
- xmm_xq_Q14_7654 = _mm_blend_epi16( xmm_xq_Q14_7654, xmm_xq_Q14_x7x5, 0xCC );
-
- /* silk_RSHIFT_ROUND(xq, 8) */
- xmm_xq_Q14_3210 = _mm_add_epi32( xmm_xq_Q14_3210, xmm_tempa );
- xmm_xq_Q14_7654 = _mm_add_epi32( xmm_xq_Q14_7654, xmm_tempa );
-
- xmm_xq_Q14_3210 = _mm_srai_epi32( xmm_xq_Q14_3210, 8 );
- xmm_xq_Q14_7654 = _mm_srai_epi32( xmm_xq_Q14_7654, 8 );
-
- /* silk_SAT16 */
- xmm_xq_Q14_3210 = _mm_packs_epi32( xmm_xq_Q14_3210, xmm_xq_Q14_7654 );
-
- /* save to xq */
- _mm_storeu_si128( (__m128i *)(&xq[ i ] ), xmm_xq_Q14_3210 );
- }
- }
- for ( ; i < length; i++)
- {- xq[i] = (opus_int16)silk_SAT16( silk_RSHIFT_ROUND( silk_SMULWW( psLPC_Q14[ i ], Gain_Q10 ), 8 ) );
- }
-
- /* Update LPC synth buffer */
- silk_memcpy( NSQ->sLPC_Q14, &NSQ->sLPC_Q14[ length ], NSQ_LPC_BUF_LENGTH * sizeof( opus_int32 ) );
-}
-
-static OPUS_INLINE void silk_nsq_scale_states_sse4_1(
- const silk_encoder_state *psEncC, /* I Encoder State */
- silk_nsq_state *NSQ, /* I/O NSQ state */
- const opus_int32 x_Q3[], /* I input in Q3 */
- opus_int32 x_sc_Q10[], /* O input scaled with 1/Gain */
- const opus_int16 sLTP[], /* I re-whitened LTP state in Q0 */
- opus_int32 sLTP_Q15[], /* O LTP state matching scaled input */
- opus_int subfr, /* I subframe number */
- const opus_int LTP_scale_Q14, /* I */
- const opus_int32 Gains_Q16[ MAX_NB_SUBFR ], /* I */
- const opus_int pitchL[ MAX_NB_SUBFR ], /* I Pitch lag */
- const opus_int signal_type /* I Signal type */
-)
-{- opus_int i, lag;
- opus_int32 gain_adj_Q16, inv_gain_Q31, inv_gain_Q23;
- __m128i xmm_inv_gain_Q23, xmm_x_Q3_x2x0, xmm_x_Q3_x3x1;
-
- lag = pitchL[ subfr ];
- inv_gain_Q31 = silk_INVERSE32_varQ( silk_max( Gains_Q16[ subfr ], 1 ), 47 );
- silk_assert( inv_gain_Q31 != 0 );
-
- /* Calculate gain adjustment factor */
- if( Gains_Q16[ subfr ] != NSQ->prev_gain_Q16 ) {- gain_adj_Q16 = silk_DIV32_varQ( NSQ->prev_gain_Q16, Gains_Q16[ subfr ], 16 );
- } else {- gain_adj_Q16 = (opus_int32)1 << 16;
- }
-
- /* Scale input */
- inv_gain_Q23 = silk_RSHIFT_ROUND( inv_gain_Q31, 8 );
-
- /* prepare inv_gain_Q23 in packed 4 32-bits */
- xmm_inv_gain_Q23 = _mm_set1_epi32(inv_gain_Q23);
-
- for( i = 0; i < psEncC->subfr_length - 3; i += 4 ) {- xmm_x_Q3_x2x0 = _mm_loadu_si128( (__m128i *)(&(x_Q3[ i ] ) ) );
-
- /* equal shift right 4 bytes*/
- xmm_x_Q3_x3x1 = _mm_shuffle_epi32( xmm_x_Q3_x2x0, _MM_SHUFFLE( 0, 3, 2, 1 ) );
-
- xmm_x_Q3_x2x0 = _mm_mul_epi32( xmm_x_Q3_x2x0, xmm_inv_gain_Q23 );
- xmm_x_Q3_x3x1 = _mm_mul_epi32( xmm_x_Q3_x3x1, xmm_inv_gain_Q23 );
-
- xmm_x_Q3_x2x0 = _mm_srli_epi64( xmm_x_Q3_x2x0, 16 );
- xmm_x_Q3_x3x1 = _mm_slli_epi64( xmm_x_Q3_x3x1, 16 );
-
- xmm_x_Q3_x2x0 = _mm_blend_epi16( xmm_x_Q3_x2x0, xmm_x_Q3_x3x1, 0xCC );
-
- _mm_storeu_si128( (__m128i *)(&(x_sc_Q10[ i ] ) ), xmm_x_Q3_x2x0 );
- }
-
- for( ; i < psEncC->subfr_length; i++ ) {- x_sc_Q10[ i ] = silk_SMULWW( x_Q3[ i ], inv_gain_Q23 );
- }
-
- /* Save inverse gain */
- NSQ->prev_gain_Q16 = Gains_Q16[ subfr ];
-
- /* After rewhitening the LTP state is un-scaled, so scale with inv_gain_Q16 */
- if( NSQ->rewhite_flag ) {- if( subfr == 0 ) {- /* Do LTP downscaling */
- inv_gain_Q31 = silk_LSHIFT( silk_SMULWB( inv_gain_Q31, LTP_scale_Q14 ), 2 );
- }
- for( i = NSQ->sLTP_buf_idx - lag - LTP_ORDER / 2; i < NSQ->sLTP_buf_idx; i++ ) {- silk_assert( i < MAX_FRAME_LENGTH );
- sLTP_Q15[ i ] = silk_SMULWB( inv_gain_Q31, sLTP[ i ] );
- }
- }
-
- /* Adjust for changing gain */
- if( gain_adj_Q16 != (opus_int32)1 << 16 ) {- /* Scale long-term shaping state */
- __m128i xmm_gain_adj_Q16, xmm_sLTP_shp_Q14_x2x0, xmm_sLTP_shp_Q14_x3x1;
-
- /* prepare gain_adj_Q16 in packed 4 32-bits */
- xmm_gain_adj_Q16 = _mm_set1_epi32(gain_adj_Q16);
-
- for( i = NSQ->sLTP_shp_buf_idx - psEncC->ltp_mem_length; i < NSQ->sLTP_shp_buf_idx - 3; i += 4 )
- {- xmm_sLTP_shp_Q14_x2x0 = _mm_loadu_si128( (__m128i *)(&(NSQ->sLTP_shp_Q14[ i ] ) ) );
- /* equal shift right 4 bytes*/
- xmm_sLTP_shp_Q14_x3x1 = _mm_shuffle_epi32( xmm_sLTP_shp_Q14_x2x0, _MM_SHUFFLE( 0, 3, 2, 1 ) );
-
- xmm_sLTP_shp_Q14_x2x0 = _mm_mul_epi32( xmm_sLTP_shp_Q14_x2x0, xmm_gain_adj_Q16 );
- xmm_sLTP_shp_Q14_x3x1 = _mm_mul_epi32( xmm_sLTP_shp_Q14_x3x1, xmm_gain_adj_Q16 );
-
- xmm_sLTP_shp_Q14_x2x0 = _mm_srli_epi64( xmm_sLTP_shp_Q14_x2x0, 16 );
- xmm_sLTP_shp_Q14_x3x1 = _mm_slli_epi64( xmm_sLTP_shp_Q14_x3x1, 16 );
-
- xmm_sLTP_shp_Q14_x2x0 = _mm_blend_epi16( xmm_sLTP_shp_Q14_x2x0, xmm_sLTP_shp_Q14_x3x1, 0xCC );
-
- _mm_storeu_si128( (__m128i *)(&(NSQ->sLTP_shp_Q14[ i ] ) ), xmm_sLTP_shp_Q14_x2x0 );
- }
-
- for( ; i < NSQ->sLTP_shp_buf_idx; i++ ) {- NSQ->sLTP_shp_Q14[ i ] = silk_SMULWW( gain_adj_Q16, NSQ->sLTP_shp_Q14[ i ] );
- }
-
- /* Scale long-term prediction state */
- if( signal_type == TYPE_VOICED && NSQ->rewhite_flag == 0 ) {- for( i = NSQ->sLTP_buf_idx - lag - LTP_ORDER / 2; i < NSQ->sLTP_buf_idx; i++ ) {- sLTP_Q15[ i ] = silk_SMULWW( gain_adj_Q16, sLTP_Q15[ i ] );
- }
- }
-
- NSQ->sLF_AR_shp_Q14 = silk_SMULWW( gain_adj_Q16, NSQ->sLF_AR_shp_Q14 );
-
- /* Scale short-term prediction and shaping states */
- for( i = 0; i < NSQ_LPC_BUF_LENGTH; i++ ) {- NSQ->sLPC_Q14[ i ] = silk_SMULWW( gain_adj_Q16, NSQ->sLPC_Q14[ i ] );
- }
- for( i = 0; i < MAX_SHAPE_LPC_ORDER; i++ ) {- NSQ->sAR2_Q14[ i ] = silk_SMULWW( gain_adj_Q16, NSQ->sAR2_Q14[ i ] );
- }
- }
-}
--- /dev/null
+++ b/silk/x86/NSQ_sse4_1.c
@@ -1,0 +1,719 @@
+/* Copyright (c) 2014, Cisco Systems, INC
+ Written by XiangMingZhu WeiZhou MinPeng YanWang
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+
+ - Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+
+ - Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER
+ OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+ LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+ NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*/
+
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include <xmmintrin.h>
+#include <emmintrin.h>
+#include <smmintrin.h>
+#include "main.h"
+#include "celt/x86/x86cpu.h"
+#include "stack_alloc.h"
+
+static OPUS_INLINE void silk_nsq_scale_states_sse4_1(
+ const silk_encoder_state *psEncC, /* I Encoder State */
+ silk_nsq_state *NSQ, /* I/O NSQ state */
+ const opus_int32 x_Q3[], /* I input in Q3 */
+ opus_int32 x_sc_Q10[], /* O input scaled with 1/Gain */
+ const opus_int16 sLTP[], /* I re-whitened LTP state in Q0 */
+ opus_int32 sLTP_Q15[], /* O LTP state matching scaled input */
+ opus_int subfr, /* I subframe number */
+ const opus_int LTP_scale_Q14, /* I */
+ const opus_int32 Gains_Q16[ MAX_NB_SUBFR ], /* I */
+ const opus_int pitchL[ MAX_NB_SUBFR ], /* I Pitch lag */
+ const opus_int signal_type /* I Signal type */
+);
+
+static OPUS_INLINE void silk_noise_shape_quantizer_10_16_sse4_1(
+ silk_nsq_state *NSQ, /* I/O NSQ state */
+ opus_int signalType, /* I Signal type */
+ const opus_int32 x_sc_Q10[], /* I */
+ opus_int8 pulses[], /* O */
+ opus_int16 xq[], /* O */
+ opus_int32 sLTP_Q15[], /* I/O LTP state */
+ const opus_int16 a_Q12[], /* I Short term prediction coefs */
+ const opus_int16 b_Q14[], /* I Long term prediction coefs */
+ const opus_int16 AR_shp_Q13[], /* I Noise shaping AR coefs */
+ opus_int lag, /* I Pitch lag */
+ opus_int32 HarmShapeFIRPacked_Q14, /* I */
+ opus_int Tilt_Q14, /* I Spectral tilt */
+ opus_int32 LF_shp_Q14, /* I */
+ opus_int32 Gain_Q16, /* I */
+ opus_int offset_Q10, /* I */
+ opus_int length, /* I Input length */
+ opus_int32 table[][4] /* I */
+);
+
+void silk_NSQ_sse4_1(
+ const silk_encoder_state *psEncC, /* I Encoder State */
+ silk_nsq_state *NSQ, /* I/O NSQ state */
+ SideInfoIndices *psIndices, /* I/O Quantization Indices */
+ const opus_int32 x_Q3[], /* I Prefiltered input signal */
+ opus_int8 pulses[], /* O Quantized pulse signal */
+ const opus_int16 PredCoef_Q12[ 2 * MAX_LPC_ORDER ], /* I Short term prediction coefs */
+ const opus_int16 LTPCoef_Q14[ LTP_ORDER * MAX_NB_SUBFR ], /* I Long term prediction coefs */
+ const opus_int16 AR2_Q13[ MAX_NB_SUBFR * MAX_SHAPE_LPC_ORDER ], /* I Noise shaping coefs */
+ const opus_int HarmShapeGain_Q14[ MAX_NB_SUBFR ], /* I Long term shaping coefs */
+ const opus_int Tilt_Q14[ MAX_NB_SUBFR ], /* I Spectral tilt */
+ const opus_int32 LF_shp_Q14[ MAX_NB_SUBFR ], /* I Low frequency shaping coefs */
+ const opus_int32 Gains_Q16[ MAX_NB_SUBFR ], /* I Quantization step sizes */
+ const opus_int pitchL[ MAX_NB_SUBFR ], /* I Pitch lags */
+ const opus_int Lambda_Q10, /* I Rate/distortion tradeoff */
+ const opus_int LTP_scale_Q14 /* I LTP state scaling */
+)
+{+ opus_int k, lag, start_idx, LSF_interpolation_flag;
+ const opus_int16 *A_Q12, *B_Q14, *AR_shp_Q13;
+ opus_int16 *pxq;
+ VARDECL( opus_int32, sLTP_Q15 );
+ VARDECL( opus_int16, sLTP );
+ opus_int32 HarmShapeFIRPacked_Q14;
+ opus_int offset_Q10;
+ VARDECL( opus_int32, x_sc_Q10 );
+
+ opus_int32 table[ 64 ][ 4 ];
+ opus_int32 tmp1;
+ opus_int32 q1_Q10, q2_Q10, rd1_Q20, rd2_Q20;
+
+ SAVE_STACK;
+
+ NSQ->rand_seed = psIndices->Seed;
+
+ /* Set unvoiced lag to the previous one, overwrite later for voiced */
+ lag = NSQ->lagPrev;
+
+ silk_assert( NSQ->prev_gain_Q16 != 0 );
+
+ offset_Q10 = silk_Quantization_Offsets_Q10[ psIndices->signalType >> 1 ][ psIndices->quantOffsetType ];
+
+ /* 0 */
+ q1_Q10 = offset_Q10;
+ q2_Q10 = offset_Q10 + ( 1024 - QUANT_LEVEL_ADJUST_Q10 );
+ rd1_Q20 = q1_Q10 * Lambda_Q10;
+ rd2_Q20 = q2_Q10 * Lambda_Q10;
+
+ table[ 32 ][ 0 ] = q1_Q10;
+ table[ 32 ][ 1 ] = q2_Q10;
+ table[ 32 ][ 2 ] = 2 * (q1_Q10 - q2_Q10);
+ table[ 32 ][ 3 ] = (rd1_Q20 - rd2_Q20) + (q1_Q10 * q1_Q10 - q2_Q10 * q2_Q10);
+
+ /* -1 */
+ q1_Q10 = offset_Q10 - ( 1024 - QUANT_LEVEL_ADJUST_Q10 );
+ q2_Q10 = offset_Q10;
+ rd1_Q20 = - q1_Q10 * Lambda_Q10;
+ rd2_Q20 = q2_Q10 * Lambda_Q10;
+
+ table[ 31 ][ 0 ] = q1_Q10;
+ table[ 31 ][ 1 ] = q2_Q10;
+ table[ 31 ][ 2 ] = 2 * (q1_Q10 - q2_Q10);
+ table[ 31 ][ 3 ] = (rd1_Q20 - rd2_Q20) + (q1_Q10 * q1_Q10 - q2_Q10 * q2_Q10);
+
+ /* > 0 */
+ for (k = 1; k <= 31; k++)
+ {+ tmp1 = offset_Q10 + silk_LSHIFT( k, 10 );
+
+ q1_Q10 = tmp1 - QUANT_LEVEL_ADJUST_Q10;
+ q2_Q10 = tmp1 - QUANT_LEVEL_ADJUST_Q10 + 1024;
+ rd1_Q20 = q1_Q10 * Lambda_Q10;
+ rd2_Q20 = q2_Q10 * Lambda_Q10;
+
+ table[ 32 + k ][ 0 ] = q1_Q10;
+ table[ 32 + k ][ 1 ] = q2_Q10;
+ table[ 32 + k ][ 2 ] = 2 * (q1_Q10 - q2_Q10);
+ table[ 32 + k ][ 3 ] = (rd1_Q20 - rd2_Q20) + (q1_Q10 * q1_Q10 - q2_Q10 * q2_Q10);
+ }
+
+ /* < -1 */
+ for (k = -32; k <= -2; k++)
+ {+ tmp1 = offset_Q10 + silk_LSHIFT( k, 10 );
+
+ q1_Q10 = tmp1 + QUANT_LEVEL_ADJUST_Q10;
+ q2_Q10 = tmp1 + QUANT_LEVEL_ADJUST_Q10 + 1024;
+ rd1_Q20 = - q1_Q10 * Lambda_Q10;
+ rd2_Q20 = - q2_Q10 * Lambda_Q10;
+
+ table[ 32 + k ][ 0 ] = q1_Q10;
+ table[ 32 + k ][ 1 ] = q2_Q10;
+ table[ 32 + k ][ 2 ] = 2 * (q1_Q10 - q2_Q10);
+ table[ 32 + k ][ 3 ] = (rd1_Q20 - rd2_Q20) + (q1_Q10 * q1_Q10 - q2_Q10 * q2_Q10);
+ }
+
+ if( psIndices->NLSFInterpCoef_Q2 == 4 ) {+ LSF_interpolation_flag = 0;
+ } else {+ LSF_interpolation_flag = 1;
+ }
+
+ ALLOC( sLTP_Q15,
+ psEncC->ltp_mem_length + psEncC->frame_length, opus_int32 );
+ ALLOC( sLTP, psEncC->ltp_mem_length + psEncC->frame_length, opus_int16 );
+ ALLOC( x_sc_Q10, psEncC->subfr_length, opus_int32 );
+ /* Set up pointers to start of sub frame */
+ NSQ->sLTP_shp_buf_idx = psEncC->ltp_mem_length;
+ NSQ->sLTP_buf_idx = psEncC->ltp_mem_length;
+ pxq = &NSQ->xq[ psEncC->ltp_mem_length ];
+ for( k = 0; k < psEncC->nb_subfr; k++ ) {+ A_Q12 = &PredCoef_Q12[ (( k >> 1 ) | ( 1 - LSF_interpolation_flag )) * MAX_LPC_ORDER ];
+ B_Q14 = <PCoef_Q14[ k * LTP_ORDER ];
+ AR_shp_Q13 = &AR2_Q13[ k * MAX_SHAPE_LPC_ORDER ];
+
+ /* Noise shape parameters */
+ silk_assert( HarmShapeGain_Q14[ k ] >= 0 );
+ HarmShapeFIRPacked_Q14 = silk_RSHIFT( HarmShapeGain_Q14[ k ], 2 );
+ HarmShapeFIRPacked_Q14 |= silk_LSHIFT( (opus_int32)silk_RSHIFT( HarmShapeGain_Q14[ k ], 1 ), 16 );
+
+ NSQ->rewhite_flag = 0;
+ if( psIndices->signalType == TYPE_VOICED ) {+ /* Voiced */
+ lag = pitchL[ k ];
+
+ /* Re-whitening */
+ if( ( k & ( 3 - silk_LSHIFT( LSF_interpolation_flag, 1 ) ) ) == 0 ) {+ /* Rewhiten with new A coefs */
+ start_idx = psEncC->ltp_mem_length - lag - psEncC->predictLPCOrder - LTP_ORDER / 2;
+ silk_assert( start_idx > 0 );
+
+ silk_LPC_analysis_filter( &sLTP[ start_idx ], &NSQ->xq[ start_idx + k * psEncC->subfr_length ],
+ A_Q12, psEncC->ltp_mem_length - start_idx, psEncC->predictLPCOrder, psEncC->arch );
+
+ NSQ->rewhite_flag = 1;
+ NSQ->sLTP_buf_idx = psEncC->ltp_mem_length;
+ }
+ }
+
+ silk_nsq_scale_states_sse4_1( psEncC, NSQ, x_Q3, x_sc_Q10, sLTP, sLTP_Q15, k, LTP_scale_Q14, Gains_Q16, pitchL, psIndices->signalType );
+
+ if ( opus_likely( ( 10 == psEncC->shapingLPCOrder ) && ( 16 == psEncC->predictLPCOrder) ) )
+ {+ silk_noise_shape_quantizer_10_16_sse4_1( NSQ, psIndices->signalType, x_sc_Q10, pulses, pxq, sLTP_Q15, A_Q12, B_Q14,
+ AR_shp_Q13, lag, HarmShapeFIRPacked_Q14, Tilt_Q14[ k ], LF_shp_Q14[ k ], Gains_Q16[ k ],
+ offset_Q10, psEncC->subfr_length, &(table[32]) );
+ }
+ else
+ {+ silk_noise_shape_quantizer( NSQ, psIndices->signalType, x_sc_Q10, pulses, pxq, sLTP_Q15, A_Q12, B_Q14,
+ AR_shp_Q13, lag, HarmShapeFIRPacked_Q14, Tilt_Q14[ k ], LF_shp_Q14[ k ], Gains_Q16[ k ], Lambda_Q10,
+ offset_Q10, psEncC->subfr_length, psEncC->shapingLPCOrder, psEncC->predictLPCOrder, psEncC->arch );
+ }
+
+ x_Q3 += psEncC->subfr_length;
+ pulses += psEncC->subfr_length;
+ pxq += psEncC->subfr_length;
+ }
+
+ /* Update lagPrev for next frame */
+ NSQ->lagPrev = pitchL[ psEncC->nb_subfr - 1 ];
+
+ /* Save quantized speech and noise shaping signals */
+ silk_memmove( NSQ->xq, &NSQ->xq[ psEncC->frame_length ], psEncC->ltp_mem_length * sizeof( opus_int16 ) );
+ silk_memmove( NSQ->sLTP_shp_Q14, &NSQ->sLTP_shp_Q14[ psEncC->frame_length ], psEncC->ltp_mem_length * sizeof( opus_int32 ) );
+ RESTORE_STACK;
+}
+
+/***********************************/
+/* silk_noise_shape_quantizer_10_16 */
+/***********************************/
+static OPUS_INLINE void silk_noise_shape_quantizer_10_16_sse4_1(
+ silk_nsq_state *NSQ, /* I/O NSQ state */
+ opus_int signalType, /* I Signal type */
+ const opus_int32 x_sc_Q10[], /* I */
+ opus_int8 pulses[], /* O */
+ opus_int16 xq[], /* O */
+ opus_int32 sLTP_Q15[], /* I/O LTP state */
+ const opus_int16 a_Q12[], /* I Short term prediction coefs */
+ const opus_int16 b_Q14[], /* I Long term prediction coefs */
+ const opus_int16 AR_shp_Q13[], /* I Noise shaping AR coefs */
+ opus_int lag, /* I Pitch lag */
+ opus_int32 HarmShapeFIRPacked_Q14, /* I */
+ opus_int Tilt_Q14, /* I Spectral tilt */
+ opus_int32 LF_shp_Q14, /* I */
+ opus_int32 Gain_Q16, /* I */
+ opus_int offset_Q10, /* I */
+ opus_int length, /* I Input length */
+ opus_int32 table[][4] /* I */
+)
+{+ opus_int i;
+ opus_int32 LTP_pred_Q13, LPC_pred_Q10, n_AR_Q12, n_LTP_Q13;
+ opus_int32 n_LF_Q12, r_Q10, q1_Q0, q1_Q10, q2_Q10;
+ opus_int32 exc_Q14, LPC_exc_Q14, xq_Q14, Gain_Q10;
+ opus_int32 tmp1, tmp2, sLF_AR_shp_Q14;
+ opus_int32 *psLPC_Q14, *shp_lag_ptr, *pred_lag_ptr;
+
+ __m128i xmm_tempa, xmm_tempb;
+
+ __m128i xmm_one;
+
+ __m128i psLPC_Q14_hi_01234567, psLPC_Q14_hi_89ABCDEF;
+ __m128i psLPC_Q14_lo_01234567, psLPC_Q14_lo_89ABCDEF;
+ __m128i a_Q12_01234567, a_Q12_89ABCDEF;
+
+ __m128i sAR2_Q14_hi_76543210, sAR2_Q14_lo_76543210;
+ __m128i AR_shp_Q13_76543210;
+
+ shp_lag_ptr = &NSQ->sLTP_shp_Q14[ NSQ->sLTP_shp_buf_idx - lag + HARM_SHAPE_FIR_TAPS / 2 ];
+ pred_lag_ptr = &sLTP_Q15[ NSQ->sLTP_buf_idx - lag + LTP_ORDER / 2 ];
+ Gain_Q10 = silk_RSHIFT( Gain_Q16, 6 );
+
+ /* Set up short term AR state */
+ psLPC_Q14 = &NSQ->sLPC_Q14[ NSQ_LPC_BUF_LENGTH - 1 ];
+
+ sLF_AR_shp_Q14 = NSQ->sLF_AR_shp_Q14;
+ xq_Q14 = psLPC_Q14[ 0 ];
+ LTP_pred_Q13 = 0;
+
+ /* load a_Q12 */
+ xmm_one = _mm_set_epi8( 1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14 );
+
+ /* load a_Q12[0] - a_Q12[7] */
+ a_Q12_01234567 = _mm_loadu_si128( (__m128i *)(&a_Q12[ 0 ] ) );
+ /* load a_Q12[ 8 ] - a_Q12[ 15 ] */
+ a_Q12_89ABCDEF = _mm_loadu_si128( (__m128i *)(&a_Q12[ 8 ] ) );
+
+ a_Q12_01234567 = _mm_shuffle_epi8( a_Q12_01234567, xmm_one );
+ a_Q12_89ABCDEF = _mm_shuffle_epi8( a_Q12_89ABCDEF, xmm_one );
+
+ /* load AR_shp_Q13 */
+ AR_shp_Q13_76543210 = _mm_loadu_si128( (__m128i *)(&AR_shp_Q13[0] ) );
+
+ /* load psLPC_Q14 */
+ xmm_one = _mm_set_epi8(15, 14, 11, 10, 7, 6, 3, 2, 13, 12, 9, 8, 5, 4, 1, 0 );
+
+ xmm_tempa = _mm_loadu_si128( (__m128i *)(&psLPC_Q14[-16]) );
+ xmm_tempb = _mm_loadu_si128( (__m128i *)(&psLPC_Q14[-12]) );
+
+ xmm_tempa = _mm_shuffle_epi8( xmm_tempa, xmm_one );
+ xmm_tempb = _mm_shuffle_epi8( xmm_tempb, xmm_one );
+
+ psLPC_Q14_hi_89ABCDEF = _mm_unpackhi_epi64( xmm_tempa, xmm_tempb );
+ psLPC_Q14_lo_89ABCDEF = _mm_unpacklo_epi64( xmm_tempa, xmm_tempb );
+
+ xmm_tempa = _mm_loadu_si128( (__m128i *)(&psLPC_Q14[ -8 ]) );
+ xmm_tempb = _mm_loadu_si128( (__m128i *)(&psLPC_Q14[ -4 ]) );
+
+ xmm_tempa = _mm_shuffle_epi8( xmm_tempa, xmm_one );
+ xmm_tempb = _mm_shuffle_epi8( xmm_tempb, xmm_one );
+
+ psLPC_Q14_hi_01234567 = _mm_unpackhi_epi64( xmm_tempa, xmm_tempb );
+ psLPC_Q14_lo_01234567 = _mm_unpacklo_epi64( xmm_tempa, xmm_tempb );
+
+ /* load sAR2_Q14 */
+ xmm_tempa = _mm_loadu_si128( (__m128i *)(&(NSQ->sAR2_Q14[ 0 ]) ) );
+ xmm_tempb = _mm_loadu_si128( (__m128i *)(&(NSQ->sAR2_Q14[ 4 ]) ) );
+
+ xmm_tempa = _mm_shuffle_epi8( xmm_tempa, xmm_one );
+ xmm_tempb = _mm_shuffle_epi8( xmm_tempb, xmm_one );
+
+ sAR2_Q14_hi_76543210 = _mm_unpackhi_epi64( xmm_tempa, xmm_tempb );
+ sAR2_Q14_lo_76543210 = _mm_unpacklo_epi64( xmm_tempa, xmm_tempb );
+
+ /* prepare 1 in 8 * 16bit */
+ xmm_one = _mm_set1_epi16(1);
+
+ for( i = 0; i < length; i++ )
+ {+ /* Short-term prediction */
+ __m128i xmm_hi_07, xmm_hi_8F, xmm_lo_07, xmm_lo_8F;
+
+ /* Avoids introducing a bias because silk_SMLAWB() always rounds to -inf */
+ LPC_pred_Q10 = 8; /* silk_RSHIFT( predictLPCOrder, 1 ); */
+
+ /* shift psLPC_Q14 */
+ psLPC_Q14_hi_89ABCDEF = _mm_alignr_epi8( psLPC_Q14_hi_01234567, psLPC_Q14_hi_89ABCDEF, 2 );
+ psLPC_Q14_lo_89ABCDEF = _mm_alignr_epi8( psLPC_Q14_lo_01234567, psLPC_Q14_lo_89ABCDEF, 2 );
+
+ psLPC_Q14_hi_01234567 = _mm_srli_si128( psLPC_Q14_hi_01234567, 2 );
+ psLPC_Q14_lo_01234567 = _mm_srli_si128( psLPC_Q14_lo_01234567, 2 );
+
+ psLPC_Q14_hi_01234567 = _mm_insert_epi16( psLPC_Q14_hi_01234567, (xq_Q14 >> 16), 7 );
+ psLPC_Q14_lo_01234567 = _mm_insert_epi16( psLPC_Q14_lo_01234567, (xq_Q14), 7 );
+
+ /* high part, use pmaddwd, results in 4 32-bit */
+ xmm_hi_07 = _mm_madd_epi16( psLPC_Q14_hi_01234567, a_Q12_01234567 );
+ xmm_hi_8F = _mm_madd_epi16( psLPC_Q14_hi_89ABCDEF, a_Q12_89ABCDEF );
+
+ /* low part, use pmulhw, results in 8 16-bit, note we need simulate unsigned * signed, _mm_srai_epi16(psLPC_Q14_lo_01234567, 15) */
+ xmm_tempa = _mm_cmpgt_epi16( _mm_setzero_si128(), psLPC_Q14_lo_01234567 );
+ xmm_tempb = _mm_cmpgt_epi16( _mm_setzero_si128(), psLPC_Q14_lo_89ABCDEF );
+
+ xmm_tempa = _mm_and_si128( xmm_tempa, a_Q12_01234567 );
+ xmm_tempb = _mm_and_si128( xmm_tempb, a_Q12_89ABCDEF );
+
+ xmm_lo_07 = _mm_mulhi_epi16( psLPC_Q14_lo_01234567, a_Q12_01234567 );
+ xmm_lo_8F = _mm_mulhi_epi16( psLPC_Q14_lo_89ABCDEF, a_Q12_89ABCDEF );
+
+ xmm_lo_07 = _mm_add_epi16( xmm_lo_07, xmm_tempa );
+ xmm_lo_8F = _mm_add_epi16( xmm_lo_8F, xmm_tempb );
+
+ xmm_lo_07 = _mm_madd_epi16( xmm_lo_07, xmm_one );
+ xmm_lo_8F = _mm_madd_epi16( xmm_lo_8F, xmm_one );
+
+ /* accumulate */
+ xmm_hi_07 = _mm_add_epi32( xmm_hi_07, xmm_hi_8F );
+ xmm_lo_07 = _mm_add_epi32( xmm_lo_07, xmm_lo_8F );
+
+ xmm_hi_07 = _mm_add_epi32( xmm_hi_07, xmm_lo_07 );
+
+ xmm_hi_07 = _mm_add_epi32( xmm_hi_07, _mm_unpackhi_epi64(xmm_hi_07, xmm_hi_07 ) );
+ xmm_hi_07 = _mm_add_epi32( xmm_hi_07, _mm_shufflelo_epi16(xmm_hi_07, 0x0E ) );
+
+ LPC_pred_Q10 += _mm_cvtsi128_si32( xmm_hi_07 );
+
+ /* Long-term prediction */
+ if ( opus_likely( signalType == TYPE_VOICED ) ) {+ /* Unrolled loop */
+ /* Avoids introducing a bias because silk_SMLAWB() always rounds to -inf */
+ LTP_pred_Q13 = 2;
+ {+ __m128i b_Q14_3210, b_Q14_0123, pred_lag_ptr_0123;
+
+ b_Q14_3210 = OP_CVTEPI16_EPI32_M64( b_Q14 );
+ b_Q14_0123 = _mm_shuffle_epi32( b_Q14_3210, 0x1B );
+
+ /* loaded: [0] [-1] [-2] [-3] */
+ pred_lag_ptr_0123 = _mm_loadu_si128( (__m128i *)(&pred_lag_ptr[ -3 ] ) );
+ /* shuffle to [-3] [-2] [-1] [0] and to new xmm */
+ xmm_tempa = _mm_shuffle_epi32( pred_lag_ptr_0123, 0x1B );
+ /*64-bit multiply, a[2] * b[-2], a[0] * b[0] */
+ xmm_tempa = _mm_mul_epi32( xmm_tempa, b_Q14_3210 );
+ /* right shift 2 bytes (16 bits), zero extended */
+ xmm_tempa = _mm_srli_si128( xmm_tempa, 2 );
+
+ /* a[1] * b[-1], a[3] * b[-3] */
+ pred_lag_ptr_0123 = _mm_mul_epi32( pred_lag_ptr_0123, b_Q14_0123 );
+ pred_lag_ptr_0123 = _mm_srli_si128( pred_lag_ptr_0123, 2 );
+
+ pred_lag_ptr_0123 = _mm_add_epi32( pred_lag_ptr_0123, xmm_tempa );
+ /* equal shift right 8 bytes*/
+ xmm_tempa = _mm_shuffle_epi32( pred_lag_ptr_0123, _MM_SHUFFLE( 0, 0, 3, 2 ) );
+ xmm_tempa = _mm_add_epi32( xmm_tempa, pred_lag_ptr_0123 );
+
+ LTP_pred_Q13 += _mm_cvtsi128_si32( xmm_tempa );
+
+ LTP_pred_Q13 = silk_SMLAWB( LTP_pred_Q13, pred_lag_ptr[ -4 ], b_Q14[ 4 ] );
+ pred_lag_ptr++;
+ }
+ }
+
+ /* Noise shape feedback */
+ NSQ->sAR2_Q14[ 9 ] = NSQ->sAR2_Q14[ 8 ];
+ NSQ->sAR2_Q14[ 8 ] = _mm_cvtsi128_si32( _mm_srli_si128(_mm_unpackhi_epi16( sAR2_Q14_lo_76543210, sAR2_Q14_hi_76543210 ), 12 ) );
+
+ sAR2_Q14_hi_76543210 = _mm_slli_si128( sAR2_Q14_hi_76543210, 2 );
+ sAR2_Q14_lo_76543210 = _mm_slli_si128( sAR2_Q14_lo_76543210, 2 );
+
+ sAR2_Q14_hi_76543210 = _mm_insert_epi16( sAR2_Q14_hi_76543210, (xq_Q14 >> 16), 0 );
+ sAR2_Q14_lo_76543210 = _mm_insert_epi16( sAR2_Q14_lo_76543210, (xq_Q14), 0 );
+
+ /* high part, use pmaddwd, results in 4 32-bit */
+ xmm_hi_07 = _mm_madd_epi16( sAR2_Q14_hi_76543210, AR_shp_Q13_76543210 );
+
+ /* low part, use pmulhw, results in 8 16-bit, note we need simulate unsigned * signed,_mm_srai_epi16(sAR2_Q14_lo_76543210, 15) */
+ xmm_tempa = _mm_cmpgt_epi16( _mm_setzero_si128(), sAR2_Q14_lo_76543210 );
+ xmm_tempa = _mm_and_si128( xmm_tempa, AR_shp_Q13_76543210 );
+
+ xmm_lo_07 = _mm_mulhi_epi16( sAR2_Q14_lo_76543210, AR_shp_Q13_76543210 );
+ xmm_lo_07 = _mm_add_epi16( xmm_lo_07, xmm_tempa );
+
+ xmm_lo_07 = _mm_madd_epi16( xmm_lo_07, xmm_one );
+
+ /* accumulate */
+ xmm_hi_07 = _mm_add_epi32( xmm_hi_07, xmm_lo_07 );
+
+ xmm_hi_07 = _mm_add_epi32( xmm_hi_07, _mm_unpackhi_epi64(xmm_hi_07, xmm_hi_07 ) );
+ xmm_hi_07 = _mm_add_epi32( xmm_hi_07, _mm_shufflelo_epi16(xmm_hi_07, 0x0E ) );
+
+ n_AR_Q12 = 5 + _mm_cvtsi128_si32( xmm_hi_07 );
+
+ n_AR_Q12 = silk_SMLAWB( n_AR_Q12, NSQ->sAR2_Q14[ 8 ], AR_shp_Q13[ 8 ] );
+ n_AR_Q12 = silk_SMLAWB( n_AR_Q12, NSQ->sAR2_Q14[ 9 ], AR_shp_Q13[ 9 ] );
+
+ n_AR_Q12 = silk_LSHIFT32( n_AR_Q12, 1 ); /* Q11 -> Q12 */
+ n_AR_Q12 = silk_SMLAWB( n_AR_Q12, sLF_AR_shp_Q14, Tilt_Q14 );
+
+ n_LF_Q12 = silk_SMULWB( NSQ->sLTP_shp_Q14[ NSQ->sLTP_shp_buf_idx - 1 ], LF_shp_Q14 );
+ n_LF_Q12 = silk_SMLAWT( n_LF_Q12, sLF_AR_shp_Q14, LF_shp_Q14 );
+
+ silk_assert( lag > 0 || signalType != TYPE_VOICED );
+
+ /* Combine prediction and noise shaping signals */
+ tmp1 = silk_SUB32( silk_LSHIFT32( LPC_pred_Q10, 2 ), n_AR_Q12 ); /* Q12 */
+ tmp1 = silk_SUB32( tmp1, n_LF_Q12 ); /* Q12 */
+ if( lag > 0 ) {+ /* Symmetric, packed FIR coefficients */
+ n_LTP_Q13 = silk_SMULWB( silk_ADD32( shp_lag_ptr[ 0 ], shp_lag_ptr[ -2 ] ), HarmShapeFIRPacked_Q14 );
+ n_LTP_Q13 = silk_SMLAWT( n_LTP_Q13, shp_lag_ptr[ -1 ], HarmShapeFIRPacked_Q14 );
+ n_LTP_Q13 = silk_LSHIFT( n_LTP_Q13, 1 );
+ shp_lag_ptr++;
+
+ tmp2 = silk_SUB32( LTP_pred_Q13, n_LTP_Q13 ); /* Q13 */
+ tmp1 = silk_ADD_LSHIFT32( tmp2, tmp1, 1 ); /* Q13 */
+ tmp1 = silk_RSHIFT_ROUND( tmp1, 3 ); /* Q10 */
+ } else {+ tmp1 = silk_RSHIFT_ROUND( tmp1, 2 ); /* Q10 */
+ }
+
+ r_Q10 = silk_SUB32( x_sc_Q10[ i ], tmp1 ); /* residual error Q10 */
+
+ /* Generate dither */
+ NSQ->rand_seed = silk_RAND( NSQ->rand_seed );
+
+ /* Flip sign depending on dither */
+ tmp2 = -r_Q10;
+ if ( NSQ->rand_seed < 0 ) r_Q10 = tmp2;
+
+ r_Q10 = silk_LIMIT_32( r_Q10, -(31 << 10), 30 << 10 );
+
+ /* Find two quantization level candidates and measure their rate-distortion */
+ q1_Q10 = silk_SUB32( r_Q10, offset_Q10 );
+ q1_Q0 = silk_RSHIFT( q1_Q10, 10 );
+
+ q1_Q10 = table[q1_Q0][0];
+ q2_Q10 = table[q1_Q0][1];
+
+ if (r_Q10 * table[q1_Q0][2] - table[q1_Q0][3] < 0)
+ {+ q1_Q10 = q2_Q10;
+ }
+
+ pulses[ i ] = (opus_int8)silk_RSHIFT_ROUND( q1_Q10, 10 );
+
+ /* Excitation */
+ exc_Q14 = silk_LSHIFT( q1_Q10, 4 );
+
+ tmp2 = -exc_Q14;
+ if ( NSQ->rand_seed < 0 ) exc_Q14 = tmp2;
+
+ /* Add predictions */
+ LPC_exc_Q14 = silk_ADD_LSHIFT32( exc_Q14, LTP_pred_Q13, 1 );
+ xq_Q14 = silk_ADD_LSHIFT32( LPC_exc_Q14, LPC_pred_Q10, 4 );
+
+ /* Update states */
+ psLPC_Q14++;
+ *psLPC_Q14 = xq_Q14;
+ sLF_AR_shp_Q14 = silk_SUB_LSHIFT32( xq_Q14, n_AR_Q12, 2 );
+
+ NSQ->sLTP_shp_Q14[ NSQ->sLTP_shp_buf_idx ] = silk_SUB_LSHIFT32( sLF_AR_shp_Q14, n_LF_Q12, 2 );
+ sLTP_Q15[ NSQ->sLTP_buf_idx ] = silk_LSHIFT( LPC_exc_Q14, 1 );
+ NSQ->sLTP_shp_buf_idx++;
+ NSQ->sLTP_buf_idx++;
+
+ /* Make dither dependent on quantized signal */
+ NSQ->rand_seed = silk_ADD32_ovflw( NSQ->rand_seed, pulses[ i ] );
+ }
+
+ NSQ->sLF_AR_shp_Q14 = sLF_AR_shp_Q14;
+
+ /* Scale XQ back to normal level before saving */
+ psLPC_Q14 = &NSQ->sLPC_Q14[ NSQ_LPC_BUF_LENGTH ];
+
+ /* write back sAR2_Q14 */
+ xmm_tempa = _mm_unpackhi_epi16( sAR2_Q14_lo_76543210, sAR2_Q14_hi_76543210 );
+ xmm_tempb = _mm_unpacklo_epi16( sAR2_Q14_lo_76543210, sAR2_Q14_hi_76543210 );
+ _mm_storeu_si128( (__m128i *)(&NSQ->sAR2_Q14[ 4 ]), xmm_tempa );
+ _mm_storeu_si128( (__m128i *)(&NSQ->sAR2_Q14[ 0 ]), xmm_tempb );
+
+ /* xq[ i ] = (opus_int16)silk_SAT16( silk_RSHIFT_ROUND( silk_SMULWW( psLPC_Q14[ i ], Gain_Q10 ), 8 ) ); */
+ {+ __m128i xmm_Gain_Q10;
+ __m128i xmm_xq_Q14_3210, xmm_xq_Q14_x3x1, xmm_xq_Q14_7654, xmm_xq_Q14_x7x5;
+
+ /* prepare (1 << 7) in packed 4 32-bits */
+ xmm_tempa = _mm_set1_epi32( (1 << 7) );
+
+ /* prepare Gain_Q10 in packed 4 32-bits */
+ xmm_Gain_Q10 = _mm_set1_epi32( Gain_Q10 );
+
+ /* process xq */
+ for (i = 0; i < length - 7; i += 8)
+ {+ xmm_xq_Q14_3210 = _mm_loadu_si128( (__m128i *)(&(psLPC_Q14[ i + 0 ] ) ) );
+ xmm_xq_Q14_7654 = _mm_loadu_si128( (__m128i *)(&(psLPC_Q14[ i + 4 ] ) ) );
+
+ /* equal shift right 4 bytes*/
+ xmm_xq_Q14_x3x1 = _mm_shuffle_epi32( xmm_xq_Q14_3210, _MM_SHUFFLE( 0, 3, 2, 1 ) );
+ /* equal shift right 4 bytes*/
+ xmm_xq_Q14_x7x5 = _mm_shuffle_epi32( xmm_xq_Q14_7654, _MM_SHUFFLE( 0, 3, 2, 1 ) );
+
+ xmm_xq_Q14_3210 = _mm_mul_epi32( xmm_xq_Q14_3210, xmm_Gain_Q10 );
+ xmm_xq_Q14_x3x1 = _mm_mul_epi32( xmm_xq_Q14_x3x1, xmm_Gain_Q10 );
+ xmm_xq_Q14_7654 = _mm_mul_epi32( xmm_xq_Q14_7654, xmm_Gain_Q10 );
+ xmm_xq_Q14_x7x5 = _mm_mul_epi32( xmm_xq_Q14_x7x5, xmm_Gain_Q10 );
+
+ xmm_xq_Q14_3210 = _mm_srli_epi64( xmm_xq_Q14_3210, 16 );
+ xmm_xq_Q14_x3x1 = _mm_slli_epi64( xmm_xq_Q14_x3x1, 16 );
+ xmm_xq_Q14_7654 = _mm_srli_epi64( xmm_xq_Q14_7654, 16 );
+ xmm_xq_Q14_x7x5 = _mm_slli_epi64( xmm_xq_Q14_x7x5, 16 );
+
+ xmm_xq_Q14_3210 = _mm_blend_epi16( xmm_xq_Q14_3210, xmm_xq_Q14_x3x1, 0xCC );
+ xmm_xq_Q14_7654 = _mm_blend_epi16( xmm_xq_Q14_7654, xmm_xq_Q14_x7x5, 0xCC );
+
+ /* silk_RSHIFT_ROUND(xq, 8) */
+ xmm_xq_Q14_3210 = _mm_add_epi32( xmm_xq_Q14_3210, xmm_tempa );
+ xmm_xq_Q14_7654 = _mm_add_epi32( xmm_xq_Q14_7654, xmm_tempa );
+
+ xmm_xq_Q14_3210 = _mm_srai_epi32( xmm_xq_Q14_3210, 8 );
+ xmm_xq_Q14_7654 = _mm_srai_epi32( xmm_xq_Q14_7654, 8 );
+
+ /* silk_SAT16 */
+ xmm_xq_Q14_3210 = _mm_packs_epi32( xmm_xq_Q14_3210, xmm_xq_Q14_7654 );
+
+ /* save to xq */
+ _mm_storeu_si128( (__m128i *)(&xq[ i ] ), xmm_xq_Q14_3210 );
+ }
+ }
+ for ( ; i < length; i++)
+ {+ xq[i] = (opus_int16)silk_SAT16( silk_RSHIFT_ROUND( silk_SMULWW( psLPC_Q14[ i ], Gain_Q10 ), 8 ) );
+ }
+
+ /* Update LPC synth buffer */
+ silk_memcpy( NSQ->sLPC_Q14, &NSQ->sLPC_Q14[ length ], NSQ_LPC_BUF_LENGTH * sizeof( opus_int32 ) );
+}
+
+static OPUS_INLINE void silk_nsq_scale_states_sse4_1(
+ const silk_encoder_state *psEncC, /* I Encoder State */
+ silk_nsq_state *NSQ, /* I/O NSQ state */
+ const opus_int32 x_Q3[], /* I input in Q3 */
+ opus_int32 x_sc_Q10[], /* O input scaled with 1/Gain */
+ const opus_int16 sLTP[], /* I re-whitened LTP state in Q0 */
+ opus_int32 sLTP_Q15[], /* O LTP state matching scaled input */
+ opus_int subfr, /* I subframe number */
+ const opus_int LTP_scale_Q14, /* I */
+ const opus_int32 Gains_Q16[ MAX_NB_SUBFR ], /* I */
+ const opus_int pitchL[ MAX_NB_SUBFR ], /* I Pitch lag */
+ const opus_int signal_type /* I Signal type */
+)
+{+ opus_int i, lag;
+ opus_int32 gain_adj_Q16, inv_gain_Q31, inv_gain_Q23;
+ __m128i xmm_inv_gain_Q23, xmm_x_Q3_x2x0, xmm_x_Q3_x3x1;
+
+ lag = pitchL[ subfr ];
+ inv_gain_Q31 = silk_INVERSE32_varQ( silk_max( Gains_Q16[ subfr ], 1 ), 47 );
+ silk_assert( inv_gain_Q31 != 0 );
+
+ /* Calculate gain adjustment factor */
+ if( Gains_Q16[ subfr ] != NSQ->prev_gain_Q16 ) {+ gain_adj_Q16 = silk_DIV32_varQ( NSQ->prev_gain_Q16, Gains_Q16[ subfr ], 16 );
+ } else {+ gain_adj_Q16 = (opus_int32)1 << 16;
+ }
+
+ /* Scale input */
+ inv_gain_Q23 = silk_RSHIFT_ROUND( inv_gain_Q31, 8 );
+
+ /* prepare inv_gain_Q23 in packed 4 32-bits */
+ xmm_inv_gain_Q23 = _mm_set1_epi32(inv_gain_Q23);
+
+ for( i = 0; i < psEncC->subfr_length - 3; i += 4 ) {+ xmm_x_Q3_x2x0 = _mm_loadu_si128( (__m128i *)(&(x_Q3[ i ] ) ) );
+
+ /* equal shift right 4 bytes*/
+ xmm_x_Q3_x3x1 = _mm_shuffle_epi32( xmm_x_Q3_x2x0, _MM_SHUFFLE( 0, 3, 2, 1 ) );
+
+ xmm_x_Q3_x2x0 = _mm_mul_epi32( xmm_x_Q3_x2x0, xmm_inv_gain_Q23 );
+ xmm_x_Q3_x3x1 = _mm_mul_epi32( xmm_x_Q3_x3x1, xmm_inv_gain_Q23 );
+
+ xmm_x_Q3_x2x0 = _mm_srli_epi64( xmm_x_Q3_x2x0, 16 );
+ xmm_x_Q3_x3x1 = _mm_slli_epi64( xmm_x_Q3_x3x1, 16 );
+
+ xmm_x_Q3_x2x0 = _mm_blend_epi16( xmm_x_Q3_x2x0, xmm_x_Q3_x3x1, 0xCC );
+
+ _mm_storeu_si128( (__m128i *)(&(x_sc_Q10[ i ] ) ), xmm_x_Q3_x2x0 );
+ }
+
+ for( ; i < psEncC->subfr_length; i++ ) {+ x_sc_Q10[ i ] = silk_SMULWW( x_Q3[ i ], inv_gain_Q23 );
+ }
+
+ /* Save inverse gain */
+ NSQ->prev_gain_Q16 = Gains_Q16[ subfr ];
+
+ /* After rewhitening the LTP state is un-scaled, so scale with inv_gain_Q16 */
+ if( NSQ->rewhite_flag ) {+ if( subfr == 0 ) {+ /* Do LTP downscaling */
+ inv_gain_Q31 = silk_LSHIFT( silk_SMULWB( inv_gain_Q31, LTP_scale_Q14 ), 2 );
+ }
+ for( i = NSQ->sLTP_buf_idx - lag - LTP_ORDER / 2; i < NSQ->sLTP_buf_idx; i++ ) {+ silk_assert( i < MAX_FRAME_LENGTH );
+ sLTP_Q15[ i ] = silk_SMULWB( inv_gain_Q31, sLTP[ i ] );
+ }
+ }
+
+ /* Adjust for changing gain */
+ if( gain_adj_Q16 != (opus_int32)1 << 16 ) {+ /* Scale long-term shaping state */
+ __m128i xmm_gain_adj_Q16, xmm_sLTP_shp_Q14_x2x0, xmm_sLTP_shp_Q14_x3x1;
+
+ /* prepare gain_adj_Q16 in packed 4 32-bits */
+ xmm_gain_adj_Q16 = _mm_set1_epi32(gain_adj_Q16);
+
+ for( i = NSQ->sLTP_shp_buf_idx - psEncC->ltp_mem_length; i < NSQ->sLTP_shp_buf_idx - 3; i += 4 )
+ {+ xmm_sLTP_shp_Q14_x2x0 = _mm_loadu_si128( (__m128i *)(&(NSQ->sLTP_shp_Q14[ i ] ) ) );
+ /* equal shift right 4 bytes*/
+ xmm_sLTP_shp_Q14_x3x1 = _mm_shuffle_epi32( xmm_sLTP_shp_Q14_x2x0, _MM_SHUFFLE( 0, 3, 2, 1 ) );
+
+ xmm_sLTP_shp_Q14_x2x0 = _mm_mul_epi32( xmm_sLTP_shp_Q14_x2x0, xmm_gain_adj_Q16 );
+ xmm_sLTP_shp_Q14_x3x1 = _mm_mul_epi32( xmm_sLTP_shp_Q14_x3x1, xmm_gain_adj_Q16 );
+
+ xmm_sLTP_shp_Q14_x2x0 = _mm_srli_epi64( xmm_sLTP_shp_Q14_x2x0, 16 );
+ xmm_sLTP_shp_Q14_x3x1 = _mm_slli_epi64( xmm_sLTP_shp_Q14_x3x1, 16 );
+
+ xmm_sLTP_shp_Q14_x2x0 = _mm_blend_epi16( xmm_sLTP_shp_Q14_x2x0, xmm_sLTP_shp_Q14_x3x1, 0xCC );
+
+ _mm_storeu_si128( (__m128i *)(&(NSQ->sLTP_shp_Q14[ i ] ) ), xmm_sLTP_shp_Q14_x2x0 );
+ }
+
+ for( ; i < NSQ->sLTP_shp_buf_idx; i++ ) {+ NSQ->sLTP_shp_Q14[ i ] = silk_SMULWW( gain_adj_Q16, NSQ->sLTP_shp_Q14[ i ] );
+ }
+
+ /* Scale long-term prediction state */
+ if( signal_type == TYPE_VOICED && NSQ->rewhite_flag == 0 ) {+ for( i = NSQ->sLTP_buf_idx - lag - LTP_ORDER / 2; i < NSQ->sLTP_buf_idx; i++ ) {+ sLTP_Q15[ i ] = silk_SMULWW( gain_adj_Q16, sLTP_Q15[ i ] );
+ }
+ }
+
+ NSQ->sLF_AR_shp_Q14 = silk_SMULWW( gain_adj_Q16, NSQ->sLF_AR_shp_Q14 );
+
+ /* Scale short-term prediction and shaping states */
+ for( i = 0; i < NSQ_LPC_BUF_LENGTH; i++ ) {+ NSQ->sLPC_Q14[ i ] = silk_SMULWW( gain_adj_Q16, NSQ->sLPC_Q14[ i ] );
+ }
+ for( i = 0; i < MAX_SHAPE_LPC_ORDER; i++ ) {+ NSQ->sAR2_Q14[ i ] = silk_SMULWW( gain_adj_Q16, NSQ->sAR2_Q14[ i ] );
+ }
+ }
+}
--- a/silk/x86/VAD_sse.c
+++ /dev/null
@@ -1,277 +1,0 @@
-/* Copyright (c) 2014, Cisco Systems, INC
- Written by XiangMingZhu WeiZhou MinPeng YanWang
-
- Redistribution and use in source and binary forms, with or without
- modification, are permitted provided that the following conditions
- are met:
-
- - Redistributions of source code must retain the above copyright
- notice, this list of conditions and the following disclaimer.
-
- - Redistributions in binary form must reproduce the above copyright
- notice, this list of conditions and the following disclaimer in the
- documentation and/or other materials provided with the distribution.
-
- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER
- OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
- EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
- PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
- PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
- LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
- NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-*/
-
-#ifdef HAVE_CONFIG_H
-#include "config.h"
-#endif
-
-#include <xmmintrin.h>
-#include <emmintrin.h>
-#include <smmintrin.h>
-
-#include "main.h"
-#include "stack_alloc.h"
-
-/* Weighting factors for tilt measure */
-static const opus_int32 tiltWeights[ VAD_N_BANDS ] = { 30000, 6000, -12000, -12000 };-
-/***************************************/
-/* Get the speech activity level in Q8 */
-/***************************************/
-opus_int silk_VAD_GetSA_Q8_sse4_1( /* O Return value, 0 if success */
- silk_encoder_state *psEncC, /* I/O Encoder state */
- const opus_int16 pIn[] /* I PCM input */
-)
-{- opus_int SA_Q15, pSNR_dB_Q7, input_tilt;
- opus_int decimated_framelength1, decimated_framelength2;
- opus_int decimated_framelength;
- opus_int dec_subframe_length, dec_subframe_offset, SNR_Q7, i, b, s;
- opus_int32 sumSquared, smooth_coef_Q16;
- opus_int16 HPstateTmp;
- VARDECL( opus_int16, X );
- opus_int32 Xnrg[ VAD_N_BANDS ];
- opus_int32 NrgToNoiseRatio_Q8[ VAD_N_BANDS ];
- opus_int32 speech_nrg, x_tmp;
- opus_int X_offset[ VAD_N_BANDS ];
- opus_int ret = 0;
- silk_VAD_state *psSilk_VAD = &psEncC->sVAD;
-
- SAVE_STACK;
-
- /* Safety checks */
- silk_assert( VAD_N_BANDS == 4 );
- silk_assert( MAX_FRAME_LENGTH >= psEncC->frame_length );
- silk_assert( psEncC->frame_length <= 512 );
- silk_assert( psEncC->frame_length == 8 * silk_RSHIFT( psEncC->frame_length, 3 ) );
-
- /***********************/
- /* Filter and Decimate */
- /***********************/
- decimated_framelength1 = silk_RSHIFT( psEncC->frame_length, 1 );
- decimated_framelength2 = silk_RSHIFT( psEncC->frame_length, 2 );
- decimated_framelength = silk_RSHIFT( psEncC->frame_length, 3 );
- /* Decimate into 4 bands:
- 0 L 3L L 3L 5L
- - -- - -- --
- 8 8 2 4 4
-
- [0-1 kHz| temp. |1-2 kHz| 2-4 kHz | 4-8 kHz |
-
- They're arranged to allow the minimal ( frame_length / 4 ) extra
- scratch space during the downsampling process */
- X_offset[ 0 ] = 0;
- X_offset[ 1 ] = decimated_framelength + decimated_framelength2;
- X_offset[ 2 ] = X_offset[ 1 ] + decimated_framelength;
- X_offset[ 3 ] = X_offset[ 2 ] + decimated_framelength2;
- ALLOC( X, X_offset[ 3 ] + decimated_framelength1, opus_int16 );
-
- /* 0-8 kHz to 0-4 kHz and 4-8 kHz */
- silk_ana_filt_bank_1( pIn, &psSilk_VAD->AnaState[ 0 ],
- X, &X[ X_offset[ 3 ] ], psEncC->frame_length );
-
- /* 0-4 kHz to 0-2 kHz and 2-4 kHz */
- silk_ana_filt_bank_1( X, &psSilk_VAD->AnaState1[ 0 ],
- X, &X[ X_offset[ 2 ] ], decimated_framelength1 );
-
- /* 0-2 kHz to 0-1 kHz and 1-2 kHz */
- silk_ana_filt_bank_1( X, &psSilk_VAD->AnaState2[ 0 ],
- X, &X[ X_offset[ 1 ] ], decimated_framelength2 );
-
- /*********************************************/
- /* HP filter on lowest band (differentiator) */
- /*********************************************/
- X[ decimated_framelength - 1 ] = silk_RSHIFT( X[ decimated_framelength - 1 ], 1 );
- HPstateTmp = X[ decimated_framelength - 1 ];
- for( i = decimated_framelength - 1; i > 0; i-- ) {- X[ i - 1 ] = silk_RSHIFT( X[ i - 1 ], 1 );
- X[ i ] -= X[ i - 1 ];
- }
- X[ 0 ] -= psSilk_VAD->HPstate;
- psSilk_VAD->HPstate = HPstateTmp;
-
- /*************************************/
- /* Calculate the energy in each band */
- /*************************************/
- for( b = 0; b < VAD_N_BANDS; b++ ) {- /* Find the decimated framelength in the non-uniformly divided bands */
- decimated_framelength = silk_RSHIFT( psEncC->frame_length, silk_min_int( VAD_N_BANDS - b, VAD_N_BANDS - 1 ) );
-
- /* Split length into subframe lengths */
- dec_subframe_length = silk_RSHIFT( decimated_framelength, VAD_INTERNAL_SUBFRAMES_LOG2 );
- dec_subframe_offset = 0;
-
- /* Compute energy per sub-frame */
- /* initialize with summed energy of last subframe */
- Xnrg[ b ] = psSilk_VAD->XnrgSubfr[ b ];
- for( s = 0; s < VAD_INTERNAL_SUBFRAMES; s++ ) {- __m128i xmm_X, xmm_acc;
- sumSquared = 0;
-
- xmm_acc = _mm_setzero_si128();
-
- for( i = 0; i < dec_subframe_length - 7; i += 8 )
- {- xmm_X = _mm_loadu_si128( (__m128i *)&(X[ X_offset[ b ] + i + dec_subframe_offset ] ) );
- xmm_X = _mm_srai_epi16( xmm_X, 3 );
- xmm_X = _mm_madd_epi16( xmm_X, xmm_X );
- xmm_acc = _mm_add_epi32( xmm_acc, xmm_X );
- }
-
- xmm_acc = _mm_add_epi32( xmm_acc, _mm_unpackhi_epi64( xmm_acc, xmm_acc ) );
- xmm_acc = _mm_add_epi32( xmm_acc, _mm_shufflelo_epi16( xmm_acc, 0x0E ) );
-
- sumSquared += _mm_cvtsi128_si32( xmm_acc );
-
- for( ; i < dec_subframe_length; i++ ) {- /* The energy will be less than dec_subframe_length * ( silk_int16_MIN / 8 ) ^ 2. */
- /* Therefore we can accumulate with no risk of overflow (unless dec_subframe_length > 128) */
- x_tmp = silk_RSHIFT(
- X[ X_offset[ b ] + i + dec_subframe_offset ], 3 );
- sumSquared = silk_SMLABB( sumSquared, x_tmp, x_tmp );
-
- /* Safety check */
- silk_assert( sumSquared >= 0 );
- }
-
- /* Add/saturate summed energy of current subframe */
- if( s < VAD_INTERNAL_SUBFRAMES - 1 ) {- Xnrg[ b ] = silk_ADD_POS_SAT32( Xnrg[ b ], sumSquared );
- } else {- /* Look-ahead subframe */
- Xnrg[ b ] = silk_ADD_POS_SAT32( Xnrg[ b ], silk_RSHIFT( sumSquared, 1 ) );
- }
-
- dec_subframe_offset += dec_subframe_length;
- }
- psSilk_VAD->XnrgSubfr[ b ] = sumSquared;
- }
-
- /********************/
- /* Noise estimation */
- /********************/
- silk_VAD_GetNoiseLevels( &Xnrg[ 0 ], psSilk_VAD );
-
- /***********************************************/
- /* Signal-plus-noise to noise ratio estimation */
- /***********************************************/
- sumSquared = 0;
- input_tilt = 0;
- for( b = 0; b < VAD_N_BANDS; b++ ) {- speech_nrg = Xnrg[ b ] - psSilk_VAD->NL[ b ];
- if( speech_nrg > 0 ) {- /* Divide, with sufficient resolution */
- if( ( Xnrg[ b ] & 0xFF800000 ) == 0 ) {- NrgToNoiseRatio_Q8[ b ] = silk_DIV32( silk_LSHIFT( Xnrg[ b ], 8 ), psSilk_VAD->NL[ b ] + 1 );
- } else {- NrgToNoiseRatio_Q8[ b ] = silk_DIV32( Xnrg[ b ], silk_RSHIFT( psSilk_VAD->NL[ b ], 8 ) + 1 );
- }
-
- /* Convert to log domain */
- SNR_Q7 = silk_lin2log( NrgToNoiseRatio_Q8[ b ] ) - 8 * 128;
-
- /* Sum-of-squares */
- sumSquared = silk_SMLABB( sumSquared, SNR_Q7, SNR_Q7 ); /* Q14 */
-
- /* Tilt measure */
- if( speech_nrg < ( (opus_int32)1 << 20 ) ) {- /* Scale down SNR value for small subband speech energies */
- SNR_Q7 = silk_SMULWB( silk_LSHIFT( silk_SQRT_APPROX( speech_nrg ), 6 ), SNR_Q7 );
- }
- input_tilt = silk_SMLAWB( input_tilt, tiltWeights[ b ], SNR_Q7 );
- } else {- NrgToNoiseRatio_Q8[ b ] = 256;
- }
- }
-
- /* Mean-of-squares */
- sumSquared = silk_DIV32_16( sumSquared, VAD_N_BANDS ); /* Q14 */
-
- /* Root-mean-square approximation, scale to dBs, and write to output pointer */
- pSNR_dB_Q7 = (opus_int16)( 3 * silk_SQRT_APPROX( sumSquared ) ); /* Q7 */
-
- /*********************************/
- /* Speech Probability Estimation */
- /*********************************/
- SA_Q15 = silk_sigm_Q15( silk_SMULWB( VAD_SNR_FACTOR_Q16, pSNR_dB_Q7 ) - VAD_NEGATIVE_OFFSET_Q5 );
-
- /**************************/
- /* Frequency Tilt Measure */
- /**************************/
- psEncC->input_tilt_Q15 = silk_LSHIFT( silk_sigm_Q15( input_tilt ) - 16384, 1 );
-
- /**************************************************/
- /* Scale the sigmoid output based on power levels */
- /**************************************************/
- speech_nrg = 0;
- for( b = 0; b < VAD_N_BANDS; b++ ) {- /* Accumulate signal-without-noise energies, higher frequency bands have more weight */
- speech_nrg += ( b + 1 ) * silk_RSHIFT( Xnrg[ b ] - psSilk_VAD->NL[ b ], 4 );
- }
-
- /* Power scaling */
- if( speech_nrg <= 0 ) {- SA_Q15 = silk_RSHIFT( SA_Q15, 1 );
- } else if( speech_nrg < 32768 ) {- if( psEncC->frame_length == 10 * psEncC->fs_kHz ) {- speech_nrg = silk_LSHIFT_SAT32( speech_nrg, 16 );
- } else {- speech_nrg = silk_LSHIFT_SAT32( speech_nrg, 15 );
- }
-
- /* square-root */
- speech_nrg = silk_SQRT_APPROX( speech_nrg );
- SA_Q15 = silk_SMULWB( 32768 + speech_nrg, SA_Q15 );
- }
-
- /* Copy the resulting speech activity in Q8 */
- psEncC->speech_activity_Q8 = silk_min_int( silk_RSHIFT( SA_Q15, 7 ), silk_uint8_MAX );
-
- /***********************************/
- /* Energy Level and SNR estimation */
- /***********************************/
- /* Smoothing coefficient */
- smooth_coef_Q16 = silk_SMULWB( VAD_SNR_SMOOTH_COEF_Q18, silk_SMULWB( (opus_int32)SA_Q15, SA_Q15 ) );
-
- if( psEncC->frame_length == 10 * psEncC->fs_kHz ) {- smooth_coef_Q16 >>= 1;
- }
-
- for( b = 0; b < VAD_N_BANDS; b++ ) {- /* compute smoothed energy-to-noise ratio per band */
- psSilk_VAD->NrgRatioSmth_Q8[ b ] = silk_SMLAWB( psSilk_VAD->NrgRatioSmth_Q8[ b ],
- NrgToNoiseRatio_Q8[ b ] - psSilk_VAD->NrgRatioSmth_Q8[ b ], smooth_coef_Q16 );
-
- /* signal to noise ratio in dB per band */
- SNR_Q7 = 3 * ( silk_lin2log( psSilk_VAD->NrgRatioSmth_Q8[b] ) - 8 * 128 );
- /* quality = sigmoid( 0.25 * ( SNR_dB - 16 ) ); */
- psEncC->input_quality_bands_Q15[ b ] = silk_sigm_Q15( silk_RSHIFT( SNR_Q7 - 16 * 128, 4 ) );
- }
-
- RESTORE_STACK;
- return( ret );
-}
--- /dev/null
+++ b/silk/x86/VAD_sse4_1.c
@@ -1,0 +1,277 @@
+/* Copyright (c) 2014, Cisco Systems, INC
+ Written by XiangMingZhu WeiZhou MinPeng YanWang
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+
+ - Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+
+ - Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER
+ OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+ LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+ NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*/
+
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include <xmmintrin.h>
+#include <emmintrin.h>
+#include <smmintrin.h>
+
+#include "main.h"
+#include "stack_alloc.h"
+
+/* Weighting factors for tilt measure */
+static const opus_int32 tiltWeights[ VAD_N_BANDS ] = { 30000, 6000, -12000, -12000 };+
+/***************************************/
+/* Get the speech activity level in Q8 */
+/***************************************/
+opus_int silk_VAD_GetSA_Q8_sse4_1( /* O Return value, 0 if success */
+ silk_encoder_state *psEncC, /* I/O Encoder state */
+ const opus_int16 pIn[] /* I PCM input */
+)
+{+ opus_int SA_Q15, pSNR_dB_Q7, input_tilt;
+ opus_int decimated_framelength1, decimated_framelength2;
+ opus_int decimated_framelength;
+ opus_int dec_subframe_length, dec_subframe_offset, SNR_Q7, i, b, s;
+ opus_int32 sumSquared, smooth_coef_Q16;
+ opus_int16 HPstateTmp;
+ VARDECL( opus_int16, X );
+ opus_int32 Xnrg[ VAD_N_BANDS ];
+ opus_int32 NrgToNoiseRatio_Q8[ VAD_N_BANDS ];
+ opus_int32 speech_nrg, x_tmp;
+ opus_int X_offset[ VAD_N_BANDS ];
+ opus_int ret = 0;
+ silk_VAD_state *psSilk_VAD = &psEncC->sVAD;
+
+ SAVE_STACK;
+
+ /* Safety checks */
+ silk_assert( VAD_N_BANDS == 4 );
+ silk_assert( MAX_FRAME_LENGTH >= psEncC->frame_length );
+ silk_assert( psEncC->frame_length <= 512 );
+ silk_assert( psEncC->frame_length == 8 * silk_RSHIFT( psEncC->frame_length, 3 ) );
+
+ /***********************/
+ /* Filter and Decimate */
+ /***********************/
+ decimated_framelength1 = silk_RSHIFT( psEncC->frame_length, 1 );
+ decimated_framelength2 = silk_RSHIFT( psEncC->frame_length, 2 );
+ decimated_framelength = silk_RSHIFT( psEncC->frame_length, 3 );
+ /* Decimate into 4 bands:
+ 0 L 3L L 3L 5L
+ - -- - -- --
+ 8 8 2 4 4
+
+ [0-1 kHz| temp. |1-2 kHz| 2-4 kHz | 4-8 kHz |
+
+ They're arranged to allow the minimal ( frame_length / 4 ) extra
+ scratch space during the downsampling process */
+ X_offset[ 0 ] = 0;
+ X_offset[ 1 ] = decimated_framelength + decimated_framelength2;
+ X_offset[ 2 ] = X_offset[ 1 ] + decimated_framelength;
+ X_offset[ 3 ] = X_offset[ 2 ] + decimated_framelength2;
+ ALLOC( X, X_offset[ 3 ] + decimated_framelength1, opus_int16 );
+
+ /* 0-8 kHz to 0-4 kHz and 4-8 kHz */
+ silk_ana_filt_bank_1( pIn, &psSilk_VAD->AnaState[ 0 ],
+ X, &X[ X_offset[ 3 ] ], psEncC->frame_length );
+
+ /* 0-4 kHz to 0-2 kHz and 2-4 kHz */
+ silk_ana_filt_bank_1( X, &psSilk_VAD->AnaState1[ 0 ],
+ X, &X[ X_offset[ 2 ] ], decimated_framelength1 );
+
+ /* 0-2 kHz to 0-1 kHz and 1-2 kHz */
+ silk_ana_filt_bank_1( X, &psSilk_VAD->AnaState2[ 0 ],
+ X, &X[ X_offset[ 1 ] ], decimated_framelength2 );
+
+ /*********************************************/
+ /* HP filter on lowest band (differentiator) */
+ /*********************************************/
+ X[ decimated_framelength - 1 ] = silk_RSHIFT( X[ decimated_framelength - 1 ], 1 );
+ HPstateTmp = X[ decimated_framelength - 1 ];
+ for( i = decimated_framelength - 1; i > 0; i-- ) {+ X[ i - 1 ] = silk_RSHIFT( X[ i - 1 ], 1 );
+ X[ i ] -= X[ i - 1 ];
+ }
+ X[ 0 ] -= psSilk_VAD->HPstate;
+ psSilk_VAD->HPstate = HPstateTmp;
+
+ /*************************************/
+ /* Calculate the energy in each band */
+ /*************************************/
+ for( b = 0; b < VAD_N_BANDS; b++ ) {+ /* Find the decimated framelength in the non-uniformly divided bands */
+ decimated_framelength = silk_RSHIFT( psEncC->frame_length, silk_min_int( VAD_N_BANDS - b, VAD_N_BANDS - 1 ) );
+
+ /* Split length into subframe lengths */
+ dec_subframe_length = silk_RSHIFT( decimated_framelength, VAD_INTERNAL_SUBFRAMES_LOG2 );
+ dec_subframe_offset = 0;
+
+ /* Compute energy per sub-frame */
+ /* initialize with summed energy of last subframe */
+ Xnrg[ b ] = psSilk_VAD->XnrgSubfr[ b ];
+ for( s = 0; s < VAD_INTERNAL_SUBFRAMES; s++ ) {+ __m128i xmm_X, xmm_acc;
+ sumSquared = 0;
+
+ xmm_acc = _mm_setzero_si128();
+
+ for( i = 0; i < dec_subframe_length - 7; i += 8 )
+ {+ xmm_X = _mm_loadu_si128( (__m128i *)&(X[ X_offset[ b ] + i + dec_subframe_offset ] ) );
+ xmm_X = _mm_srai_epi16( xmm_X, 3 );
+ xmm_X = _mm_madd_epi16( xmm_X, xmm_X );
+ xmm_acc = _mm_add_epi32( xmm_acc, xmm_X );
+ }
+
+ xmm_acc = _mm_add_epi32( xmm_acc, _mm_unpackhi_epi64( xmm_acc, xmm_acc ) );
+ xmm_acc = _mm_add_epi32( xmm_acc, _mm_shufflelo_epi16( xmm_acc, 0x0E ) );
+
+ sumSquared += _mm_cvtsi128_si32( xmm_acc );
+
+ for( ; i < dec_subframe_length; i++ ) {+ /* The energy will be less than dec_subframe_length * ( silk_int16_MIN / 8 ) ^ 2. */
+ /* Therefore we can accumulate with no risk of overflow (unless dec_subframe_length > 128) */
+ x_tmp = silk_RSHIFT(
+ X[ X_offset[ b ] + i + dec_subframe_offset ], 3 );
+ sumSquared = silk_SMLABB( sumSquared, x_tmp, x_tmp );
+
+ /* Safety check */
+ silk_assert( sumSquared >= 0 );
+ }
+
+ /* Add/saturate summed energy of current subframe */
+ if( s < VAD_INTERNAL_SUBFRAMES - 1 ) {+ Xnrg[ b ] = silk_ADD_POS_SAT32( Xnrg[ b ], sumSquared );
+ } else {+ /* Look-ahead subframe */
+ Xnrg[ b ] = silk_ADD_POS_SAT32( Xnrg[ b ], silk_RSHIFT( sumSquared, 1 ) );
+ }
+
+ dec_subframe_offset += dec_subframe_length;
+ }
+ psSilk_VAD->XnrgSubfr[ b ] = sumSquared;
+ }
+
+ /********************/
+ /* Noise estimation */
+ /********************/
+ silk_VAD_GetNoiseLevels( &Xnrg[ 0 ], psSilk_VAD );
+
+ /***********************************************/
+ /* Signal-plus-noise to noise ratio estimation */
+ /***********************************************/
+ sumSquared = 0;
+ input_tilt = 0;
+ for( b = 0; b < VAD_N_BANDS; b++ ) {+ speech_nrg = Xnrg[ b ] - psSilk_VAD->NL[ b ];
+ if( speech_nrg > 0 ) {+ /* Divide, with sufficient resolution */
+ if( ( Xnrg[ b ] & 0xFF800000 ) == 0 ) {+ NrgToNoiseRatio_Q8[ b ] = silk_DIV32( silk_LSHIFT( Xnrg[ b ], 8 ), psSilk_VAD->NL[ b ] + 1 );
+ } else {+ NrgToNoiseRatio_Q8[ b ] = silk_DIV32( Xnrg[ b ], silk_RSHIFT( psSilk_VAD->NL[ b ], 8 ) + 1 );
+ }
+
+ /* Convert to log domain */
+ SNR_Q7 = silk_lin2log( NrgToNoiseRatio_Q8[ b ] ) - 8 * 128;
+
+ /* Sum-of-squares */
+ sumSquared = silk_SMLABB( sumSquared, SNR_Q7, SNR_Q7 ); /* Q14 */
+
+ /* Tilt measure */
+ if( speech_nrg < ( (opus_int32)1 << 20 ) ) {+ /* Scale down SNR value for small subband speech energies */
+ SNR_Q7 = silk_SMULWB( silk_LSHIFT( silk_SQRT_APPROX( speech_nrg ), 6 ), SNR_Q7 );
+ }
+ input_tilt = silk_SMLAWB( input_tilt, tiltWeights[ b ], SNR_Q7 );
+ } else {+ NrgToNoiseRatio_Q8[ b ] = 256;
+ }
+ }
+
+ /* Mean-of-squares */
+ sumSquared = silk_DIV32_16( sumSquared, VAD_N_BANDS ); /* Q14 */
+
+ /* Root-mean-square approximation, scale to dBs, and write to output pointer */
+ pSNR_dB_Q7 = (opus_int16)( 3 * silk_SQRT_APPROX( sumSquared ) ); /* Q7 */
+
+ /*********************************/
+ /* Speech Probability Estimation */
+ /*********************************/
+ SA_Q15 = silk_sigm_Q15( silk_SMULWB( VAD_SNR_FACTOR_Q16, pSNR_dB_Q7 ) - VAD_NEGATIVE_OFFSET_Q5 );
+
+ /**************************/
+ /* Frequency Tilt Measure */
+ /**************************/
+ psEncC->input_tilt_Q15 = silk_LSHIFT( silk_sigm_Q15( input_tilt ) - 16384, 1 );
+
+ /**************************************************/
+ /* Scale the sigmoid output based on power levels */
+ /**************************************************/
+ speech_nrg = 0;
+ for( b = 0; b < VAD_N_BANDS; b++ ) {+ /* Accumulate signal-without-noise energies, higher frequency bands have more weight */
+ speech_nrg += ( b + 1 ) * silk_RSHIFT( Xnrg[ b ] - psSilk_VAD->NL[ b ], 4 );
+ }
+
+ /* Power scaling */
+ if( speech_nrg <= 0 ) {+ SA_Q15 = silk_RSHIFT( SA_Q15, 1 );
+ } else if( speech_nrg < 32768 ) {+ if( psEncC->frame_length == 10 * psEncC->fs_kHz ) {+ speech_nrg = silk_LSHIFT_SAT32( speech_nrg, 16 );
+ } else {+ speech_nrg = silk_LSHIFT_SAT32( speech_nrg, 15 );
+ }
+
+ /* square-root */
+ speech_nrg = silk_SQRT_APPROX( speech_nrg );
+ SA_Q15 = silk_SMULWB( 32768 + speech_nrg, SA_Q15 );
+ }
+
+ /* Copy the resulting speech activity in Q8 */
+ psEncC->speech_activity_Q8 = silk_min_int( silk_RSHIFT( SA_Q15, 7 ), silk_uint8_MAX );
+
+ /***********************************/
+ /* Energy Level and SNR estimation */
+ /***********************************/
+ /* Smoothing coefficient */
+ smooth_coef_Q16 = silk_SMULWB( VAD_SNR_SMOOTH_COEF_Q18, silk_SMULWB( (opus_int32)SA_Q15, SA_Q15 ) );
+
+ if( psEncC->frame_length == 10 * psEncC->fs_kHz ) {+ smooth_coef_Q16 >>= 1;
+ }
+
+ for( b = 0; b < VAD_N_BANDS; b++ ) {+ /* compute smoothed energy-to-noise ratio per band */
+ psSilk_VAD->NrgRatioSmth_Q8[ b ] = silk_SMLAWB( psSilk_VAD->NrgRatioSmth_Q8[ b ],
+ NrgToNoiseRatio_Q8[ b ] - psSilk_VAD->NrgRatioSmth_Q8[ b ], smooth_coef_Q16 );
+
+ /* signal to noise ratio in dB per band */
+ SNR_Q7 = 3 * ( silk_lin2log( psSilk_VAD->NrgRatioSmth_Q8[b] ) - 8 * 128 );
+ /* quality = sigmoid( 0.25 * ( SNR_dB - 16 ) ); */
+ psEncC->input_quality_bands_Q15[ b ] = silk_sigm_Q15( silk_RSHIFT( SNR_Q7 - 16 * 128, 4 ) );
+ }
+
+ RESTORE_STACK;
+ return( ret );
+}
--- a/silk/x86/VQ_WMat_EC_sse.c
+++ /dev/null
@@ -1,142 +1,0 @@
-/* Copyright (c) 2014, Cisco Systems, INC
- Written by XiangMingZhu WeiZhou MinPeng YanWang
-
- Redistribution and use in source and binary forms, with or without
- modification, are permitted provided that the following conditions
- are met:
-
- - Redistributions of source code must retain the above copyright
- notice, this list of conditions and the following disclaimer.
-
- - Redistributions in binary form must reproduce the above copyright
- notice, this list of conditions and the following disclaimer in the
- documentation and/or other materials provided with the distribution.
-
- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER
- OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
- EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
- PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
- PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
- LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
- NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-*/
-
-#ifdef HAVE_CONFIG_H
-#include "config.h"
-#endif
-
-#include <xmmintrin.h>
-#include <emmintrin.h>
-#include <smmintrin.h>
-#include "main.h"
-#include "celt/x86/x86cpu.h"
-
-/* Entropy constrained matrix-weighted VQ, hard-coded to 5-element vectors, for a single input data vector */
-void silk_VQ_WMat_EC_sse4_1(
- opus_int8 *ind, /* O index of best codebook vector */
- opus_int32 *rate_dist_Q14, /* O best weighted quant error + mu * rate */
- opus_int *gain_Q7, /* O sum of absolute LTP coefficients */
- const opus_int16 *in_Q14, /* I input vector to be quantized */
- const opus_int32 *W_Q18, /* I weighting matrix */
- const opus_int8 *cb_Q7, /* I codebook */
- const opus_uint8 *cb_gain_Q7, /* I codebook effective gain */
- const opus_uint8 *cl_Q5, /* I code length for each codebook vector */
- const opus_int mu_Q9, /* I tradeoff betw. weighted error and rate */
- const opus_int32 max_gain_Q7, /* I maximum sum of absolute LTP coefficients */
- opus_int L /* I number of vectors in codebook */
-)
-{- opus_int k, gain_tmp_Q7;
- const opus_int8 *cb_row_Q7;
- opus_int16 diff_Q14[ 5 ];
- opus_int32 sum1_Q14, sum2_Q16;
-
- __m128i C_tmp1, C_tmp2, C_tmp3, C_tmp4, C_tmp5;
- /* Loop over codebook */
- *rate_dist_Q14 = silk_int32_MAX;
- cb_row_Q7 = cb_Q7;
- for( k = 0; k < L; k++ ) {- gain_tmp_Q7 = cb_gain_Q7[k];
-
- diff_Q14[ 0 ] = in_Q14[ 0 ] - silk_LSHIFT( cb_row_Q7[ 0 ], 7 );
-
- C_tmp1 = OP_CVTEPI16_EPI32_M64( &in_Q14[ 1 ] );
- C_tmp2 = OP_CVTEPI8_EPI32_M32( &cb_row_Q7[ 1 ] );
- C_tmp2 = _mm_slli_epi32( C_tmp2, 7 );
- C_tmp1 = _mm_sub_epi32( C_tmp1, C_tmp2 );
-
- diff_Q14[ 1 ] = _mm_extract_epi16( C_tmp1, 0 );
- diff_Q14[ 2 ] = _mm_extract_epi16( C_tmp1, 2 );
- diff_Q14[ 3 ] = _mm_extract_epi16( C_tmp1, 4 );
- diff_Q14[ 4 ] = _mm_extract_epi16( C_tmp1, 6 );
-
- /* Weighted rate */
- sum1_Q14 = silk_SMULBB( mu_Q9, cl_Q5[ k ] );
-
- /* Penalty for too large gain */
- sum1_Q14 = silk_ADD_LSHIFT32( sum1_Q14, silk_max( silk_SUB32( gain_tmp_Q7, max_gain_Q7 ), 0 ), 10 );
-
- silk_assert( sum1_Q14 >= 0 );
-
- /* first row of W_Q18 */
- C_tmp3 = _mm_loadu_si128( (__m128i *)(&W_Q18[ 1 ] ) );
- C_tmp4 = _mm_mul_epi32( C_tmp3, C_tmp1 );
- C_tmp4 = _mm_srli_si128( C_tmp4, 2 );
-
- C_tmp1 = _mm_shuffle_epi32( C_tmp1, _MM_SHUFFLE( 0, 3, 2, 1 ) ); /* shift right 4 bytes */
- C_tmp3 = _mm_shuffle_epi32( C_tmp3, _MM_SHUFFLE( 0, 3, 2, 1 ) ); /* shift right 4 bytes */
-
- C_tmp5 = _mm_mul_epi32( C_tmp3, C_tmp1 );
- C_tmp5 = _mm_srli_si128( C_tmp5, 2 );
-
- C_tmp5 = _mm_add_epi32( C_tmp4, C_tmp5 );
- C_tmp5 = _mm_slli_epi32( C_tmp5, 1 );
-
- C_tmp5 = _mm_add_epi32( C_tmp5, _mm_shuffle_epi32( C_tmp5, _MM_SHUFFLE( 0, 0, 0, 2 ) ) );
- sum2_Q16 = _mm_cvtsi128_si32( C_tmp5 );
-
- sum2_Q16 = silk_SMLAWB( sum2_Q16, W_Q18[ 0 ], diff_Q14[ 0 ] );
- sum1_Q14 = silk_SMLAWB( sum1_Q14, sum2_Q16, diff_Q14[ 0 ] );
-
- /* second row of W_Q18 */
- sum2_Q16 = silk_SMULWB( W_Q18[ 7 ], diff_Q14[ 2 ] );
- sum2_Q16 = silk_SMLAWB( sum2_Q16, W_Q18[ 8 ], diff_Q14[ 3 ] );
- sum2_Q16 = silk_SMLAWB( sum2_Q16, W_Q18[ 9 ], diff_Q14[ 4 ] );
- sum2_Q16 = silk_LSHIFT( sum2_Q16, 1 );
- sum2_Q16 = silk_SMLAWB( sum2_Q16, W_Q18[ 6 ], diff_Q14[ 1 ] );
- sum1_Q14 = silk_SMLAWB( sum1_Q14, sum2_Q16, diff_Q14[ 1 ] );
-
- /* third row of W_Q18 */
- sum2_Q16 = silk_SMULWB( W_Q18[ 13 ], diff_Q14[ 3 ] );
- sum2_Q16 = silk_SMLAWB( sum2_Q16, W_Q18[ 14 ], diff_Q14[ 4 ] );
- sum2_Q16 = silk_LSHIFT( sum2_Q16, 1 );
- sum2_Q16 = silk_SMLAWB( sum2_Q16, W_Q18[ 12 ], diff_Q14[ 2 ] );
- sum1_Q14 = silk_SMLAWB( sum1_Q14, sum2_Q16, diff_Q14[ 2 ] );
-
- /* fourth row of W_Q18 */
- sum2_Q16 = silk_SMULWB( W_Q18[ 19 ], diff_Q14[ 4 ] );
- sum2_Q16 = silk_LSHIFT( sum2_Q16, 1 );
- sum2_Q16 = silk_SMLAWB( sum2_Q16, W_Q18[ 18 ], diff_Q14[ 3 ] );
- sum1_Q14 = silk_SMLAWB( sum1_Q14, sum2_Q16, diff_Q14[ 3 ] );
-
- /* last row of W_Q18 */
- sum2_Q16 = silk_SMULWB( W_Q18[ 24 ], diff_Q14[ 4 ] );
- sum1_Q14 = silk_SMLAWB( sum1_Q14, sum2_Q16, diff_Q14[ 4 ] );
-
- silk_assert( sum1_Q14 >= 0 );
-
- /* find best */
- if( sum1_Q14 < *rate_dist_Q14 ) {- *rate_dist_Q14 = sum1_Q14;
- *ind = (opus_int8)k;
- *gain_Q7 = gain_tmp_Q7;
- }
-
- /* Go to next cbk vector */
- cb_row_Q7 += LTP_ORDER;
- }
-}
--- /dev/null
+++ b/silk/x86/VQ_WMat_EC_sse4_1.c
@@ -1,0 +1,142 @@
+/* Copyright (c) 2014, Cisco Systems, INC
+ Written by XiangMingZhu WeiZhou MinPeng YanWang
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+
+ - Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+
+ - Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER
+ OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+ LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+ NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*/
+
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include <xmmintrin.h>
+#include <emmintrin.h>
+#include <smmintrin.h>
+#include "main.h"
+#include "celt/x86/x86cpu.h"
+
+/* Entropy constrained matrix-weighted VQ, hard-coded to 5-element vectors, for a single input data vector */
+void silk_VQ_WMat_EC_sse4_1(
+ opus_int8 *ind, /* O index of best codebook vector */
+ opus_int32 *rate_dist_Q14, /* O best weighted quant error + mu * rate */
+ opus_int *gain_Q7, /* O sum of absolute LTP coefficients */
+ const opus_int16 *in_Q14, /* I input vector to be quantized */
+ const opus_int32 *W_Q18, /* I weighting matrix */
+ const opus_int8 *cb_Q7, /* I codebook */
+ const opus_uint8 *cb_gain_Q7, /* I codebook effective gain */
+ const opus_uint8 *cl_Q5, /* I code length for each codebook vector */
+ const opus_int mu_Q9, /* I tradeoff betw. weighted error and rate */
+ const opus_int32 max_gain_Q7, /* I maximum sum of absolute LTP coefficients */
+ opus_int L /* I number of vectors in codebook */
+)
+{+ opus_int k, gain_tmp_Q7;
+ const opus_int8 *cb_row_Q7;
+ opus_int16 diff_Q14[ 5 ];
+ opus_int32 sum1_Q14, sum2_Q16;
+
+ __m128i C_tmp1, C_tmp2, C_tmp3, C_tmp4, C_tmp5;
+ /* Loop over codebook */
+ *rate_dist_Q14 = silk_int32_MAX;
+ cb_row_Q7 = cb_Q7;
+ for( k = 0; k < L; k++ ) {+ gain_tmp_Q7 = cb_gain_Q7[k];
+
+ diff_Q14[ 0 ] = in_Q14[ 0 ] - silk_LSHIFT( cb_row_Q7[ 0 ], 7 );
+
+ C_tmp1 = OP_CVTEPI16_EPI32_M64( &in_Q14[ 1 ] );
+ C_tmp2 = OP_CVTEPI8_EPI32_M32( &cb_row_Q7[ 1 ] );
+ C_tmp2 = _mm_slli_epi32( C_tmp2, 7 );
+ C_tmp1 = _mm_sub_epi32( C_tmp1, C_tmp2 );
+
+ diff_Q14[ 1 ] = _mm_extract_epi16( C_tmp1, 0 );
+ diff_Q14[ 2 ] = _mm_extract_epi16( C_tmp1, 2 );
+ diff_Q14[ 3 ] = _mm_extract_epi16( C_tmp1, 4 );
+ diff_Q14[ 4 ] = _mm_extract_epi16( C_tmp1, 6 );
+
+ /* Weighted rate */
+ sum1_Q14 = silk_SMULBB( mu_Q9, cl_Q5[ k ] );
+
+ /* Penalty for too large gain */
+ sum1_Q14 = silk_ADD_LSHIFT32( sum1_Q14, silk_max( silk_SUB32( gain_tmp_Q7, max_gain_Q7 ), 0 ), 10 );
+
+ silk_assert( sum1_Q14 >= 0 );
+
+ /* first row of W_Q18 */
+ C_tmp3 = _mm_loadu_si128( (__m128i *)(&W_Q18[ 1 ] ) );
+ C_tmp4 = _mm_mul_epi32( C_tmp3, C_tmp1 );
+ C_tmp4 = _mm_srli_si128( C_tmp4, 2 );
+
+ C_tmp1 = _mm_shuffle_epi32( C_tmp1, _MM_SHUFFLE( 0, 3, 2, 1 ) ); /* shift right 4 bytes */
+ C_tmp3 = _mm_shuffle_epi32( C_tmp3, _MM_SHUFFLE( 0, 3, 2, 1 ) ); /* shift right 4 bytes */
+
+ C_tmp5 = _mm_mul_epi32( C_tmp3, C_tmp1 );
+ C_tmp5 = _mm_srli_si128( C_tmp5, 2 );
+
+ C_tmp5 = _mm_add_epi32( C_tmp4, C_tmp5 );
+ C_tmp5 = _mm_slli_epi32( C_tmp5, 1 );
+
+ C_tmp5 = _mm_add_epi32( C_tmp5, _mm_shuffle_epi32( C_tmp5, _MM_SHUFFLE( 0, 0, 0, 2 ) ) );
+ sum2_Q16 = _mm_cvtsi128_si32( C_tmp5 );
+
+ sum2_Q16 = silk_SMLAWB( sum2_Q16, W_Q18[ 0 ], diff_Q14[ 0 ] );
+ sum1_Q14 = silk_SMLAWB( sum1_Q14, sum2_Q16, diff_Q14[ 0 ] );
+
+ /* second row of W_Q18 */
+ sum2_Q16 = silk_SMULWB( W_Q18[ 7 ], diff_Q14[ 2 ] );
+ sum2_Q16 = silk_SMLAWB( sum2_Q16, W_Q18[ 8 ], diff_Q14[ 3 ] );
+ sum2_Q16 = silk_SMLAWB( sum2_Q16, W_Q18[ 9 ], diff_Q14[ 4 ] );
+ sum2_Q16 = silk_LSHIFT( sum2_Q16, 1 );
+ sum2_Q16 = silk_SMLAWB( sum2_Q16, W_Q18[ 6 ], diff_Q14[ 1 ] );
+ sum1_Q14 = silk_SMLAWB( sum1_Q14, sum2_Q16, diff_Q14[ 1 ] );
+
+ /* third row of W_Q18 */
+ sum2_Q16 = silk_SMULWB( W_Q18[ 13 ], diff_Q14[ 3 ] );
+ sum2_Q16 = silk_SMLAWB( sum2_Q16, W_Q18[ 14 ], diff_Q14[ 4 ] );
+ sum2_Q16 = silk_LSHIFT( sum2_Q16, 1 );
+ sum2_Q16 = silk_SMLAWB( sum2_Q16, W_Q18[ 12 ], diff_Q14[ 2 ] );
+ sum1_Q14 = silk_SMLAWB( sum1_Q14, sum2_Q16, diff_Q14[ 2 ] );
+
+ /* fourth row of W_Q18 */
+ sum2_Q16 = silk_SMULWB( W_Q18[ 19 ], diff_Q14[ 4 ] );
+ sum2_Q16 = silk_LSHIFT( sum2_Q16, 1 );
+ sum2_Q16 = silk_SMLAWB( sum2_Q16, W_Q18[ 18 ], diff_Q14[ 3 ] );
+ sum1_Q14 = silk_SMLAWB( sum1_Q14, sum2_Q16, diff_Q14[ 3 ] );
+
+ /* last row of W_Q18 */
+ sum2_Q16 = silk_SMULWB( W_Q18[ 24 ], diff_Q14[ 4 ] );
+ sum1_Q14 = silk_SMLAWB( sum1_Q14, sum2_Q16, diff_Q14[ 4 ] );
+
+ silk_assert( sum1_Q14 >= 0 );
+
+ /* find best */
+ if( sum1_Q14 < *rate_dist_Q14 ) {+ *rate_dist_Q14 = sum1_Q14;
+ *ind = (opus_int8)k;
+ *gain_Q7 = gain_tmp_Q7;
+ }
+
+ /* Go to next cbk vector */
+ cb_row_Q7 += LTP_ORDER;
+ }
+}
--- a/silk_sources.mk
+++ b/silk_sources.mk
@@ -77,11 +77,11 @@
silk/stereo_quant_pred.c \
silk/LPC_fit.c
-SILK_SOURCES_SSE4_1 = silk/x86/NSQ_sse.c \
-silk/x86/NSQ_del_dec_sse.c \
+SILK_SOURCES_SSE4_1 = silk/x86/NSQ_sse4_1.c \
+silk/x86/NSQ_del_dec_sse4_1.c \
silk/x86/x86_silk_map.c \
-silk/x86/VAD_sse.c \
-silk/x86/VQ_WMat_EC_sse.c
+silk/x86/VAD_sse4_1.c \
+silk/x86/VQ_WMat_EC_sse4_1.c
SILK_SOURCES_ARM_NEON_INTR = \
silk/arm/arm_silk_map.c \
@@ -115,8 +115,8 @@
silk/fixed/schur64_FIX.c \
silk/fixed/schur_FIX.c
-SILK_SOURCES_FIXED_SSE4_1 = silk/fixed/x86/vector_ops_FIX_sse.c \
-silk/fixed/x86/burg_modified_FIX_sse.c
+SILK_SOURCES_FIXED_SSE4_1 = silk/fixed/x86/vector_ops_FIX_sse4_1.c \
+silk/fixed/x86/burg_modified_FIX_sse4_1.c
SILK_SOURCES_FIXED_ARM_NEON_INTR = \
silk/fixed/arm/warped_autocorrelation_FIX_neon_intr.c
--- a/win32/VS2015/opus.vcxproj
+++ b/win32/VS2015/opus.vcxproj
@@ -257,7 +257,7 @@
<ClCompile Include="..\..\celt\quant_bands.c" />
<ClCompile Include="..\..\celt\rate.c" />
<ClCompile Include="..\..\celt\vq.c" />
- <ClCompile Include="..\..\celt\x86\celt_lpc_sse.c" />
+ <ClCompile Include="..\..\celt\x86\celt_lpc_sse4_1.c" />
<ClCompile Include="..\..\celt\x86\pitch_sse.c" />
<ClCompile Include="..\..\celt\x86\pitch_sse2.c" />
<ClCompile Include="..\..\celt\x86\pitch_sse4_1.c" />
@@ -341,10 +341,10 @@
<ClCompile Include="..\..\silk\table_LSF_cos.c" />
<ClCompile Include="..\..\silk\VAD.c" />
<ClCompile Include="..\..\silk\VQ_WMat_EC.c" />
- <ClCompile Include="..\..\silk\x86\NSQ_del_dec_sse.c" />
- <ClCompile Include="..\..\silk\x86\NSQ_sse.c" />
- <ClCompile Include="..\..\silk\x86\VAD_sse.c" />
- <ClCompile Include="..\..\silk\x86\VQ_WMat_EC_sse.c" />
+ <ClCompile Include="..\..\silk\x86\NSQ_del_dec_sse4_1.c" />
+ <ClCompile Include="..\..\silk\x86\NSQ_sse4_1.c" />
+ <ClCompile Include="..\..\silk\x86\VAD_sse4_1.c" />
+ <ClCompile Include="..\..\silk\x86\VQ_WMat_EC_sse4_1.c" />
<ClCompile Include="..\..\silk\x86\x86_silk_map.c" />
<ClCompile Include="..\..\src\analysis.c" />
<ClCompile Include="..\..\src\mlp.c" />
@@ -391,4 +391,4 @@
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
<ImportGroup Label="ExtensionTargets">
</ImportGroup>
-</Project>
\ No newline at end of file
+</Project>
--- a/win32/VS2015/opus.vcxproj.filters
+++ b/win32/VS2015/opus.vcxproj.filters
@@ -230,7 +230,7 @@
<ClCompile Include="..\..\celt\celt_lpc.c">
<Filter>Source Files</Filter>
</ClCompile>
- <ClCompile Include="..\..\celt\x86\celt_lpc_sse.c">
+ <ClCompile Include="..\..\celt\x86\celt_lpc_sse4_1.c">
<Filter>Source Files</Filter>
</ClCompile>
<ClCompile Include="..\..\celt\cwrs.c">
@@ -449,10 +449,10 @@
<ClCompile Include="..\..\silk\NSQ_del_dec.c">
<Filter>Source Files</Filter>
</ClCompile>
- <ClCompile Include="..\..\silk\x86\NSQ_del_dec_sse.c">
+ <ClCompile Include="..\..\silk\x86\NSQ_del_dec_sse4_1.c">
<Filter>Source Files</Filter>
</ClCompile>
- <ClCompile Include="..\..\silk\x86\NSQ_sse.c">
+ <ClCompile Include="..\..\silk\x86\NSQ_sse4_1.c">
<Filter>Source Files</Filter>
</ClCompile>
<ClCompile Include="..\..\silk\pitch_est_tables.c">
@@ -548,13 +548,13 @@
<ClCompile Include="..\..\silk\VAD.c">
<Filter>Source Files</Filter>
</ClCompile>
- <ClCompile Include="..\..\silk\x86\VAD_sse.c">
+ <ClCompile Include="..\..\silk\x86\VAD_sse4_1.c">
<Filter>Source Files</Filter>
</ClCompile>
<ClCompile Include="..\..\silk\VQ_WMat_EC.c">
<Filter>Source Files</Filter>
</ClCompile>
- <ClCompile Include="..\..\silk\x86\VQ_WMat_EC_sse.c">
+ <ClCompile Include="..\..\silk\x86\VQ_WMat_EC_sse4_1.c">
<Filter>Source Files</Filter>
</ClCompile>
<ClCompile Include="..\..\silk\x86\x86_silk_map.c">
@@ -567,4 +567,4 @@
<Filter>Source Files</Filter>
</ClCompile>
</ItemGroup>
-</Project>
\ No newline at end of file
+</Project>