ref: 0c0c5f940a3fc58c84b51765fc04f0294b42fd2d
parent: f2b86588cd6db584968a2bace3007734e4fe6d1d
author: Jean-Marc Valin <[email protected]>
date: Mon Mar 7 15:54:33 EST 2011
Support for glitchles mode switching Uses a 5ms redundant CELT frame embedded into the SILK or hybrid packet to handle the switching. It's still possible to use the PLC-based method when no redundant packet is included.
--- a/src/opus_decoder.c
+++ b/src/opus_decoder.c
@@ -71,6 +71,17 @@
return st;
}
+static void smooth_fade(const short *in1, const short *in2, short *out, int overlap, int channels)
+{+ int i, c;
+ for (c=0;c<channels;c++)
+ {+ /* FIXME: Make this 16-bit safe, remove division */
+ for (i=0;i<overlap;i++)
+ out[i*channels+c] = (i*in2[i*channels+c] + (overlap-i)*in1[i*channels+c])/overlap;
+ }
+}
+
int opus_decode(OpusDecoder *st, const unsigned char *data,
int len, short *pcm, int frame_size, int decode_fec)
{@@ -85,6 +96,10 @@
int transition=0;
int start_band;
int redundancy=0;
+ int redundancy_bytes = 0;
+ int celt_to_silk=0;
+ short redundant_audio[240*2];
+ int c;
/* Payloads of 1 (2 including ToC) or 0 trigger the PLC/DTX */
if (len<=2)
@@ -127,13 +142,13 @@
mode = st->prev_mode;
}
- if (mode != st->prev_mode && st->prev_mode > 0
+ if (data!=NULL && !st->prev_redundancy && mode != st->prev_mode && st->prev_mode > 0
&& !(mode == MODE_SILK_ONLY && st->prev_mode == MODE_HYBRID)
&& !(mode == MODE_HYBRID && st->prev_mode == MODE_SILK_ONLY))
{transition = 1;
- if (mode == MODE_CELT_ONLY && !st->prev_redundancy)
- opus_decode(st, NULL, 0, pcm_transition, IMAX(480, audiosize), 0);
+ if (mode == MODE_CELT_ONLY)
+ opus_decode(st, NULL, 0, pcm_transition, IMAX(st->Fs/100, audiosize), 0);
}
if (audiosize > frame_size)
{@@ -162,6 +177,7 @@
} else if( st->bandwidth == BANDWIDTH_WIDEBAND ) {DecControl.internalSampleRate = 16000;
} else {+ DecControl.internalSampleRate = 16000;
SKP_assert( 0 );
}
} else {@@ -189,47 +205,68 @@
}
start_band = 0;
- if (mode == MODE_HYBRID && data != NULL)
+ if (mode != MODE_CELT_ONLY && data != NULL)
{/* Check if we have a redundant 0-8 kHz band */
redundancy = ec_dec_bit_logp(&dec, 12);
- if (!redundancy)
- start_band = 17;
+ if (redundancy)
+ {+ celt_to_silk = ec_dec_bit_logp(&dec, 1);
+ if (mode == MODE_HYBRID)
+ redundancy_bytes = 2 + ec_dec_uint(&dec, 256);
+ else
+ redundancy_bytes = len - ((ec_tell(&dec)+7)>>3);
+ len -= redundancy_bytes;
+ /* Shrink decoder because of raw bits */
+ dec.storage -= redundancy_bytes;
+ }
+ start_band = 17;
}
+
+ if (mode != MODE_SILK_ONLY)
+ {+ int endband=21;
+
+ switch(st->bandwidth)
+ {+ case BANDWIDTH_NARROWBAND:
+ endband = 13;
+ break;
+ case BANDWIDTH_WIDEBAND:
+ endband = 17;
+ break;
+ case BANDWIDTH_SUPERWIDEBAND:
+ endband = 19;
+ break;
+ case BANDWIDTH_FULLBAND:
+ endband = 21;
+ break;
+ }
+ celt_decoder_ctl(st->celt_dec, CELT_SET_END_BAND(endband));
+ celt_decoder_ctl(st->celt_dec, CELT_SET_CHANNELS(st->stream_channels));
+ }
+
if (redundancy)
transition = 0;
if (transition && mode != MODE_CELT_ONLY)
- opus_decode(st, NULL, 0, pcm_transition, IMAX(480, audiosize), 0);
+ opus_decode(st, NULL, 0, pcm_transition, IMAX(st->Fs/100, audiosize), 0);
+ /* 5 ms redundant frame for CELT->SILK*/
+ if (redundancy && celt_to_silk)
+ {+ celt_decode(st->celt_dec, data+len, redundancy_bytes, redundant_audio, st->Fs/200);
+ celt_decoder_ctl(st->celt_dec, CELT_RESET_STATE);
+ }
+
/* MUST be after PLC */
celt_decoder_ctl(st->celt_dec, CELT_SET_START_BAND(start_band));
+ if (transition)
+ celt_decoder_ctl(st->celt_dec, CELT_RESET_STATE);
if (mode != MODE_SILK_ONLY)
{- int endband;
-
- switch(st->bandwidth)
- {- case BANDWIDTH_NARROWBAND:
- endband = 13;
- break;
- case BANDWIDTH_WIDEBAND:
- endband = 17;
- break;
- case BANDWIDTH_SUPERWIDEBAND:
- endband = 19;
- break;
- case BANDWIDTH_FULLBAND:
- endband = 21;
- break;
- }
- celt_decoder_ctl(st->celt_dec, CELT_SET_END_BAND(endband));
- celt_decoder_ctl(st->celt_dec, CELT_SET_CHANNELS(st->stream_channels));
-
- if (st->prev_mode == MODE_SILK_ONLY)
- celt_decoder_ctl(st->celt_dec, CELT_RESET_STATE);
/* Decode CELT */
celt_ret = celt_decode_with_ec(st->celt_dec, decode_fec?NULL:data, len, pcm_celt, frame_size, &dec);
for (i=0;i<frame_size*st->channels;i++)
@@ -236,6 +273,32 @@
pcm[i] = ADD_SAT16(pcm[i], pcm_celt[i]);
}
+ /* 5 ms redundant frame for SILK->CELT */
+ if (redundancy && !celt_to_silk)
+ {+ int N2, N4;
+ N2 = st->Fs/200;
+ N4 = st->Fs/400;
+ celt_decoder_ctl(st->celt_dec, CELT_RESET_STATE);
+ celt_decoder_ctl(st->celt_dec, CELT_SET_START_BAND(0));
+
+ celt_decode(st->celt_dec, data+len, redundancy_bytes, redundant_audio, N2);
+ smooth_fade(pcm+st->channels*(frame_size-N4), redundant_audio+st->channels*N4,
+ pcm+st->channels*(frame_size-N4), N4, st->channels);
+ }
+ if (redundancy && celt_to_silk)
+ {+ int N2, N4;
+ N2 = st->Fs/200;
+ N4 = st->Fs/400;
+
+ for (c=0;c<st->channels;c++)
+ {+ for (i=0;i<N4;i++)
+ pcm[st->channels*i+c] = redundant_audio[st->channels*i];
+ }
+ smooth_fade(redundant_audio+st->channels*N4, pcm+st->channels*N4, pcm+st->channels*N4, N4, st->channels);
+ }
if (transition)
{int plc_length, overlap;
@@ -246,9 +309,8 @@
for (i=0;i<plc_length;i++)
pcm[i] = pcm_transition[i];
- overlap = IMIN(480, IMAX(0, audiosize-plc_length));
- for (i=0;i<overlap;i++)
- pcm[plc_length+i] = (i*pcm[plc_length+i] + (overlap-i)*pcm_transition[plc_length+i])/overlap;
+ overlap = IMIN(st->Fs/100, IMAX(0, audiosize-plc_length));
+ smooth_fade(pcm_transition+plc_length, pcm+plc_length, pcm+plc_length, overlap, st->channels);
}
#if OPUS_TEST_RANGE_CODER_STATE
st->rangeFinal = dec.rng;
--- a/src/opus_encoder.c
+++ b/src/opus_encoder.c
@@ -81,6 +81,10 @@
st->use_vbr = 0;
st->bitrate_bps = 32000;
+ st->encoder_buffer = st->Fs/100;
+ st->delay_compensation = st->Fs/400;
+ if (st->Fs > 16000)
+ st->delay_compensation += 10;
return st;
}
@@ -95,8 +99,13 @@
int silk_internal_bandwidth;
int bytes_target;
int prefill=0;
- int start_band;
+ int start_band = 0;
int redundancy = 0;
+ int redundancy_bytes = 0;
+ int celt_to_silk = 0;
+ short pcm_buf[960*2];
+ int nb_compr_bytes;
+ int to_celt = 0;
bytes_target = st->bitrate_bps * frame_size / (st->Fs * 8) - 1;
@@ -107,6 +116,18 @@
SKP_Silk_SDK_InitEncoder( st->silk_enc, &dummy);
prefill=1;
}
+ if (st->prev_mode >0 &&
+ ((st->mode != MODE_CELT_ONLY && st->prev_mode == MODE_CELT_ONLY)
+ || (st->mode == MODE_CELT_ONLY && st->prev_mode != MODE_CELT_ONLY)))
+ {+ redundancy = 1;
+ celt_to_silk = (st->mode != MODE_CELT_ONLY);
+ if (!celt_to_silk)
+ {+ st->mode = st->prev_mode;
+ to_celt = 1;
+ }
+ }
ec_enc_init(&enc, data, max_data_bytes-1);
@@ -159,7 +180,7 @@
if (prefill)
{int zero=0;
- SKP_Silk_SDK_Encode( st->silk_enc, &st->silk_mode, st->delay_buffer, ENCODER_BUFFER, NULL, &zero, 1 );
+ SKP_Silk_SDK_Encode( st->silk_enc, &st->silk_mode, st->delay_buffer, st->encoder_buffer, NULL, &zero, 1 );
}
ret = SKP_Silk_SDK_Encode( st->silk_enc, &st->silk_mode, pcm, frame_size, &enc, &nBytes, 0 );
@@ -182,11 +203,8 @@
}
/* CELT processing */
- if (st->mode != MODE_SILK_ONLY)
{- int endband;
- short pcm_buf[960*2];
- int nb_compr_bytes;
+ int endband=21;
switch(st->bandwidth)
{@@ -205,7 +223,9 @@
}
celt_encoder_ctl(st->celt_enc, CELT_SET_END_BAND(endband));
celt_encoder_ctl(st->celt_enc, CELT_SET_CHANNELS(st->stream_channels));
-
+ }
+ if (st->mode != MODE_SILK_ONLY)
+ {celt_encoder_ctl(st->celt_enc, CELT_SET_VBR(0));
celt_encoder_ctl(st->celt_enc, CELT_SET_BITRATE(510000));
if (st->prev_mode == MODE_SILK_ONLY)
@@ -212,25 +232,16 @@
{unsigned char dummy[2];
celt_encoder_ctl(st->celt_enc, CELT_RESET_STATE);
+ celt_encoder_ctl(st->celt_enc, CELT_SET_START_BAND(0));
celt_encoder_ctl(st->celt_enc, CELT_SET_PREDICTION(0));
/* FIXME: This wastes CPU a bit compared to just prefilling the buffer */
- celt_encode(st->celt_enc, &st->delay_buffer[(ENCODER_BUFFER-ENCODER_DELAY_COMPENSATION-120)*st->channels], 120, dummy, 10);
+ celt_encode(st->celt_enc, &st->delay_buffer[(st->encoder_buffer-st->delay_compensation-120)*st->channels], 120, dummy, 10);
} else {celt_encoder_ctl(st->celt_enc, CELT_SET_PREDICTION(2));
}
- start_band = 0;
if (st->mode == MODE_HYBRID)
{- /* Check if we have a redundant 0-8 kHz band */
- ec_enc_bit_logp(&enc, redundancy, 12);
- if (!redundancy)
- start_band = 17;
- }
- celt_encoder_ctl(st->celt_enc, CELT_SET_START_BAND(start_band));
-
- if (st->mode == MODE_HYBRID)
- {int len;
len = (ec_tell(&enc)+7)>>3;
@@ -251,32 +262,86 @@
}
}
- for (i=0;i<IMIN(frame_size, ENCODER_DELAY_COMPENSATION)*st->channels;i++)
- pcm_buf[i] = st->delay_buffer[(ENCODER_BUFFER-ENCODER_DELAY_COMPENSATION)*st->channels+i];
- for (;i<frame_size*st->channels;i++)
- pcm_buf[i] = pcm[i-ENCODER_DELAY_COMPENSATION*st->channels];
ec_enc_shrink(&enc, nb_compr_bytes);
+ } else {+ nb_compr_bytes = 0;
+ }
+ for (i=0;i<IMIN(frame_size, st->delay_compensation)*st->channels;i++)
+ pcm_buf[i] = st->delay_buffer[(st->encoder_buffer-st->delay_compensation)*st->channels+i];
+ for (;i<frame_size*st->channels;i++)
+ pcm_buf[i] = pcm[i-st->delay_compensation*st->channels];
+ if (st->mode != MODE_CELT_ONLY)
+ {+ /* Check if we have a redundant 0-8 kHz band */
+ ec_enc_bit_logp(&enc, redundancy, 12);
+ if (redundancy)
+ {+ redundancy_bytes = 40;
+ ec_enc_bit_logp(&enc, celt_to_silk, 1);
+ if (st->mode == MODE_HYBRID)
+ ec_enc_uint(&enc, redundancy_bytes-2, 256);
+ }
+ start_band = 17;
+ }
+
+ if (st->mode == MODE_SILK_ONLY)
+ {+ ret = (ec_tell(&enc)+7)>>3;
+ ec_enc_done(&enc);
+ nb_compr_bytes = ret;
+ }
+
+ /* 5 ms redundant frame for CELT->SILK */
+ if (redundancy && celt_to_silk)
+ {+ celt_encoder_ctl(st->celt_enc, CELT_SET_START_BAND(0));
+ /* FIXME: This is wrong -- we need to set the flags properly */
+ celt_encoder_ctl(st->celt_enc, CELT_SET_VBR(0));
+ /* FIXME: Make sure not to overflow here */
+ celt_encode(st->celt_enc, pcm_buf, st->Fs/200, data+nb_compr_bytes, redundancy_bytes);
+ celt_encoder_ctl(st->celt_enc, CELT_RESET_STATE);
+ }
+
+ celt_encoder_ctl(st->celt_enc, CELT_SET_START_BAND(start_band));
+
+ if (st->mode != MODE_SILK_ONLY)
+ {/* Encode high band with CELT */
ret = celt_encode_with_ec(st->celt_enc, pcm_buf, frame_size, NULL, nb_compr_bytes, &enc);
-
- if (frame_size>ENCODER_BUFFER)
- {- for (i=0;i<ENCODER_BUFFER*st->channels;i++)
- st->delay_buffer[i] = pcm[(frame_size-ENCODER_BUFFER)*st->channels+i];
- } else {- int tmp = ENCODER_BUFFER-frame_size;
- for (i=0;i<tmp*st->channels;i++)
- st->delay_buffer[i] = st->delay_buffer[i+frame_size*st->channels];
- for (i=0;i<frame_size*st->channels;i++)
- st->delay_buffer[tmp*st->channels+i] = pcm[i];
- }
- } else {- ret = (ec_tell(&enc)+7)>>3;
- ec_enc_done(&enc);
}
+ /* 5 ms redundant frame for SILK->CELT */
+ if (redundancy && !celt_to_silk)
+ {+ int N2, N4;
+ N2 = st->Fs/200;
+ N4 = st->Fs/400;
+
+ celt_encoder_ctl(st->celt_enc, CELT_RESET_STATE);
+ celt_encoder_ctl(st->celt_enc, CELT_SET_START_BAND(0));
+ celt_encoder_ctl(st->celt_enc, CELT_SET_PREDICTION(0));
+
+ /* FIXME: Do proper prefilling here */
+ celt_encode(st->celt_enc, pcm_buf+st->channels*(frame_size-N2-N4), N4, data+nb_compr_bytes, redundancy_bytes);
+
+ celt_encode(st->celt_enc, pcm_buf+st->channels*(frame_size-N2), N2, data+nb_compr_bytes, redundancy_bytes);
+ }
+
+
+ if (frame_size>st->encoder_buffer)
+ {+ for (i=0;i<st->encoder_buffer*st->channels;i++)
+ st->delay_buffer[i] = pcm[(frame_size-st->encoder_buffer)*st->channels+i];
+ } else {+ int tmp = st->encoder_buffer-frame_size;
+ for (i=0;i<tmp*st->channels;i++)
+ st->delay_buffer[i] = st->delay_buffer[i+frame_size*st->channels];
+ for (i=0;i<frame_size*st->channels;i++)
+ st->delay_buffer[tmp*st->channels+i] = pcm[i];
+ }
+
/* Signalling the mode in the first byte */
data--;
framerate = st->Fs/frame_size;
@@ -310,8 +375,11 @@
#if OPUS_TEST_RANGE_CODER_STATE
st->rangeFinal = enc.rng;
#endif
- st->prev_mode = st->mode;
- return ret+1;
+ if (to_celt)
+ st->prev_mode = MODE_CELT_ONLY;
+ else
+ st->prev_mode = st->mode;
+ return ret+1+redundancy_bytes;
}
void opus_encoder_ctl(OpusEncoder *st, int request, ...)
--- a/src/opus_encoder.h
+++ b/src/opus_encoder.h
@@ -33,8 +33,7 @@
#include "SKP_Silk_SDK_API.h"
/* FIXME: This is only valid for 48 kHz */
-#define ENCODER_DELAY_COMPENSATION 130
-#define ENCODER_BUFFER 480
+#define MAX_ENCODER_BUFFER 480
struct OpusEncoder {CELTEncoder *celt_enc;
@@ -50,8 +49,10 @@
int Fs;
int use_vbr;
int bitrate_bps;
+ int encoder_buffer;
+ int delay_compensation;
- short delay_buffer[ENCODER_BUFFER*2];
+ short delay_buffer[MAX_ENCODER_BUFFER*2];
#ifdef OPUS_TEST_RANGE_CODER_STATE
int rangeFinal;