Refactoring jitter buffer

Features are: * Packet based buffer * Random in, first out * Adaptive delay compensation (voice) * Fixed delay (data, optionally MODEM/FAX) * Interpolation of missing frames * Any sample size
2022-07-31 07:55:14 +02:00
parent 4fc92eba45
commit 2b7efedc48
42 changed files with 544 additions and 206 deletions
--- a/src/libjitter/jitter.c
+++ b/src/libjitter/jitter.c
@@ -1,6 +1,6 @@
 /* Jitter buffering functions
 *
- * (C) 2016 by Andreas Eversberg <jolly@eversberg.eu>
+ * (C) 2022 by Andreas Eversberg <jolly@eversberg.eu>
 * All Rights Reserved
 *
 * This program is free software: you can redistribute it and/or modify
@@ -17,6 +17,59 @@
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

+/* How does it work:
+ *
+ * Storing:
+ *
+ * Each saved frame is sorted into the list of packages by their sequence
+ * number.
+ *
+ * The first packet will be stored with a delay of minimum jitter window size.
+ * 
+ * Packets with the same sequence are dropped.
+ *
+ * Early packts that exceed maximum jitter window size cause jitter
+ * window to shift into the future.
+ *
+ * Late packets cause jitter window to shift into the past (allowing more
+ * delay). Minimum jitter window size is added also, to prevent subsequent
+ * packets from beeing late too.
+ *
+ * If no sequence is provided (autosequence), the sequence number is generated
+ * by a counter. Also the timestamp is generated by counting the length of each
+ * frame.
+ *
+ * If ssrc changes, the buffer is reset.
+ *
+ *
+ * Playout:
+ *
+ * The caller of the playout function can request any length of samples from
+ * the packet list. The packt's time stamp and the jitter window time stamp
+ * indicate what portion of a packet is already provided to the caller.
+ * Complete packet, sent to the caller, are removed.
+ *
+ * Missing packets are interpolated by repeating last 20ms of audio (optional)
+ * or by inserting zeroes (sample size > 1 byte) or by inserting 0xff (sample
+ * size = 1).
+ *
+ * Optionally the constant delay will be measured continuously and lowered if
+ * greater than minimum window size. (adaptive jitter buffer size)
+ *
+ * Note that the delay is measured with time stamp of frame, no matter what
+ * the length is. Length is an extra delay, but not considered here.
+ *
+ *
+ * Unlocking:
+ *
+ * If the buffer is created or reset, the buffer is locked, so no packets are
+ * stored. When the playout routine is called, the buffer is unlocked. This
+ * prevents from filling the buffer before playout is performed, which would
+ * cause high delay.
+ *
+ */
+
+#include <stdio.h>
 #include <stdint.h>
 #include <stdlib.h>
 #include <string.h>
@@ -26,35 +79,97 @@
 #include "../libdebug/debug.h"
 #include "jitter.h"

+#define INITIAL_DELAY_INTERVAL	0.5
+#define REPEAT_DELAY_INTERVAL	3.0
+#define EXTRA_BUFFER		0.020	// 20 ms
+
+/* uncomment to enable heavy debugging */
+//#define HEAVY_DEBUG
+
+static int unnamed_count = 1;
+
 /* create jitter buffer */
-int jitter_create(jitter_t *jitter, int length)
+int jitter_create(jitter_t *jb, const char *name, double samplerate, int sample_size, double target_window_duration, double max_window_duration, uint32_t window_flags)
 {
-	memset(jitter, 0, sizeof(*jitter));
-	jitter->spl = malloc(length * sizeof(sample_t));
-	if (!jitter->spl) {
-		PDEBUG(DDSP, DEBUG_ERROR, "No memory for jitter buffer.\n");
-		return -ENOMEM;
+	int rc = 0;
+	memset(jb, 0, sizeof(*jb));
+	jb->sample_duration = 1.0 / samplerate;
+	jb->sample_size = sample_size;
+	jb->target_window_size = (int)(samplerate * target_window_duration);
+	jb->max_window_size = (int)(samplerate * max_window_duration);
+	jb->window_flags = window_flags;
+
+	jb->extra_size = (int)(EXTRA_BUFFER * samplerate);
+	jb->extra_samples = calloc(sample_size, jb->extra_size);
+	if (!jb->extra_samples) {
+		PDEBUG(DJITTER, DEBUG_ERROR, "No memory for frame.\n");
+		rc = -ENOMEM;
+		goto error;
 	}
-	jitter->len = length;

-	jitter_reset(jitter);

-	return 0;
+	/* optionally give a string to be show with the debug */
+	if (name && *name)
+		snprintf(jb->name, sizeof(jb->name) - 1, "(%s) ", name);
+	else
+		snprintf(jb->name, sizeof(jb->name) - 1, "(unnamed %d) ", unnamed_count++);
+
+	jitter_reset(jb);
+
+	PDEBUG(DJITTER, DEBUG_INFO, "%sCreated jitter buffer. (samplerate=%.0f, target_window=%.0fms, max_window=%.0fms, flag:latency=%s flag:repeat=%s)\n", jb->name, samplerate, target_window_duration * 1000.0, max_window_duration * 1000.0, (window_flags & JITTER_FLAG_LATENCY) ? "true" : "false", (window_flags & JITTER_FLAG_REPEAT) ? "true" : "false");
+
+error:
+	if (rc)
+		jitter_destroy(jb);
+	return rc;
 }

-void jitter_reset(jitter_t *jitter)
+/* reset jitter buffer */
+void jitter_reset(jitter_t *jb)
 {
-	memset(jitter->spl, 0, jitter->len * sizeof(sample_t));
+	jitter_frame_t *jf, *temp;

-	/* put write pointer ahead by half of the buffer length */
-	jitter->inptr = jitter->len / 2;
+	PDEBUG(DJITTER, DEBUG_INFO, "%sReset jitter buffer.\n", jb->name);
+
+	/* jitter buffer locked */
+	jb->unlocked = 0;
+
+	/* window becomes invalid */
+	jb->window_valid = 0;
+
+	/* remove all pending frames */
+	jf = jb->frame_list;
+	while(jf) {
+		temp = jf;
+		jf = jf->next;
+		free(temp);
+	}
+	jb->frame_list = NULL;
+
+	/* clear extrapolation buffer */
+	if (jb->extra_samples) {
+		if (jb->sample_size == 1)
+			memset(jb->extra_samples, 0xff, jb->sample_size * jb->extra_size);
+		else
+			memset(jb->extra_samples, 0, jb->sample_size * jb->extra_size);
+	}
+	jb->extra_index = 0;
+
+	/* delay measurement and reduction */
+	jb->delay_counter = 0.0;
+	jb->delay_interval = INITIAL_DELAY_INTERVAL;
+	jb->min_delay_value = -1;
 }

-void jitter_destroy(jitter_t *jitter)
+void jitter_destroy(jitter_t *jb)
 {
-	if (jitter->spl) {
-		free(jitter->spl);
-		jitter->spl = NULL;
+	jitter_reset(jb);
+
+	PDEBUG(DJITTER, DEBUG_INFO, "%sDestroying jitter buffer.\n", jb->name);
+
+	if (jb->extra_samples) {
+		free(jb->extra_samples);
+		jb->extra_samples = NULL;
 	}
 }

@@ -62,64 +177,230 @@ void jitter_destroy(jitter_t *jitter)
 *
 * stop if buffer is completely filled
 */
-void jitter_save(jitter_t *jb, sample_t *samples, int length)
+void jitter_save(jitter_t *jb, void *samples, int length, int has_sequence, uint16_t sequence, uint32_t timestamp, uint32_t ssrc)
 {
-	sample_t *spl;
-	int inptr, outptr, len, space;
-	int i;
+	jitter_frame_t *jf, **jfp;
+	int16_t offset_sequence;
+	int32_t offset_timestamp;

-	spl = jb->spl;
-	inptr = jb->inptr;
-	outptr = jb->outptr;
-	len = jb->len;
-	space = (outptr - inptr + len - 1) % len;
+	/* ignore frames until the buffer is unlocked by jitter_load() */
+	if (!jb->unlocked)
+		return;

-	if (space < length)
-		length = space;
-	for (i = 0; i < length; i++) {
-		spl[inptr++] = *samples++;
-		if (inptr == len)
-			inptr = 0;
+	/* omit frames with no data */
+	if (length < 1)
+		return;
+
+	/* generate sequence and timestamp automatically, if enabled */
+	if (!has_sequence) {
+#ifdef DEBUG_JITTER
+		PDEBUG(DJITTER, DEBUG_DEBUG, "%sSave frame of %d samples (no seqence).\n", jb->name, length);
+#endif
+		sequence = jb->next_sequence;
+		jb->next_sequence++;
+		timestamp = jb->next_timestamp;
+		jb->next_timestamp += length;
+		ssrc = jb->window_ssrc;
+	} else {
+#ifdef HEAVY_DEBUG
+		PDEBUG(DJITTER, DEBUG_DEBUG, "%sSave frame of %d samples (seqence=%u timestamp=%u ssrc=0x%02x).\n", jb->name, length, sequence, timestamp, ssrc);
+#endif
+		jb->next_sequence = sequence + 1;
+		jb->next_timestamp = timestamp + length;
 	}

-	jb->inptr = inptr;
+	/* first packet (with this ssrc) sets window size to target_window_size */
+	if (!jb->window_valid || jb->window_ssrc != ssrc) {
+		if (!jb->window_valid)
+			PDEBUG(DJITTER, DEBUG_DEBUG, "%s Initial frame after init or reset.\n", jb->name);
+		else
+			PDEBUG(DJITTER, DEBUG_DEBUG, "%s SSRC changed.\n", jb->name);
+		// NOTE: Reset must be called before finding the frame location below, because there will be no frame in list anymore!
+		jitter_reset(jb);
+		jb->unlocked = 1;
+		/* when using dynamic jitter buffer, we use half of the target delay */
+		if ((jb->window_flags & JITTER_FLAG_LATENCY)) {
+			jb->window_timestamp = timestamp - (uint32_t)jb->target_window_size / 2;
+		} else {
+			jb->window_timestamp = timestamp - (uint32_t)jb->target_window_size;
+	       	}
+		jb->window_valid = 1;
+		jb->window_ssrc = ssrc;
+	}
+
+	/* find location where to put frame into the list, depending on sequence number */
+	jfp = &jb->frame_list;
+	while(*jfp) {
+		offset_sequence = (int16_t)(sequence - (*jfp)->sequence);
+		/* found double entry */
+		if (offset_sequence == 0) {
+			PDEBUG(DJITTER, DEBUG_DEBUG, "%s Dropping double packet (sequence = %d)\n", jb->name, sequence);
+			return;
+		}
+		/* offset is negative, so we found the position to insert frame */
+		if (offset_sequence < 0)
+			break;
+		jfp = &((*jfp)->next);
+	}
+
+	offset_timestamp = timestamp - jb->window_timestamp;
+#ifdef HEAVY_DEBUG
+	PDEBUG(DJITTER, DEBUG_DEBUG, "%sFrame has offset of %.0fms in jitter buffer.\n", jb->name, (double)offset_timestamp * jb->sample_duration * 1000.0);
+#endif
+
+	/* measure delay */
+	if (jb->min_delay_value < 0 || offset_timestamp < jb->min_delay_value)
+		jb->min_delay_value = offset_timestamp;
+
+	/* if frame is too early (delay ceases), shift window to the future */
+	if (offset_timestamp > jb->max_window_size) {
+		if ((jb->window_flags & JITTER_FLAG_LATENCY)) {
+			PDEBUG(DJITTER, DEBUG_DEBUG, "%s Frame too early: Shift jitter buffer to the future, to make the frame fit to the end. (offset_timestamp(%d) > max_window_size(%d))\n", jb->name, offset_timestamp, jb->max_window_size);
+			/* shift window so it fits to the end of window */
+			jb->window_timestamp = timestamp - jb->max_window_size;
+		} else {
+			PDEBUG(DJITTER, DEBUG_DEBUG, "%s Frame too early: Shift jitter buffer to the future, to make the frame fit to the target delay. (offset_timestamp(%d) > max_window_size(%d))\n", jb->name, offset_timestamp, jb->max_window_size);
+			/* shift window so frame fits to the start of window + target delay */
+			jb->window_timestamp = timestamp - (uint32_t)(jb->target_window_size);
+		}
+	}
+
+	/* is frame is too late, shift window to the past. */
+	if (offset_timestamp < 0) {
+		if ((jb->window_flags & JITTER_FLAG_LATENCY)) {
+			PDEBUG(DJITTER, DEBUG_DEBUG, "%s Frame too late: Shift jitter buffer to the past, and add target window size. (offset_timestamp(%d) < 0)\n", jb->name, offset_timestamp);
+			/* shift window so frame fits to the start of window + half of target delay */
+			jb->window_timestamp = timestamp - (uint32_t)(jb->target_window_size) / 2;
+		} else {
+			PDEBUG(DJITTER, DEBUG_DEBUG, "%s Frame too late: Shift jitter buffer to the past, and add half target window size. (offset_timestamp(%d) < 0)\n", jb->name, offset_timestamp);
+			/* shift window so frame fits to the start of window + target delay */
+			jb->window_timestamp = timestamp - (uint32_t)(jb->target_window_size);
+		}
+	}
+
+	/* insert or append frame */
+#ifdef HEAVY_DEBUG
+	PDEBUG(DJITTER, DEBUG_DEBUG, "%s Store frame\n", jb->name);
+#endif
+	jf = malloc(sizeof(*jf) + length * jb->sample_size);
+	if (!jf) {
+		PDEBUG(DJITTER, DEBUG_ERROR, "No memory for frame.\n");
+		return;
+	}
+	memset(jf, 0, sizeof(*jf)); // note: clear header only
+	jf->sequence = sequence;
+	jf->timestamp = timestamp;
+	memcpy(jf->samples, samples, length * jb->sample_size);
+	jf->length = length;
+	jf->next = *jfp;
+	*jfp = jf;
 }

 /* get audio from jitterbuffer
 */
-void jitter_load(jitter_t *jb, sample_t *samples, int length)
+void jitter_load(jitter_t *jb, void *samples, int length)
 {
-	sample_t *spl;
-	int inptr, outptr, len, fill;
-	int i, ii;
+	jitter_frame_t *jf;
+	int32_t count, count2, index;

-	spl = jb->spl;
-	inptr = jb->inptr;
-	outptr = jb->outptr;
-	len = jb->len;
-	fill = (inptr - outptr + len) % len;
+#ifdef HEAVY_DEBUG
+	PDEBUG(DJITTER, DEBUG_DEBUG, "%sLoad chunk of %d samples.\n", jb->name, length);
+#endif

-	if (fill < length)
-		ii = fill;
-	else
-		ii = length;
+	/* now unlock jitter buffer */
+	jb->unlocked = 1;

-	/* fill what we got */
-	for (i = 0; i < ii; i++) {
-		*samples++ = spl[outptr++];
-		if (outptr == len)
-			outptr = 0;
-	}
-	/* on underrun, fill with silence */
-	for (; i < length; i++) {
-		*samples++ = 0;
+	/* reduce delay */
+	jb->delay_counter += jb->sample_duration * (double)length;
+	if (jb->delay_counter >= jb->delay_interval) {
+		if (jb->min_delay_value >= 0)
+			PDEBUG(DJITTER, DEBUG_DEBUG, "%s Statistics: target_window_delay=%.0fms max_window_delay=%.0fms  current min_delay=%.0fms\n", jb->name, (double)jb->target_window_size * jb->sample_duration * 1000.0, (double)jb->max_window_size * jb->sample_duration * 1000.0, (double)jb->min_delay_value * jb->sample_duration * 1000.0);
+		/* delay reduction, if maximum delay is greater than target jitter window size */
+		if ((jb->window_flags & JITTER_FLAG_LATENCY) && jb->min_delay_value > jb->target_window_size) {
+			PDEBUG(DJITTER, DEBUG_DEBUG, "%s Reducing current minimum delay of %.0fms, because maximum delay is greater than target window size of %.0fms.\n", jb->name, (double)jb->min_delay_value * jb->sample_duration * 1000.0, (double)jb->target_window_size * jb->sample_duration * 1000.0);
+			/* only reduce delay to half of the target window size */
+			jb->window_timestamp += jb->min_delay_value - jb->target_window_size / 2;
+
+		}
+		jb->delay_counter -= jb->delay_interval;
+		jb->delay_interval = REPEAT_DELAY_INTERVAL;
+		jb->min_delay_value = -1;
 	}

-	jb->outptr = outptr;
-}
-
-void jitter_clear(jitter_t *jb)
-{
-	jb->inptr = jb->outptr = 0;
+	/* process all frames until output buffer is loaded */
+	while (length) {
+		/* always get frame with the lowest sequence number (1st frame) */
+		jf = jb->frame_list;
+
+		if (jf) {
+			count = jf->timestamp - jb->window_timestamp;
+			if (count > length)
+				count = length;
+		} else
+			count = length;
+		/* if there is no frame or we have not reached frame's time stamp, extrapolate */
+		if (count > 0) {
+#ifdef HEAVY_DEBUG
+			if (jf)
+				PDEBUG(DJITTER, DEBUG_DEBUG, "%s There is a frame ahead in buffer after %d samples. Interpolating gap.\n", jb->name, jf->timestamp - jb->window_timestamp);
+			else
+				PDEBUG(DJITTER, DEBUG_DEBUG, "%s There is no frame ahead in buffer. Interpolating gap.\n", jb->name);
+#endif
+			/* extrapolate by playing the extrapolation buffer */
+			while (count) {
+				count2 = count;
+				if (count2 > jb->extra_size - jb->extra_index)
+					count2 = jb->extra_size - jb->extra_index;
+				memcpy(samples, (uint8_t *)jb->extra_samples + jb->extra_index * jb->sample_size, count2 * jb->sample_size);
+				jb->extra_index += count2;
+				if (jb->extra_index == jb->extra_size)
+					jb->extra_index = 0;
+				samples = (uint8_t *)samples + count2 * jb->sample_size;
+				length -= count2;
+				jb->window_timestamp += count2;
+				count -= count2;
+			}
+			if (length == 0)
+				return;
+		}
+
+		/* copy samples from frame (what is not in the past) */
+		index = jb->window_timestamp - jf->timestamp;
+		while (index < jf->length) {
+			/* use the lowest value of 'playout length' or 'remaining packet length' */
+			count = length;
+			if (jf->length - index < count)
+				count = jf->length - index;
+			/* if extrapolation is used, limit count to what we can store into buffer */
+			if (jb->extra_samples && jb->extra_size - jb->extra_index < count)
+				count = jb->extra_size - jb->extra_index;
+			/* copy samples from packet to play out, increment sample pointer and decrement length */
+#ifdef HEAVY_DEBUG
+			PDEBUG(DJITTER, DEBUG_DEBUG, "%s Copy data (offset=%u count=%u) from frame (sequence=%u timestamp=%u length=%u).\n", jb->name, index, count, jf->sequence, jf->timestamp, jf->length);
+#endif
+			memcpy(samples, (uint8_t *)jf->samples + index * jb->sample_size, count * jb->sample_size);
+			samples = (uint8_t *)samples + count * jb->sample_size;
+			length -= count;
+			/* copy frame data to extrapolation buffer also, increment index */
+			if ((jb->window_flags & JITTER_FLAG_REPEAT)) {
+				memcpy((uint8_t *)jb->extra_samples + jb->extra_index * jb->sample_size, (uint8_t *)jf->samples + index * jb->sample_size, count * jb->sample_size);
+				jb->extra_index += count;
+				if (jb->extra_index == jb->extra_size)
+					jb->extra_index = 0;
+			}
+			/* increment time stamp */
+			jb->window_timestamp += count;
+			index += count;
+			/* if there was enough to play out, we are done */
+			if (length == 0)
+				return;
+		}
+
+		/* free frame, because all samples are now in the past */
+		jb->frame_list = jf->next;
+		free(jf);
+
+		/* now go for next loop, in case there is still date to play out */
+	}
 }