path: root/drivers/gpu/arm/midgard/mali_kbase_tlstream.c

/*
 *
 * (C) COPYRIGHT 2015 ARM Limited. All rights reserved.
 *
 * This program is free software and is provided to you under the terms of the
 * GNU General Public License version 2 as published by the Free Software
 * Foundation, and any use by you of this program is subject to the terms
 * of such GNU licence.
 *
 * A copy of the licence is included with the program, and can also be obtained
 * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
 * Boston, MA  02110-1301, USA.
 *
 */



#include <linux/anon_inodes.h>
#include <linux/atomic.h>
#include <linux/file.h>
#include <linux/mutex.h>
#include <linux/poll.h>
#include <linux/spinlock.h>
#include <linux/string.h>
#include <linux/stringify.h>
#include <linux/timer.h>
#include <linux/wait.h>

#include <mali_kbase.h>
#include <mali_kbase_jm.h>
#include <mali_kbase_tlstream.h>

/*****************************************************************************/

/* The version of timeline stream. */
#define KBASEP_TLSTREAM_VERSION 1

/* The maximum expected length of string in tracepoint descriptor. */
#define STRLEN_MAX         64 /* bytes */

/* The number of nanoseconds in a second. */
#define NSECS_IN_SEC       1000000000ull /* ns */

/* The number of nanoseconds to wait before autoflushing the stream. */
#define AUTOFLUSH_TIMEOUT  (2ull * NSECS_IN_SEC) /* ns */

/* The period of autoflush checker execution in milliseconds. */
#define AUTOFLUSH_INTERVAL 1000 /* ms */

/* The maximum size of a single packet used by timeline. */
#define PACKET_SIZE        2048 /* bytes */

/* The number of packets used by one timeline stream. */
#define PACKET_COUNT       16

/* The number of bytes reserved for packet header.
 * These value must be defined according to MIPE documentation. */
#define PACKET_HEADER_SIZE 8 /* bytes */

/* The number of bytes reserved for packet sequence number.
 * These value must be defined according to MIPE documentation. */
#define PACKET_NUMBER_SIZE 4 /* bytes */

/* Packet header - first word.
 * These values must be defined according to MIPE documentation. */
#define PACKET_STREAMID_POS  0
#define PACKET_STREAMID_LEN  8
#define PACKET_RSVD1_POS     (PACKET_STREAMID_POS + PACKET_STREAMID_LEN)
#define PACKET_RSVD1_LEN     8
#define PACKET_TYPE_POS      (PACKET_RSVD1_POS + PACKET_RSVD1_LEN)
#define PACKET_TYPE_LEN      3
#define PACKET_CLASS_POS     (PACKET_TYPE_POS + PACKET_TYPE_LEN)
#define PACKET_CLASS_LEN     7
#define PACKET_FAMILY_POS    (PACKET_CLASS_POS + PACKET_CLASS_LEN)
#define PACKET_FAMILY_LEN    6

/* Packet header - second word
 * These values must be defined according to MIPE documentation. */
#define PACKET_LENGTH_POS    0
#define PACKET_LENGTH_LEN    24
#define PACKET_SEQBIT_POS    (PACKET_LENGTH_POS + PACKET_LENGTH_LEN)
#define PACKET_SEQBIT_LEN    1
#define PACKET_RSVD2_POS     (PACKET_SEQBIT_POS + PACKET_SEQBIT_LEN)
#define PACKET_RSVD2_LEN     7

/* Types of streams generated by timeline.
 * Order is significant! Header streams must precede respective body streams. */
enum tl_stream_type {
	TL_STREAM_TYPE_OBJ_HEADER,
	TL_STREAM_TYPE_OBJ_SUMMARY,
	TL_STREAM_TYPE_OBJ,
	TL_STREAM_TYPE_AUX_HEADER,
	TL_STREAM_TYPE_AUX,

	TL_STREAM_TYPE_COUNT
};

/* Timeline packet family ids.
 * Values are significant! Check MIPE documentation. */
enum tl_packet_family {
	TL_PACKET_FAMILY_CTRL = 0, /* control packets */
	TL_PACKET_FAMILY_TL   = 1, /* timeline packets */

	TL_PACKET_FAMILY_COUNT
};

/* Packet classes used in timeline streams.
 * Values are significant! Check MIPE documentation. */
enum tl_packet_class {
	TL_PACKET_CLASS_OBJ = 0, /* timeline objects packet */
	TL_PACKET_CLASS_AUX = 1, /* auxiliary events packet */
};

/* Packet types used in timeline streams.
 * Values are significant! Check MIPE documentation. */
enum tl_packet_type {
	TL_PACKET_TYPE_HEADER  = 0, /* stream's header/directory */
	TL_PACKET_TYPE_BODY    = 1, /* stream's body */
	TL_PACKET_TYPE_SUMMARY = 2, /* stream's summary */
};

/* Message ids of trace events that are recorded in the timeline stream. */
enum tl_msg_id {
	/* Timeline object events. */
	KBASE_TL_NEW_CTX,
	KBASE_TL_NEW_GPU,
	KBASE_TL_NEW_LPU,
	KBASE_TL_NEW_ATOM,
	KBASE_TL_DEL_CTX,
	KBASE_TL_DEL_ATOM,
	KBASE_TL_LIFELINK_LPU_GPU,
	KBASE_TL_RET_GPU_CTX,
	KBASE_TL_RET_ATOM_CTX,
	KBASE_TL_RET_ATOM_LPU,
	KBASE_TL_NRET_GPU_CTX,
	KBASE_TL_NRET_ATOM_CTX,
	KBASE_TL_NRET_ATOM_LPU,

	/* Timeline non-object events. */
	KBASE_AUX_PM_STATE,
	KBASE_AUX_JOB_SOFTSTOP,
	KBASE_AUX_PAGEFAULT,
	KBASE_AUX_PAGESALLOC
};

/*****************************************************************************/

/**
 * struct tl_stream - timeline stream structure
 * @lock: message order lock
 * @buffer: array of buffers
 * @wbi: write buffer index
 * @rbi: read buffer index
 * @numbered: if non-zero stream's packets are sequentially numbered
 * @last_write_time: timestamp indicating last write
 *
 * This structure holds information needed to construct proper packets in the
 * timeline stream. Each message in sequence must bear timestamp that is greater
 * to one in previous message in the same stream. For this reason lock is held
 * throughout the process of message creation. Each stream contains set of
 * buffers. Each buffer will hold one MIPE packet. In case there is no free
 * space required to store incoming message the oldest buffer is discarded.
 * Each packet in timeline body stream has sequence number embedded (this value
 * must increment monotonically and is used by packets receiver to discover
 * buffer overflows.
 */
struct tl_stream {
	spinlock_t lock;

	struct {
		atomic_t size;              /* number of bytes in buffer */
		char     data[PACKET_SIZE]; /* buffer's data */
	} buffer[PACKET_COUNT];

	atomic_t wbi;
	atomic_t rbi;

	int      numbered;
	u64      last_write_time;
};

/**
 * struct tp_desc - tracepoint message descriptor structure
 * @id:        tracepoint ID identifying message in stream
 * @id_str:    human readable version of tracepoint ID
 * @name:      tracepoint description
 * @arg_types: tracepoint's arguments types declaration
 * @arg_names: comma separated list of tracepoint's arguments names
 */
struct tp_desc {
	u32        id;
	const char *id_str;
	const char *name;
	const char *arg_types;
	const char *arg_names;
};

/*****************************************************************************/

/* Configuration of timeline streams generated by kernel.
 * Kernel emit only streams containing either timeline object events or
 * auxiliary events. All streams have stream id value of 1 (as opposed to user
 * space streams that have value of 0). */
static const struct {
	enum tl_packet_family pkt_family;
	enum tl_packet_class  pkt_class;
	enum tl_packet_type   pkt_type;
	unsigned int          stream_id;
} tl_stream_cfg[TL_STREAM_TYPE_COUNT] = {
	{TL_PACKET_FAMILY_TL, TL_PACKET_CLASS_OBJ, TL_PACKET_TYPE_HEADER,  1},
	{TL_PACKET_FAMILY_TL, TL_PACKET_CLASS_OBJ, TL_PACKET_TYPE_SUMMARY, 1},
	{TL_PACKET_FAMILY_TL, TL_PACKET_CLASS_OBJ, TL_PACKET_TYPE_BODY,    1},
	{TL_PACKET_FAMILY_TL, TL_PACKET_CLASS_AUX, TL_PACKET_TYPE_HEADER,  1},
	{TL_PACKET_FAMILY_TL, TL_PACKET_CLASS_AUX, TL_PACKET_TYPE_BODY,    1}
};

/* The timeline streams generated by kernel. */
static struct tl_stream *tl_stream[TL_STREAM_TYPE_COUNT];

/* Autoflush timer. */
static struct timer_list autoflush_timer;

/* If non-zero autoflush timer is active. */
static atomic_t autoflush_timer_active;

/* Reader lock. Only one reader is allowed to have access to the timeline
 * streams at any given time. */
static DEFINE_MUTEX(tl_reader_lock);

/* Indicator of whether the timeline stream file descriptor is already used. */
static atomic_t tlstream_busy = {0};

/* Timeline stream event queue. */
static DECLARE_WAIT_QUEUE_HEAD(tl_event_queue);

/* The timeline stream file operations functions. */
static ssize_t kbasep_tlstream_read(
		struct file *filp,
		char __user *buffer,
		size_t      size,
		loff_t      *f_pos);
static unsigned int kbasep_tlstream_poll(struct file *filp, poll_table *wait);
static int kbasep_tlstream_release(struct inode *inode, struct file *filp);

/* The timeline stream file operations structure. */
static const struct file_operations kbasep_tlstream_fops = {
	.release = kbasep_tlstream_release,
	.read    = kbasep_tlstream_read,
	.poll    = kbasep_tlstream_poll,
};

/* Descriptors of timeline messages transmitted in object events stream. */
static const struct tp_desc tp_desc_obj[] = {
	{
		KBASE_TL_NEW_CTX,
		__stringify(KBASE_TL_NEW_CTX),
		"object ctx is created",
		"@pI",
		"ctx,ctx_nr"
	},
	{
		KBASE_TL_NEW_GPU,
		__stringify(KBASE_TL_NEW_GPU),
		"object gpu is created",
		"@pII",
		"gpu,gpu_id,core_count"
	},
	{
		KBASE_TL_NEW_LPU,
		__stringify(KBASE_TL_NEW_LPU),
		"object lpu is created",
		"@pII",
		"lpu,lpu_nr,lpu_fn"
	},
	{
		KBASE_TL_NEW_ATOM,
		__stringify(KBASE_TL_NEW_ATOM),
		"object atom is created",
		"@pI",
		"atom,atom_nr"
	},
	{
		KBASE_TL_DEL_CTX,
		__stringify(KBASE_TL_DEL_CTX),
		"context is destroyed",
		"@p",
		"context"
	},
	{
		KBASE_TL_DEL_ATOM,
		__stringify(KBASE_TL_DEL_ATOM),
		"atom is destroyed",
		"@p",
		"atom"
	},
	{
		KBASE_TL_LIFELINK_LPU_GPU,
		__stringify(KBASE_TL_LIFELINK_LPU_GPU),
		"lpu is deleted with gpu",
		"@pp",
		"lpu,gpu"
	},
	{
		KBASE_TL_RET_GPU_CTX,
		__stringify(KBASE_TL_RET_GPU_CTX),
		"gpu is retained by context",
		"@pp",
		"gpu,ctx"
	},
	{
		KBASE_TL_RET_ATOM_CTX,
		__stringify(KBASE_TL_RET_ATOM_CTX),
		"atom is retained by context",
		"@pp",
		"atom,ctx"
	},
	{
		KBASE_TL_RET_ATOM_LPU,
		__stringify(KBASE_TL_RET_ATOM_LPU),
		"atom is retained by lpu",
		"@pp",
		"atom,lpu"
	},
	{
		KBASE_TL_NRET_GPU_CTX,
		__stringify(KBASE_TL_NRET_GPU_CTX),
		"gpu is released by context",
		"@pp",
		"gpu,ctx"
	},
	{
		KBASE_TL_NRET_ATOM_CTX,
		__stringify(KBASE_TL_NRET_ATOM_CTX),
		"atom is released by context",
		"@pp",
		"atom,context"
	},
	{
		KBASE_TL_NRET_ATOM_LPU,
		__stringify(KBASE_TL_NRET_ATOM_LPU),
		"atom is released by lpu",
		"@pp",
		"atom,lpu"
	},
};

/* Descriptors of timeline messages transmitted in auxiliary events stream. */
static const struct tp_desc tp_desc_aux[] = {
	{
		KBASE_AUX_PM_STATE,
		__stringify(KBASE_AUX_PM_STATE),
		"PM state",
		"@IL",
		"core_type,core_state_bitset"
	},
	{
		KBASE_AUX_JOB_SOFTSTOP,
		__stringify(KBASE_AUX_JOB_SOFTSTOP),
		"Job soft stop",
		"@I",
		"tag_id"
	},
	{
		KBASE_AUX_PAGEFAULT,
		__stringify(KBASE_AUX_PAGEFAULT),
		"Page fault",
		"@II",
		"as_id,page_cnt"
	},
	{
		KBASE_AUX_PAGESALLOC,
		__stringify(KBASE_AUX_PAGESALLOC),
		"Total alloc pages change",
		"@l",
		"page_cnt_change"
	}
};

#if MALI_UNIT_TEST
/* Number of bytes read by user. */
static atomic_t tlstream_bytes_collected = {0};

/* Number of bytes generated by tracepoint messages. */
static atomic_t tlstream_bytes_generated = {0};
#endif /* MALI_UNIT_TEST */

/*****************************************************************************/

/**
 * kbasep_tlstream_get_timestamp - return timestamp
 *
 * Function returns timestamp value based on raw monotonic timer. Value will
 * wrap around zero in case of overflow.
 * Return: timestamp value
 */
static u64 kbasep_tlstream_get_timestamp(void)
{
	struct timespec ts;
	u64             timestamp;

	getrawmonotonic(&ts);
	timestamp = (u64)ts.tv_sec * NSECS_IN_SEC + ts.tv_nsec;
	return timestamp;
}

/**
 * kbasep_tlstream_write_bytes - write data to message buffer
 * @buffer: buffer where data will be written
 * @pos:    position in the buffer where to place data
 * @bytes:  pointer to buffer holding data
 * @len:    length of data to be written
 *
 * Return: updated position in the buffer
 */
static size_t kbasep_tlstream_write_bytes(
		char       *buffer,
		size_t     pos,
		const void *bytes,
		size_t     len)
{
	KBASE_DEBUG_ASSERT(buffer);
	KBASE_DEBUG_ASSERT(bytes);

	memcpy(&buffer[pos], bytes, len);

	return pos + len;
}

/**
 * kbasep_tlstream_write_string - write string to message buffer
 * @buffer:         buffer where data will be written
 * @pos:            position in the buffer where to place data
 * @string:         pointer to buffer holding the source string
 * @max_write_size: number of bytes that can be stored in buffer
 *
 * Return: updated position in the buffer
 */
static size_t kbasep_tlstream_write_string(
		char       *buffer,
		size_t     pos,
		const char *string,
		size_t     max_write_size)
{
	u32 string_len;

	KBASE_DEBUG_ASSERT(buffer);
	KBASE_DEBUG_ASSERT(string);
	/* Timeline string consists of at least string length and nul
	 * terminator. */
	KBASE_DEBUG_ASSERT(max_write_size >= sizeof(string_len) + sizeof(char));
	max_write_size -= sizeof(string_len);

	string_len = strlcpy(
			&buffer[pos + sizeof(string_len)],
			string,
			max_write_size);
	string_len += sizeof(char);

	/* Make sure that the source string fit into the buffer. */
	KBASE_DEBUG_ASSERT(string_len <= max_write_size);

	/* Update string length. */
	memcpy(&buffer[pos], &string_len, sizeof(string_len));

	return pos + sizeof(string_len) + string_len;
}

/**
 * kbasep_tlstream_write_timestamp - write timestamp to message buffer
 * @buffer: buffer where data will be written
 * @pos:    position in the buffer where to place data
 *
 * Return: updated position in the buffer
 */
static size_t kbasep_tlstream_write_timestamp(void *buffer, size_t pos)
{
	u64 timestamp = kbasep_tlstream_get_timestamp();

	return kbasep_tlstream_write_bytes(
			buffer, pos,
			&timestamp, sizeof(timestamp));
}

/**
 * kbasep_tlstream_put_bits - put bits in a word
 * @word:   pointer to the words being modified
 * @value:  value that shall be written to given position
 * @bitpos: position where value shall be written (in bits)
 * @bitlen: length of value (in bits)
 */
static void kbasep_tlstream_put_bits(
		u32          *word,
		u32          value,
		unsigned int bitpos,
		unsigned int bitlen)
{
	const u32 mask = ((1 << bitlen) - 1) << bitpos;

	KBASE_DEBUG_ASSERT(word);
	KBASE_DEBUG_ASSERT((0 != bitlen) && (32 >= bitlen));
	KBASE_DEBUG_ASSERT((bitpos + bitlen) <= 32);

	*word &= ~mask;
	*word |= ((value << bitpos) & mask);
}

/**
 * kbasep_tlstream_packet_header_setup - setup the packet header
 * @buffer:     pointer to the buffer
 * @pkt_family: packet's family
 * @pkt_type:   packet's type
 * @pkt_class:  packet's class
 * @stream_id:  stream id
 * @numbered:   non-zero if this stream is numbered
 *
 * Function sets up immutable part of packet header in the given buffer.
 */
static void kbasep_tlstream_packet_header_setup(
		char                  *buffer,
		enum tl_packet_family pkt_family,
		enum tl_packet_class  pkt_class,
		enum tl_packet_type   pkt_type,
		unsigned int          stream_id,
		int                   numbered)
{
	u32 word0 = 0;
	u32 word1 = 0;

	KBASE_DEBUG_ASSERT(buffer);
	KBASE_DEBUG_ASSERT(pkt_family == TL_PACKET_FAMILY_TL);
	KBASE_DEBUG_ASSERT(
			(pkt_type == TL_PACKET_TYPE_HEADER)  ||
			(pkt_type == TL_PACKET_TYPE_SUMMARY) ||
			(pkt_type == TL_PACKET_TYPE_BODY));
	KBASE_DEBUG_ASSERT(
			(pkt_class == TL_PACKET_CLASS_OBJ) ||
			(pkt_class == TL_PACKET_CLASS_AUX));

	kbasep_tlstream_put_bits(
			&word0, pkt_family,
			PACKET_FAMILY_POS, PACKET_FAMILY_LEN);
	kbasep_tlstream_put_bits(
			&word0, pkt_class,
			PACKET_CLASS_POS, PACKET_CLASS_LEN);
	kbasep_tlstream_put_bits(
			&word0, pkt_type,
			PACKET_TYPE_POS, PACKET_TYPE_LEN);
	kbasep_tlstream_put_bits(
			&word0, stream_id,
			PACKET_STREAMID_POS, PACKET_STREAMID_LEN);

	if (numbered)
		kbasep_tlstream_put_bits(
				&word1, 1,
				PACKET_SEQBIT_POS, PACKET_SEQBIT_LEN);

	memcpy(&buffer[0],             &word0, sizeof(word0));
	memcpy(&buffer[sizeof(word0)], &word1, sizeof(word1));
}

/**
 * kbasep_tlstream_packet_header_update - update the packet header
 * @buffer:    pointer to the buffer
 * @data_size: amount of data carried in this packet
 *
 * Function updates mutable part of packet header in the given buffer.
 * Note that value of data_size must not including size of the header.
 */
static void kbasep_tlstream_packet_header_update(
		char   *buffer,
		size_t data_size)
{
	u32 word0;
	u32 word1;

	KBASE_DEBUG_ASSERT(buffer);
	CSTD_UNUSED(word0);

	memcpy(&word1, &buffer[sizeof(word0)], sizeof(word1));

	kbasep_tlstream_put_bits(
			&word1, data_size,
			PACKET_LENGTH_POS, PACKET_LENGTH_LEN);

	memcpy(&buffer[sizeof(word0)], &word1, sizeof(word1));
}

/**
 * kbasep_tlstream_packet_number_update - update the packet number
 * @buffer:  pointer to the buffer
 * @counter: value of packet counter for this packet's stream
 *
 * Function updates packet number embedded within the packet placed in the
 * given buffer.
 */
static void kbasep_tlstream_packet_number_update(char *buffer, u32 counter)
{
	KBASE_DEBUG_ASSERT(buffer);

	memcpy(&buffer[PACKET_HEADER_SIZE], &counter, sizeof(counter));
}

/**
 * kbasep_timeline_stream_reset - reset stream
 * @stream:  pointer to the stream structure
 *
 * Function discards all pending messages and resets packet counters.
 */
static void kbasep_timeline_stream_reset(struct tl_stream *stream)
{
	unsigned int i;

	for (i = 0; i < PACKET_COUNT; i++) {
		if (stream->numbered)
			atomic_set(
					&stream->buffer[i].size,
					PACKET_HEADER_SIZE +
					PACKET_NUMBER_SIZE);
		else
			atomic_set(&stream->buffer[i].size, PACKET_HEADER_SIZE);
	}

	atomic_set(&stream->wbi, 0);
	atomic_set(&stream->rbi, 0);
}

/**
 * kbasep_timeline_stream_init - initialize timeline stream
 * @stream:      pointer to the stream structure
 * @stream_type: stream type
 */
static void kbasep_timeline_stream_init(
		struct tl_stream    *stream,
		enum tl_stream_type stream_type)
{
	unsigned int i;

	KBASE_DEBUG_ASSERT(stream);
	KBASE_DEBUG_ASSERT(TL_STREAM_TYPE_COUNT > stream_type);

	spin_lock_init(&stream->lock);

	/* All packets carrying tracepoints shall be numbered. */
	if (TL_PACKET_TYPE_BODY == tl_stream_cfg[stream_type].pkt_type)
		stream->numbered = 1;
	else
		stream->numbered = 0;

	for (i = 0; i < PACKET_COUNT; i++)
		kbasep_tlstream_packet_header_setup(
				stream->buffer[i].data,
				tl_stream_cfg[stream_type].pkt_family,
				tl_stream_cfg[stream_type].pkt_class,
				tl_stream_cfg[stream_type].pkt_type,
				tl_stream_cfg[stream_type].stream_id,
				stream->numbered);

	kbasep_timeline_stream_reset(tl_stream[stream_type]);
}

/**
 * kbasep_timeline_stream_term - terminate timeline stream
 * @stream: pointer to the stream structure
 */
static void kbasep_timeline_stream_term(struct tl_stream *stream)
{
	KBASE_DEBUG_ASSERT(stream);
}

/**
 * kbasep_tlstream_msgbuf_submit - submit packet to the user space
 * @stream:     pointer to the stream structure
 * @wb_idx_raw: write buffer index
 * @wb_size:    length of data stored in current buffer
 *
 * Function updates currently written buffer with packet header. Then write
 * index is incremented and buffer is handled to user space. Parameters
 * of new buffer are returned using provided arguments.
 *
 * Return: length of data in new buffer
 *
 * Warning:  User must update the stream structure with returned value.
 */
static size_t kbasep_tlstream_msgbuf_submit(
		struct tl_stream *stream,
		unsigned int      wb_idx_raw,
		unsigned int      wb_size)
{
	unsigned int rb_idx_raw = atomic_read(&stream->rbi);
	unsigned int wb_idx = wb_idx_raw % PACKET_COUNT;

	kbasep_tlstream_packet_header_update(
			stream->buffer[wb_idx].data,
			wb_size - PACKET_HEADER_SIZE);

	if (stream->numbered)
		kbasep_tlstream_packet_number_update(
				stream->buffer[wb_idx].data,
				wb_idx_raw);

	/* Increasing write buffer index will expose this packet to the reader.
	 * As stream->lock is not taken on reader side we must make sure memory
	 * is updated correctly before this will happen. */
	smp_wmb();
	wb_idx_raw++;
	atomic_set(&stream->wbi, wb_idx_raw);

	/* Inform user that packets are ready for reading. */
	wake_up_interruptible(&tl_event_queue);

	/* Detect and mark overflow in this stream. */
	if (PACKET_COUNT == wb_idx_raw - rb_idx_raw) {
		/* Reader side depends on this increment to correctly handle
		 * overflows. The value shall be updated only if it was not
		 * modified by the reader. The data holding buffer will not be
		 * updated before stream->lock is released, however size of the
		 * buffer will. Make sure this increment is globally visible
		 * before information about selected write buffer size. */
		atomic_cmpxchg(&stream->rbi, rb_idx_raw, rb_idx_raw + 1);
	}

	wb_size = PACKET_HEADER_SIZE;
	if (stream->numbered)
		wb_size += PACKET_NUMBER_SIZE;

	return wb_size;
}

/**
 * kbasep_tlstream_msgbuf_acquire - lock selected stream and reserves buffer
 * @stream_type: type of the stream that shall be locked
 * @msg_size:    message size
 * @flags:       pointer to store flags passed back on stream release
 *
 * Function will lock the stream and reserve the number of bytes requested
 * in msg_size for the user.
 *
 * Return: pointer to the buffer where message can be stored
 *
 * Warning: Stream must be relased with kbasep_tlstream_msgbuf_release().
 *          Only atomic operations are allowed while stream is locked
 *          (i.e. do not use any operation that may sleep).
 */
static char *kbasep_tlstream_msgbuf_acquire(
		enum tl_stream_type stream_type,
		size_t              msg_size,
		unsigned long       *flags)
{
	struct tl_stream *stream;
	unsigned int     wb_idx_raw;
	unsigned int     wb_idx;
	size_t           wb_size;

	KBASE_DEBUG_ASSERT(TL_STREAM_TYPE_COUNT > stream_type);
	KBASE_DEBUG_ASSERT(
			PACKET_SIZE - PACKET_HEADER_SIZE - PACKET_NUMBER_SIZE >=
			msg_size);

	stream = tl_stream[stream_type];

	spin_lock_irqsave(&stream->lock, *flags);

	wb_idx_raw = atomic_read(&stream->wbi);
	wb_idx     = wb_idx_raw % PACKET_COUNT;
	wb_size    = atomic_read(&stream->buffer[wb_idx].size);

	/* Select next buffer if data will not fit into current one. */
	if (PACKET_SIZE < wb_size + msg_size) {
		wb_size = kbasep_tlstream_msgbuf_submit(
				stream, wb_idx_raw, wb_size);
		wb_idx  = (wb_idx_raw + 1) % PACKET_COUNT;
	}

	/* Reserve space in selected buffer. */
	atomic_set(&stream->buffer[wb_idx].size, wb_size + msg_size);

#if MALI_UNIT_TEST
	atomic_add(msg_size, &tlstream_bytes_generated);
#endif /* MALI_UNIT_TEST */

	return &stream->buffer[wb_idx].data[wb_size];
}

/**
 * kbasep_tlstream_msgbuf_release - unlock selected stream
 * @stream_type:  type of the stream that shall be locked
 * @flags:        value obtained during stream acquire
 *
 * Function releases stream that has been previously locked with a call to
 * kbasep_tlstream_msgbuf_acquire().
 */
static void kbasep_tlstream_msgbuf_release(
		enum tl_stream_type stream_type,
		unsigned long       flags)
{
	struct tl_stream *stream;

	KBASE_DEBUG_ASSERT(TL_STREAM_TYPE_COUNT > stream_type);

	stream = tl_stream[stream_type];
	stream->last_write_time = kbasep_tlstream_get_timestamp();

	spin_unlock_irqrestore(&stream->lock, flags);
}

/*****************************************************************************/

/**
 * kbasep_tlstream_flush_stream - flush stream
 * @stype:  type of stream to be flushed
 *
 * Flush pending data in timeline stream.
 */
static void kbasep_tlstream_flush_stream(enum tl_stream_type stype)
{
	struct tl_stream *stream = tl_stream[stype];
	unsigned long    flags;
	unsigned int     wb_idx_raw;
	unsigned int     wb_idx;
	size_t           wb_size;
	size_t           min_size = PACKET_HEADER_SIZE;

	if (stream->numbered)
		min_size += PACKET_NUMBER_SIZE;

	spin_lock_irqsave(&stream->lock, flags);

	wb_idx_raw = atomic_read(&stream->wbi);
	wb_idx     = wb_idx_raw % PACKET_COUNT;
	wb_size    = atomic_read(&stream->buffer[wb_idx].size);

	if (wb_size > min_size) {
		wb_size = kbasep_tlstream_msgbuf_submit(
				stream, wb_idx_raw, wb_size);
		wb_idx = (wb_idx_raw + 1) % PACKET_COUNT;
		atomic_set(&stream->buffer[wb_idx].size, wb_size);
	}
	spin_unlock_irqrestore(&stream->lock, flags);
}

/**
 * kbasep_tlstream_autoflush_timer_callback - autoflush timer callback
 * @data:  unused
 *
 * Timer is executed periodically to check if any of the stream contains
 * buffer ready to be submitted to user space.
 */
static void kbasep_tlstream_autoflush_timer_callback(unsigned long data)
{
	u64                 timestamp = kbasep_tlstream_get_timestamp();
	enum tl_stream_type stype;
	int                 rcode;

	CSTD_UNUSED(data);

	for (stype = 0; stype < TL_STREAM_TYPE_COUNT; stype++) {
		struct tl_stream *stream = tl_stream[stype];
		unsigned long    flags;
		unsigned int     wb_idx_raw;
		unsigned int     wb_idx;
		size_t           wb_size;
		size_t           min_size = PACKET_HEADER_SIZE;

		if (stream->numbered)
			min_size += PACKET_NUMBER_SIZE;

		spin_lock_irqsave(&stream->lock, flags);

		wb_idx_raw = atomic_read(&stream->wbi);
		wb_idx     = wb_idx_raw % PACKET_COUNT;
		wb_size    = atomic_read(&stream->buffer[wb_idx].size);

		if (
				(wb_size > min_size) &&
				(
				 timestamp - stream->last_write_time >
				 AUTOFLUSH_TIMEOUT)) {

			wb_size = kbasep_tlstream_msgbuf_submit(
					stream, wb_idx_raw, wb_size);
			wb_idx = (wb_idx_raw + 1) % PACKET_COUNT;
			atomic_set(&stream->buffer[wb_idx].size, wb_size);
		}
		spin_unlock_irqrestore(&stream->lock, flags);
	}

	if (atomic_read(&autoflush_timer_active))
		rcode = mod_timer(
				&autoflush_timer,
				jiffies + msecs_to_jiffies(AUTOFLUSH_INTERVAL));
	CSTD_UNUSED(rcode);
}

/**
 * kbasep_tlstream_packet_pending - check timeline streams for pending packets
 * @stype:      pointer to variable where stream type will be placed
 * @rb_idx_raw: pointer to variable where read buffer index will be placed
 *
 * Function checks all streams for pending packets. It will stop as soon as
 * packet ready to be submitted to user space is detected. Variables under
 * pointers, passed as the parameters to this function will be updated with
 * values pointing to right stream and buffer.
 *
 * Return: non-zero if any of timeline streams has at last one packet ready
 */
static int kbasep_tlstream_packet_pending(
		enum tl_stream_type *stype,
		unsigned int        *rb_idx_raw)
{
	int pending = 0;

	KBASE_DEBUG_ASSERT(stype);
	KBASE_DEBUG_ASSERT(rb_idx_raw);

	for (
			*stype = 0;
			(*stype < TL_STREAM_TYPE_COUNT) && !pending;
			(*stype)++) {
		if (NULL != tl_stream[*stype]) {
			*rb_idx_raw = atomic_read(&tl_stream[*stype]->rbi);
			/* Read buffer index may be updated by writer in case of
			 * overflow. Read and write buffer indexes must be
			 * loaded in correct order. */
			smp_rmb();
			if (atomic_read(&tl_stream[*stype]->wbi) != *rb_idx_raw)
				pending = 1;
		}
	}
	(*stype)--;

	return pending;
}

/**
 * kbasep_tlstream_read - copy data from streams to buffer provided by user
 * @filp:   pointer to file structure (unused)
 * @buffer: pointer to the buffer provided by user
 * @size:   maximum amount of data that can be stored in the buffer
 * @f_pos:  pointer to file offset (unused)
 *
 * Return: number of bytes stored in the buffer
 */
static ssize_t kbasep_tlstream_read(
		struct file *filp,
		char __user *buffer,
		size_t      size,
		loff_t      *f_pos)
{
	ssize_t copy_len = 0;

	KBASE_DEBUG_ASSERT(filp);
	KBASE_DEBUG_ASSERT(buffer);
	KBASE_DEBUG_ASSERT(f_pos);
	CSTD_UNUSED(filp);

	if ((0 > *f_pos) || (PACKET_SIZE > size))
		return -EINVAL;

	mutex_lock(&tl_reader_lock);

	while (copy_len < size) {
		enum tl_stream_type stype;
		unsigned int        rb_idx_raw;
		unsigned int        rb_idx;
		size_t              rb_size;

		/* If we don't have any data yet, wait for packet to be
		 * submitted. If we already read some packets and there is no
		 * packet pending return back to user. */
		if (0 < copy_len) {
			if (!kbasep_tlstream_packet_pending(
						&stype,
						&rb_idx_raw))
				break;
		} else {
			if (wait_event_interruptible(
						tl_event_queue,
						kbasep_tlstream_packet_pending(
							&stype,
							&rb_idx_raw))) {
				copy_len = -ERESTARTSYS;
				break;
			}
		}

		/* Check if this packet fits into the user buffer.
		 * If so copy its content. */
		rb_idx = rb_idx_raw % PACKET_COUNT;
		rb_size = atomic_read(&tl_stream[stype]->buffer[rb_idx].size);
		if (rb_size > size - copy_len)
			break;
		if (copy_to_user(
					&buffer[copy_len],
					tl_stream[stype]->buffer[rb_idx].data,
					rb_size)) {
			copy_len = -EFAULT;
			break;
		}

		/* Verify if there was no overflow in selected stream. Make sure
		 * that if incorrect size was used we will know about it. */
		smp_rmb();
		if (atomic_read(&tl_stream[stype]->rbi) == rb_idx_raw) {
			copy_len += rb_size;
			atomic_inc(&tl_stream[stype]->rbi);

#if MALI_UNIT_TEST
			atomic_add(rb_size, &tlstream_bytes_collected);
#endif /* MALI_UNIT_TEST */
		}
	}

	mutex_unlock(&tl_reader_lock);

	return copy_len;
}

/**
 * kbasep_tlstream_poll - poll timeline stream for packets
 * @filp: pointer to file structure
 * @wait: pointer to poll table
 * Return: POLLIN if data can be read without blocking, otherwise zero
 */
static unsigned int kbasep_tlstream_poll(struct file *filp, poll_table *wait)
{
	enum tl_stream_type stream_type;
	unsigned int        rb_idx;

	KBASE_DEBUG_ASSERT(filp);
	KBASE_DEBUG_ASSERT(wait);

	poll_wait(filp, &tl_event_queue, wait);
	if (kbasep_tlstream_packet_pending(&stream_type, &rb_idx))
		return POLLIN;
	return 0;
}

/**
 * kbasep_tlstream_release - release timeline stream descriptor
 * @inode: pointer to inode structure
 * @filp:  pointer to file structure
 *
 * Return always return zero
 */
static int kbasep_tlstream_release(struct inode *inode, struct file *filp)
{
	KBASE_DEBUG_ASSERT(inode);
	KBASE_DEBUG_ASSERT(filp);
	CSTD_UNUSED(inode);
	CSTD_UNUSED(filp);
	atomic_set(&tlstream_busy, 0);
	return 0;
}

/**
 * kbasep_tlstream_timeline_header - prepare timeline header stream packet
 * @stream_type: type of the stream that will carry header data
 * @tp_desc:     pointer to array with tracepoint descriptors
 * @tp_count:    number of descriptors in the given array
 *
 * Functions fills in information about tracepoints stored in body stream
 * associated with this header stream.
 */
static void kbasep_tlstream_timeline_header(
		enum tl_stream_type  stream_type,
		const struct tp_desc *tp_desc,
		u32                  tp_count)
{
	const u8      tv = KBASEP_TLSTREAM_VERSION; /* tlstream version */
	const u8      ps = sizeof(void *); /* pointer size */
	size_t        msg_size = sizeof(tv) + sizeof(ps) + sizeof(tp_count);
	char          *buffer;
	size_t        pos = 0;
	unsigned long flags;
	unsigned int  i;

	KBASE_DEBUG_ASSERT(TL_STREAM_TYPE_COUNT > stream_type);
	KBASE_DEBUG_ASSERT(tp_desc);

	/* Calculate the size of the timeline message. */
	for (i = 0; i < tp_count; i++) {
		msg_size += sizeof(tp_desc[i].id);
		msg_size +=
			strnlen(tp_desc[i].id_str,    STRLEN_MAX) +
			sizeof(char) + sizeof(u32);
		msg_size +=
			strnlen(tp_desc[i].name,      STRLEN_MAX) +
			sizeof(char) + sizeof(u32);
		msg_size +=
			strnlen(tp_desc[i].arg_types, STRLEN_MAX) +
			sizeof(char) + sizeof(u32);
		msg_size +=
			strnlen(tp_desc[i].arg_names, STRLEN_MAX) +
			sizeof(char) + sizeof(u32);
	}

	KBASE_DEBUG_ASSERT(PACKET_SIZE - PACKET_HEADER_SIZE >= msg_size);

	buffer = kbasep_tlstream_msgbuf_acquire(stream_type, msg_size, &flags);
	KBASE_DEBUG_ASSERT(buffer);

	pos = kbasep_tlstream_write_bytes(buffer, pos, &tv, sizeof(tv));
	pos = kbasep_tlstream_write_bytes(buffer, pos, &ps, sizeof(ps));
	pos = kbasep_tlstream_write_bytes(
			buffer, pos, &tp_count, sizeof(tp_count));

	for (i = 0; i < tp_count; i++) {
		pos = kbasep_tlstream_write_bytes(
				buffer, pos,
				&tp_desc[i].id, sizeof(tp_desc[i].id));
		pos = kbasep_tlstream_write_string(
				buffer, pos,
				tp_desc[i].id_str, msg_size - pos);
		pos = kbasep_tlstream_write_string(
				buffer, pos,
				tp_desc[i].name, msg_size - pos);
		pos = kbasep_tlstream_write_string(
				buffer, pos,
				tp_desc[i].arg_types, msg_size - pos);
		pos = kbasep_tlstream_write_string(
				buffer, pos,
				tp_desc[i].arg_names, msg_size - pos);
	}

	KBASE_DEBUG_ASSERT(msg_size == pos);

	kbasep_tlstream_msgbuf_release(stream_type, flags);

	/* We don't expect any more data to be read in this stream.
	 * As header stream must be read before its associated body stream,
	 * make this packet visible to the user straightaway. */
	kbasep_tlstream_flush_stream(stream_type);
}

/*****************************************************************************/

int kbase_tlstream_init(void)
{
	enum tl_stream_type i;
	int                 rcode;

	/* Prepare stream structures. */
	for (i = 0; i < TL_STREAM_TYPE_COUNT; i++) {
		tl_stream[i] = kmalloc(sizeof(**tl_stream), GFP_KERNEL);
		if (!tl_stream[i])
			break;
		kbasep_timeline_stream_init(tl_stream[i], i);
	}
	if (TL_STREAM_TYPE_COUNT > i) {
		for (; i > 0; i--) {
			kbasep_timeline_stream_term(tl_stream[i - 1]);
			kfree(tl_stream[i - 1]);
		}
		return -ENOMEM;
	}

	/* Initialize autoflush timer. */
	atomic_set(&autoflush_timer_active, 1);
	setup_timer(&autoflush_timer,
			kbasep_tlstream_autoflush_timer_callback,
			0);
	rcode = mod_timer(
			&autoflush_timer,
			jiffies + msecs_to_jiffies(AUTOFLUSH_INTERVAL));
	CSTD_UNUSED(rcode);

	return 0;
}

void kbase_tlstream_term(void)
{
	enum tl_stream_type i;

	atomic_set(&autoflush_timer_active, 0);
	del_timer_sync(&autoflush_timer);

	for (i = 0; i < TL_STREAM_TYPE_COUNT; i++) {
		kbasep_timeline_stream_term(tl_stream[i]);
		kfree(tl_stream[i]);
	}
}

int kbase_tlstream_acquire(struct kbase_context *kctx, int *fd)
{
	if (0 == atomic_cmpxchg(&tlstream_busy, 0, 1)) {
		*fd = anon_inode_getfd(
				"[mali_tlstream]",
				&kbasep_tlstream_fops,
				kctx,
				O_RDONLY | O_CLOEXEC);
		if (0 > *fd) {
			atomic_set(&tlstream_busy, 0);
			return *fd;
		}

		/* Reset and initialize header streams. */
		kbasep_timeline_stream_reset(
				tl_stream[TL_STREAM_TYPE_OBJ_HEADER]);
		kbasep_timeline_stream_reset(
				tl_stream[TL_STREAM_TYPE_OBJ_SUMMARY]);
		kbasep_timeline_stream_reset(
				tl_stream[TL_STREAM_TYPE_AUX_HEADER]);
		kbasep_tlstream_timeline_header(
				TL_STREAM_TYPE_OBJ_HEADER,
				tp_desc_obj,
				ARRAY_SIZE(tp_desc_obj));
		kbasep_tlstream_timeline_header(
				TL_STREAM_TYPE_AUX_HEADER,
				tp_desc_aux,
				ARRAY_SIZE(tp_desc_aux));
	} else {
		*fd = -EBUSY;
	}

	return 0;
}

void kbase_tlstream_flush_streams(void)
{
	enum tl_stream_type stype;

	for (stype = 0; stype < TL_STREAM_TYPE_COUNT; stype++)
		kbasep_tlstream_flush_stream(stype);
}

void kbase_tlstream_reset_body_streams(void)
{
	kbasep_timeline_stream_reset(
			tl_stream[TL_STREAM_TYPE_OBJ]);
	kbasep_timeline_stream_reset(
			tl_stream[TL_STREAM_TYPE_AUX]);
}

#if MALI_UNIT_TEST
void kbase_tlstream_stats(u32 *bytes_collected, u32 *bytes_generated)
{
	KBASE_DEBUG_ASSERT(bytes_collected);
	KBASE_DEBUG_ASSERT(bytes_generated);
	*bytes_collected = atomic_read(&tlstream_bytes_collected);
	*bytes_generated = atomic_read(&tlstream_bytes_generated);
}
#endif /* MALI_UNIT_TEST */

/*****************************************************************************/

void kbase_tlstream_tl_summary_new_ctx(void *context, u32 nr)
{
	const u32     msg_id = KBASE_TL_NEW_CTX;
	const size_t  msg_size =
		sizeof(msg_id) + sizeof(u64) + sizeof(context) + sizeof(nr);
	unsigned long flags;
	char          *buffer;
	size_t        pos = 0;

	buffer = kbasep_tlstream_msgbuf_acquire(
			TL_STREAM_TYPE_OBJ_SUMMARY,
			msg_size, &flags);
	KBASE_DEBUG_ASSERT(buffer);

	pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id));
	pos = kbasep_tlstream_write_timestamp(buffer, pos);
	pos = kbasep_tlstream_write_bytes(
			buffer, pos, &context, sizeof(context));
	pos = kbasep_tlstream_write_bytes(
			buffer, pos, &nr, sizeof(nr));
	KBASE_DEBUG_ASSERT(msg_size == pos);

	kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ_SUMMARY, flags);
}

void kbase_tlstream_tl_summary_new_gpu(void *gpu, u32 id, u32 core_count)
{
	const u32     msg_id = KBASE_TL_NEW_GPU;
	const size_t  msg_size =
		sizeof(msg_id) + sizeof(u64) + sizeof(gpu) + sizeof(id) +
		sizeof(core_count);
	unsigned long flags;
	char          *buffer;
	size_t        pos = 0;

	buffer = kbasep_tlstream_msgbuf_acquire(
			TL_STREAM_TYPE_OBJ_SUMMARY,
			msg_size, &flags);
	KBASE_DEBUG_ASSERT(buffer);

	pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id));
	pos = kbasep_tlstream_write_timestamp(buffer, pos);
	pos = kbasep_tlstream_write_bytes(
			buffer, pos, &gpu, sizeof(gpu));
	pos = kbasep_tlstream_write_bytes(
			buffer, pos, &id, sizeof(id));
	pos = kbasep_tlstream_write_bytes(
			buffer, pos, &core_count, sizeof(core_count));
	KBASE_DEBUG_ASSERT(msg_size == pos);

	kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ_SUMMARY, flags);
}

void kbase_tlstream_tl_summary_new_lpu(void *lpu, u32 nr, u32 fn)
{
	const u32     msg_id = KBASE_TL_NEW_LPU;
	const size_t  msg_size =
		sizeof(msg_id) + sizeof(u64) + sizeof(lpu) + sizeof(nr) +
		sizeof(fn);
	unsigned long flags;
	char          *buffer;
	size_t        pos = 0;

	buffer = kbasep_tlstream_msgbuf_acquire(
			TL_STREAM_TYPE_OBJ_SUMMARY,
			msg_size, &flags);
	KBASE_DEBUG_ASSERT(buffer);

	pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id));
	pos = kbasep_tlstream_write_timestamp(buffer, pos);
	pos = kbasep_tlstream_write_bytes(
			buffer, pos, &lpu, sizeof(lpu));
	pos = kbasep_tlstream_write_bytes(
			buffer, pos, &nr, sizeof(nr));
	pos = kbasep_tlstream_write_bytes(
			buffer, pos, &fn, sizeof(fn));
	KBASE_DEBUG_ASSERT(msg_size == pos);

	kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ_SUMMARY, flags);
}

void kbase_tlstream_tl_summary_lifelink_lpu_gpu(void *lpu, void *gpu)
{
	const u32     msg_id = KBASE_TL_LIFELINK_LPU_GPU;
	const size_t  msg_size =
		sizeof(msg_id) + sizeof(u64) + sizeof(lpu) + sizeof(gpu);
	unsigned long flags;
	char          *buffer;
	size_t        pos = 0;

	buffer = kbasep_tlstream_msgbuf_acquire(
			TL_STREAM_TYPE_OBJ_SUMMARY,
			msg_size, &flags);
	KBASE_DEBUG_ASSERT(buffer);

	pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id));
	pos = kbasep_tlstream_write_timestamp(buffer, pos);
	pos = kbasep_tlstream_write_bytes(
			buffer, pos, &lpu, sizeof(lpu));
	pos = kbasep_tlstream_write_bytes(
			buffer, pos, &gpu, sizeof(gpu));
	KBASE_DEBUG_ASSERT(msg_size == pos);

	kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ_SUMMARY, flags);
}

/*****************************************************************************/

void kbase_tlstream_tl_new_ctx(void *context, u32 nr)
{
	const u32     msg_id = KBASE_TL_NEW_CTX;
	const size_t  msg_size =
		sizeof(msg_id) + sizeof(u64) + sizeof(context) + sizeof(nr);
	unsigned long flags;
	char          *buffer;
	size_t        pos = 0;

	buffer = kbasep_tlstream_msgbuf_acquire(
			TL_STREAM_TYPE_OBJ,
			msg_size, &flags);
	KBASE_DEBUG_ASSERT(buffer);

	pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id));
	pos = kbasep_tlstream_write_timestamp(buffer, pos);
	pos = kbasep_tlstream_write_bytes(
			buffer, pos, &context, sizeof(context));
	pos = kbasep_tlstream_write_bytes(
			buffer, pos, &nr, sizeof(nr));
	KBASE_DEBUG_ASSERT(msg_size == pos);

	kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags);
}

void kbase_tlstream_tl_new_atom(void *atom, u32 nr)
{
	const u32     msg_id = KBASE_TL_NEW_ATOM;
	const size_t  msg_size =
		sizeof(msg_id) + sizeof(u64) + sizeof(atom) + sizeof(nr);
	unsigned long flags;
	char          *buffer;
	size_t        pos = 0;

	buffer = kbasep_tlstream_msgbuf_acquire(
			TL_STREAM_TYPE_OBJ,
			msg_size, &flags);
	KBASE_DEBUG_ASSERT(buffer);

	pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id));
	pos = kbasep_tlstream_write_timestamp(buffer, pos);
	pos = kbasep_tlstream_write_bytes(
			buffer, pos, &atom, sizeof(atom));
	pos = kbasep_tlstream_write_bytes(
			buffer, pos, &nr, sizeof(nr));
	KBASE_DEBUG_ASSERT(msg_size == pos);

	kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags);
}

void kbase_tlstream_tl_del_ctx(void *context)
{
	const u32     msg_id = KBASE_TL_DEL_CTX;
	const size_t  msg_size =
		sizeof(msg_id) + sizeof(u64) + sizeof(context);
	unsigned long flags;
	char          *buffer;
	size_t        pos = 0;

	buffer = kbasep_tlstream_msgbuf_acquire(
			TL_STREAM_TYPE_OBJ,
			msg_size, &flags);
	KBASE_DEBUG_ASSERT(buffer);

	pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id));
	pos = kbasep_tlstream_write_timestamp(buffer, pos);
	pos = kbasep_tlstream_write_bytes(
			buffer, pos, &context, sizeof(context));
	KBASE_DEBUG_ASSERT(msg_size == pos);

	kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags);
}

void kbase_tlstream_tl_del_atom(void *atom)
{
	const u32     msg_id = KBASE_TL_DEL_ATOM;
	const size_t  msg_size =
		sizeof(msg_id) + sizeof(u64) + sizeof(atom);
	unsigned long flags;
	char          *buffer;
	size_t        pos = 0;

	buffer = kbasep_tlstream_msgbuf_acquire(
			TL_STREAM_TYPE_OBJ,
			msg_size, &flags);
	KBASE_DEBUG_ASSERT(buffer);

	pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id));
	pos = kbasep_tlstream_write_timestamp(buffer, pos);
	pos = kbasep_tlstream_write_bytes(
			buffer, pos, &atom, sizeof(atom));
	KBASE_DEBUG_ASSERT(msg_size == pos);

	kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags);
}

void kbase_tlstream_tl_ret_gpu_ctx(void *gpu, void *context)
{
	const u32     msg_id = KBASE_TL_RET_GPU_CTX;
	const size_t  msg_size =
		sizeof(msg_id) + sizeof(u64) + sizeof(gpu) + sizeof(context);
	unsigned long flags;
	char          *buffer;
	size_t        pos = 0;

	buffer = kbasep_tlstream_msgbuf_acquire(
			TL_STREAM_TYPE_OBJ,
			msg_size, &flags);
	KBASE_DEBUG_ASSERT(buffer);

	pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id));
	pos = kbasep_tlstream_write_timestamp(buffer, pos);
	pos = kbasep_tlstream_write_bytes(
			buffer, pos, &gpu, sizeof(gpu));
	pos = kbasep_tlstream_write_bytes(
			buffer, pos, &context, sizeof(context));
	KBASE_DEBUG_ASSERT(msg_size == pos);

	kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags);
}

void kbase_tlstream_tl_ret_atom_ctx(void *atom, void *context)
{
	const u32     msg_id = KBASE_TL_RET_ATOM_CTX;
	const size_t  msg_size =
		sizeof(msg_id) + sizeof(u64) + sizeof(atom) + sizeof(context);
	unsigned long flags;
	char          *buffer;
	size_t        pos = 0;

	buffer = kbasep_tlstream_msgbuf_acquire(
			TL_STREAM_TYPE_OBJ,
			msg_size, &flags);
	KBASE_DEBUG_ASSERT(buffer);

	pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id));
	pos = kbasep_tlstream_write_timestamp(buffer, pos);
	pos = kbasep_tlstream_write_bytes(
			buffer, pos, &atom, sizeof(atom));
	pos = kbasep_tlstream_write_bytes(
			buffer, pos, &context, sizeof(context));
	KBASE_DEBUG_ASSERT(msg_size == pos);

	kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags);
}

void kbase_tlstream_tl_ret_atom_lpu(void *atom, void *lpu)
{
	const u32     msg_id = KBASE_TL_RET_ATOM_LPU;
	const size_t  msg_size =
		sizeof(msg_id) + sizeof(u64) + sizeof(atom) + sizeof(lpu);
	unsigned long flags;
	char          *buffer;
	size_t        pos = 0;

	buffer = kbasep_tlstream_msgbuf_acquire(
			TL_STREAM_TYPE_OBJ,
			msg_size, &flags);
	KBASE_DEBUG_ASSERT(buffer);

	pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id));
	pos = kbasep_tlstream_write_timestamp(buffer, pos);
	pos = kbasep_tlstream_write_bytes(
			buffer, pos, &atom, sizeof(atom));
	pos = kbasep_tlstream_write_bytes(
			buffer, pos, &lpu, sizeof(lpu));
	KBASE_DEBUG_ASSERT(msg_size == pos);

	kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags);
}

void kbase_tlstream_tl_nret_gpu_ctx(void *gpu, void *context)
{
	const u32     msg_id = KBASE_TL_NRET_GPU_CTX;
	const size_t  msg_size =
		sizeof(msg_id) + sizeof(u64) + sizeof(gpu) + sizeof(context);
	unsigned long flags;
	char          *buffer;
	size_t        pos = 0;

	buffer = kbasep_tlstream_msgbuf_acquire(
			TL_STREAM_TYPE_OBJ,
			msg_size, &flags);
	KBASE_DEBUG_ASSERT(buffer);

	pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id));
	pos = kbasep_tlstream_write_timestamp(buffer, pos);
	pos = kbasep_tlstream_write_bytes(
			buffer, pos, &gpu, sizeof(gpu));
	pos = kbasep_tlstream_write_bytes(
			buffer, pos, &context, sizeof(context));
	KBASE_DEBUG_ASSERT(msg_size == pos);

	kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags);
}

void kbase_tlstream_tl_nret_atom_ctx(void *atom, void *context)
{
	const u32     msg_id = KBASE_TL_NRET_ATOM_CTX;
	const size_t  msg_size =
		sizeof(msg_id) + sizeof(u64) + sizeof(atom) + sizeof(context);
	unsigned long flags;
	char          *buffer;
	size_t        pos = 0;

	buffer = kbasep_tlstream_msgbuf_acquire(
			TL_STREAM_TYPE_OBJ,
			msg_size, &flags);
	KBASE_DEBUG_ASSERT(buffer);

	pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id));
	pos = kbasep_tlstream_write_timestamp(buffer, pos);
	pos = kbasep_tlstream_write_bytes(
			buffer, pos, &atom, sizeof(atom));
	pos = kbasep_tlstream_write_bytes(
			buffer, pos, &context, sizeof(context));
	KBASE_DEBUG_ASSERT(msg_size == pos);

	kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags);
}

void kbase_tlstream_tl_nret_atom_lpu(void *atom, void *lpu)
{
	const u32     msg_id = KBASE_TL_NRET_ATOM_LPU;
	const size_t  msg_size =
		sizeof(msg_id) + sizeof(u64) + sizeof(atom) + sizeof(lpu);
	unsigned long flags;
	char          *buffer;
	size_t        pos = 0;

	buffer = kbasep_tlstream_msgbuf_acquire(
			TL_STREAM_TYPE_OBJ,
			msg_size, &flags);
	KBASE_DEBUG_ASSERT(buffer);

	pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id));
	pos = kbasep_tlstream_write_timestamp(buffer, pos);
	pos = kbasep_tlstream_write_bytes(
			buffer, pos, &atom, sizeof(atom));
	pos = kbasep_tlstream_write_bytes(
			buffer, pos, &lpu, sizeof(lpu));
	KBASE_DEBUG_ASSERT(msg_size == pos);

	kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags);
}

/*****************************************************************************/

void kbase_tlstream_aux_pm_state(u32 core_type, u64 state)
{
	const u32     msg_id = KBASE_AUX_PM_STATE;
	const size_t  msg_size =
		sizeof(msg_id) + sizeof(u64) + sizeof(core_type) +
		sizeof(state);
	unsigned long flags;
	char          *buffer;
	size_t        pos = 0;

	buffer = kbasep_tlstream_msgbuf_acquire(
			TL_STREAM_TYPE_AUX,
			msg_size, &flags);
	KBASE_DEBUG_ASSERT(buffer);

	pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id));
	pos = kbasep_tlstream_write_timestamp(buffer, pos);
	pos = kbasep_tlstream_write_bytes(
			buffer, pos, &core_type, sizeof(core_type));
	pos = kbasep_tlstream_write_bytes(buffer, pos, &state, sizeof(state));
	KBASE_DEBUG_ASSERT(msg_size == pos);

	kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_AUX, flags);
}

void kbase_tlstream_aux_job_softstop(u32 js_id)
{
	const u32     msg_id = KBASE_AUX_JOB_SOFTSTOP;
	const size_t  msg_size =
		sizeof(msg_id) + sizeof(u64) + sizeof(js_id);
	unsigned long flags;
	char          *buffer;
	size_t        pos = 0;

	buffer = kbasep_tlstream_msgbuf_acquire(
			TL_STREAM_TYPE_AUX,
			msg_size, &flags);
	KBASE_DEBUG_ASSERT(buffer);

	pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id));
	pos = kbasep_tlstream_write_timestamp(buffer, pos);
	pos = kbasep_tlstream_write_bytes(buffer, pos, &js_id, sizeof(js_id));
	KBASE_DEBUG_ASSERT(msg_size == pos);

	kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_AUX, flags);
}

void kbase_tlstream_aux_pagefault(u32 mmu_as, u32 page_count)
{
	const u32     msg_id = KBASE_AUX_PAGEFAULT;
	const size_t  msg_size =
		sizeof(msg_id) + sizeof(u64) + sizeof(mmu_as) +
		sizeof(page_count);
	unsigned long flags;
	char          *buffer;
	size_t        pos = 0;

	buffer = kbasep_tlstream_msgbuf_acquire(
			TL_STREAM_TYPE_AUX, msg_size, &flags);
	KBASE_DEBUG_ASSERT(buffer);

	pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id));
	pos = kbasep_tlstream_write_timestamp(buffer, pos);
	pos = kbasep_tlstream_write_bytes(buffer, pos, &mmu_as, sizeof(mmu_as));
	pos = kbasep_tlstream_write_bytes(
			buffer, pos, &page_count, sizeof(page_count));
	KBASE_DEBUG_ASSERT(msg_size == pos);

	kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_AUX, flags);
}

void kbase_tlstream_aux_pagesalloc(s64 page_count_change)
{
	const u32     msg_id = KBASE_AUX_PAGESALLOC;
	const size_t  msg_size =
		sizeof(msg_id) + sizeof(u64) + sizeof(page_count_change);
	unsigned long flags;
	char          *buffer;
	size_t        pos = 0;

	buffer = kbasep_tlstream_msgbuf_acquire(
			TL_STREAM_TYPE_AUX, msg_size, &flags);
	KBASE_DEBUG_ASSERT(buffer);

	pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id));
	pos = kbasep_tlstream_write_timestamp(buffer, pos);
	pos = kbasep_tlstream_write_bytes(
			buffer, pos,
			&page_count_change, sizeof(page_count_change));
	KBASE_DEBUG_ASSERT(msg_size == pos);

	kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_AUX, flags);
}