path: root/platform/linux-dpdk/odp_packet.c

/* Copyright (c) 2013-2018, Linaro Limited
 * Copyright (c) 2019-2023, Nokia
 * All rights reserved.
 *
 * SPDX-License-Identifier:     BSD-3-Clause
 */

#include <odp/api/align.h>
#include <odp/api/buffer.h>
#include <odp/api/byteorder.h>
#include <odp/api/hash.h>
#include <odp/api/hints.h>
#include <odp/api/packet.h>
#include <odp/api/packet_flags.h>
#include <odp/api/packet_io.h>
#include <odp/api/proto_stats.h>
#include <odp/api/timer.h>

#include <odp/api/plat/byteorder_inlines.h>
#include <odp/api/plat/event_inlines.h>
#include <odp/api/plat/packet_inlines.h>
#include <odp/api/plat/packet_io_inlines.h>

#include <odp_chksum_internal.h>
#include <odp_debug_internal.h>
#include <odp_event_internal.h>
#include <odp_event_validation_internal.h>
#include <odp_macros_internal.h>
#include <odp_packet_internal.h>
#include <odp_packet_io_internal.h>
#include <odp_parse_internal.h>
#include <odp_pool_internal.h>
#include <odp_print_internal.h>

#include <rte_version.h>

#include <protocols/eth.h>
#include <protocols/ip.h>
#include <protocols/sctp.h>
#include <protocols/tcp.h>
#include <protocols/udp.h>

#include <string.h>
#include <stdio.h>
#include <stddef.h>
#include <inttypes.h>

#if RTE_VERSION < RTE_VERSION_NUM(21, 11, 0, 0)
	#define RTE_MBUF_F_RX_RSS_HASH PKT_RX_RSS_HASH
#endif

#include <odp/visibility_begin.h>

/* Fill in packet header field offsets for inline functions */

const _odp_packet_inline_offset_t _odp_packet_inline ODP_ALIGNED_CACHE = {
	.mb               = offsetof(odp_packet_hdr_t, event_hdr.mb),
	.pool             = offsetof(odp_packet_hdr_t, event_hdr.pool),
	.input            = offsetof(odp_packet_hdr_t, input),
	.user_ptr         = offsetof(odp_packet_hdr_t, user_ptr),
	.l2_offset        = offsetof(odp_packet_hdr_t, p.l2_offset),
	.l3_offset        = offsetof(odp_packet_hdr_t, p.l3_offset),
	.l4_offset        = offsetof(odp_packet_hdr_t, p.l4_offset),
	.timestamp        = offsetof(odp_packet_hdr_t, timestamp),
	.input_flags      = offsetof(odp_packet_hdr_t, p.input_flags),
	.flags            = offsetof(odp_packet_hdr_t, p.flags),
	.subtype          = offsetof(odp_packet_hdr_t, subtype),
	.cls_mark         = offsetof(odp_packet_hdr_t, cls_mark),
	.ipsec_ctx        = offsetof(odp_packet_hdr_t, ipsec_ctx),
	.crypto_op        = offsetof(odp_packet_hdr_t, crypto_op_result),
	.buf_addr         = offsetof(odp_packet_hdr_t, event_hdr.mb.buf_addr),
	.data             = offsetof(odp_packet_hdr_t, event_hdr.mb.data_off),
	.pkt_len          = offsetof(odp_packet_hdr_t, event_hdr.mb.pkt_len),
	.seg_len          = offsetof(odp_packet_hdr_t, event_hdr.mb.data_len),
	.nb_segs          = offsetof(odp_packet_hdr_t, event_hdr.mb.nb_segs),
	.user_area        = offsetof(odp_packet_hdr_t, uarea_addr),
	.rss              = offsetof(odp_packet_hdr_t, event_hdr.mb.hash.rss),
	.ol_flags         = offsetof(odp_packet_hdr_t, event_hdr.mb.ol_flags),
	.rss_flag         = RTE_MBUF_F_RX_RSS_HASH
};

#include <odp/visibility_end.h>

/* Catch if DPDK mbuf members sizes have changed */
struct rte_mbuf _odp_dummy_mbuf;
ODP_STATIC_ASSERT(sizeof(_odp_dummy_mbuf.data_off) == sizeof(uint16_t),
		  "data_off should be uint16_t");
ODP_STATIC_ASSERT(sizeof(_odp_dummy_mbuf.pkt_len) == sizeof(uint32_t),
		  "pkt_len should be uint32_t");
ODP_STATIC_ASSERT(sizeof(_odp_dummy_mbuf.data_len) == sizeof(uint16_t),
		  "data_len should be uint16_t");
ODP_STATIC_ASSERT(sizeof(_odp_dummy_mbuf.nb_segs) == sizeof(uint16_t),
		  "nb_segs should be uint16_t");
ODP_STATIC_ASSERT(sizeof(_odp_dummy_mbuf.hash.rss) == sizeof(uint32_t),
		  "hash.rss should be uint32_t");
ODP_STATIC_ASSERT(sizeof(_odp_dummy_mbuf.ol_flags) == sizeof(uint64_t),
		  "ol_flags should be uint64_t");

/* Check that invalid values are the same. Some versions of Clang  and pedantic
 * build have trouble with the strong type casting, and complain that these
 * invalid values are not integral constants.
 *
 * Invalid values are required to be equal for _odp_buffer_is_valid() to work
 * properly. */
#ifndef __clang__
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wpedantic"
ODP_STATIC_ASSERT(ODP_PACKET_INVALID == 0, "Packet invalid not 0");
ODP_STATIC_ASSERT(ODP_BUFFER_INVALID == 0, "Buffer invalid not 0");
ODP_STATIC_ASSERT(ODP_EVENT_INVALID  == 0, "Event invalid not 0");
ODP_STATIC_ASSERT(ODP_PACKET_VECTOR_INVALID == 0, "Packet vector invalid not 0");
ODP_STATIC_ASSERT(ODP_PACKET_TX_COMPL_INVALID == 0, "Packet TX completion invalid not 0");
ODP_STATIC_ASSERT(ODP_TIMEOUT_INVALID == 0, "Timeout invalid not 0");
#pragma GCC diagnostic pop
#endif

/* Calculate the number of segments */
static inline int num_segments(uint32_t len, uint32_t seg_len)
{
	int num = 1;

	if (odp_unlikely(len > seg_len)) {
		num = len / seg_len;

		if (odp_likely((num * seg_len) != len))
			num += 1;
	}

	return num;
}

static inline int packet_reset(odp_packet_t pkt, uint32_t len)
{
	odp_packet_hdr_t *const pkt_hdr = packet_hdr(pkt);
	struct rte_mbuf *ms, *mb = &pkt_hdr->event_hdr.mb;
	uint8_t nb_segs = 0;
	int32_t lenleft = len;

	packet_init(pkt_hdr, ODP_PKTIO_INVALID);

	mb->port = 0xff;
	mb->pkt_len = len;
	mb->data_off = RTE_PKTMBUF_HEADROOM;
	mb->vlan_tci = 0;
	nb_segs = 1;

	if (RTE_PKTMBUF_HEADROOM + lenleft <= mb->buf_len) {
		mb->data_len = lenleft;
	} else {
		mb->data_len = mb->buf_len - RTE_PKTMBUF_HEADROOM;
		lenleft -= mb->data_len;
		ms = mb->next;
		while (lenleft > 0) {
			nb_segs++;
			ms->data_len = lenleft <= ms->buf_len ?
				lenleft : ms->buf_len;
			lenleft -= ms->buf_len;
			ms = ms->next;
		}
	}

	mb->nb_segs = nb_segs;
	return 0;
}

/* Reset unmodified single segment packet after rte_pktmbuf_alloc(), which has already called
 * rte_pktmbuf_reset() internally. */
static inline void packet_reset_fresh(odp_packet_t pkt, uint32_t len)
{
	struct rte_mbuf *mb = pkt_to_mbuf(pkt);

	packet_init(packet_hdr(pkt), ODP_PKTIO_INVALID);

	mb->pkt_len = len;
	mb->data_len = len;
}

static inline int pktmbuf_alloc_multi(struct rte_mempool *mp, struct rte_mbuf **mbufs, int num)
{
	if (odp_likely(rte_pktmbuf_alloc_bulk(mp, mbufs, num) == 0))
		return num;

	for (int i = 0; i < num; i++) {
		mbufs[i] = rte_pktmbuf_alloc(mp);

		if (odp_unlikely(mbufs[i] == NULL))
			return i;
	}

	return num;
}

static odp_packet_t packet_alloc(pool_t *pool, uint32_t len)
{
	odp_packet_t pkt;
	uintmax_t totsize = RTE_PKTMBUF_HEADROOM + len;
	odp_packet_hdr_t *pkt_hdr;
	uint16_t seg_len = pool->seg_len;
	int num_seg;

	num_seg = num_segments(totsize, seg_len);

	if (odp_likely(num_seg == 1)) {
		struct rte_mbuf *mbuf = rte_pktmbuf_alloc(pool->rte_mempool);

		if (odp_unlikely(mbuf == NULL))
			return ODP_PACKET_INVALID;

		pkt_hdr = (odp_packet_hdr_t *)mbuf;
		odp_prefetch((uint8_t *)mbuf + sizeof(struct rte_mbuf));
		odp_prefetch((uint8_t *)mbuf + sizeof(struct rte_mbuf) +
			     ODP_CACHE_LINE_SIZE);

		pkt = packet_handle(pkt_hdr);
		packet_reset_fresh(pkt, len);

		return pkt;
	}

	/* Create segmented packet */

	struct rte_mbuf *mbufs[num_seg];
	struct rte_mbuf *head;
	int ret;

	/* Check num_seg here so rte_pktmbuf_chain() always succeeds */
	if (odp_unlikely(num_seg > RTE_MBUF_MAX_NB_SEGS))
		return ODP_PACKET_INVALID;

	/* Avoid invalid 'maybe-uninitialized' warning with GCC 12 */
	mbufs[0] = NULL;

	ret = pktmbuf_alloc_multi(pool->rte_mempool, mbufs, num_seg);
	if (odp_unlikely(ret != num_seg)) {
		for (int i = 0; i < ret; i++)
			rte_pktmbuf_free(mbufs[i]);

		return ODP_PACKET_INVALID;
	}

	head = mbufs[0];
	pkt_hdr = (odp_packet_hdr_t *)head;
	odp_prefetch((uint8_t *)head + sizeof(struct rte_mbuf));
	odp_prefetch((uint8_t *)head + sizeof(struct rte_mbuf) +
			ODP_CACHE_LINE_SIZE);

	for (int i = 1; i < num_seg; i++) {
		struct rte_mbuf *nextseg = mbufs[i];

		nextseg->data_off = 0;

		rte_pktmbuf_chain(head, nextseg);
	}

	pkt = packet_handle(pkt_hdr);
	packet_reset(pkt, len);

	return pkt;
}

static int packet_alloc_multi(pool_t *pool, uint32_t len, odp_packet_t pkt[], int num)
{
	uintmax_t totsize = RTE_PKTMBUF_HEADROOM + len;
	int num_seg, i;

	num_seg = num_segments(totsize, pool->seg_len);

	if (odp_likely(num_seg == 1)) {
		int ret;

		ret = pktmbuf_alloc_multi(pool->rte_mempool, (struct rte_mbuf **)pkt, num);

		for (i = 0; i < ret; i++) {
			struct rte_mbuf *mbuf = pkt_to_mbuf(pkt[i]);

			odp_prefetch((uint8_t *)mbuf + sizeof(struct rte_mbuf));
			odp_prefetch((uint8_t *)mbuf + sizeof(struct rte_mbuf) +
				ODP_CACHE_LINE_SIZE);

			packet_reset_fresh(pkt[i], len);
		}
		return ret;
	}

	/* Fall back to using packet_alloc() for segmented packets */

	for (i = 0; i < num; i++) {
		pkt[i] = packet_alloc(pool, len);
		if (odp_unlikely(pkt[i] == ODP_PACKET_INVALID))
			return i;
	}
	return i;
}

odp_packet_t odp_packet_alloc(odp_pool_t pool_hdl, uint32_t len)
{
	pool_t *pool = _odp_pool_entry(pool_hdl);

	_ODP_ASSERT(pool->type == ODP_POOL_PACKET);

	if (odp_unlikely(len == 0))
		return ODP_PACKET_INVALID;

	return packet_alloc(pool, len);
}

int odp_packet_alloc_multi(odp_pool_t pool_hdl, uint32_t len,
			   odp_packet_t pkt[], int num)
{
	pool_t *pool = _odp_pool_entry(pool_hdl);

	_ODP_ASSERT(pool->type == ODP_POOL_PACKET);

	if (odp_unlikely(len == 0))
		return -1;

	return packet_alloc_multi(pool, len, pkt, num);
}

int odp_packet_reset(odp_packet_t pkt, uint32_t len)
{
	if (odp_unlikely(len == 0))
		return -1;

	if (RTE_PKTMBUF_HEADROOM + len > odp_packet_buf_len(pkt)) {
		_ODP_DBG("Not enough head room for that packet %d/%d\n",
			 RTE_PKTMBUF_HEADROOM + len, odp_packet_buf_len(pkt));
		return -1;
	}

	return packet_reset(pkt, len);
}

int odp_event_filter_packet(const odp_event_t event[],
			    odp_packet_t packet[],
			    odp_event_t remain[], int num)
{
	int i;
	int num_pkt = 0;
	int num_rem = 0;

	for (i = 0; i < num; i++) {
		if (odp_event_type(event[i]) == ODP_EVENT_PACKET) {
			packet[num_pkt] = odp_packet_from_event(event[i]);
			num_pkt++;
		} else {
			remain[num_rem] = event[i];
			num_rem++;
		}
	}

	return num_pkt;
}

void *odp_packet_tail(odp_packet_t pkt)
{
	struct rte_mbuf *mb = &(packet_hdr(pkt)->event_hdr.mb);

	mb = rte_pktmbuf_lastseg(mb);
	return (void *)(rte_pktmbuf_mtod(mb, char *) + mb->data_len);
}

void *odp_packet_push_head(odp_packet_t pkt, uint32_t len)
{
	struct rte_mbuf *mb = &(packet_hdr(pkt)->event_hdr.mb);

	return (void *)rte_pktmbuf_prepend(mb, len);
}

static void _copy_head_metadata(struct rte_mbuf *newhead,
				struct rte_mbuf *oldhead)
{
	rte_mbuf_refcnt_set(newhead, rte_mbuf_refcnt_read(oldhead));

	_odp_packet_copy_md((odp_packet_hdr_t *)newhead, (odp_packet_hdr_t *)oldhead, 0);
}

int odp_packet_extend_head(odp_packet_t *pkt, uint32_t len, void **data_ptr,
			   uint32_t *seg_len)
{
	struct rte_mbuf *mb = &(packet_hdr(*pkt)->event_hdr.mb);
	int addheadsize = len - rte_pktmbuf_headroom(mb);

	if (addheadsize > 0) {
		struct rte_mbuf *newhead, *t;
		int i;

		newhead = rte_pktmbuf_alloc(mb->pool);
		if (newhead == NULL)
			return -1;

		newhead->data_len = addheadsize % newhead->buf_len;
		newhead->pkt_len = addheadsize;
		newhead->data_off = newhead->buf_len - newhead->data_len;
		newhead->nb_segs = addheadsize / newhead->buf_len + 1;
		t = newhead;

		for (i = 0; i < newhead->nb_segs - 1; --i) {
			t->next = rte_pktmbuf_alloc(mb->pool);

			if (t->next == NULL) {
				rte_pktmbuf_free(newhead);
				return -1;
			}
			/* The intermediate segments are fully used */
			t->data_len = t->buf_len;
			t->data_off = 0;
		}
		if (rte_pktmbuf_chain(newhead, mb)) {
			rte_pktmbuf_free(newhead);
			return -1;
		}
		/* Expand the original head segment*/
		newhead->pkt_len += rte_pktmbuf_headroom(mb);
		mb->data_len += rte_pktmbuf_headroom(mb);
		mb->data_off = 0;
		_copy_head_metadata(newhead, mb);
		mb = newhead;
		*pkt = (odp_packet_t)newhead;
	} else {
		rte_pktmbuf_prepend(mb, len);
	}

	if (data_ptr)
		*data_ptr = odp_packet_data(*pkt);
	if (seg_len)
		*seg_len = mb->data_len;

	return 0;
}

void *odp_packet_pull_head(odp_packet_t pkt, uint32_t len)
{
	struct rte_mbuf *mb = pkt_to_mbuf(pkt);

	if (odp_unlikely(len >= mb->data_len))
		return NULL;

	return (void *)rte_pktmbuf_adj(mb, len);
}

int odp_packet_trunc_head(odp_packet_t *pkt, uint32_t len, void **data_ptr,
			  uint32_t *seg_len)
{
	struct rte_mbuf *mb = pkt_to_mbuf(*pkt);

	if (odp_unlikely(len >= odp_packet_len(*pkt)))
		return -1;

	if (len > mb->data_len) {
		struct rte_mbuf *newhead = mb, *prev = NULL;
		uint32_t left = len;

		while (newhead->next != NULL) {
			if (newhead->data_len > left)
				break;
			left -= newhead->data_len;
			prev = newhead;
			newhead = newhead->next;
			--mb->nb_segs;
		}
		newhead->data_off += left;
		newhead->nb_segs = mb->nb_segs;
		newhead->pkt_len = mb->pkt_len - len;
		newhead->data_len -= left;
		_copy_head_metadata(newhead, mb);
		prev->next = NULL;
		rte_pktmbuf_free(mb);
		mb = newhead;
		*pkt = (odp_packet_t)newhead;
	} else {
		rte_pktmbuf_adj(mb, len);
	}

	if (data_ptr)
		*data_ptr = odp_packet_data(*pkt);
	if (seg_len)
		*seg_len = mb->data_len;

	return 0;
}

void *odp_packet_push_tail(odp_packet_t pkt, uint32_t len)
{
	struct rte_mbuf *mb = &(packet_hdr(pkt)->event_hdr.mb);

	return (void *)rte_pktmbuf_append(mb, len);
}

int odp_packet_extend_tail(odp_packet_t *pkt, uint32_t len, void **data_ptr,
			   uint32_t *seg_len)
{
	struct rte_mbuf *mb = &(packet_hdr(*pkt)->event_hdr.mb);
	int newtailsize = len - odp_packet_tailroom(*pkt);
	uint32_t old_pkt_len = odp_packet_len(*pkt);

	if (data_ptr)
		*data_ptr = odp_packet_tail(*pkt);

	if (newtailsize > 0) {
		struct rte_mbuf *newtail = rte_pktmbuf_alloc(mb->pool);
		struct rte_mbuf *t;
		struct rte_mbuf *m_last = rte_pktmbuf_lastseg(mb);
		int i;

		if (newtail == NULL)
			return -1;
		newtail->data_off = 0;
		newtail->pkt_len = newtailsize;
		if (newtailsize > newtail->buf_len)
			newtail->data_len = newtail->buf_len;
		else
			newtail->data_len = newtailsize;
		newtail->nb_segs = newtailsize / newtail->buf_len + 1;
		t = newtail;

		for (i = 0; i < newtail->nb_segs - 1; ++i) {
			t->next = rte_pktmbuf_alloc(mb->pool);

			if (t->next == NULL) {
				rte_pktmbuf_free(newtail);
				return -1;
			}
			t = t->next;
			t->data_off = 0;
			/* The last segment's size is not trivial*/
			t->data_len = i == newtail->nb_segs - 2 ?
				      newtailsize % newtail->buf_len :
				      t->buf_len;
		}
		if (rte_pktmbuf_chain(mb, newtail)) {
			rte_pktmbuf_free(newtail);
			return -1;
		}
		/* Expand the original tail */
		m_last->data_len = m_last->buf_len - m_last->data_off;
		mb->pkt_len += len - newtailsize;
	} else {
		rte_pktmbuf_append(mb, len);
	}

	if (seg_len)
		odp_packet_offset(*pkt, old_pkt_len, seg_len, NULL);

	return 0;
}

void *odp_packet_pull_tail(odp_packet_t pkt, uint32_t len)
{
	struct rte_mbuf *mb = pkt_to_mbuf(pkt);
	struct rte_mbuf *mb_last = rte_pktmbuf_lastseg(mb);

	if (odp_unlikely(len >= mb_last->data_len))
		return NULL;

	if (rte_pktmbuf_trim(mb, len))
		return NULL;
	else
		return odp_packet_tail(pkt);
}

int odp_packet_trunc_tail(odp_packet_t *pkt, uint32_t len, void **tail_ptr,
			  uint32_t *tailroom)
{
	struct rte_mbuf *mb = pkt_to_mbuf(*pkt);
	struct rte_mbuf *last_mb = rte_pktmbuf_lastseg(mb);

	if (odp_unlikely(len >= odp_packet_len(*pkt)))
		return -1;

	/*
	 * Trim only if the last segment does not become zero length.
	 */
	if (odp_likely(len < last_mb->data_len)) {
		if (odp_unlikely(rte_pktmbuf_trim(mb, len)))
			return -1;
	} else {
		struct rte_mbuf *reverse[mb->nb_segs];
		struct rte_mbuf *t = mb;
		int i;

		for (i = 0; i < mb->nb_segs; ++i) {
			reverse[i] = t;
			t = t->next;
		}
		for (i = mb->nb_segs - 1; i >= 0 && len > 0; --i) {
			t = reverse[i];
			if (len >= t->data_len) {
				len -= t->data_len;
				mb->pkt_len -= t->data_len;
				t->data_len = 0;
				if (i > 0) {
					rte_pktmbuf_free_seg(t);
					--mb->nb_segs;
					reverse[i - 1]->next = NULL;
				}
			} else {
				t->data_len -= len;
				mb->pkt_len -= len;
				len = 0;
			}
		}
	}

	if (tail_ptr)
		*tail_ptr = odp_packet_tail(*pkt);
	if (tailroom)
		*tailroom = odp_packet_tailroom(*pkt);

	return 0;
}

/*
 *
 * Meta-data
 * ********************************************************
 *
 */

uint16_t odp_packet_ones_comp(odp_packet_t pkt, odp_packet_data_range_t *range)
{
	(void)pkt;
	range->length = 0;
	range->offset = 0;
	return 0;
}

/*
 *
 * Manipulation
 * ********************************************************
 *
 */

int odp_packet_add_data(odp_packet_t *pkt_ptr, uint32_t offset, uint32_t len)
{
	odp_packet_t pkt = *pkt_ptr;
	odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt);
	uint32_t pktlen = odp_packet_len(pkt);
	odp_pool_t pool = pkt_hdr->event_hdr.pool;
	odp_packet_t newpkt;

	if (offset > pktlen)
		return -1;

	newpkt = odp_packet_alloc(pool, pktlen + len);

	if (newpkt == ODP_PACKET_INVALID)
		return -1;

	if (odp_packet_copy_from_pkt(newpkt, 0, pkt, 0, offset) != 0 ||
	    odp_packet_copy_from_pkt(newpkt, offset + len, pkt, offset,
				     pktlen - offset) != 0) {
		odp_packet_free(newpkt);
		return -1;
	}

	_odp_packet_copy_md(packet_hdr(newpkt), pkt_hdr, 0);
	odp_packet_free(pkt);
	*pkt_ptr = newpkt;

	return 1;
}

int odp_packet_rem_data(odp_packet_t *pkt_ptr, uint32_t offset, uint32_t len)
{
	odp_packet_t pkt = *pkt_ptr;
	odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt);
	uint32_t pktlen = odp_packet_len(pkt);
	odp_pool_t pool = pkt_hdr->event_hdr.pool;
	odp_packet_t newpkt;

	if (odp_unlikely(offset + len >= pktlen))
		return -1;

	newpkt = odp_packet_alloc(pool, pktlen - len);

	if (newpkt == ODP_PACKET_INVALID)
		return -1;

	if (odp_packet_copy_from_pkt(newpkt, 0, pkt, 0, offset) != 0 ||
	    odp_packet_copy_from_pkt(newpkt, offset, pkt, offset + len,
				     pktlen - offset - len) != 0) {
		odp_packet_free(newpkt);
		return -1;
	}

	_odp_packet_copy_md(packet_hdr(newpkt), pkt_hdr, 0);
	odp_packet_free(pkt);
	*pkt_ptr = newpkt;

	return 1;
}

int odp_packet_align(odp_packet_t *pkt, uint32_t offset, uint32_t len,
		     uint32_t align)
{
	int rc;
	uint32_t shift;
	uint32_t seglen = 0;  /* GCC */
	void *addr = odp_packet_offset(*pkt, offset, &seglen, NULL);
	uint64_t uaddr = (uint64_t)(uintptr_t)addr;
	uint64_t misalign;

	if (align > ODP_CACHE_LINE_SIZE)
		return -1;

	if (seglen >= len) {
		misalign = align <= 1 ? 0 :
			_ODP_ROUNDUP_ALIGN(uaddr, align) - uaddr;
		if (misalign == 0)
			return 0;
		shift = align - misalign;
	} else {
		if (len > odp_packet_seg_len(*pkt))
			return -1;
		shift  = len - seglen;
		uaddr -= shift;
		misalign = align <= 1 ? 0 :
			_ODP_ROUNDUP_ALIGN(uaddr, align) - uaddr;
		if (misalign)
			shift += align - misalign;
	}

	rc = odp_packet_extend_head(pkt, shift, NULL, NULL);
	if (rc < 0)
		return rc;

	(void)odp_packet_move_data(*pkt, 0, shift,
				   odp_packet_len(*pkt) - shift);

	(void)odp_packet_trunc_tail(pkt, shift, NULL, NULL);
	return 1;
}

int odp_packet_concat(odp_packet_t *dst, odp_packet_t src)
{
	struct rte_mbuf *mb_dst = pkt_to_mbuf(*dst);
	struct rte_mbuf *mb_src = pkt_to_mbuf(src);
	odp_packet_t new_dst;
	odp_pool_t pool;
	uint32_t dst_len;
	uint32_t src_len;

	/* Copy if packets are from different pools */
	if (odp_likely(mb_dst->pool == mb_src->pool)) {
		if (odp_likely(!rte_pktmbuf_chain(mb_dst, mb_src)))
			return 0;
	} else {
		odp_packet_t new_src = odp_packet_copy_part(src, 0, odp_packet_len(src),
							    odp_packet_pool(*dst));

		if (odp_unlikely(new_src == ODP_PACKET_INVALID))
			return -1;

		if (odp_likely(!rte_pktmbuf_chain(mb_dst, pkt_to_mbuf(new_src)))) {
			odp_packet_free(src);
			return 1;
		}
		odp_packet_free(new_src);
	}

	/* Fall back to using standard copy operations after maximum number of
	 * segments has been reached. */
	dst_len = odp_packet_len(*dst);
	src_len = odp_packet_len(src);
	pool = odp_packet_pool(*dst);

	new_dst = odp_packet_copy(*dst, pool);
	if (odp_unlikely(new_dst == ODP_PACKET_INVALID))
		return -1;

	if (odp_packet_extend_tail(&new_dst, src_len, NULL, NULL) >= 0) {
		(void)odp_packet_copy_from_pkt(new_dst, dst_len,
					       src, 0, src_len);
		odp_packet_free(*dst);
		odp_packet_free(src);
		*dst = new_dst;
		return 1;
	}

	odp_packet_free(new_dst);
	return -1;
}

int odp_packet_split(odp_packet_t *pkt, uint32_t len, odp_packet_t *tail)
{
	uint32_t pktlen = odp_packet_len(*pkt);

	if (odp_unlikely(len == 0 || len >= pktlen || tail == NULL))
		return -1;

	*tail = odp_packet_copy_part(*pkt, len, pktlen - len,
				     odp_packet_pool(*pkt));

	if (*tail == ODP_PACKET_INVALID)
		return -1;

	return odp_packet_trunc_tail(pkt, pktlen - len, NULL, NULL);
}

/*
 *
 * Copy
 * ********************************************************
 *
 */

odp_packet_t odp_packet_copy(odp_packet_t pkt, odp_pool_t pool)
{
	uint32_t pktlen = odp_packet_len(pkt);
	odp_packet_t newpkt;

	if (odp_unlikely(_odp_packet_copy_md_possible(pool, odp_packet_pool(pkt)) < 0)) {
		_ODP_ERR("Unable to copy packet metadata\n");
		return ODP_PACKET_INVALID;
	}

	newpkt = odp_packet_alloc(pool, pktlen);
	if (odp_unlikely(newpkt == ODP_PACKET_INVALID))
		return ODP_PACKET_INVALID;

	if (odp_unlikely(odp_packet_copy_from_pkt(newpkt, 0, pkt, 0, pktlen))) {
		odp_packet_free(newpkt);
		newpkt = ODP_PACKET_INVALID;
	}

	_odp_packet_copy_md(packet_hdr(newpkt), packet_hdr(pkt), 1);

	return newpkt;
}

odp_packet_t odp_packet_copy_part(odp_packet_t pkt, uint32_t offset,
				  uint32_t len, odp_pool_t pool)
{
	uint32_t pktlen = odp_packet_len(pkt);
	odp_packet_t newpkt;

	if (offset >= pktlen || offset + len > pktlen)
		return ODP_PACKET_INVALID;

	newpkt = odp_packet_alloc(pool, len);
	if (newpkt != ODP_PACKET_INVALID)
		odp_packet_copy_from_pkt(newpkt, 0, pkt, offset, len);

	return newpkt;
}

int odp_packet_copy_from_pkt(odp_packet_t dst, uint32_t dst_offset,
			     odp_packet_t src, uint32_t src_offset,
			     uint32_t len)
{
	odp_packet_hdr_t *dst_hdr = packet_hdr(dst);
	odp_packet_hdr_t *src_hdr = packet_hdr(src);
	void *dst_map;
	void *src_map;
	uint32_t cpylen, minseg;
	uint32_t dst_seglen = 0; /* GCC */
	uint32_t src_seglen = 0; /* GCC */
	int overlap;

	if (dst_offset + len > odp_packet_len(dst) ||
	    src_offset + len > odp_packet_len(src))
		return -1;

	overlap = (dst_hdr == src_hdr &&
		   ((dst_offset <= src_offset &&
		     dst_offset + len >= src_offset) ||
		    (src_offset <= dst_offset &&
		     src_offset + len >= dst_offset)));

	if (overlap && src_offset < dst_offset) {
		odp_packet_t temp =
			odp_packet_copy_part(src, src_offset, len,
					     odp_packet_pool(src));
		if (temp == ODP_PACKET_INVALID)
			return -1;
		odp_packet_copy_from_pkt(dst, dst_offset, temp, 0, len);
		odp_packet_free(temp);
		return 0;
	}

	while (len > 0) {
		dst_map = odp_packet_offset(dst, dst_offset, &dst_seglen, NULL);
		src_map = odp_packet_offset(src, src_offset, &src_seglen, NULL);

		minseg = dst_seglen > src_seglen ? src_seglen : dst_seglen;
		cpylen = len > minseg ? minseg : len;

		if (overlap)
			memmove(dst_map, src_map, cpylen);
		else
			memcpy(dst_map, src_map, cpylen);

		dst_offset += cpylen;
		src_offset += cpylen;
		len        -= cpylen;
	}

	return 0;
}

int odp_packet_copy_data(odp_packet_t pkt, uint32_t dst_offset,
			 uint32_t src_offset, uint32_t len)
{
	return odp_packet_copy_from_pkt(pkt, dst_offset,
					pkt, src_offset, len);
}

int odp_packet_move_data(odp_packet_t pkt, uint32_t dst_offset,
			 uint32_t src_offset, uint32_t len)
{
	return odp_packet_copy_from_pkt(pkt, dst_offset,
					pkt, src_offset, len);
}

int _odp_packet_set_data(odp_packet_t pkt, uint32_t offset,
			 uint8_t c, uint32_t len)
{
	void *mapaddr;
	uint32_t seglen = 0; /* GCC */
	uint32_t setlen;

	if (offset + len > odp_packet_len(pkt))
		return -1;

	while (len > 0) {
		mapaddr = odp_packet_offset(pkt, offset, &seglen, NULL);
		setlen = len > seglen ? seglen : len;
		if (odp_unlikely(setlen == 0))
			return -1;
		memset(mapaddr, c, setlen);
		offset  += setlen;
		len     -= setlen;
	}

	return 0;
}

int _odp_packet_cmp_data(odp_packet_t pkt, uint32_t offset,
			 const void *s, uint32_t len)
{
	const uint8_t *ptr = s;
	void *mapaddr;
	uint32_t seglen = 0; /* GCC */
	uint32_t cmplen;
	int ret;

	_ODP_ASSERT(offset + len <= odp_packet_len(pkt));

	while (len > 0) {
		mapaddr = odp_packet_offset(pkt, offset, &seglen, NULL);
		cmplen = len > seglen ? seglen : len;
		ret = memcmp(mapaddr, ptr, cmplen);
		if (ret != 0)
			return ret;
		offset  += cmplen;
		len     -= cmplen;
		ptr     += cmplen;
	}

	return 0;
}

/*
 *
 * Debugging
 * ********************************************************
 *
 */
static int packet_print_input_flags(odp_packet_hdr_t *hdr, char *str, int max)
{
	int len = 0;

	if (hdr->p.input_flags.l2)
		len += _odp_snprint(&str[len], max - len, "l2 ");
	if (hdr->p.input_flags.l3)
		len += _odp_snprint(&str[len], max - len, "l3 ");
	if (hdr->p.input_flags.l4)
		len += _odp_snprint(&str[len], max - len, "l4 ");
	if (hdr->p.input_flags.eth)
		len += _odp_snprint(&str[len], max - len, "eth ");
	if (hdr->p.input_flags.vlan)
		len += _odp_snprint(&str[len], max - len, "vlan ");
	if (hdr->p.input_flags.arp)
		len += _odp_snprint(&str[len], max - len, "arp ");
	if (hdr->p.input_flags.ipv4)
		len += _odp_snprint(&str[len], max - len, "ipv4 ");
	if (hdr->p.input_flags.ipv6)
		len += _odp_snprint(&str[len], max - len, "ipv6 ");
	if (hdr->p.input_flags.ipsec)
		len += _odp_snprint(&str[len], max - len, "ipsec ");
	if (hdr->p.input_flags.udp)
		len += _odp_snprint(&str[len], max - len, "udp ");
	if (hdr->p.input_flags.tcp)
		len += _odp_snprint(&str[len], max - len, "tcp ");
	if (hdr->p.input_flags.sctp)
		len += _odp_snprint(&str[len], max - len, "sctp ");
	if (hdr->p.input_flags.icmp)
		len += _odp_snprint(&str[len], max - len, "icmp ");

	return len;
}

void odp_packet_print(odp_packet_t pkt)
{
	odp_packet_seg_t seg;
	int max_len = 1024;
	char str[max_len];
	int len = 0;
	int n = max_len - 1;
	odp_packet_hdr_t *hdr = packet_hdr(pkt);
	pool_t *pool = _odp_pool_entry(hdr->event_hdr.pool);

	len += _odp_snprint(&str[len], n - len, "Packet info\n");
	len += _odp_snprint(&str[len], n - len, "-----------\n");
	len += _odp_snprint(&str[len], n - len, "  handle         0x%" PRIx64 "\n",
			    odp_packet_to_u64(pkt));
	len += _odp_snprint(&str[len], n - len, "  pool index     %u\n", pool->pool_idx);
	len += _odp_snprint(&str[len], n - len, "  buf index      %u\n", hdr->event_hdr.index);
	len += _odp_snprint(&str[len], n - len, "  ev subtype     %i\n", hdr->subtype);
	len += _odp_snprint(&str[len], n - len, "  input_flags    0x%" PRIx64 "\n",
			    hdr->p.input_flags.all);
	if (hdr->p.input_flags.all) {
		len += _odp_snprint(&str[len], n - len, "               ");
		len += packet_print_input_flags(hdr, &str[len], n - len);
		len += _odp_snprint(&str[len], n - len, "\n");
	}
	len += _odp_snprint(&str[len], n - len,
			    "  flags          0x%" PRIx32 "\n", hdr->p.flags.all_flags);
	len += _odp_snprint(&str[len], n - len,
			    "  cls_mark       %" PRIu64 "\n", odp_packet_cls_mark(pkt));
	len += _odp_snprint(&str[len], n - len,
			    "  user ptr       %p\n", hdr->user_ptr);
	len += _odp_snprint(&str[len], n - len,
			    "  user area      %p\n", hdr->uarea_addr);
	len += _odp_snprint(&str[len], n - len,
			    "  l2_offset      %" PRIu32 "\n", hdr->p.l2_offset);
	len += _odp_snprint(&str[len], n - len,
			    "  l3_offset      %" PRIu32 "\n", hdr->p.l3_offset);
	len += _odp_snprint(&str[len], n - len,
			    "  l4_offset      %" PRIu32 "\n", hdr->p.l4_offset);
	len += _odp_snprint(&str[len], n - len,
			    "  frame_len      %" PRIu32 "\n", hdr->event_hdr.mb.pkt_len);
	len += _odp_snprint(&str[len], n - len,
			    "  input          %" PRIu64 "\n", odp_pktio_to_u64(hdr->input));
	len += _odp_snprint(&str[len], n - len,
			    "  headroom       %" PRIu32 "\n", odp_packet_headroom(pkt));
	len += _odp_snprint(&str[len], n - len,
			    "  tailroom       %" PRIu32 "\n", odp_packet_tailroom(pkt));
	len += _odp_snprint(&str[len], n - len,
			    "  num_segs       %i\n", odp_packet_num_segs(pkt));

	seg = odp_packet_first_seg(pkt);

	for (int seg_idx = 0; seg != ODP_PACKET_SEG_INVALID; seg_idx++) {
		len += _odp_snprint(&str[len], n - len,
				    "    [%d] seg_len    %-4" PRIu32 "  seg_data %p\n",
				    seg_idx, odp_packet_seg_data_len(pkt, seg),
				    odp_packet_seg_data(pkt, seg));

		seg = odp_packet_next_seg(pkt, seg);
	}

	str[len] = '\0';

	_ODP_PRINT("%s\n", str);
}

void odp_packet_print_data(odp_packet_t pkt, uint32_t offset,
			   uint32_t byte_len)
{
	odp_packet_hdr_t *hdr = packet_hdr(pkt);
	uint32_t bytes_per_row = 16;
	int num_rows = (byte_len + bytes_per_row - 1) / bytes_per_row;
	int max_len = 256 + (3 * byte_len) + (3 * num_rows);
	char str[max_len];
	int len = 0;
	int n = max_len - 1;
	uint32_t data_len = odp_packet_len(pkt);
	pool_t *pool = _odp_pool_entry(hdr->event_hdr.pool);

	len += _odp_snprint(&str[len], n - len, "Packet data\n");
	len += _odp_snprint(&str[len], n - len, "-----------\n");
	len += _odp_snprint(&str[len], n - len,
			    "  handle         0x%" PRIx64 "\n", odp_packet_to_u64(pkt));
	len += _odp_snprint(&str[len], n - len,
			    "  pool name      %s\n", pool->name);
	len += _odp_snprint(&str[len], n - len,
			    "  buf index      %" PRIu32 "\n", hdr->event_hdr.index);
	len += _odp_snprint(&str[len], n - len,
			    "  segcount       %" PRIu8 "\n", hdr->event_hdr.mb.nb_segs);
	len += _odp_snprint(&str[len], n - len,
			    "  data len       %" PRIu32 "\n", data_len);
	len += _odp_snprint(&str[len], n - len,
			    "  data ptr       %p\n", odp_packet_data(pkt));
	len += _odp_snprint(&str[len], n - len,
			    "  print offset   %" PRIu32 "\n", offset);
	len += _odp_snprint(&str[len], n - len,
			    "  print length   %" PRIu32 "\n", byte_len);

	if (offset + byte_len > data_len) {
		len += _odp_snprint(&str[len], n - len, " BAD OFFSET OR LEN\n");
		_ODP_PRINT("%s\n", str);
		return;
	}

	while (byte_len) {
		uint32_t copy_len;
		uint8_t data[bytes_per_row];
		uint32_t i;

		if (byte_len > bytes_per_row)
			copy_len = bytes_per_row;
		else
			copy_len = byte_len;

		odp_packet_copy_to_mem(pkt, offset, copy_len, data);

		len += _odp_snprint(&str[len], n - len, " ");

		for (i = 0; i < copy_len; i++)
			len += _odp_snprint(&str[len], n - len, " %02x", data[i]);

		len += _odp_snprint(&str[len], n - len, "\n");

		byte_len -= copy_len;
		offset   += copy_len;
	}

	_ODP_PRINT("%s\n", str);
}

int odp_packet_is_valid(odp_packet_t pkt)
{
	odp_event_t ev;

	if (pkt == ODP_PACKET_INVALID)
		return 0;

	ev = odp_packet_to_event(pkt);

	if (_odp_event_is_valid(ev) == 0)
		return 0;

	if (odp_event_type(ev) != ODP_EVENT_PACKET)
		return 0;

	if (odp_unlikely(_odp_packet_validate(pkt, _ODP_EV_PACKET_IS_VALID)))
		return 0;

	switch (odp_event_subtype(ev)) {
	case ODP_EVENT_PACKET_BASIC:
		/* Fall through */
	case ODP_EVENT_PACKET_COMP:
		/* Fall through */
	case ODP_EVENT_PACKET_CRYPTO:
		/* Fall through */
	case ODP_EVENT_PACKET_IPSEC:
		/* Fall through */
		break;
	default:
		return 0;
	}

	return 1;
}

/*
 *
 * Internal Use Routines
 * ********************************************************
 *
 */

static uint64_t packet_sum_partial(odp_packet_hdr_t *pkt_hdr,
				   uint32_t l3_offset,
				   uint32_t offset,
				   uint32_t len)
{
	uint64_t sum = 0;
	uint32_t frame_len = odp_packet_len(packet_handle(pkt_hdr));

	if (offset + len > frame_len)
		return 0;

	while (len > 0) {
		uint32_t seglen = 0; /* GCC */
		void *mapaddr = odp_packet_offset(packet_handle(pkt_hdr), offset, &seglen, NULL);

		if (seglen > len)
			seglen = len;

		sum += chksum_partial(mapaddr, seglen, offset - l3_offset);
		len -= seglen;
		offset += seglen;
	}

	return sum;
}

static inline uint16_t packet_sum(odp_packet_hdr_t *pkt_hdr,
				  uint32_t l3_offset,
				  uint32_t offset,
				  uint32_t len,
				  uint64_t sum)
{
	sum += packet_sum_partial(pkt_hdr, l3_offset, offset, len);
	return chksum_finalize(sum);
}

static uint32_t packet_sum_crc32c(odp_packet_hdr_t *pkt_hdr,
				  uint32_t offset,
				  uint32_t len,
				  uint32_t init_val)
{
	uint32_t sum = init_val;

	if (offset + len > odp_packet_len(packet_handle(pkt_hdr)))
		return sum;

	while (len > 0) {
		uint32_t seglen = 0; /* GCC */

		void *mapaddr = odp_packet_offset(packet_handle(pkt_hdr),
						  offset, &seglen, NULL);

		if (seglen > len)
			seglen = len;

		sum = odp_hash_crc32c(mapaddr, seglen, sum);
		len -= seglen;
		offset += seglen;
	}

	return sum;
}

static inline int packet_ipv4_chksum(odp_packet_t pkt,
				     uint32_t offset,
				     _odp_ipv4hdr_t *ip,
				     odp_u16sum_t *chksum)
{
	unsigned int nleft = _ODP_IPV4HDR_IHL(ip->ver_ihl) * 4;
	uint16_t buf[nleft / 2];
	int res;

	if (odp_unlikely(nleft < sizeof(*ip)))
		return -1;
	ip->chksum = 0;
	memcpy(buf, ip, sizeof(*ip));
	res = odp_packet_copy_to_mem(pkt, offset + sizeof(*ip),
				     nleft - sizeof(*ip),
				     buf + sizeof(*ip) / 2);
	if (odp_unlikely(res < 0))
		return res;

	*chksum = ~chksum_finalize(chksum_partial(buf, nleft, 0));

	return 0;
}

#define _ODP_IPV4HDR_CSUM_OFFSET ODP_OFFSETOF(_odp_ipv4hdr_t, chksum)
#define _ODP_IPV4ADDR_OFFSSET ODP_OFFSETOF(_odp_ipv4hdr_t, src_addr)
#define _ODP_IPV6ADDR_OFFSSET ODP_OFFSETOF(_odp_ipv6hdr_t, src_addr)
#define _ODP_IPV4HDR_CSUM_OFFSET ODP_OFFSETOF(_odp_ipv4hdr_t, chksum)
#define _ODP_UDP_LEN_OFFSET ODP_OFFSETOF(_odp_udphdr_t, length)
#define _ODP_UDP_CSUM_OFFSET ODP_OFFSETOF(_odp_udphdr_t, chksum)

/**
 * Calculate and fill in IPv4 checksum
 *
 * @param pkt  ODP packet
 *
 * @retval 0 on success
 * @retval <0 on failure
 */
int _odp_packet_ipv4_chksum_insert(odp_packet_t pkt)
{
	uint32_t offset;
	_odp_ipv4hdr_t ip;
	odp_u16sum_t chksum;
	int res;

	offset = odp_packet_l3_offset(pkt);
	if (offset == ODP_PACKET_OFFSET_INVALID)
		return -1;

	res = odp_packet_copy_to_mem(pkt, offset, sizeof(ip), &ip);
	if (odp_unlikely(res < 0))
		return res;

	res = packet_ipv4_chksum(pkt, offset, &ip, &chksum);
	if (odp_unlikely(res < 0))
		return res;

	return odp_packet_copy_from_mem(pkt,
					offset + _ODP_IPV4HDR_CSUM_OFFSET,
					2, &chksum);
}

static int _odp_packet_tcp_udp_chksum_insert(odp_packet_t pkt, uint16_t proto)
{
	odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt);
	uint32_t zero = 0;
	uint64_t sum;
	uint16_t l3_ver = 0; /* GCC */
	uint16_t chksum;
	uint32_t chksum_offset;
	uint32_t frame_len = odp_packet_len(pkt);

	if (pkt_hdr->p.l3_offset == ODP_PACKET_OFFSET_INVALID)
		return -1;
	if (pkt_hdr->p.l4_offset == ODP_PACKET_OFFSET_INVALID)
		return -1;

	odp_packet_copy_to_mem(pkt, pkt_hdr->p.l3_offset, 2, &l3_ver);

	if (_ODP_IPV4HDR_VER(l3_ver) == _ODP_IPV4)
		sum = packet_sum_partial(pkt_hdr,
					 pkt_hdr->p.l3_offset,
					 pkt_hdr->p.l3_offset +
					 _ODP_IPV4ADDR_OFFSSET,
					 2 * _ODP_IPV4ADDR_LEN);
	else
		sum = packet_sum_partial(pkt_hdr,
					 pkt_hdr->p.l3_offset,
					 pkt_hdr->p.l3_offset +
					 _ODP_IPV6ADDR_OFFSSET,
					 2 * _ODP_IPV6ADDR_LEN);
#if ODP_BYTE_ORDER == ODP_BIG_ENDIAN
	sum += proto;
#else
	sum += proto << 8;
#endif

	if (proto == _ODP_IPPROTO_TCP) {
		sum += odp_cpu_to_be_16(frame_len -
					pkt_hdr->p.l4_offset);
		chksum_offset = pkt_hdr->p.l4_offset + _ODP_UDP_CSUM_OFFSET;
	} else {
		sum += packet_sum_partial(pkt_hdr,
					  pkt_hdr->p.l3_offset,
					  pkt_hdr->p.l4_offset +
					  _ODP_UDP_LEN_OFFSET,
					  2);
		chksum_offset = pkt_hdr->p.l4_offset + _ODP_UDP_CSUM_OFFSET;
	}
	odp_packet_copy_from_mem(pkt, chksum_offset, 2, &zero);

	sum += packet_sum_partial(pkt_hdr,
				  pkt_hdr->p.l3_offset,
				  pkt_hdr->p.l4_offset,
				  frame_len -
				  pkt_hdr->p.l4_offset);

	chksum = ~chksum_finalize(sum);

	if (proto == _ODP_IPPROTO_UDP && chksum == 0)
		chksum = 0xffff;

	return odp_packet_copy_from_mem(pkt,
					chksum_offset,
					2, &chksum);
}

/**
 * Calculate and fill in TCP checksum
 *
 * @param pkt  ODP packet
 *
 * @retval 0 on success
 * @retval <0 on failure
 */
int _odp_packet_tcp_chksum_insert(odp_packet_t pkt)
{
	return _odp_packet_tcp_udp_chksum_insert(pkt, _ODP_IPPROTO_TCP);
}

/**
 * Calculate and fill in UDP checksum
 *
 * @param pkt  ODP packet
 *
 * @retval 0 on success
 * @retval <0 on failure
 */
int _odp_packet_udp_chksum_insert(odp_packet_t pkt)
{
	return _odp_packet_tcp_udp_chksum_insert(pkt, _ODP_IPPROTO_UDP);
}

/**
 * Calculate and fill in SCTP checksum
 *
 * @param pkt  ODP packet
 *
 * @retval 0 on success
 * @retval <0 on failure
 */
int _odp_packet_sctp_chksum_insert(odp_packet_t pkt)
{
	odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt);
	uint32_t sum;
	uint32_t frame_len = odp_packet_len(pkt);

	if (pkt_hdr->p.l4_offset == ODP_PACKET_OFFSET_INVALID)
		return -1;

	sum = 0;
	odp_packet_copy_from_mem(pkt, pkt_hdr->p.l4_offset + 8, 4, &sum);
	sum = ~packet_sum_crc32c(pkt_hdr, pkt_hdr->p.l4_offset,
				 frame_len - pkt_hdr->p.l4_offset,
				 ~0);
	return odp_packet_copy_from_mem(pkt, pkt_hdr->p.l4_offset + 8, 4, &sum);
}

int _odp_packet_l4_chksum(odp_packet_hdr_t *pkt_hdr,
			  odp_pktin_config_opt_t opt, uint64_t l4_part_sum)

{
	uint32_t frame_len = odp_packet_len(packet_handle(pkt_hdr));

	/* UDP chksum == 0 case is covered in parse_udp() */
	if (opt.bit.udp_chksum &&
	    pkt_hdr->p.input_flags.udp &&
	    !pkt_hdr->p.input_flags.ipfrag &&
	    !pkt_hdr->p.input_flags.udp_chksum_zero) {
		uint16_t sum = ~packet_sum(pkt_hdr,
					   pkt_hdr->p.l3_offset,
					   pkt_hdr->p.l4_offset,
					   frame_len -
					   pkt_hdr->p.l4_offset,
					   l4_part_sum);

		pkt_hdr->p.input_flags.l4_chksum_done = 1;
		if (sum != 0) {
			pkt_hdr->p.flags.l4_chksum_err = 1;
			pkt_hdr->p.flags.udp_err = 1;
			_ODP_DBG("UDP chksum fail (%x)!\n", sum);
			if (opt.bit.drop_udp_err)
				return -1;
		}
	}

	if (opt.bit.tcp_chksum &&
	    pkt_hdr->p.input_flags.tcp &&
	    !pkt_hdr->p.input_flags.ipfrag) {
		uint16_t sum = ~packet_sum(pkt_hdr,
					   pkt_hdr->p.l3_offset,
					   pkt_hdr->p.l4_offset,
					   frame_len -
					   pkt_hdr->p.l4_offset,
					   l4_part_sum);

		pkt_hdr->p.input_flags.l4_chksum_done = 1;
		if (sum != 0) {
			pkt_hdr->p.flags.l4_chksum_err = 1;
			pkt_hdr->p.flags.tcp_err = 1;
			_ODP_DBG("TCP chksum fail (%x)!\n", sum);
			if (opt.bit.drop_tcp_err)
				return -1;
		}
	}

	if (opt.bit.sctp_chksum &&
	    pkt_hdr->p.input_flags.sctp &&
	    !pkt_hdr->p.input_flags.ipfrag) {
		uint32_t seg_len = 0;
		_odp_sctphdr_t hdr_copy;
		uint32_t sum = ~packet_sum_crc32c(pkt_hdr,
						  pkt_hdr->p.l4_offset +
						  _ODP_SCTPHDR_LEN,
						  frame_len -
						  pkt_hdr->p.l4_offset -
						  _ODP_SCTPHDR_LEN,
						  l4_part_sum);
		_odp_sctphdr_t *sctp = odp_packet_offset(packet_handle(pkt_hdr),
							 pkt_hdr->p.l4_offset,
							 &seg_len, NULL);
		if (odp_unlikely(seg_len < sizeof(*sctp))) {
			odp_packet_t pkt = packet_handle(pkt_hdr);

			sctp = &hdr_copy;
			odp_packet_copy_to_mem(pkt, pkt_hdr->p.l4_offset,
					       sizeof(*sctp), sctp);
		}
		pkt_hdr->p.input_flags.l4_chksum_done = 1;
		if (sum != sctp->chksum) {
			pkt_hdr->p.flags.l4_chksum_err = 1;
			pkt_hdr->p.flags.sctp_err = 1;
			_ODP_DBG("SCTP chksum fail (%x/%x)!\n", sum, sctp->chksum);
			if (opt.bit.drop_sctp_err)
				return -1;
		}
	}

	return pkt_hdr->p.flags.all.error != 0;
}

int odp_packet_parse(odp_packet_t pkt, uint32_t offset,
		     const odp_packet_parse_param_t *param)
{
	odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt);
	const uint8_t *data;
	uint32_t seg_len;
	uint32_t seg_end;
	uint32_t packet_len = odp_packet_len(pkt);
	odp_proto_t proto = param->proto;
	odp_proto_layer_t layer = param->last_layer;
	int ret;
	uint16_t ethtype;
	uint64_t l4_part_sum = 0;
	const uint32_t min_seglen = PARSE_ETH_BYTES + PARSE_L3_L4_BYTES;
	uint8_t buf[min_seglen];
	odp_pktin_config_opt_t opt;

	if (proto == ODP_PROTO_NONE || layer == ODP_PROTO_LAYER_NONE)
		return -1;

	data = odp_packet_offset(pkt, offset, &seg_len, NULL);

	if (data == NULL)
		return -1;

	/*
	 * We must not have a packet segment boundary within the parsed
	 * packet data range. Copy enough data to a temporary buffer for
	 * parsing if necessary.
	 */
	if (odp_unlikely(pkt_hdr->event_hdr.mb.nb_segs > 1) &&
	    odp_unlikely(seg_len < min_seglen)) {
		seg_len = min_seglen;
		if (seg_len > packet_len - offset)
			seg_len = packet_len - offset;
		odp_packet_copy_to_mem(pkt, offset, seg_len, buf);
		data = buf;
	}

	seg_end = offset + seg_len; /* one past the maximum offset */

	/* Reset parser flags, keep other flags */
	packet_parse_reset(pkt_hdr, 0);

	if (proto == ODP_PROTO_ETH) {
		/* Assume valid L2 header, no CRC/FCS check in SW */
		pkt_hdr->p.l2_offset = offset;

		ethtype = _odp_parse_eth(&pkt_hdr->p, &data, &offset, packet_len);
	} else if (proto == ODP_PROTO_IPV4) {
		ethtype = _ODP_ETHTYPE_IPV4;
	} else if (proto == ODP_PROTO_IPV6) {
		ethtype = _ODP_ETHTYPE_IPV6;
	} else {
		ethtype = 0; /* Invalid */
	}

	opt.all_bits = 0;
	opt.bit.ipv4_chksum = param->chksums.chksum.ipv4;
	opt.bit.udp_chksum = param->chksums.chksum.udp;
	opt.bit.tcp_chksum = param->chksums.chksum.tcp;
	opt.bit.sctp_chksum = param->chksums.chksum.sctp;

	ret = _odp_packet_parse_common_l3_l4(&pkt_hdr->p, data, offset,
					     packet_len, seg_end, layer,
					     ethtype, &l4_part_sum, opt);

	if (ret)
		return -1;

	if (layer >= ODP_PROTO_LAYER_L4) {
		ret = _odp_packet_l4_chksum(pkt_hdr, opt, l4_part_sum);
		if (ret)
			return -1;
	}

	return 0;
}

int odp_packet_parse_multi(const odp_packet_t pkt[], const uint32_t offset[],
			   int num, const odp_packet_parse_param_t *param)
{
	int i;

	for (i = 0; i < num; i++)
		if (odp_packet_parse(pkt[i], offset[i], param))
			return i;

	return num;
}

void odp_packet_parse_result(odp_packet_t pkt,
			     odp_packet_parse_result_t *result)
{
	/* TODO: optimize to single word copy when packet header stores bits
	 * directly into odp_packet_parse_result_flag_t */
	result->flag.all           = 0;
	result->flag.has_error     = odp_packet_has_error(pkt);
	result->flag.has_l2_error  = odp_packet_has_l2_error(pkt);
	result->flag.has_l3_error  = odp_packet_has_l3_error(pkt);
	result->flag.has_l4_error  = odp_packet_has_l4_error(pkt);
	result->flag.has_l2        = odp_packet_has_l2(pkt);
	result->flag.has_l3        = odp_packet_has_l3(pkt);
	result->flag.has_l4        = odp_packet_has_l4(pkt);
	result->flag.has_eth       = odp_packet_has_eth(pkt);
	result->flag.has_eth_bcast = odp_packet_has_eth_bcast(pkt);
	result->flag.has_eth_mcast = odp_packet_has_eth_mcast(pkt);
	result->flag.has_jumbo     = odp_packet_has_jumbo(pkt);
	result->flag.has_vlan      = odp_packet_has_vlan(pkt);
	result->flag.has_vlan_qinq = odp_packet_has_vlan_qinq(pkt);
	result->flag.has_arp       = odp_packet_has_arp(pkt);
	result->flag.has_ipv4      = odp_packet_has_ipv4(pkt);
	result->flag.has_ipv6      = odp_packet_has_ipv6(pkt);
	result->flag.has_ip_bcast  = odp_packet_has_ip_bcast(pkt);
	result->flag.has_ip_mcast  = odp_packet_has_ip_mcast(pkt);
	result->flag.has_ipfrag    = odp_packet_has_ipfrag(pkt);
	result->flag.has_ipopt     = odp_packet_has_ipopt(pkt);
	result->flag.has_ipsec     = odp_packet_has_ipsec(pkt);
	result->flag.has_udp       = odp_packet_has_udp(pkt);
	result->flag.has_tcp       = odp_packet_has_tcp(pkt);
	result->flag.has_sctp      = odp_packet_has_sctp(pkt);
	result->flag.has_icmp      = odp_packet_has_icmp(pkt);

	result->packet_len       = odp_packet_len(pkt);
	result->l2_offset        = odp_packet_l2_offset(pkt);
	result->l3_offset        = odp_packet_l3_offset(pkt);
	result->l4_offset        = odp_packet_l4_offset(pkt);
	result->l3_chksum_status = odp_packet_l3_chksum_status(pkt);
	result->l4_chksum_status = odp_packet_l4_chksum_status(pkt);
	result->l2_type          = odp_packet_l2_type(pkt);
	result->l3_type          = odp_packet_l3_type(pkt);
	result->l4_type          = odp_packet_l4_type(pkt);
}

void odp_packet_parse_result_multi(const odp_packet_t pkt[],
				   odp_packet_parse_result_t *result[],
				   int num)
{
	int i;

	for (i = 0; i < num; i++)
		odp_packet_parse_result(pkt[i], result[i]);
}

uint64_t odp_packet_to_u64(odp_packet_t hdl)
{
	return _odp_pri(hdl);
}

uint64_t odp_packet_seg_to_u64(odp_packet_seg_t hdl)
{
	return _odp_pri(hdl);
}

odp_packet_t odp_packet_ref(odp_packet_t pkt, uint32_t offset)
{
	odp_packet_t new;
	int ret;

	_ODP_ASSERT(!odp_packet_has_ref(pkt));

	new = odp_packet_copy(pkt, odp_packet_pool(pkt));

	if (new == ODP_PACKET_INVALID) {
		_ODP_ERR("copy failed\n");
		return ODP_PACKET_INVALID;
	}

	ret = odp_packet_trunc_head(&new, offset, NULL, NULL);

	if (ret < 0) {
		_ODP_ERR("trunk_head failed\n");
		odp_packet_free(new);
		return ODP_PACKET_INVALID;
	}

	return new;
}

odp_packet_t odp_packet_ref_pkt(odp_packet_t pkt, uint32_t offset,
				odp_packet_t hdr)
{
	odp_packet_t new;
	int ret;

	_ODP_ASSERT(!odp_packet_has_ref(pkt));

	new = odp_packet_copy(pkt, odp_packet_pool(pkt));

	if (new == ODP_PACKET_INVALID) {
		_ODP_ERR("copy failed\n");
		return ODP_PACKET_INVALID;
	}

	if (offset) {
		ret = odp_packet_trunc_head(&new, offset, NULL, NULL);

		if (ret < 0) {
			_ODP_ERR("trunk_head failed\n");
			odp_packet_free(new);
			return ODP_PACKET_INVALID;
		}
	}

	ret = odp_packet_concat(&hdr, new);

	if (ret < 0) {
		_ODP_ERR("concat failed\n");
		odp_packet_free(new);
		return ODP_PACKET_INVALID;
	}

	return hdr;
}

void odp_packet_lso_request_clr(odp_packet_t pkt)
{
	odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt);

	pkt_hdr->p.flags.lso = 0;
}

int odp_packet_has_lso_request(odp_packet_t pkt)
{
	odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt);

	return pkt_hdr->p.flags.lso;
}

uint32_t odp_packet_payload_offset(odp_packet_t pkt)
{
	odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt);

	if (pkt_hdr->p.flags.payload_off)
		return pkt_hdr->payload_offset;

	return ODP_PACKET_OFFSET_INVALID;
}

int odp_packet_payload_offset_set(odp_packet_t pkt, uint32_t offset)
{
	odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt);

	pkt_hdr->p.flags.payload_off = 1;
	pkt_hdr->payload_offset      = offset;

	return 0;
}

void odp_packet_aging_tmo_set(odp_packet_t pkt, uint64_t tmo_ns)
{
	odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt);

	pkt_hdr->p.flags.tx_aging = tmo_ns ? 1 : 0;
	pkt_hdr->tx_aging_ns = tmo_ns;
}

uint64_t odp_packet_aging_tmo(odp_packet_t pkt)
{
	odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt);

	return pkt_hdr->p.flags.tx_aging ? pkt_hdr->tx_aging_ns : 0;
}

int odp_packet_tx_compl_request(odp_packet_t pkt, const odp_packet_tx_compl_opt_t *opt)
{
	odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt);

	pkt_hdr->p.flags.tx_compl = opt->mode == ODP_PACKET_TX_COMPL_ALL ? 1 : 0;
	pkt_hdr->dst_queue = opt->queue;

	return 0;
}

int odp_packet_has_tx_compl_request(odp_packet_t pkt)
{
	odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt);

	return pkt_hdr->p.flags.tx_compl;
}

void odp_packet_tx_compl_free(odp_packet_tx_compl_t tx_compl)
{
	if (odp_unlikely(tx_compl == ODP_PACKET_TX_COMPL_INVALID)) {
		_ODP_ERR("Bad TX completion event handle\n");
		return;
	}

	odp_buffer_free((odp_buffer_t)tx_compl);
}

void *odp_packet_tx_compl_user_ptr(odp_packet_tx_compl_t tx_compl)
{
	if (odp_unlikely(tx_compl == ODP_PACKET_TX_COMPL_INVALID)) {
		_ODP_ERR("Bad TX completion event handle\n");
		return NULL;
	}

	_odp_pktio_tx_compl_t *data = odp_buffer_addr((odp_buffer_t)tx_compl);

	return (void *)(uintptr_t)data->user_ptr;
}

odp_packet_reass_status_t odp_packet_reass_status(odp_packet_t pkt)
{
	(void)pkt;
	return ODP_PACKET_REASS_NONE;
}

int odp_packet_reass_info(odp_packet_t pkt, odp_packet_reass_info_t *info)
{
	(void)pkt;
	(void)info;
	return -1;
}

int odp_packet_reass_partial_state(odp_packet_t pkt, odp_packet_t frags[],
				   odp_packet_reass_partial_state_t *res)
{
	(void)pkt;
	(void)frags;
	(void)res;
	return -ENOTSUP;
}

uint32_t odp_packet_disassemble(odp_packet_t pkt, odp_packet_buf_t pkt_buf[],
				uint32_t num)
{
	uint32_t i;
	odp_packet_seg_t seg;
	odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt);
	pool_t *pool = _odp_pool_entry(pkt_hdr->event_hdr.pool);
	uint32_t num_segs = odp_packet_num_segs(pkt);

	if (odp_unlikely(pool->type != ODP_POOL_PACKET)) {
		_ODP_ERR("Not a packet pool\n");
		return 0;
	}

	if (odp_unlikely(pool->pool_ext == 0)) {
		_ODP_ERR("Not an external memory pool\n");
		return 0;
	}

	if (odp_unlikely(num < num_segs)) {
		_ODP_ERR("Not enough buffer handles %u. Packet has %u segments.\n", num, num_segs);
		return 0;
	}

	seg = odp_packet_first_seg(pkt);

	for (i = 0; i < num_segs; i++) {
		pkt_buf[i] = (odp_packet_buf_t)(uintptr_t)seg;
		seg = odp_packet_next_seg(pkt, seg);
	}

	return num_segs;
}

odp_packet_t odp_packet_reassemble(odp_pool_t pool_hdl,
				   odp_packet_buf_t pkt_buf[], uint32_t num)
{
	uint32_t i, data_len;
	odp_packet_hdr_t *cur_seg, *next_seg;
	odp_packet_hdr_t *pkt_hdr = (odp_packet_hdr_t *)(uintptr_t)pkt_buf[0];
	uint32_t headroom = odp_packet_buf_data_offset(pkt_buf[0]);

	pool_t *pool = _odp_pool_entry(pool_hdl);

	if (odp_unlikely(pool->type != ODP_POOL_PACKET)) {
		_ODP_ERR("Not a packet pool\n");
		return ODP_PACKET_INVALID;
	}

	if (odp_unlikely(pool->pool_ext == 0)) {
		_ODP_ERR("Not an external memory pool\n");
		return ODP_PACKET_INVALID;
	}

	if (odp_unlikely(num == 0)) {
		_ODP_ERR("Bad number of buffers: %u\n", num);
		return ODP_PACKET_INVALID;
	}

	cur_seg = pkt_hdr;
	data_len = 0;

	for (i = 0; i < num; i++) {
		struct rte_mbuf *mb;

		next_seg = NULL;
		if (i < num - 1)
			next_seg = (odp_packet_hdr_t *)(uintptr_t)pkt_buf[i + 1];

		data_len += cur_seg->event_hdr.mb.data_len;
		mb = (struct rte_mbuf *)(uintptr_t)cur_seg;
		mb->next = (struct rte_mbuf *)next_seg;
		cur_seg = next_seg;
	}

	pkt_hdr->event_hdr.mb.nb_segs = num;
	pkt_hdr->event_hdr.mb.pkt_len = data_len;
	pkt_hdr->event_hdr.mb.data_off = headroom;

	/* Reset metadata */
	pkt_hdr->subtype = ODP_EVENT_PACKET_BASIC;
	pkt_hdr->input = ODP_PKTIO_INVALID;
	packet_parse_reset(pkt_hdr, 1);

	return packet_handle(pkt_hdr);
}

void odp_packet_proto_stats_request(odp_packet_t pkt, odp_packet_proto_stats_opt_t *opt)
{
	(void)pkt;
	(void)opt;
}

odp_proto_stats_t odp_packet_proto_stats(odp_packet_t pkt)
{
	(void)pkt;

	return ODP_PROTO_STATS_INVALID;
}