1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
|
/* Copyright (c) 2019-2023, Nokia
* All rights reserved.
*
* SPDX-License-Identifier: BSD-3-Clause
*/
#ifndef ODP_PLAT_BUFFER_INLINES_H_
#define ODP_PLAT_BUFFER_INLINES_H_
#include <odp/api/buffer_types.h>
#include <odp/api/event.h>
#include <odp/api/hints.h>
#include <odp/api/pool_types.h>
#include <odp/api/plat/buffer_inline_types.h>
#include <odp/api/plat/debug_inlines.h>
#include <odp/api/plat/event_inline_types.h>
#include <odp/api/plat/event_validation_external.h>
#include <rte_mbuf.h>
#include <rte_mempool.h>
#if defined(__PPC64__) && defined(bool)
#undef bool
#define bool _Bool
#endif
#if defined(__PPC64__) && defined(vector)
#undef vector
#endif
/** @cond _ODP_HIDE_FROM_DOXYGEN_ */
#ifndef _ODP_NO_INLINE
/* Inline functions by default */
#define _ODP_INLINE static inline
#define odp_buffer_from_event __odp_buffer_from_event
#define odp_buffer_from_event_multi __odp_buffer_from_event_multi
#define odp_buffer_to_event __odp_buffer_to_event
#define odp_buffer_to_event_multi __odp_buffer_to_event_multi
#define odp_buffer_addr __odp_buffer_addr
#define odp_buffer_size __odp_buffer_size
#define odp_buffer_pool __odp_buffer_pool
#define odp_buffer_user_area __odp_buffer_user_area
#define odp_buffer_free __odp_buffer_free
#define odp_buffer_free_multi __odp_buffer_free_multi
#define odp_buffer_is_valid __odp_buffer_is_valid
#else
#define _ODP_INLINE
#endif
_ODP_INLINE odp_buffer_t odp_buffer_from_event(odp_event_t ev)
{
_ODP_ASSERT(odp_event_type(ev) == ODP_EVENT_BUFFER);
return (odp_buffer_t)ev;
}
_ODP_INLINE void odp_buffer_from_event_multi(odp_buffer_t buf[], const odp_event_t ev[], int num)
{
for (int i = 0; i < num; i++)
buf[i] = odp_buffer_from_event(ev[i]);
}
_ODP_INLINE odp_event_t odp_buffer_to_event(odp_buffer_t buf)
{
return (odp_event_t)buf;
}
_ODP_INLINE void odp_buffer_to_event_multi(const odp_buffer_t buf[], odp_event_t ev[], int num)
{
for (int i = 0; i < num; i++)
ev[i] = odp_buffer_to_event(buf[i]);
}
_ODP_INLINE void *odp_buffer_addr(odp_buffer_t buf)
{
return _odp_event_hdr_field(buf, void *, base_data);
}
_ODP_INLINE uint32_t odp_buffer_size(odp_buffer_t buf)
{
return _odp_event_hdr_field(buf, uint16_t, buf_len);
}
_ODP_INLINE odp_pool_t odp_buffer_pool(odp_buffer_t buf)
{
return (odp_pool_t)(uintptr_t)_odp_event_hdr_field(buf, void *, pool);
}
_ODP_INLINE void *odp_buffer_user_area(odp_buffer_t buf)
{
return _odp_buffer_get(buf, void *, uarea_addr);
}
_ODP_INLINE void odp_buffer_free(odp_buffer_t buf)
{
struct rte_mbuf *mbuf = (struct rte_mbuf *)buf;
_odp_buffer_validate(buf, _ODP_EV_BUFFER_FREE);
rte_mempool_put(mbuf->pool, mbuf);
}
_ODP_INLINE void odp_buffer_free_multi(const odp_buffer_t buf[], int num)
{
struct rte_mbuf *mbuf_tbl[num];
struct rte_mempool *mp_pending;
unsigned int num_pending;
if (odp_unlikely(num <= 0))
return;
_odp_buffer_validate_multi(buf, num, _ODP_EV_BUFFER_FREE_MULTI);
mbuf_tbl[0] = (struct rte_mbuf *)buf[0];
mp_pending = mbuf_tbl[0]->pool;
num_pending = 1;
for (int i = 1; i < num; i++) {
struct rte_mbuf *mbuf = (struct rte_mbuf *)buf[i];
if (mbuf->pool != mp_pending) {
rte_mempool_put_bulk(mp_pending, (void **)mbuf_tbl, num_pending);
mbuf_tbl[0] = mbuf;
num_pending = 1;
mp_pending = mbuf->pool;
} else {
mbuf_tbl[num_pending++] = mbuf;
}
}
rte_mempool_put_bulk(mp_pending, (void **)mbuf_tbl, num_pending);
}
_ODP_INLINE int odp_buffer_is_valid(odp_buffer_t buf)
{
if (odp_event_is_valid(odp_buffer_to_event(buf)) == 0)
return 0;
if (odp_event_type(odp_buffer_to_event(buf)) != ODP_EVENT_BUFFER)
return 0;
if (odp_unlikely(_odp_buffer_validate(buf, _ODP_EV_BUFFER_IS_VALID)))
return 0;
return 1;
}
/** @endcond */
#endif
|
