1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
|
// SPDX-License-Identifier: GPL-2.0
/*
* Shadow Call Stack support.
*
* Copyright (C) 2019 Google LLC
*/
#include <linux/cpuhotplug.h>
#include <linux/kasan.h>
#include <linux/mm.h>
#include <linux/mmzone.h>
#include <linux/scs.h>
#include <linux/slab.h>
#include <linux/vmalloc.h>
#include <asm/scs.h>
static inline void *__scs_base(struct task_struct *tsk)
{
/*
* To minimize risk the of exposure, architectures may clear a
* task's thread_info::shadow_call_stack while that task is
* running, and only save/restore the active shadow call stack
* pointer when the usual register may be clobbered (e.g. across
* context switches).
*
* The shadow call stack is aligned to SCS_SIZE, and grows
* upwards, so we can mask out the low bits to extract the base
* when the task is not running.
*/
return (void *)((unsigned long)task_scs(tsk) & ~(SCS_SIZE - 1));
}
static inline unsigned long *scs_magic(void *s)
{
return (unsigned long *)(s + SCS_SIZE) - 1;
}
static inline void scs_set_magic(void *s)
{
*scs_magic(s) = SCS_END_MAGIC;
}
#ifdef CONFIG_SHADOW_CALL_STACK_VMAP
/* Matches NR_CACHED_STACKS for VMAP_STACK */
#define NR_CACHED_SCS 2
static DEFINE_PER_CPU(void *, scs_cache[NR_CACHED_SCS]);
static void *scs_alloc(int node)
{
int i;
void *s;
for (i = 0; i < NR_CACHED_SCS; i++) {
s = this_cpu_xchg(scs_cache[i], NULL);
if (s) {
memset(s, 0, SCS_SIZE);
goto out;
}
}
/*
* We allocate a full page for the shadow stack, which should be
* more than we need. Check the assumption nevertheless.
*/
BUILD_BUG_ON(SCS_SIZE > PAGE_SIZE);
s = __vmalloc_node_range(PAGE_SIZE, SCS_SIZE,
VMALLOC_START, VMALLOC_END,
GFP_SCS, PAGE_KERNEL, 0,
node, __builtin_return_address(0));
out:
if (s)
scs_set_magic(s);
/* TODO: poison for KASAN, unpoison in scs_free */
return s;
}
static void scs_free(void *s)
{
int i;
for (i = 0; i < NR_CACHED_SCS; i++)
if (this_cpu_cmpxchg(scs_cache[i], 0, s) == NULL)
return;
vfree_atomic(s);
}
static int scs_cleanup(unsigned int cpu)
{
int i;
void **cache = per_cpu_ptr(scs_cache, cpu);
for (i = 0; i < NR_CACHED_SCS; i++) {
vfree(cache[i]);
cache[i] = NULL;
}
return 0;
}
void __init scs_init(void)
{
WARN_ON(cpuhp_setup_state(CPUHP_BP_PREPARE_DYN, "scs:scs_cache", NULL,
scs_cleanup) < 0);
}
#else /* !CONFIG_SHADOW_CALL_STACK_VMAP */
static struct kmem_cache *scs_cache;
static inline void *scs_alloc(int node)
{
void *s;
s = kmem_cache_alloc_node(scs_cache, GFP_SCS, node);
if (s) {
scs_set_magic(s);
/*
* Poison the allocation to catch unintentional accesses to
* the shadow stack when KASAN is enabled.
*/
kasan_poison_object_data(scs_cache, s);
}
return s;
}
static inline void scs_free(void *s)
{
kasan_unpoison_object_data(scs_cache, s);
kmem_cache_free(scs_cache, s);
}
void __init scs_init(void)
{
scs_cache = kmem_cache_create("scs_cache", SCS_SIZE, SCS_SIZE,
0, NULL);
WARN_ON(!scs_cache);
}
#endif /* CONFIG_SHADOW_CALL_STACK_VMAP */
void scs_task_reset(struct task_struct *tsk)
{
/*
* Reset the shadow stack to the base address in case the task
* is reused.
*/
task_set_scs(tsk, __scs_base(tsk));
}
int scs_prepare(struct task_struct *tsk, int node)
{
void *s;
s = scs_alloc(node);
if (!s)
return -ENOMEM;
task_set_scs(tsk, s);
return 0;
}
bool scs_corrupted(struct task_struct *tsk)
{
unsigned long *magic = scs_magic(__scs_base(tsk));
return READ_ONCE_NOCHECK(*magic) != SCS_END_MAGIC;
}
void scs_release(struct task_struct *tsk)
{
void *s;
s = __scs_base(tsk);
if (!s)
return;
WARN_ON(scs_corrupted(tsk));
task_set_scs(tsk, NULL);
scs_free(s);
}
|
