1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
|
/*
* This file and its contents are supplied under the terms of the
* Common Development and Distribution License ("CDDL"), version 1.0.
* You may only use this file in accordance with the terms of version
* 1.0 of the CDDL.
*
* A full copy of the text of the CDDL should have accompanied this
* source. A copy of the CDDL is also available via the Internet at
* http://www.illumos.org/license/CDDL.
*/
/*
* Copyright 2019 Joyent, Inc.
*/
.file "retpoline.s"
/*
* This file implements the various hooks that are needed for retpolines and
* return stack buffer (RSB) stuffing. For more information, please see the
* 'Speculative Execution CPU Side Channel Security' section of the
* uts/i86pc/os/cpuid.c big theory statement.
*/
#include <sys/asm_linkage.h>
#include <sys/x86_archext.h>
#if defined(__amd64)
/*
* This macro generates the default retpoline entry point that the compiler
* expects. It implements the expected retpoline form.
*/
#define RETPOLINE_MKTHUNK(reg) \
ENTRY(__x86_indirect_thunk_/**/reg) \
call 2f; \
1: \
pause; \
lfence; \
jmp 1b; \
2: \
movq %/**/reg, (%rsp); \
ret; \
SET_SIZE(__x86_indirect_thunk_/**/reg)
/*
* This macro generates the default retpoline form. It exists in addition to the
* thunk so if we need to restore the default retpoline behavior to the thunk
* we can.
*/
#define RETPOLINE_MKGENERIC(reg) \
ENTRY(__x86_indirect_thunk_gen_/**/reg) \
call 2f; \
1: \
pause; \
lfence; \
jmp 1b; \
2: \
movq %/**/reg, (%rsp); \
ret; \
SET_SIZE(__x86_indirect_thunk_gen_/**/reg)
/*
* This macro generates the no-op form of the retpoline which will be used if we
* either need to disable retpolines because we have enhanced IBRS or because we
* have been asked to disable mitigations.
*/
#define RETPOLINE_MKJUMP(reg) \
ENTRY(__x86_indirect_thunk_jmp_/**/reg) \
jmp *%/**/reg; \
SET_SIZE(__x86_indirect_thunk_jmp_/**/reg)
RETPOLINE_MKTHUNK(rax)
RETPOLINE_MKTHUNK(rbx)
RETPOLINE_MKTHUNK(rcx)
RETPOLINE_MKTHUNK(rdx)
RETPOLINE_MKTHUNK(rdi)
RETPOLINE_MKTHUNK(rsi)
RETPOLINE_MKTHUNK(rbp)
RETPOLINE_MKTHUNK(r8)
RETPOLINE_MKTHUNK(r9)
RETPOLINE_MKTHUNK(r10)
RETPOLINE_MKTHUNK(r11)
RETPOLINE_MKTHUNK(r12)
RETPOLINE_MKTHUNK(r13)
RETPOLINE_MKTHUNK(r14)
RETPOLINE_MKTHUNK(r15)
RETPOLINE_MKGENERIC(rax)
RETPOLINE_MKGENERIC(rbx)
RETPOLINE_MKGENERIC(rcx)
RETPOLINE_MKGENERIC(rdx)
RETPOLINE_MKGENERIC(rdi)
RETPOLINE_MKGENERIC(rsi)
RETPOLINE_MKGENERIC(rbp)
RETPOLINE_MKGENERIC(r8)
RETPOLINE_MKGENERIC(r9)
RETPOLINE_MKGENERIC(r10)
RETPOLINE_MKGENERIC(r11)
RETPOLINE_MKGENERIC(r12)
RETPOLINE_MKGENERIC(r13)
RETPOLINE_MKGENERIC(r14)
RETPOLINE_MKGENERIC(r15)
RETPOLINE_MKJUMP(rax)
RETPOLINE_MKJUMP(rbx)
RETPOLINE_MKJUMP(rcx)
RETPOLINE_MKJUMP(rdx)
RETPOLINE_MKJUMP(rdi)
RETPOLINE_MKJUMP(rsi)
RETPOLINE_MKJUMP(rbp)
RETPOLINE_MKJUMP(r8)
RETPOLINE_MKJUMP(r9)
RETPOLINE_MKJUMP(r10)
RETPOLINE_MKJUMP(r11)
RETPOLINE_MKJUMP(r12)
RETPOLINE_MKJUMP(r13)
RETPOLINE_MKJUMP(r14)
RETPOLINE_MKJUMP(r15)
/*
* The x86_rsb_stuff function is called from pretty arbitrary
* contexts. It's much easier for us to save and restore all the
* registers we touch rather than clobber them for callers. You must
* preserve this property or the system will panic at best.
*/
ENTRY(x86_rsb_stuff)
/*
* These nops are present so we can patch a ret instruction if we need
* to disable RSB stuffing because enhanced IBRS is present or we're
* disabling mitigations.
*/
nop
nop
pushq %rdi
pushq %rax
movl $16, %edi
movq %rsp, %rax
rsb_loop:
call 2f
1:
pause
call 1b
2:
call 2f
1:
pause
call 1b
2:
subl $1, %edi
jnz rsb_loop
movq %rax, %rsp
popq %rax
popq %rdi
ret
SET_SIZE(x86_rsb_stuff)
#elif defined(__i386)
/*
* While the kernel is 64-bit only, dboot is still 32-bit, so there are a
* limited number of variants that are used for 32-bit. However as dboot is
* short lived and uses them sparingly, we only do the full variant and do not
* have an AMD specific version.
*/
#define RETPOLINE_MKTHUNK(reg) \
ENTRY(__x86_indirect_thunk_/**/reg) \
call 2f; \
1: \
pause; \
lfence; \
jmp 1b; \
2: \
movl %/**/reg, (%esp); \
ret; \
SET_SIZE(__x86_indirect_thunk_/**/reg)
RETPOLINE_MKTHUNK(edi)
RETPOLINE_MKTHUNK(eax)
#else
#error "Your architecture is in another castle."
#endif
|
