1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
|
/*
* This file and its contents are supplied under the terms of the
* Common Development and Distribution License ("CDDL"), version 1.0.
* You may only use this file in accordance with the terms of version
* 1.0 of the CDDL.
*
* A full copy of the text of the CDDL should have accompanied this
* source. A copy of the CDDL is also available via the Internet at
* http://www.illumos.org/license/CDDL.
*/
/*
* Copyright 2016 Joyent, Inc.
*/
#include <sys/asm_linkage.h>
#include <sys/segments.h>
#include <sys/time_impl.h>
#include <sys/tsc.h>
#define GETCPU_GDT_OFFSET SEL_GDT(GDT_CPUID, SEL_UPL)
.file "cp_subr.s"
/*
* hrtime_t
* __cp_tsc_read(uint_t cp_tsc_type)
*/
ENTRY_NP(__cp_tsc_read)
cmpl $TSC_TSCP, %edi
je 1f
cmpl $TSC_RDTSC_MFENCE, %edi
je 2f
cmpl $TSC_RDTSC_LFENCE, %edi
je 3f
cmpl $TSC_RDTSC_CPUID, %edi
je 4f
ud2a /* abort with SIGILL */
1:
rdtscp
jmp 5f
2:
mfence
rdtsc
jmp 5f
3:
lfence
rdtsc
jmp 5f
4:
movq %rbx, %r11
xorl %eax, %eax
cpuid
rdtsc
movq %r11, %rbx
5:
shlq $0x20, %rdx
orq %rdx, %rax
ret
SET_SIZE(__cp_tsc_read)
/*
* hrtime_t
* __cp_tsc_readcpu(uint_t cp_tsc_type, uint_t *cpu_id)
*/
ENTRY_NP(__cp_tsc_readcpu)
/*
* Both time and cpu_id can be queried quickly (using few registers) on
* systems which support RDTSCP. On each cpu, the cpu_id is stored in
* the TSC_AUX MSR by the kernel.
*/
cmpl $TSC_TSCP, %edi
jne 1f
rdtscp
movl %ecx, (%rsi)
shlq $0x20, %rdx
orq %rdx, %rax
ret
1:
mov $GETCPU_GDT_OFFSET, %eax
lsl %ax, %eax
movq %rax, %r11
cmpl $TSC_RDTSC_MFENCE, %edi
je 2f
cmpl $TSC_RDTSC_LFENCE, %edi
je 3f
cmpl $TSC_RDTSC_CPUID, %edi
je 4f
ud2a /* abort with SIGILL */
2:
mfence
rdtsc
jmp 5f
3:
lfence
rdtsc
jmp 5f
4:
movq %rbx, %r10
xorl %eax, %eax
cpuid
rdtsc
movq %r10, %rbx
5:
shlq %rdx
orq %rax, %rdx
/*
* With a TSC reading in-hand, confirm that the thread has not migrated
* since the cpu_id was first checked.
*/
mov $GETCPU_GDT_OFFSET, %eax
lsl %ax, %eax
cmpq %rax, %r11
jne 1b
movl %eax, (%rsi)
movq %rdx, %rax
ret
SET_SIZE(__cp_tsc_readcpu)
/*
* uint_t
* __cp_do_getcpu(uint_t cp_tsc_type)
*/
ENTRY_NP(__cp_do_getcpu)
/*
* If RDTSCP is available, it is a quick way to grab the cpu_id which
* is stored in the TSC_AUX MSR by the kernel.
*/
cmpl $TSC_TSCP, %edi
jne 1f
rdtscp
movl %ecx, %eax
ret
1:
mov $GETCPU_GDT_OFFSET, %eax
lsl %ax, %eax
ret
SET_SIZE(__cp_do_getcpu)
|
