9059 Simplify SMAP relocations with krtld

Portions contributed by: John Levon <john.levon@joyent.com>
Reviewed by: Patrick Mooney <pmooney@pfmooney.com>
Approved by: Dan McDonald <danmcd@joyent.com>

9 files changed, 171 insertions, 57 deletions

diff --git a/usr/src/uts/common/krtld/kobj.c b/usr/src/uts/common/krtld/kobj.c
index 0e03425c8b..e6a91bcb33 100644
--- a/usr/src/uts/common/krtld/kobj.c
+++ b/usr/src/uts/common/krtld/kobj.c
@@ -2179,6 +2179,7 @@ static void
 free_module_data(struct module *mp)
 {
 	struct module_list *lp, *tmp;
+	hotinline_desc_t *hid, *next;
 	int ksyms_exported = 0;
 
 	lp = mp->head;
@@ -2188,6 +2189,15 @@ free_module_data(struct module *mp)
 		kobj_free((char *)tmp, sizeof (*tmp));
 	}
 
+	/* release hotinlines */
+	hid = mp->hi_calls;
+	while (hid != NULL) {
+		next = hid->hid_next;
+		kobj_free(hid->hid_symname, strlen(hid->hid_symname) + 1);
+		kobj_free(hid, sizeof (hotinline_desc_t));
+		hid = next;
+	}
+
 	rw_enter(&ksyms_lock, RW_WRITER);
 	if (mp->symspace) {
 		if (vmem_contains(ksyms_arena, mp->symspace, mp->symsize)) {
@@ -3033,8 +3043,18 @@ do_symbols(struct module *mp, Elf64_Addr bss_base)
 		if (sp->st_shndx == SHN_UNDEF) {
 			resolved = 0;
 
+			/*
+			 * Skip over sdt probes and smap calls,
+			 * they're relocated later.
+			 */
 			if (strncmp(name, sdt_prefix, strlen(sdt_prefix)) == 0)
 				continue;
+#if defined(__x86)
+			if (strcmp(name, "smap_enable") == 0 ||
+			    strcmp(name, "smap_disable") == 0)
+				continue;
+#endif /* defined(__x86) */
+
 
 			/*
 			 * If it's not a weak reference and it's
diff --git a/usr/src/uts/common/os/modctl.c b/usr/src/uts/common/os/modctl.c
index 7463cdb92f..d8782b320e 100644
--- a/usr/src/uts/common/os/modctl.c
+++ b/usr/src/uts/common/os/modctl.c
@@ -21,6 +21,7 @@
 
 /*
  * Copyright (c) 1990, 2010, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2017 Joyent, Inc.
  */
 
 /*
@@ -3473,6 +3474,11 @@ mod_load(struct modctl *mp, int usepath)
 		retval = install_stubs_by_name(mp, mp->mod_modname);
 
 		/*
+		 * Perform hotinlines before module is started.
+		 */
+		do_hotinlines(mp->mod_mp);
+
+		/*
 		 * Now that the module is loaded, we need to give DTrace
 		 * a chance to notify its providers.  This is done via
 		 * the dtrace_modload function pointer.
diff --git a/usr/src/uts/common/sys/kobj.h b/usr/src/uts/common/sys/kobj.h
index 2396ef4625..d52a54f6b7 100644
--- a/usr/src/uts/common/sys/kobj.h
+++ b/usr/src/uts/common/sys/kobj.h
@@ -24,6 +24,9 @@
  *
  * Copyright 2017 RackTop Systems.
  */
+/*
+ * Copyright (c) 2017 Joyent, Inc.
+ */
 
 #ifndef _SYS_KOBJ_H
 #define	_SYS_KOBJ_H
@@ -47,6 +50,12 @@ struct module_list {
 	struct module *mp;
 };
 
+typedef struct hotinline_desc {
+	char	*hid_symname;		/* symbol name */
+	uintptr_t hid_instr_offset;	/* offset of call in text */
+	struct hotinline_desc *hid_next;	/* next hotinline */
+} hotinline_desc_t;
+
 typedef unsigned short	symid_t;		/* symbol table index */
 typedef unsigned char	*reloc_dest_t;
 
@@ -99,6 +108,8 @@ struct module {
 	caddr_t textwin;
 	caddr_t textwin_base;
 
+	hotinline_desc_t *hi_calls;
+
 	sdt_probedesc_t *sdt_probes;
 	size_t sdt_nprobes;
 	char *sdt_tab;
@@ -187,6 +198,7 @@ extern int kobj_read_file(struct _buf *, char *, unsigned, unsigned);
 extern int kobj_get_filesize(struct _buf *, uint64_t *size);
 extern uintptr_t kobj_getelfsym(char *, void *, int *);
 extern void kobj_set_ctf(struct module *, caddr_t data, size_t size);
+extern void do_hotinlines(struct module *);
 
 extern int kobj_filbuf(struct _buf *);
 extern void kobj_sync(void);
diff --git a/usr/src/uts/i86pc/Makefile.rules b/usr/src/uts/i86pc/Makefile.rules
index 317c22d55d..e4f2fee0a0 100644
--- a/usr/src/uts/i86pc/Makefile.rules
+++ b/usr/src/uts/i86pc/Makefile.rules
@@ -325,7 +325,8 @@ $(OBJS_DIR)/%.o:		$(SRC)/common/atomic/$(ATOMIC_SUBDIR)/%.s
 #
 
 $(OBJS_DIR)/dtracestubs.s:	$(UNIX_O) $(LIBS)
-	$(NM) -u $(UNIX_O) $(LIBS) | $(GREP) __dtrace_probe_ | $(SORT) | \
+	$(NM) -u $(UNIX_O) $(LIBS) | \
+	$(EGREP) '(__dtrace_probe_|smap_(disable|enable))' | $(SORT) | \
 	    $(UNIQ) | $(AWK) '{ \
 	    printf("\t.globl %s\n\t.type %s,@function\n%s:\n", \
 	    $$1, $$1, $$1); }' > $(OBJS_DIR)/dtracestubs.s
diff --git a/usr/src/uts/i86pc/os/machdep.c b/usr/src/uts/i86pc/os/machdep.c
index d2230dd0a0..7dbf63e2c7 100644
--- a/usr/src/uts/i86pc/os/machdep.c
+++ b/usr/src/uts/i86pc/os/machdep.c
@@ -21,7 +21,7 @@
 
 /*
  * Copyright (c) 1992, 2010, Oracle and/or its affiliates. All rights reserved.
- * Copyright 2017, Joyent, Inc.
+ * Copyright 2020 Joyent, Inc.
  */
 /*
  * Copyright (c) 2010, Intel Corporation.
@@ -190,6 +190,12 @@ extern void pm_cfb_rele(void);
 extern fastboot_info_t newkernel;
 
 /*
+ * Instructions to enable or disable SMAP, respectively.
+ */
+static const uint8_t clac_instr[3] = { 0x0f, 0x01, 0xca };
+static const uint8_t stac_instr[3] = { 0x0f, 0x01, 0xcb };
+
+/*
  * Machine dependent code to reboot.
  * "mdep" is interpreted as a character pointer; if non-null, it is a pointer
  * to a string to be used as the argument string when rebooting.
@@ -1455,3 +1461,37 @@ plat_dr_disable_capability(uint64_t features)
 {
 	atomic_and_64(&plat_dr_options, ~features);
 }
+
+/*
+ * If SMAP is supported, look through hi_calls and inline
+ * calls to smap_enable() to clac and smap_disable() to stac.
+ */
+void
+hotinline_smap(hotinline_desc_t *hid)
+{
+	if (is_x86_feature(x86_featureset, X86FSET_SMAP) == B_FALSE)
+		return;
+
+	if (strcmp(hid->hid_symname, "smap_enable") == 0) {
+		bcopy(clac_instr, (void *)hid->hid_instr_offset,
+		    sizeof (clac_instr));
+	} else if (strcmp(hid->hid_symname, "smap_disable") == 0) {
+		bcopy(stac_instr, (void *)hid->hid_instr_offset,
+		    sizeof (stac_instr));
+	}
+}
+
+/*
+ * Loop through hi_calls and hand off the inlining to
+ * the appropriate calls.
+ */
+void
+do_hotinlines(struct module *mp)
+{
+	for (hotinline_desc_t *hid = mp->hi_calls; hid != NULL;
+	    hid = hid->hid_next) {
+#if !defined(__xpv)
+		hotinline_smap(hid);
+#endif	/* __xpv */
+	}
+}
diff --git a/usr/src/uts/i86pc/os/startup.c b/usr/src/uts/i86pc/os/startup.c
index dfbb83cd15..07a08658c9 100644
--- a/usr/src/uts/i86pc/os/startup.c
+++ b/usr/src/uts/i86pc/os/startup.c
@@ -697,6 +697,7 @@ startup_smap(void)
 	uint32_t inst;
 	uint8_t *instp;
 	char sym[128];
+	struct modctl *modp;
 
 	extern int _smap_enable_patch_count;
 	extern int _smap_disable_patch_count;
@@ -730,8 +731,15 @@ startup_smap(void)
 		hot_patch_kernel_text((caddr_t)instp, inst, 4);
 	}
 
-	hot_patch_kernel_text((caddr_t)smap_enable, SMAP_CLAC_INSTR, 4);
-	hot_patch_kernel_text((caddr_t)smap_disable, SMAP_STAC_INSTR, 4);
+	/*
+	 * Hotinline calls to smap_enable and smap_disable within
+	 * unix module. Hotinlines in other modules are done on
+	 * mod_load().
+	 */
+	modp = mod_hold_by_name("unix");
+	do_hotinlines(modp->mod_mp);
+	mod_release_mod(modp);
+
 	setcr4(getcr4() | CR4_SMAP);
 	smap_enable();
 }
diff --git a/usr/src/uts/intel/amd64/krtld/kobj_reloc.c b/usr/src/uts/intel/amd64/krtld/kobj_reloc.c
index b75e1300e1..146a5148a1 100644
--- a/usr/src/uts/intel/amd64/krtld/kobj_reloc.c
+++ b/usr/src/uts/intel/amd64/krtld/kobj_reloc.c
@@ -119,6 +119,57 @@ sdt_reloc_resolve(struct module *mp, char *symname, uint8_t *instr)
 	return (0);
 }
 
+
+/*
+ * We're relying on the fact that the call we're replacing is
+ * call (e8) plus 4 bytes of address, making a 5 byte instruction
+ */
+#define	NOP_INSTR	0x90
+#define	SMAP_NOPS	5
+
+/*
+ * Currently the only call replaced as a hot inline
+ * is smap_enable() and smap_disable(). If more are needed
+ * we should probably come up with an sdt probe like prefix
+ * and look for those instead of exact call names.
+ */
+static int
+smap_reloc_resolve(struct module *mp, char *symname, uint8_t *instr)
+{
+	uint_t symlen;
+	hotinline_desc_t *hid;
+
+	if (strcmp(symname, "smap_enable") == 0 ||
+	    strcmp(symname, "smap_disable") == 0) {
+
+#ifdef	KOBJ_DEBUG
+		if (kobj_debug & D_RELOCATIONS) {
+			_kobj_printf(ops, "smap_reloc_resolve: %s relocating "
+			    "enable/disable_smap\n", mp->filename);
+		}
+#endif
+
+		hid = kobj_alloc(sizeof (hotinline_desc_t), KM_WAIT);
+		symlen = strlen(symname) + 1;
+		hid->hid_symname = kobj_alloc(symlen, KM_WAIT);
+		bcopy(symname, hid->hid_symname, symlen);
+
+		/*
+		 * We backtrack one byte here to consume the call
+		 * instruction itself.
+		 */
+		hid->hid_instr_offset = (uintptr_t)instr - 1;
+		hid->hid_next = mp->hi_calls;
+		mp->hi_calls = hid;
+
+		memset((void *)hid->hid_instr_offset, NOP_INSTR, SMAP_NOPS);
+
+		return (0);
+	}
+
+	return (1);
+}
+
 int
 do_relocate(struct module *mp, char *reltbl, int nreloc, int relocsize,
     Addr baseaddr)
@@ -222,6 +273,11 @@ do_relocate(struct module *mp, char *reltbl, int nreloc, int relocsize,
 					continue;
 
 				if (symref->st_shndx == SHN_UNDEF &&
+				    smap_reloc_resolve(mp, mp->strings +
+				    symref->st_name, (uint8_t *)off) == 0)
+					continue;
+
+				if (symref->st_shndx == SHN_UNDEF &&
 				    tnf_reloc_resolve(mp->strings +
 				    symref->st_name, &symref->st_value,
 				    &addend, off, &probelist, &taglist) != 0) {
diff --git a/usr/src/uts/intel/ia32/ml/copy.s b/usr/src/uts/intel/ia32/ml/copy.s
index d02637e5fe..5e5f822518 100644
--- a/usr/src/uts/intel/ia32/ml/copy.s
+++ b/usr/src/uts/intel/ia32/ml/copy.s
@@ -36,7 +36,7 @@
 /*         All Rights Reserved						*/
 
 /*
- * Copyright 2019 Joyent, Inc.
+ * Copyright 2020 Joyent, Inc.
  */
 
 #include <sys/errno.h>
@@ -66,11 +66,10 @@
  *
  * Rules and Constraints:
  *
- * 1. For anything that's not in copy.s, we have it do explicit calls to the
- * smap related code. It usually is in a position where it is able to. This is
- * restricted to the following three places: DTrace, resume() in swtch.s and
- * on_fault/no_fault. If you want to add it somewhere else, we should be
- * thinking twice.
+ * 1. For anything that's not in copy.s, we have it do explicit smap_disable()
+ * or smap_enable() calls.  This is restricted to the following three places:
+ * DTrace, resume() in swtch.s and on_fault/no_fault. If you want to add it
+ * somewhere else, we should be thinking twice.
  *
  * 2. We try to toggle this at the smallest window possible. This means that if
  * we take a fault, need to try to use a copyop in copyin() or copyout(), or any
@@ -81,33 +80,25 @@
  * explicitly only allowed to be called while in an on_fault()/no_fault() handler,
  * which already takes care of ensuring that SMAP is enabled and disabled. Note
  * this means that when under an on_fault()/no_fault() handler, one must not
- * call the non-*_noeer() routines.
+ * call the non-*_noerr() routines.
  *
  * 4. The first thing we should do after coming out of an lofault handler is to
- * make sure that we call smap_enable again to ensure that we are safely
+ * make sure that we call smap_enable() again to ensure that we are safely
  * protected, as more often than not, we will have disabled smap to get there.
  *
- * 5. The SMAP functions, smap_enable and smap_disable may not touch any
- * registers beyond those done by the call and ret. These routines may be called
- * from arbitrary contexts in copy.s where we have slightly more special ABIs in
- * place.
+ * 5. smap_enable() and smap_disable() don't exist: calls to these functions
+ * generate runtime relocations, that are then processed into the necessary
+ * clac/stac, via the krtld hotinlines mechanism and hotinline_smap().
  *
  * 6. For any inline user of SMAP, the appropriate SMAP_ENABLE_INSTR and
- * SMAP_DISABLE_INSTR macro should be used (except for smap_enable() and
- * smap_disable()). If the number of these is changed, you must update the
- * constants SMAP_ENABLE_COUNT and SMAP_DISABLE_COUNT below.
+ * SMAP_DISABLE_INSTR macro should be used. If the number of these is changed,
+ * you must update the constants SMAP_ENABLE_COUNT and SMAP_DISABLE_COUNT below.
  *
- * 7. Note, at this time SMAP is not implemented for the 32-bit kernel. There is
- * no known technical reason preventing it from being enabled.
- *
- * 8. Generally this .s file is processed by a K&R style cpp. This means that it
+ * 7. Generally this .s file is processed by a K&R style cpp. This means that it
  * really has a lot of feelings about whitespace. In particular, if you have a
  * macro FOO with the arguments FOO(1, 3), the second argument is in fact ' 3'.
  *
- * 9. The smap_enable and smap_disable functions should not generally be called.
- * They exist such that DTrace and on_trap() may use them, that's it.
- *
- * 10. In general, the kernel has its own value for rflags that gets used. This
+ * 8. In general, the kernel has its own value for rflags that gets used. This
  * is maintained in a few different places which vary based on how the thread
  * comes into existence and whether it's a user thread. In general, when the
  * kernel takes a trap, it always will set ourselves to a known set of flags,
@@ -1902,30 +1893,6 @@ _flt_/**/NAME:					\
 	.string "copyout_noerr: argument not in kernel address space"
 #endif
 
-/*
- * These functions are used for SMAP, supervisor mode access protection. They
- * are hotpatched to become real instructions when the system starts up which is
- * done in mlsetup() as a part of enabling the other CR4 related features.
- *
- * Generally speaking, smap_disable() is a stac instruction and smap_enable is a
- * clac instruction. It's safe to call these any number of times, and in fact,
- * out of paranoia, the kernel will likely call it at several points.
- */
-
-	ENTRY(smap_disable)
-	nop
-	nop
-	nop
-	ret
-	SET_SIZE(smap_disable)
-
-	ENTRY(smap_enable)
-	nop
-	nop
-	nop
-	ret
-	SET_SIZE(smap_enable)
-
 .data
 .align	4
 .globl	_smap_enable_patch_count
diff --git a/usr/src/uts/intel/sys/archsystm.h b/usr/src/uts/intel/sys/archsystm.h
index 4c210f024f..55c387f9b1 100644
--- a/usr/src/uts/intel/sys/archsystm.h
+++ b/usr/src/uts/intel/sys/archsystm.h
@@ -183,13 +183,17 @@ extern void fakesoftint(void);
 extern void *plat_traceback(void *);
 
 /*
- * The following two macros are the four byte instruction sequence of stac, ret
- * and clac, ret. These are used in startup_smap() as a part of properly setting
- * up the valid instructions. For more information on SMAP, see
- * uts/intel/ia32/ml/copy.s.
+ * The following two macros are the four byte instruction sequence of stac, nop
+ * and clac, nop. These are used in startup_smap() and hotinline_smap() as a
+ * part of properly setting up the valid instructions. For more information on
+ * SMAP, see uts/intel/ia32/ml/copy.s, uts/i86pc/os/machdep.c and
+ * uts/common/os/modctl.c.
+ *
+ * Note that smap_disable and smap_enable are resolved to stubs at compile time,
+ * but inlined at runtime by do_hotinlines() in uts/i86pc/os/machdep.c.
  */
-#define	SMAP_CLAC_INSTR	0xc3ca010f
-#define	SMAP_STAC_INSTR	0xc3cb010f
+#define	SMAP_CLAC_INSTR	0x90ca010f
+#define	SMAP_STAC_INSTR	0x90cb010f
 extern void smap_disable(void);
 extern void smap_enable(void);