From 9f9230833b50b8271840dc2c12bd1e94d9df7d12 Mon Sep 17 00:00:00 2001 From: Alexander Pyhalov Date: Tue, 30 Aug 2016 12:55:15 +0300 Subject: 5969 update illumos-gate to use python2.7 Portions contributed by: Igor Kozhukhov Reviewed by: Andrew Stormont Reviewed by: Albert Lee Reviewed by: Adam Stevko Approved by: Gordon Ross --- usr/src/Makefile.master | 9 +- usr/src/Targetdirs | 12 +- usr/src/cmd/mdb/Makefile.common | 2 +- usr/src/cmd/mdb/Makefile.module | 11 +- .../cmd/mdb/common/modules/libpython/libpython.c | 461 +++++++++++ .../mdb/common/modules/libpython2.6/libpython26.c | 461 ----------- usr/src/cmd/mdb/intel/amd64/libpython/Makefile | 48 ++ usr/src/cmd/mdb/intel/amd64/libpython2.6/Makefile | 45 -- usr/src/cmd/mdb/intel/ia32/libpython/Makefile | 49 ++ usr/src/cmd/mdb/intel/ia32/libpython2.6/Makefile | 46 -- usr/src/cmd/mdb/sparc/v7/libpython/Makefile | 47 ++ usr/src/cmd/mdb/sparc/v7/libpython2.6/Makefile | 44 -- usr/src/cmd/mdb/sparc/v9/libpython/Makefile | 48 ++ usr/src/cmd/mdb/sparc/v9/libpython2.6/Makefile | 45 -- usr/src/cmd/pyzfs/Makefile | 1 - usr/src/cmd/pyzfs/pyzfs.py | 2 +- usr/src/lib/brand/solaris10/cmd/Makefile | 6 +- .../lib/brand/solaris10/cmd/s10_python_wrapper.sh | 59 -- .../brand/solaris10/cmd/s10_python_wrapper.sh.in | 59 ++ usr/src/lib/pylibbe/Makefile.com | 9 +- usr/src/lib/pysolaris/Makefile.com | 10 +- usr/src/lib/pysolaris/common/__init__.py | 2 +- usr/src/lib/pyzfs/Makefile.com | 10 +- usr/src/lib/pyzfs/common/__init__.py | 2 +- usr/src/lib/pyzfs/common/allow.py | 2 +- usr/src/lib/pyzfs/common/dataset.py | 2 +- usr/src/lib/pyzfs/common/groupspace.py | 2 +- usr/src/lib/pyzfs/common/holds.py | 2 +- usr/src/lib/pyzfs/common/table.py | 2 +- usr/src/lib/pyzfs/common/unallow.py | 2 +- usr/src/lib/pyzfs/common/userspace.py | 2 +- usr/src/lib/pyzfs/common/util.py | 2 +- usr/src/pkg/Makefile | 4 +- usr/src/pkg/manifests/developer-build-onbld.mf | 132 ++-- usr/src/pkg/manifests/developer-debug-mdb.mf | 5 +- usr/src/pkg/manifests/install-beadm.mf | 12 +- usr/src/pkg/manifests/system-file-system-zfs.mf | 46 +- usr/src/pkg/manifests/system-library.mf | 12 +- usr/src/test/test-runner/cmd/Makefile | 4 +- usr/src/test/test-runner/cmd/run | 871 +++++++++++++++++++++ usr/src/test/test-runner/cmd/run.py | 871 --------------------- usr/src/tools/Makefile | 12 +- usr/src/tools/Makefile.python | 18 +- usr/src/tools/Makefile.tools | 1 - usr/src/tools/scripts/Makefile | 5 +- usr/src/tools/scripts/cddlchk.py | 2 +- usr/src/tools/scripts/copyrightchk.py | 2 +- usr/src/tools/scripts/git-pbchk.py | 2 +- usr/src/tools/scripts/hdrchk.py | 2 +- usr/src/tools/scripts/hg-active.py | 2 +- usr/src/tools/scripts/mapfilechk.py | 2 +- usr/src/tools/scripts/onu.sh | 245 ------ usr/src/tools/scripts/onu.sh.in | 245 ++++++ usr/src/tools/scripts/validate_pkg.py | 2 +- usr/src/tools/scripts/wsdiff.py | 2 +- 55 files changed, 2032 insertions(+), 1971 deletions(-) create mode 100644 usr/src/cmd/mdb/common/modules/libpython/libpython.c delete mode 100644 usr/src/cmd/mdb/common/modules/libpython2.6/libpython26.c create mode 100644 usr/src/cmd/mdb/intel/amd64/libpython/Makefile delete mode 100644 usr/src/cmd/mdb/intel/amd64/libpython2.6/Makefile create mode 100644 usr/src/cmd/mdb/intel/ia32/libpython/Makefile delete mode 100644 usr/src/cmd/mdb/intel/ia32/libpython2.6/Makefile create mode 100644 usr/src/cmd/mdb/sparc/v7/libpython/Makefile delete mode 100644 usr/src/cmd/mdb/sparc/v7/libpython2.6/Makefile create mode 100644 usr/src/cmd/mdb/sparc/v9/libpython/Makefile delete mode 100644 usr/src/cmd/mdb/sparc/v9/libpython2.6/Makefile delete mode 100644 usr/src/lib/brand/solaris10/cmd/s10_python_wrapper.sh create mode 100644 usr/src/lib/brand/solaris10/cmd/s10_python_wrapper.sh.in create mode 100644 usr/src/test/test-runner/cmd/run delete mode 100644 usr/src/test/test-runner/cmd/run.py delete mode 100644 usr/src/tools/scripts/onu.sh create mode 100644 usr/src/tools/scripts/onu.sh.in (limited to 'usr/src') diff --git a/usr/src/Makefile.master b/usr/src/Makefile.master index 279feec79e..efda1ed727 100644 --- a/usr/src/Makefile.master +++ b/usr/src/Makefile.master @@ -177,8 +177,9 @@ PERL_VERSION= 5.10.0 PERL_PKGVERS= -510 PERL_ARCH = i86pc-solaris-64int $(SPARC_BLD)PERL_ARCH = sun4-solaris-64int -PYTHON_26= /usr/bin/python2.6 -PYTHON= $(PYTHON_26) +PYTHON_VERSION= 2.7 +PYTHON_PKGVERS= -27 +PYTHON= /usr/bin/python$(PYTHON_VERSION) SORT= /usr/bin/sort TOUCH= /usr/bin/touch WC= /usr/bin/wc @@ -246,7 +247,7 @@ INS.symlink= $(RM) $@; $(SYMLINK) $(INSLINKTARGET) $@ # files we must make certain to not adjust the mtime of the source # (.py) file. # -INS.pyfile= $(INS.file); $(TOUCH) -r $< $@ +INS.pyfile= $(RM) $@; $(SED) -e "1s:^\#!@PYTHON@:\#!/usr/bin/python$(PYTHON_VERSION):" < $< > $@; $(CHMOD) $(FILEMODE) $@; $(TOUCH) -r $< $@ # MACH must be set in the shell environment per uname -p on the build host # More specific architecture variables should be set in lower makefiles. @@ -1095,7 +1096,7 @@ PKGPUBLISHER_NONREDIST= on-extra $(CHMOD) +x $@ .py: - $(RM) $@; $(CAT) $< > $@; $(CHMOD) +x $@ + $(RM) $@; $(SED) -e "1s:^\#!@PYTHON@:\#!/usr/bin/python$(PYTHON_VERSION):" < $< > $@; $(CHMOD) +x $@ .py.pyc: $(RM) $@ diff --git a/usr/src/Targetdirs b/usr/src/Targetdirs index a015bf9b9b..1e4e0a2751 100644 --- a/usr/src/Targetdirs +++ b/usr/src/Targetdirs @@ -283,12 +283,12 @@ DIRS= \ /usr/net \ /usr/net/servers \ /usr/lib/pool \ - /usr/lib/python2.6 \ - /usr/lib/python2.6/vendor-packages \ - /usr/lib/python2.6/vendor-packages/64 \ - /usr/lib/python2.6/vendor-packages/solaris \ - /usr/lib/python2.6/vendor-packages/zfs \ - /usr/lib/python2.6/vendor-packages/beadm \ + /usr/lib/python$(PYTHON_VERSION) \ + /usr/lib/python$(PYTHON_VERSION)/vendor-packages \ + /usr/lib/python$(PYTHON_VERSION)/vendor-packages/64 \ + /usr/lib/python$(PYTHON_VERSION)/vendor-packages/solaris \ + /usr/lib/python$(PYTHON_VERSION)/vendor-packages/zfs \ + /usr/lib/python$(PYTHON_VERSION)/vendor-packages/beadm \ /usr/lib/rcap \ /usr/lib/rcap/$(MACH32) \ /usr/lib/sa \ diff --git a/usr/src/cmd/mdb/Makefile.common b/usr/src/cmd/mdb/Makefile.common index 4d79ba0a18..ebdeac9f3a 100644 --- a/usr/src/cmd/mdb/Makefile.common +++ b/usr/src/cmd/mdb/Makefile.common @@ -34,7 +34,7 @@ COMMON_MODULES_PROC = \ libfksmbsrv \ libnvpair \ libproc \ - libpython2.6 \ + libpython \ libsysevent \ libtopo \ libumem \ diff --git a/usr/src/cmd/mdb/Makefile.module b/usr/src/cmd/mdb/Makefile.module index f741cd8278..47a3e15fe2 100644 --- a/usr/src/cmd/mdb/Makefile.module +++ b/usr/src/cmd/mdb/Makefile.module @@ -31,13 +31,20 @@ include $(SRC)/cmd/mdb/Makefile.tools +# +# Make sure we're getting a consistent execution environment for the +# embedded scripts. +# +SHELL= /usr/bin/ksh93 + $(KMOD_SOURCES_DIFFERENT)KMODSRCS = $(MODSRCS) $(KMOD_SOURCES_DIFFERENT)KMODASMSRCS = $(MODASMSRCS) MODOBJS = $(MODSRCS:%.c=dmod/%.o) $(MODASMSRCS:%.s=dmod/%.o) KMODOBJS = $(KMODSRCS:%.c=kmod/%.o) $(KMODASMSRCS:%.s=kmod/%.o) -MODNAME = $(MODULE:%.so=%) +MODNAME_cmd = if [ -n "$(MODULE_NAME)" ]; then print $(MODULE_NAME); else print $(MODULE)| sed -e 's:\.so$$::'; fi +MODNAME = $(MODNAME_cmd:sh) KMODULE = $(MODNAME) MODFILE = dmod/$(MODULE) @@ -101,7 +108,7 @@ C99MODE = $(C99_ENABLE) CFLAGS += $(CCVERBOSE) CFLAGS64 += $(CCVERBOSE) -CPPFLAGS += $($(MDBTGT)_TGTFLAGS) -I../../../common +CPPFLAGS += $($(MDBTGT)_TGTFLAGS) -I../../../common -I/usr/include/python$(PYTHON_VERSION) LDFLAGS += $(ZTEXT) LDFLAGS64 += $(ZTEXT) ASFLAGS += -P diff --git a/usr/src/cmd/mdb/common/modules/libpython/libpython.c b/usr/src/cmd/mdb/common/modules/libpython/libpython.c new file mode 100644 index 0000000000..57b8eaec70 --- /dev/null +++ b/usr/src/cmd/mdb/common/modules/libpython/libpython.c @@ -0,0 +1,461 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright 2010 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +#include + +#include +#include +#include +#include +#include + +#include +#include + +/* + * Decoding Python Stack Frames + * ============================ + * + * Python2 uses a variety of objects to construct its call chain. An address + * space may have one or more PyInterpreterState objects, which are the base + * object in the interpreter's state. These objects are kept in a linked list + * with a head pointer named interp_head. This makes it possible for the + * debugger to get a toehold on data structures necessary to understand the + * interpreter. Since most of these structures are linked out of the + * InterpreterState, traversals generally start here. + * + * In order to decode a frame, the debugger needs to walk from + * PyInterpreterState down to a PyCodeObject. The diagram below shows the + * the objects that must be examined in order to reach a leaf PyCodeObject. + * + * +--------------------+ next +--------------------+ next + * interp_head -> | PyInterpreterState | ----> | PyInterpreterState | ---> ... + * +--------------------+ +--------------------+ + * | | tstate_head + * | tstate_head V + * | +---------------+ frame + * V | PyThreadState | -----> ... + * +---------------+ frame +---------------+ + * | PyThreadState | ---> ... + * +---------------+ + * | next + * V + * +---------------+ frame +---------------+ f_back +---------------+ + * | PyThreadState | ------> | PyFrameObject | -----> | PyFrameObject | + * +---------------+ +---------------+ +---------------+ + * | | + * | f_code | f_code + * V V + * +--------------+ ... + * | PyCodeObject | + * +--------------+ + * co_filename | | | co_lnotab + * +-------------+ | +-------------+ + * | co_name | | + * V V V + * +----------------+ +----------------+ +----------------+ + * | PyStringObject | | PyStringObject | | PyStringObject | + * +----------------+ +----------------+ +----------------+ + * + * The interp_head pointer is a list of one or more PyInterpreterState + * objects. Each of these objects can contain one or more PyThreadState + * objects. The PyInterpreterState object keeps a pointer to the head of the + * list of PyThreadState objects as tstate_head. + * + * Each thread keeps ahold of its stack frames. The PyThreadState object + * has a pointer to the topmost PyFrameObject, kept in frame. The + * successive frames on the stack are kept linked in the PyFrameObject's + * f_back pointer, with each frame pointing to its caller. + * + * In order to decode each call frame, our code needs to look at the + * PyCodeObject for each frame. Essentially, this is the code that is + * being executed in the frame. The PyFrameObject keeps a pointer to this + * code object in f_code. In order to print meaningful debug information, + * it's necessary to extract the Python filename (co_filename), the + * function name (co_name), and the line number within the file + * (co_lnotab). The filename and function are stored as strings, but the + * line number is a mapping of bytecode offsets to line numbers. The + * description of the lnotab algorithm lives here: + * + * http://svn.python.org/projects/python/trunk/Objects/lnotab_notes.txt + * + * In order to decode the frame, the debugger needs to walk each + * InterpreterState object. For each InterpreterState, every PyThreadState + * must be traversed. The PyThreadState objects point to the + * PyFrameObjects. For every thread, we must walk the frames backwards and + * decode the strings that are in the PyCodeObjects. + */ + +/* + * The Python-dependent debugging functionality lives in its own helper + * library. The helper agent is provided by libpython2.[67]_db.so, which + * is also used by pstack(1) for debugging Python processes. + * + * Define needed prototypes here. + */ + +#define PYDB_VERSION 1 +typedef struct pydb_agent pydb_agent_t; +typedef struct pydb_iter pydb_iter_t; + +typedef pydb_agent_t *(*pydb_agent_create_f)(struct ps_prochandle *P, int vers); +typedef void (*pydb_agent_destroy_f)(pydb_agent_t *py); +typedef int (*pydb_get_frameinfo_f)(pydb_agent_t *py, uintptr_t frame_addr, + char *fbuf, size_t bufsz, int verbose); +typedef pydb_iter_t *(*pydb_iter_init_f)(pydb_agent_t *py, uintptr_t addr); +typedef uintptr_t (*pydb_iter_next_f)(pydb_iter_t *iter); +typedef void (*pydb_iter_fini_f)(pydb_iter_t *iter); + +static pydb_agent_create_f pydb_agent_create; +static pydb_agent_destroy_f pydb_agent_destroy; +static pydb_get_frameinfo_f pydb_get_frameinfo; +static pydb_iter_init_f pydb_frame_iter_init; +static pydb_iter_init_f pydb_interp_iter_init; +static pydb_iter_init_f pydb_thread_iter_init; +static pydb_iter_next_f pydb_iter_next; +static pydb_iter_fini_f pydb_iter_fini; + +static pydb_agent_t *pydb_hdl = NULL; +static void *pydb_dlhdl = NULL; + +/*ARGSUSED*/ +static int +py_frame(uintptr_t addr, uint_t flags, int argc, const mdb_arg_t *argv) +{ + char buf[1024]; + int verbose = FALSE; + + if (mdb_getopts(argc, argv, + 'v', MDB_OPT_SETBITS, TRUE, &verbose, + NULL) != argc) { + return (DCMD_USAGE); + } + + if (flags & DCMD_PIPE_OUT) { + mdb_warn("py_stack cannot output into a pipe\n"); + return (DCMD_ERR); + } + + if (!(flags & DCMD_ADDRSPEC)) { + mdb_warn("no address"); + return (DCMD_USAGE); + } + + if (pydb_get_frameinfo(pydb_hdl, addr, buf, sizeof (buf), + verbose) < 0) { + mdb_warn("Unable to find frame at address %p\n", addr); + return (DCMD_ERR); + } + + mdb_printf("%s", buf); + + return (DCMD_OK); +} + +int +py_interp_walk_init(mdb_walk_state_t *wsp) +{ + pydb_iter_t *pdi; + + pdi = pydb_interp_iter_init(pydb_hdl, wsp->walk_addr); + + if (pdi == NULL) { + mdb_warn("unable to create interpreter iterator\n"); + return (DCMD_ERR); + } + + wsp->walk_data = pdi; + + return (WALK_NEXT); +} + +int +py_walk_step(mdb_walk_state_t *wsp) +{ + pydb_iter_t *pdi = wsp->walk_data; + uintptr_t addr; + int status; + + addr = pydb_iter_next(pdi); + + if (addr == NULL) { + return (WALK_DONE); + } + + status = wsp->walk_callback(addr, 0, wsp->walk_cbdata); + + return (status); +} + +void +py_walk_fini(mdb_walk_state_t *wsp) +{ + pydb_iter_t *pdi = wsp->walk_data; + pydb_iter_fini(pdi); +} + +int +py_thread_walk_init(mdb_walk_state_t *wsp) +{ + pydb_iter_t *pdi; + + pdi = pydb_thread_iter_init(pydb_hdl, wsp->walk_addr); + if (pdi == NULL) { + mdb_warn("unable to create thread iterator\n"); + return (DCMD_ERR); + } + + wsp->walk_data = pdi; + + return (WALK_NEXT); +} + +int +py_frame_walk_init(mdb_walk_state_t *wsp) +{ + pydb_iter_t *pdi; + + pdi = pydb_frame_iter_init(pydb_hdl, wsp->walk_addr); + if (pdi == NULL) { + mdb_warn("unable to create frame iterator\n"); + return (DCMD_ERR); + } + + wsp->walk_data = pdi; + + return (WALK_NEXT); +} + +/*ARGSUSED*/ +static int +python_stack(uintptr_t addr, const PyThreadState *ts, uint_t *verbose) +{ + mdb_arg_t nargv; + uint_t nargc = (verbose != NULL && *verbose) ? 1 : 0; + /* + * Pass the ThreadState to the frame walker. Have frame walker + * call frame dcmd. + */ + mdb_printf("PyThreadState: %0?p\n", addr); + + nargv.a_type = MDB_TYPE_STRING; + nargv.a_un.a_str = "-v"; + + if (mdb_pwalk_dcmd("pyframe", "pyframe", nargc, &nargv, addr) == -1) { + mdb_warn("can't walk 'pyframe'"); + return (WALK_ERR); + } + + return (WALK_NEXT); +} + +/*ARGSUSED*/ +static int +python_thread(uintptr_t addr, const PyInterpreterState *is, uint_t *verbose) +{ + /* + * Pass the InterpreterState to the threadstate walker. + */ + if (mdb_pwalk("pythread", (mdb_walk_cb_t)python_stack, verbose, + addr) == -1) { + mdb_warn("can't walk 'pythread'"); + return (WALK_ERR); + } + + return (WALK_NEXT); +} + +/*ARGSUSED*/ +static int +py_stack(uintptr_t addr, uint_t flags, int argc, const mdb_arg_t *argv) +{ + uint_t verbose = FALSE; + + if (mdb_getopts(argc, argv, + 'v', MDB_OPT_SETBITS, TRUE, &verbose, + NULL) != argc) + return (DCMD_USAGE); + + if (flags & DCMD_PIPE_OUT) { + mdb_warn("py_stack cannot output into a pipe\n"); + return (DCMD_ERR); + } + + if (flags & DCMD_ADDRSPEC) { + mdb_arg_t nargv; + uint_t nargc = verbose ? 1 : 0; + + nargv.a_type = MDB_TYPE_STRING; + nargv.a_un.a_str = "-v"; + + if (mdb_pwalk_dcmd("pyframe", "pyframe", nargc, &nargv, addr) + == -1) { + mdb_warn("can't walk 'pyframe'"); + return (DCMD_ERR); + } + return (DCMD_OK); + } + + if (mdb_walk("pyinterp", (mdb_walk_cb_t)python_thread, + &verbose) == -1) { + mdb_warn("can't walk 'pyinterp'"); + return (DCMD_ERR); + } + + return (DCMD_OK); +} + +static const mdb_dcmd_t dcmds[] = { + { "pystack", "[-v]", "print python stacks", py_stack }, + { "pyframe", "[-v]", "print python frames", py_frame }, + { NULL } +}; + +static const mdb_walker_t walkers[] = { + { "pyinterp", "walk python interpreter structures", + py_interp_walk_init, py_walk_step, py_walk_fini }, + { "pythread", "given an interpreter, walk the list of python threads", + py_thread_walk_init, py_walk_step, py_walk_fini }, + { "pyframe", "given a thread state, walk the list of frame objects", + py_frame_walk_init, py_walk_step, py_walk_fini }, + { NULL } +}; + +static const mdb_modinfo_t modinfo = { + MDB_API_VERSION, dcmds, walkers +}; + +/*ARGSUSED*/ +static int +python_object_iter(void *cd, const prmap_t *pmp, const char *obj) +{ + char path[PATH_MAX]; + char *name; + char *s1, *s2; + struct ps_prochandle *Pr = cd; + + name = strstr(obj, "/libpython"); + + if (name) { + (void) strcpy(path, obj); + if (Pstatus(Pr)->pr_dmodel != PR_MODEL_NATIVE) { + s1 = name; + s2 = path + (s1 - obj); + (void) strcpy(s2, "/64"); + s2 += 3; + (void) strcpy(s2, s1); + } + + s1 = strstr(obj, ".so"); + s2 = strstr(path, ".so"); + (void) strcpy(s2, "_db"); + s2 += 3; + (void) strcpy(s2, s1); + + if ((pydb_dlhdl = dlopen(path, RTLD_LAZY|RTLD_GLOBAL)) != NULL) + return (1); + } + + return (0); +} + +static int +python_db_init(void) +{ + struct ps_prochandle *Ph; + + if (mdb_get_xdata("pshandle", &Ph, sizeof (Ph)) == -1) { + mdb_warn("couldn't read pshandle xdata\n"); + dlclose(pydb_dlhdl); + pydb_dlhdl = NULL; + return (-1); + } + + (void) Pobject_iter(Ph, python_object_iter, Ph); + + pydb_agent_create = (pydb_agent_create_f) + dlsym(pydb_dlhdl, "pydb_agent_create"); + pydb_agent_destroy = (pydb_agent_destroy_f) + dlsym(pydb_dlhdl, "pydb_agent_destroy"); + pydb_get_frameinfo = (pydb_get_frameinfo_f) + dlsym(pydb_dlhdl, "pydb_get_frameinfo"); + + pydb_frame_iter_init = (pydb_iter_init_f) + dlsym(pydb_dlhdl, "pydb_frame_iter_init"); + pydb_interp_iter_init = (pydb_iter_init_f) + dlsym(pydb_dlhdl, "pydb_interp_iter_init"); + pydb_thread_iter_init = (pydb_iter_init_f) + dlsym(pydb_dlhdl, "pydb_thread_iter_init"); + pydb_iter_next = (pydb_iter_next_f)dlsym(pydb_dlhdl, "pydb_iter_next"); + pydb_iter_fini = (pydb_iter_fini_f)dlsym(pydb_dlhdl, "pydb_iter_fini"); + + + if (pydb_agent_create == NULL || pydb_agent_destroy == NULL || + pydb_get_frameinfo == NULL || pydb_frame_iter_init == NULL || + pydb_interp_iter_init == NULL || pydb_thread_iter_init == NULL || + pydb_iter_next == NULL || pydb_iter_fini == NULL) { + mdb_warn("couldn't load pydb functions"); + dlclose(pydb_dlhdl); + pydb_dlhdl = NULL; + return (-1); + } + + pydb_hdl = pydb_agent_create(Ph, PYDB_VERSION); + if (pydb_hdl == NULL) { + mdb_warn("unable to create pydb_agent"); + dlclose(pydb_dlhdl); + pydb_dlhdl = NULL; + return (-1); + } + + return (0); +} + +static void +python_db_fini(void) +{ + if (pydb_dlhdl) { + pydb_agent_destroy(pydb_hdl); + pydb_hdl = NULL; + + dlclose(pydb_dlhdl); + pydb_dlhdl = NULL; + } +} + +const mdb_modinfo_t * +_mdb_init(void) +{ + if (python_db_init() != 0) + return (NULL); + + return (&modinfo); +} + +void +_mdb_fini(void) +{ + python_db_fini(); +} diff --git a/usr/src/cmd/mdb/common/modules/libpython2.6/libpython26.c b/usr/src/cmd/mdb/common/modules/libpython2.6/libpython26.c deleted file mode 100644 index 9d4635326d..0000000000 --- a/usr/src/cmd/mdb/common/modules/libpython2.6/libpython26.c +++ /dev/null @@ -1,461 +0,0 @@ -/* - * CDDL HEADER START - * - * The contents of this file are subject to the terms of the - * Common Development and Distribution License (the "License"). - * You may not use this file except in compliance with the License. - * - * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE - * or http://www.opensolaris.org/os/licensing. - * See the License for the specific language governing permissions - * and limitations under the License. - * - * When distributing Covered Code, include this CDDL HEADER in each - * file and include the License file at usr/src/OPENSOLARIS.LICENSE. - * If applicable, add the following below this CDDL HEADER, with the - * fields enclosed by brackets "[]" replaced with your own identifying - * information: Portions Copyright [yyyy] [name of copyright owner] - * - * CDDL HEADER END - */ -/* - * Copyright 2010 Sun Microsystems, Inc. All rights reserved. - * Use is subject to license terms. - */ - -#include - -#include -#include -#include -#include -#include - -#include -#include - -/* - * Decoding Python Stack Frames - * ============================ - * - * Python2.6 uses a variety of objects to construct its call chain. An address - * space may have one or more PyInterpreterState objects, which are the base - * object in the interpreter's state. These objects are kept in a linked list - * with a head pointer named interp_head. This makes it possible for the - * debugger to get a toehold on data structures necessary to understand the - * interpreter. Since most of these structures are linked out of the - * InterpreterState, traversals generally start here. - * - * In order to decode a frame, the debugger needs to walk from - * PyInterpreterState down to a PyCodeObject. The diagram below shows the - * the objects that must be examined in order to reach a leaf PyCodeObject. - * - * +--------------------+ next +--------------------+ next - * interp_head -> | PyInterpreterState | ----> | PyInterpreterState | ---> ... - * +--------------------+ +--------------------+ - * | | tstate_head - * | tstate_head V - * | +---------------+ frame - * V | PyThreadState | -----> ... - * +---------------+ frame +---------------+ - * | PyThreadState | ---> ... - * +---------------+ - * | next - * V - * +---------------+ frame +---------------+ f_back +---------------+ - * | PyThreadState | ------> | PyFrameObject | -----> | PyFrameObject | - * +---------------+ +---------------+ +---------------+ - * | | - * | f_code | f_code - * V V - * +--------------+ ... - * | PyCodeObject | - * +--------------+ - * co_filename | | | co_lnotab - * +-------------+ | +-------------+ - * | co_name | | - * V V V - * +----------------+ +----------------+ +----------------+ - * | PyStringObject | | PyStringObject | | PyStringObject | - * +----------------+ +----------------+ +----------------+ - * - * The interp_head pointer is a list of one or more PyInterpreterState - * objects. Each of these objects can contain one or more PyThreadState - * objects. The PyInterpreterState object keeps a pointer to the head of the - * list of PyThreadState objects as tstate_head. - * - * Each thread keeps ahold of its stack frames. The PyThreadState object - * has a pointer to the topmost PyFrameObject, kept in frame. The - * successive frames on the stack are kept linked in the PyFrameObject's - * f_back pointer, with each frame pointing to its caller. - * - * In order to decode each call frame, our code needs to look at the - * PyCodeObject for each frame. Essentially, this is the code that is - * being executed in the frame. The PyFrameObject keeps a pointer to this - * code object in f_code. In order to print meaningful debug information, - * it's necessary to extract the Python filename (co_filename), the - * function name (co_name), and the line number within the file - * (co_lnotab). The filename and function are stored as strings, but the - * line number is a mapping of bytecode offsets to line numbers. The - * description of the lnotab algorithm lives here: - * - * http://svn.python.org/projects/python/trunk/Objects/lnotab_notes.txt - * - * In order to decode the frame, the debugger needs to walk each - * InterpreterState object. For each InterpreterState, every PyThreadState - * must be traversed. The PyThreadState objects point to the - * PyFrameObjects. For every thread, we must walk the frames backwards and - * decode the strings that are in the PyCodeObjects. - */ - -/* - * The Python-dependent debugging functionality lives in its own helper - * library. The helper agent is provided by libpython2.6_db.so, which is also - * used by pstack(1) for debugging Python processes. - * - * Define needed prototypes here. - */ - -#define PYDB_VERSION 1 -typedef struct pydb_agent pydb_agent_t; -typedef struct pydb_iter pydb_iter_t; - -typedef pydb_agent_t *(*pydb_agent_create_f)(struct ps_prochandle *P, int vers); -typedef void (*pydb_agent_destroy_f)(pydb_agent_t *py); -typedef int (*pydb_get_frameinfo_f)(pydb_agent_t *py, uintptr_t frame_addr, - char *fbuf, size_t bufsz, int verbose); -typedef pydb_iter_t *(*pydb_iter_init_f)(pydb_agent_t *py, uintptr_t addr); -typedef uintptr_t (*pydb_iter_next_f)(pydb_iter_t *iter); -typedef void (*pydb_iter_fini_f)(pydb_iter_t *iter); - -static pydb_agent_create_f pydb_agent_create; -static pydb_agent_destroy_f pydb_agent_destroy; -static pydb_get_frameinfo_f pydb_get_frameinfo; -static pydb_iter_init_f pydb_frame_iter_init; -static pydb_iter_init_f pydb_interp_iter_init; -static pydb_iter_init_f pydb_thread_iter_init; -static pydb_iter_next_f pydb_iter_next; -static pydb_iter_fini_f pydb_iter_fini; - -static pydb_agent_t *pydb_hdl = NULL; -static void *pydb_dlhdl = NULL; - -/*ARGSUSED*/ -static int -py_frame(uintptr_t addr, uint_t flags, int argc, const mdb_arg_t *argv) -{ - char buf[1024]; - int verbose = FALSE; - - if (mdb_getopts(argc, argv, - 'v', MDB_OPT_SETBITS, TRUE, &verbose, - NULL) != argc) { - return (DCMD_USAGE); - } - - if (flags & DCMD_PIPE_OUT) { - mdb_warn("py_stack cannot output into a pipe\n"); - return (DCMD_ERR); - } - - if (!(flags & DCMD_ADDRSPEC)) { - mdb_warn("no address"); - return (DCMD_USAGE); - } - - if (pydb_get_frameinfo(pydb_hdl, addr, buf, sizeof (buf), - verbose) < 0) { - mdb_warn("Unable to find frame at address %p\n", addr); - return (DCMD_ERR); - } - - mdb_printf("%s", buf); - - return (DCMD_OK); -} - -int -py_interp_walk_init(mdb_walk_state_t *wsp) -{ - pydb_iter_t *pdi; - - pdi = pydb_interp_iter_init(pydb_hdl, wsp->walk_addr); - - if (pdi == NULL) { - mdb_warn("unable to create interpreter iterator\n"); - return (DCMD_ERR); - } - - wsp->walk_data = pdi; - - return (WALK_NEXT); -} - -int -py_walk_step(mdb_walk_state_t *wsp) -{ - pydb_iter_t *pdi = wsp->walk_data; - uintptr_t addr; - int status; - - addr = pydb_iter_next(pdi); - - if (addr == NULL) { - return (WALK_DONE); - } - - status = wsp->walk_callback(addr, 0, wsp->walk_cbdata); - - return (status); -} - -void -py_walk_fini(mdb_walk_state_t *wsp) -{ - pydb_iter_t *pdi = wsp->walk_data; - pydb_iter_fini(pdi); -} - -int -py_thread_walk_init(mdb_walk_state_t *wsp) -{ - pydb_iter_t *pdi; - - pdi = pydb_thread_iter_init(pydb_hdl, wsp->walk_addr); - if (pdi == NULL) { - mdb_warn("unable to create thread iterator\n"); - return (DCMD_ERR); - } - - wsp->walk_data = pdi; - - return (WALK_NEXT); -} - -int -py_frame_walk_init(mdb_walk_state_t *wsp) -{ - pydb_iter_t *pdi; - - pdi = pydb_frame_iter_init(pydb_hdl, wsp->walk_addr); - if (pdi == NULL) { - mdb_warn("unable to create frame iterator\n"); - return (DCMD_ERR); - } - - wsp->walk_data = pdi; - - return (WALK_NEXT); -} - -/*ARGSUSED*/ -static int -python_stack(uintptr_t addr, const PyThreadState *ts, uint_t *verbose) -{ - mdb_arg_t nargv; - uint_t nargc = (verbose != NULL && *verbose) ? 1 : 0; - /* - * Pass the ThreadState to the frame walker. Have frame walker - * call frame dcmd. - */ - mdb_printf("PyThreadState: %0?p\n", addr); - - nargv.a_type = MDB_TYPE_STRING; - nargv.a_un.a_str = "-v"; - - if (mdb_pwalk_dcmd("pyframe", "pyframe", nargc, &nargv, addr) == -1) { - mdb_warn("can't walk 'pyframe'"); - return (WALK_ERR); - } - - return (WALK_NEXT); -} - -/*ARGSUSED*/ -static int -python_thread(uintptr_t addr, const PyInterpreterState *is, uint_t *verbose) -{ - /* - * Pass the InterpreterState to the threadstate walker. - */ - if (mdb_pwalk("pythread", (mdb_walk_cb_t)python_stack, verbose, - addr) == -1) { - mdb_warn("can't walk 'pythread'"); - return (WALK_ERR); - } - - return (WALK_NEXT); -} - -/*ARGSUSED*/ -static int -py_stack(uintptr_t addr, uint_t flags, int argc, const mdb_arg_t *argv) -{ - uint_t verbose = FALSE; - - if (mdb_getopts(argc, argv, - 'v', MDB_OPT_SETBITS, TRUE, &verbose, - NULL) != argc) - return (DCMD_USAGE); - - if (flags & DCMD_PIPE_OUT) { - mdb_warn("py_stack cannot output into a pipe\n"); - return (DCMD_ERR); - } - - if (flags & DCMD_ADDRSPEC) { - mdb_arg_t nargv; - uint_t nargc = verbose ? 1 : 0; - - nargv.a_type = MDB_TYPE_STRING; - nargv.a_un.a_str = "-v"; - - if (mdb_pwalk_dcmd("pyframe", "pyframe", nargc, &nargv, addr) - == -1) { - mdb_warn("can't walk 'pyframe'"); - return (DCMD_ERR); - } - return (DCMD_OK); - } - - if (mdb_walk("pyinterp", (mdb_walk_cb_t)python_thread, - &verbose) == -1) { - mdb_warn("can't walk 'pyinterp'"); - return (DCMD_ERR); - } - - return (DCMD_OK); -} - -static const mdb_dcmd_t dcmds[] = { - { "pystack", "[-v]", "print python stacks", py_stack }, - { "pyframe", "[-v]", "print python frames", py_frame }, - { NULL } -}; - -static const mdb_walker_t walkers[] = { - { "pyinterp", "walk python interpreter structures", - py_interp_walk_init, py_walk_step, py_walk_fini }, - { "pythread", "given an interpreter, walk the list of python threads", - py_thread_walk_init, py_walk_step, py_walk_fini }, - { "pyframe", "given a thread state, walk the list of frame objects", - py_frame_walk_init, py_walk_step, py_walk_fini }, - { NULL } -}; - -static const mdb_modinfo_t modinfo = { - MDB_API_VERSION, dcmds, walkers -}; - -/*ARGSUSED*/ -static int -python_object_iter(void *cd, const prmap_t *pmp, const char *obj) -{ - char path[PATH_MAX]; - char *name; - char *s1, *s2; - struct ps_prochandle *Pr = cd; - - name = strstr(obj, "/libpython"); - - if (name) { - (void) strcpy(path, obj); - if (Pstatus(Pr)->pr_dmodel != PR_MODEL_NATIVE) { - s1 = name; - s2 = path + (s1 - obj); - (void) strcpy(s2, "/64"); - s2 += 3; - (void) strcpy(s2, s1); - } - - s1 = strstr(obj, ".so"); - s2 = strstr(path, ".so"); - (void) strcpy(s2, "_db"); - s2 += 3; - (void) strcpy(s2, s1); - - if ((pydb_dlhdl = dlopen(path, RTLD_LAZY|RTLD_GLOBAL)) != NULL) - return (1); - } - - return (0); -} - -static int -python_db_init(void) -{ - struct ps_prochandle *Ph; - - if (mdb_get_xdata("pshandle", &Ph, sizeof (Ph)) == -1) { - mdb_warn("couldn't read pshandle xdata\n"); - dlclose(pydb_dlhdl); - pydb_dlhdl = NULL; - return (-1); - } - - (void) Pobject_iter(Ph, python_object_iter, Ph); - - pydb_agent_create = (pydb_agent_create_f) - dlsym(pydb_dlhdl, "pydb_agent_create"); - pydb_agent_destroy = (pydb_agent_destroy_f) - dlsym(pydb_dlhdl, "pydb_agent_destroy"); - pydb_get_frameinfo = (pydb_get_frameinfo_f) - dlsym(pydb_dlhdl, "pydb_get_frameinfo"); - - pydb_frame_iter_init = (pydb_iter_init_f) - dlsym(pydb_dlhdl, "pydb_frame_iter_init"); - pydb_interp_iter_init = (pydb_iter_init_f) - dlsym(pydb_dlhdl, "pydb_interp_iter_init"); - pydb_thread_iter_init = (pydb_iter_init_f) - dlsym(pydb_dlhdl, "pydb_thread_iter_init"); - pydb_iter_next = (pydb_iter_next_f)dlsym(pydb_dlhdl, "pydb_iter_next"); - pydb_iter_fini = (pydb_iter_fini_f)dlsym(pydb_dlhdl, "pydb_iter_fini"); - - - if (pydb_agent_create == NULL || pydb_agent_destroy == NULL || - pydb_get_frameinfo == NULL || pydb_frame_iter_init == NULL || - pydb_interp_iter_init == NULL || pydb_thread_iter_init == NULL || - pydb_iter_next == NULL || pydb_iter_fini == NULL) { - mdb_warn("couldn't load pydb functions"); - dlclose(pydb_dlhdl); - pydb_dlhdl = NULL; - return (-1); - } - - pydb_hdl = pydb_agent_create(Ph, PYDB_VERSION); - if (pydb_hdl == NULL) { - mdb_warn("unable to create pydb_agent"); - dlclose(pydb_dlhdl); - pydb_dlhdl = NULL; - return (-1); - } - - return (0); -} - -static void -python_db_fini(void) -{ - if (pydb_dlhdl) { - pydb_agent_destroy(pydb_hdl); - pydb_hdl = NULL; - - dlclose(pydb_dlhdl); - pydb_dlhdl = NULL; - } -} - -const mdb_modinfo_t * -_mdb_init(void) -{ - if (python_db_init() != 0) - return (NULL); - - return (&modinfo); -} - -void -_mdb_fini(void) -{ - python_db_fini(); -} diff --git a/usr/src/cmd/mdb/intel/amd64/libpython/Makefile b/usr/src/cmd/mdb/intel/amd64/libpython/Makefile new file mode 100644 index 0000000000..cd4f4c669f --- /dev/null +++ b/usr/src/cmd/mdb/intel/amd64/libpython/Makefile @@ -0,0 +1,48 @@ +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# +# +# Copyright 2010 Sun Microsystems, Inc. All rights reserved. +# Use is subject to license terms. +# + +include $(SRC)/Makefile.master + +MODULE = libpython$(PYTHON_VERSION).so +MDBTGT = proc +MODULE_NAME = libpython + +MODSRCS_DIR = ../../../common/modules/libpython + +MODSRCS = libpython.c + +include ../../../../Makefile.cmd +include ../../../../Makefile.cmd.64 +include ../../Makefile.amd64 +include ../../../Makefile.module + +dmod/$(MODULE) := LDLIBS += -lproc + +%.o: $(MODSRCS_DIR)/%.c + $(COMPILE.c) $< + $(CTFCONVERT_O) + +%.ln: $(MODSRCS_DIR)/%.c + $(LINT.c) -c $< diff --git a/usr/src/cmd/mdb/intel/amd64/libpython2.6/Makefile b/usr/src/cmd/mdb/intel/amd64/libpython2.6/Makefile deleted file mode 100644 index 7f2e268e42..0000000000 --- a/usr/src/cmd/mdb/intel/amd64/libpython2.6/Makefile +++ /dev/null @@ -1,45 +0,0 @@ -# -# CDDL HEADER START -# -# The contents of this file are subject to the terms of the -# Common Development and Distribution License (the "License"). -# You may not use this file except in compliance with the License. -# -# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE -# or http://www.opensolaris.org/os/licensing. -# See the License for the specific language governing permissions -# and limitations under the License. -# -# When distributing Covered Code, include this CDDL HEADER in each -# file and include the License file at usr/src/OPENSOLARIS.LICENSE. -# If applicable, add the following below this CDDL HEADER, with the -# fields enclosed by brackets "[]" replaced with your own identifying -# information: Portions Copyright [yyyy] [name of copyright owner] -# -# CDDL HEADER END -# -# -# Copyright 2010 Sun Microsystems, Inc. All rights reserved. -# Use is subject to license terms. -# - -MODULE = libpython2.6.so -MDBTGT = proc - -MODSRCS_DIR = ../../../common/modules/libpython2.6 - -MODSRCS = libpython26.c - -include ../../../../Makefile.cmd -include ../../../../Makefile.cmd.64 -include ../../Makefile.amd64 -include ../../../Makefile.module - -dmod/$(MODULE) := LDLIBS += -lproc - -%.o: $(MODSRCS_DIR)/%.c - $(COMPILE.c) $< - $(CTFCONVERT_O) - -%.ln: $(MODSRCS_DIR)/%.c - $(LINT.c) -c $< diff --git a/usr/src/cmd/mdb/intel/ia32/libpython/Makefile b/usr/src/cmd/mdb/intel/ia32/libpython/Makefile new file mode 100644 index 0000000000..2abfe7acb6 --- /dev/null +++ b/usr/src/cmd/mdb/intel/ia32/libpython/Makefile @@ -0,0 +1,49 @@ +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# +# +# Copyright 2010 Sun Microsystems, Inc. All rights reserved. +# Use is subject to license terms. +# + +include $(SRC)/Makefile.master + +MODULE = libpython$(PYTHON_VERSION).so +MDBTGT = proc +MODULE_NAME = libpython + +MODSRCS_DIR = ../../../common/modules/libpython + +MODSRCS = libpython.c + +include ../../../../Makefile.cmd +include ../../Makefile.ia32 +include ../../../Makefile.module + +%.o := CPPFLAGS += -_gcc=-isystem -_gcc=$(ADJUNCT_PROTO)/usr/include + +dmod/$(MODULE) := LDLIBS += -lproc + +%.o: $(MODSRCS_DIR)/%.c + $(COMPILE.c) $< + $(CTFCONVERT_O) + +%.ln: $(MODSRCS_DIR)/%.c + $(LINT.c) -c $< diff --git a/usr/src/cmd/mdb/intel/ia32/libpython2.6/Makefile b/usr/src/cmd/mdb/intel/ia32/libpython2.6/Makefile deleted file mode 100644 index 1928c56cd3..0000000000 --- a/usr/src/cmd/mdb/intel/ia32/libpython2.6/Makefile +++ /dev/null @@ -1,46 +0,0 @@ -# -# CDDL HEADER START -# -# The contents of this file are subject to the terms of the -# Common Development and Distribution License (the "License"). -# You may not use this file except in compliance with the License. -# -# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE -# or http://www.opensolaris.org/os/licensing. -# See the License for the specific language governing permissions -# and limitations under the License. -# -# When distributing Covered Code, include this CDDL HEADER in each -# file and include the License file at usr/src/OPENSOLARIS.LICENSE. -# If applicable, add the following below this CDDL HEADER, with the -# fields enclosed by brackets "[]" replaced with your own identifying -# information: Portions Copyright [yyyy] [name of copyright owner] -# -# CDDL HEADER END -# -# -# Copyright 2010 Sun Microsystems, Inc. All rights reserved. -# Use is subject to license terms. -# - -MODULE = libpython2.6.so -MDBTGT = proc - -MODSRCS_DIR = ../../../common/modules/libpython2.6 - -MODSRCS = libpython26.c - -include ../../../../Makefile.cmd -include ../../Makefile.ia32 -include ../../../Makefile.module - -%.o := CPPFLAGS += -_gcc=-isystem -_gcc=$(ADJUNCT_PROTO)/usr/include - -dmod/$(MODULE) := LDLIBS += -lproc - -%.o: $(MODSRCS_DIR)/%.c - $(COMPILE.c) $< - $(CTFCONVERT_O) - -%.ln: $(MODSRCS_DIR)/%.c - $(LINT.c) -c $< diff --git a/usr/src/cmd/mdb/sparc/v7/libpython/Makefile b/usr/src/cmd/mdb/sparc/v7/libpython/Makefile new file mode 100644 index 0000000000..f0c2120e4f --- /dev/null +++ b/usr/src/cmd/mdb/sparc/v7/libpython/Makefile @@ -0,0 +1,47 @@ +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# +# +# Copyright 2010 Sun Microsystems, Inc. All rights reserved. +# Use is subject to license terms. +# + +include $(SRC)/Makefile.master + +MODULE = libpython$(PYTHON_VERSION).so +MDBTGT = proc +MODULE_NAME = libpython + +MODSRCS_DIR = ../../../common/modules/libpython + +MODSRCS = libpython.c + +include ../../../../Makefile.cmd +include ../../Makefile.sparcv7 +include ../../../Makefile.module + +dmod/$(MODULE) := LDLIBS += -lproc + +%.o: $(MODSRCS_DIR)/%.c + $(COMPILE.c) $< + $(CTFCONVERT_O) + +%.ln: $(MODSRCS_DIR)/%.c + $(LINT.c) -c $< diff --git a/usr/src/cmd/mdb/sparc/v7/libpython2.6/Makefile b/usr/src/cmd/mdb/sparc/v7/libpython2.6/Makefile deleted file mode 100644 index 67fa9e938f..0000000000 --- a/usr/src/cmd/mdb/sparc/v7/libpython2.6/Makefile +++ /dev/null @@ -1,44 +0,0 @@ -# -# CDDL HEADER START -# -# The contents of this file are subject to the terms of the -# Common Development and Distribution License (the "License"). -# You may not use this file except in compliance with the License. -# -# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE -# or http://www.opensolaris.org/os/licensing. -# See the License for the specific language governing permissions -# and limitations under the License. -# -# When distributing Covered Code, include this CDDL HEADER in each -# file and include the License file at usr/src/OPENSOLARIS.LICENSE. -# If applicable, add the following below this CDDL HEADER, with the -# fields enclosed by brackets "[]" replaced with your own identifying -# information: Portions Copyright [yyyy] [name of copyright owner] -# -# CDDL HEADER END -# -# -# Copyright 2010 Sun Microsystems, Inc. All rights reserved. -# Use is subject to license terms. -# - -MODULE = libpython2.6.so -MDBTGT = proc - -MODSRCS_DIR = ../../../common/modules/libpython2.6 - -MODSRCS = libpython26.c - -include ../../../../Makefile.cmd -include ../../Makefile.sparcv7 -include ../../../Makefile.module - -dmod/$(MODULE) := LDLIBS += -lproc - -%.o: $(MODSRCS_DIR)/%.c - $(COMPILE.c) $< - $(CTFCONVERT_O) - -%.ln: $(MODSRCS_DIR)/%.c - $(LINT.c) -c $< diff --git a/usr/src/cmd/mdb/sparc/v9/libpython/Makefile b/usr/src/cmd/mdb/sparc/v9/libpython/Makefile new file mode 100644 index 0000000000..cb3d7a78b8 --- /dev/null +++ b/usr/src/cmd/mdb/sparc/v9/libpython/Makefile @@ -0,0 +1,48 @@ +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# +# +# Copyright 2010 Sun Microsystems, Inc. All rights reserved. +# Use is subject to license terms. +# + +include $(SRC)/Makefile.master + +MODULE = libpython$(PYTHON_VERSION).so +MDBTGT = proc +MODULE_NAME = libpython + +MODSRCS_DIR = ../../../common/modules/libpython + +MODSRCS = libpython.c + +include ../../../../Makefile.cmd +include ../../../../Makefile.cmd.64 +include ../../Makefile.sparcv9 +include ../../../Makefile.module + +dmod/$(MODULE) := LDLIBS += -lproc + +%.o: $(MODSRCS_DIR)/%.c + $(COMPILE.c) $< + $(CTFCONVERT_O) + +%.ln: $(MODSRCS_DIR)/%.c + $(LINT.c) -c $< diff --git a/usr/src/cmd/mdb/sparc/v9/libpython2.6/Makefile b/usr/src/cmd/mdb/sparc/v9/libpython2.6/Makefile deleted file mode 100644 index abccb3974c..0000000000 --- a/usr/src/cmd/mdb/sparc/v9/libpython2.6/Makefile +++ /dev/null @@ -1,45 +0,0 @@ -# -# CDDL HEADER START -# -# The contents of this file are subject to the terms of the -# Common Development and Distribution License (the "License"). -# You may not use this file except in compliance with the License. -# -# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE -# or http://www.opensolaris.org/os/licensing. -# See the License for the specific language governing permissions -# and limitations under the License. -# -# When distributing Covered Code, include this CDDL HEADER in each -# file and include the License file at usr/src/OPENSOLARIS.LICENSE. -# If applicable, add the following below this CDDL HEADER, with the -# fields enclosed by brackets "[]" replaced with your own identifying -# information: Portions Copyright [yyyy] [name of copyright owner] -# -# CDDL HEADER END -# -# -# Copyright 2010 Sun Microsystems, Inc. All rights reserved. -# Use is subject to license terms. -# - -MODULE = libpython2.6.so -MDBTGT = proc - -MODSRCS_DIR = ../../../common/modules/libpython2.6 - -MODSRCS = libpython26.c - -include ../../../../Makefile.cmd -include ../../../../Makefile.cmd.64 -include ../../Makefile.sparcv9 -include ../../../Makefile.module - -dmod/$(MODULE) := LDLIBS += -lproc - -%.o: $(MODSRCS_DIR)/%.c - $(COMPILE.c) $< - $(CTFCONVERT_O) - -%.ln: $(MODSRCS_DIR)/%.c - $(LINT.c) -c $< diff --git a/usr/src/cmd/pyzfs/Makefile b/usr/src/cmd/pyzfs/Makefile index 65d3594902..8895ccf525 100644 --- a/usr/src/cmd/pyzfs/Makefile +++ b/usr/src/cmd/pyzfs/Makefile @@ -30,7 +30,6 @@ PYSRCS= pyzfs.py PYOBJS= $(PYSRCS:%.py=%.pyc) PYFILES= $(PYSRCS) $(PYOBJS) POFILE= pyzfs.po -PYTHON= $(PYTHON_26) ROOTLIBZFSFILES= $(PYFILES:%=$(ROOTLIB)/zfs/%) diff --git a/usr/src/cmd/pyzfs/pyzfs.py b/usr/src/cmd/pyzfs/pyzfs.py index 2088993d6d..03f29062d0 100644 --- a/usr/src/cmd/pyzfs/pyzfs.py +++ b/usr/src/cmd/pyzfs/pyzfs.py @@ -1,4 +1,4 @@ -#! /usr/bin/python2.6 -S +#!@PYTHON@ -S # # CDDL HEADER START # diff --git a/usr/src/lib/brand/solaris10/cmd/Makefile b/usr/src/lib/brand/solaris10/cmd/Makefile index bad81c8f61..99ac1cce10 100644 --- a/usr/src/lib/brand/solaris10/cmd/Makefile +++ b/usr/src/lib/brand/solaris10/cmd/Makefile @@ -31,6 +31,10 @@ include ../Makefile.s10 # override the install directory ROOTBIN = $(ROOTBRANDDIR) CLOBBERFILES = $(ROOTPROGS) +CLEANFILES += s10_python_wrapper.sh + +s10_python_wrapper.sh: s10_python_wrapper.sh.in + $(SED) -e "s:@PYTHON_VERSION@:$(PYTHON_VERSION):g" < s10_python_wrapper.sh.in > $@ .KEEP_STATE: @@ -41,6 +45,6 @@ lint: install: all $(ROOTPROGS) clean: - $(RM) $(PROGS) + $(RM) $(PROGS) $(CLEANFILES) include $(SRC)/cmd/Makefile.targ diff --git a/usr/src/lib/brand/solaris10/cmd/s10_python_wrapper.sh b/usr/src/lib/brand/solaris10/cmd/s10_python_wrapper.sh deleted file mode 100644 index b884f62a92..0000000000 --- a/usr/src/lib/brand/solaris10/cmd/s10_python_wrapper.sh +++ /dev/null @@ -1,59 +0,0 @@ -#!/bin/sh -# -# CDDL HEADER START -# -# The contents of this file are subject to the terms of the -# Common Development and Distribution License (the "License"). -# You may not use this file except in compliance with the License. -# -# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE -# or http://www.opensolaris.org/os/licensing. -# See the License for the specific language governing permissions -# and limitations under the License. -# -# When distributing Covered Code, include this CDDL HEADER in each -# file and include the License file at usr/src/OPENSOLARIS.LICENSE. -# If applicable, add the following below this CDDL HEADER, with the -# fields enclosed by brackets "[]" replaced with your own identifying -# information: Portions Copyright [yyyy] [name of copyright owner] -# -# CDDL HEADER END -# -# Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved. -# - -# -# This script is invoked by python-governed executable wrappers from within -# s10-branded zones. It uses the native python and native linker. -# The only consumer is currently the zfs command which uses python for -# a few of its subcommands. -# -# All native executables must be run using the native linker. -# By default, the kernel loads the linker at /lib/ld.so.1, which -# in an s10 zone is the s10 linker. Hence when we run the native -# executable below, we explicitly specify /.SUNWnative/lib/ld.so.1 as our 32- -# bit linker and /.SUNWnative/lib/64/ld.so.1 as our 64-bit linker. -# For convience we define "n" to be the native path prefix. -# -pyname=$0 -n=/.SUNWnative - -PYTHONPATH=/.SUNWnative/usr/lib/python2.6/vendor-packages -export PYTHONPATH - -# This wrapper is running in the S10 zone so there is no L10N for the -# following error msg. -if [ ! -f $n$pyname ]; then - echo "Error: \"$pyname\" is not installed in the global zone" - exit 1 -fi - -exec $n/usr/lib/brand/solaris10/s10_native \ - $n/lib/ld.so.1 \ - -e LD_NOENVIRON=1 \ - -e LD_NOCONFIG=1 \ - -e LD_PRELOAD_32=s10_npreload.so.1 \ - -e LD_PRELOAD_64=s10_npreload.so.1 \ - -e LD_LIBRARY_PATH_32="$n/lib:$n/usr/lib:$n/usr/lib/mps" \ - -e LD_LIBRARY_PATH_64="$n/lib/64:$n/usr/lib/64:$n/usr/lib/mps/64" \ - $n/usr/bin/python2.6 $n$pyname "$@" diff --git a/usr/src/lib/brand/solaris10/cmd/s10_python_wrapper.sh.in b/usr/src/lib/brand/solaris10/cmd/s10_python_wrapper.sh.in new file mode 100644 index 0000000000..ecb7c27155 --- /dev/null +++ b/usr/src/lib/brand/solaris10/cmd/s10_python_wrapper.sh.in @@ -0,0 +1,59 @@ +#!/bin/sh +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# +# Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved. +# + +# +# This script is invoked by python-governed executable wrappers from within +# s10-branded zones. It uses the native python and native linker. +# The only consumer is currently the zfs command which uses python for +# a few of its subcommands. +# +# All native executables must be run using the native linker. +# By default, the kernel loads the linker at /lib/ld.so.1, which +# in an s10 zone is the s10 linker. Hence when we run the native +# executable below, we explicitly specify /.SUNWnative/lib/ld.so.1 as our 32- +# bit linker and /.SUNWnative/lib/64/ld.so.1 as our 64-bit linker. +# For convience we define "n" to be the native path prefix. +# +pyname=$0 +n=/.SUNWnative + +PYTHONPATH=/.SUNWnative/usr/lib/python@PYTHON_VERSION@/vendor-packages +export PYTHONPATH + +# This wrapper is running in the S10 zone so there is no L10N for the +# following error msg. +if [ ! -f $n$pyname ]; then + echo "Error: \"$pyname\" is not installed in the global zone" + exit 1 +fi + +exec $n/usr/lib/brand/solaris10/s10_native \ + $n/lib/ld.so.1 \ + -e LD_NOENVIRON=1 \ + -e LD_NOCONFIG=1 \ + -e LD_PRELOAD_32=s10_npreload.so.1 \ + -e LD_PRELOAD_64=s10_npreload.so.1 \ + -e LD_LIBRARY_PATH_32="$n/lib:$n/usr/lib:$n/usr/lib/mps" \ + -e LD_LIBRARY_PATH_64="$n/lib/64:$n/usr/lib/64:$n/usr/lib/mps/64" \ + $n/usr/bin/python@PYTHON_VERSION@ $n$pyname "$@" diff --git a/usr/src/lib/pylibbe/Makefile.com b/usr/src/lib/pylibbe/Makefile.com index 9f964ce417..19240f56a5 100644 --- a/usr/src/lib/pylibbe/Makefile.com +++ b/usr/src/lib/pylibbe/Makefile.com @@ -30,11 +30,10 @@ OBJECTS = libbe_py.o include ../../Makefile.lib -PYTHON = $(PYTHON_26) LIBLINKS = SRCDIR = ../common -ROOTLIBDIR= $(ROOT)/usr/lib/python2.6/vendor-packages -ROOTLIBDIR64= $(ROOT)/usr/lib/python2.6/vendor-packages/64 +ROOTLIBDIR= $(ROOT)/usr/lib/python$(PYTHON_VERSION)/vendor-packages +ROOTLIBDIR64= $(ROOT)/usr/lib/python$(PYTHON_VERSION)/vendor-packages/64 PYOBJS= $(PYSRCS:%.py=$(SRCDIR)/%.pyc) PYFILES= $(PYSRCS) $(PYSRCS:%.py=%.pyc) ROOTPYBEFILES= $(PYFILES:%=$(ROOTLIBDIR)/%) @@ -44,12 +43,12 @@ C99MODE= $(C99_ENABLE) LIBS = $(DYNLIB) LDLIBS += -lbe -lnvpair -lc CFLAGS += $(CCVERBOSE) -CPPFLAGS += -I$(ADJUNCT_PROTO)/usr/include/python2.6 \ +CPPFLAGS += -I$(ADJUNCT_PROTO)/usr/include/python$(PYTHON_VERSION) \ -D_FILE_OFFSET_BITS=64 -I../../libbe/common .KEEP_STATE: -all install := LDLIBS += -lpython2.6 +all install := LDLIBS += -lpython$(PYTHON_VERSION) all: $(PYOBJS) $(LIBS) diff --git a/usr/src/lib/pysolaris/Makefile.com b/usr/src/lib/pysolaris/Makefile.com index 28d68704f9..7423665381 100644 --- a/usr/src/lib/pysolaris/Makefile.com +++ b/usr/src/lib/pysolaris/Makefile.com @@ -32,20 +32,19 @@ include ../../Makefile.lib LIBLINKS = SRCDIR = ../common -ROOTLIBDIR= $(ROOT)/usr/lib/python2.6/vendor-packages/solaris +ROOTLIBDIR= $(ROOT)/usr/lib/python$(PYTHON_VERSION)/vendor-packages/solaris PYOBJS= $(PYSRCS:%.py=$(SRCDIR)/%.pyc) PYFILES= $(PYSRCS) $(PYSRCS:%.py=%.pyc) ROOTPYSOLFILES= $(PYFILES:%=$(ROOTLIBDIR)/%) -PYTHON= $(PYTHON_26) C99MODE= -xc99=%all C99LMODE= -Xc99=%all LIBS = $(DYNLIB) -LDLIBS += -lc -lsec -lidmap -lpython2.6 +LDLIBS += -lc -lsec -lidmap -lpython$(PYTHON_VERSION) CFLAGS += $(CCVERBOSE) CERRWARN += -_gcc=-Wno-unused-variable -CPPFLAGS += -I$(ADJUNCT_PROTO)/usr/include/python2.6 +CPPFLAGS += -I$(ADJUNCT_PROTO)/usr/include/python$(PYTHON_VERSION) .KEEP_STATE: @@ -56,6 +55,9 @@ install: all $(ROOTPYSOLFILES) $(ROOTLIBDIR)/%: % $(INS.pyfile) +$(ROOTLIBDIR)/%: ../common/% + $(INS.pyfile) + lint: lintcheck include ../../Makefile.targ diff --git a/usr/src/lib/pysolaris/common/__init__.py b/usr/src/lib/pysolaris/common/__init__.py index 1119f0645a..c811dbb347 100644 --- a/usr/src/lib/pysolaris/common/__init__.py +++ b/usr/src/lib/pysolaris/common/__init__.py @@ -1,4 +1,4 @@ -#! /usr/bin/python2.6 +#!@PYTHON@ # # CDDL HEADER START # diff --git a/usr/src/lib/pyzfs/Makefile.com b/usr/src/lib/pyzfs/Makefile.com index 0c996d0711..668151416d 100644 --- a/usr/src/lib/pyzfs/Makefile.com +++ b/usr/src/lib/pyzfs/Makefile.com @@ -35,8 +35,7 @@ include ../../Makefile.lib LIBLINKS = SRCDIR = ../common -ROOTLIBDIR= $(ROOT)/usr/lib/python2.6/vendor-packages/zfs -PYTHON= $(PYTHON_26) +ROOTLIBDIR= $(ROOT)/usr/lib/python$(PYTHON_VERSION)/vendor-packages/zfs PYOBJS= $(PYSRCS:%.py=$(SRCDIR)/%.pyc) PYFILES= $(PYSRCS) $(PYSRCS:%.py=%.pyc) ROOTPYZFSFILES= $(PYFILES:%=$(ROOTLIBDIR)/%) @@ -45,10 +44,10 @@ C99MODE= -xc99=%all C99LMODE= -Xc99=%all LIBS = $(DYNLIB) -LDLIBS += -lc -lnvpair -lpython2.6 -lzfs +LDLIBS += -lc -lnvpair -lpython$(PYTHON_VERSION) -lzfs CFLAGS += $(CCVERBOSE) CERRWARN += -_gcc=-Wno-unused-variable -CPPFLAGS += -I$(ADJUNCT_PROTO)/usr/include/python2.6 +CPPFLAGS += -I$(ADJUNCT_PROTO)/usr/include/python$(PYTHON_VERSION) CPPFLAGS += -I../../../uts/common/fs/zfs CPPFLAGS += -I../../../common/zfs @@ -61,6 +60,9 @@ install: all $(ROOTPYZFSFILES) $(ROOTLIBDIR)/%: % $(INS.pyfile) +$(ROOTLIBDIR)/%: ../common/% + $(INS.pyfile) + lint: lintcheck include ../../Makefile.targ diff --git a/usr/src/lib/pyzfs/common/__init__.py b/usr/src/lib/pyzfs/common/__init__.py index 76b0998a3e..5a99094658 100644 --- a/usr/src/lib/pyzfs/common/__init__.py +++ b/usr/src/lib/pyzfs/common/__init__.py @@ -1,4 +1,4 @@ -#! /usr/bin/python2.6 +#!@PYTHON@ # # CDDL HEADER START # diff --git a/usr/src/lib/pyzfs/common/allow.py b/usr/src/lib/pyzfs/common/allow.py index 7ad4b49cc3..2c01280b4b 100644 --- a/usr/src/lib/pyzfs/common/allow.py +++ b/usr/src/lib/pyzfs/common/allow.py @@ -1,4 +1,4 @@ -#! /usr/bin/python2.6 +#!@PYTHON@ # # CDDL HEADER START # diff --git a/usr/src/lib/pyzfs/common/dataset.py b/usr/src/lib/pyzfs/common/dataset.py index 26192e4075..9d4652235a 100644 --- a/usr/src/lib/pyzfs/common/dataset.py +++ b/usr/src/lib/pyzfs/common/dataset.py @@ -1,4 +1,4 @@ -#! /usr/bin/python2.6 +#!@PYTHON@ # # CDDL HEADER START # diff --git a/usr/src/lib/pyzfs/common/groupspace.py b/usr/src/lib/pyzfs/common/groupspace.py index 9f380fdb89..6193456a32 100644 --- a/usr/src/lib/pyzfs/common/groupspace.py +++ b/usr/src/lib/pyzfs/common/groupspace.py @@ -1,4 +1,4 @@ -#! /usr/bin/python2.6 +#!@PYTHON@ # # CDDL HEADER START # diff --git a/usr/src/lib/pyzfs/common/holds.py b/usr/src/lib/pyzfs/common/holds.py index 800e28f974..0a1508e76a 100644 --- a/usr/src/lib/pyzfs/common/holds.py +++ b/usr/src/lib/pyzfs/common/holds.py @@ -1,4 +1,4 @@ -#! /usr/bin/python2.6 +#!@PYTHON@ # # CDDL HEADER START # diff --git a/usr/src/lib/pyzfs/common/table.py b/usr/src/lib/pyzfs/common/table.py index d2a45a142c..87aab1d6f1 100644 --- a/usr/src/lib/pyzfs/common/table.py +++ b/usr/src/lib/pyzfs/common/table.py @@ -1,4 +1,4 @@ -#! /usr/bin/python2.6 +#!@PYTHON@ # # CDDL HEADER START # diff --git a/usr/src/lib/pyzfs/common/unallow.py b/usr/src/lib/pyzfs/common/unallow.py index cbdd4dd73f..630067946e 100644 --- a/usr/src/lib/pyzfs/common/unallow.py +++ b/usr/src/lib/pyzfs/common/unallow.py @@ -1,4 +1,4 @@ -#! /usr/bin/python2.6 +#!@PYTHON@ # # CDDL HEADER START # diff --git a/usr/src/lib/pyzfs/common/userspace.py b/usr/src/lib/pyzfs/common/userspace.py index 8464f54b89..79355e1e23 100644 --- a/usr/src/lib/pyzfs/common/userspace.py +++ b/usr/src/lib/pyzfs/common/userspace.py @@ -1,4 +1,4 @@ -#! /usr/bin/python2.6 +#!@PYTHON@ # # CDDL HEADER START # diff --git a/usr/src/lib/pyzfs/common/util.py b/usr/src/lib/pyzfs/common/util.py index a33c6693ee..cfc21ac57c 100644 --- a/usr/src/lib/pyzfs/common/util.py +++ b/usr/src/lib/pyzfs/common/util.py @@ -1,4 +1,4 @@ -#! /usr/bin/python2.6 +#!@PYTHON@ # # CDDL HEADER START # diff --git a/usr/src/pkg/Makefile b/usr/src/pkg/Makefile index 35a7f6e27d..a00bdfc452 100644 --- a/usr/src/pkg/Makefile +++ b/usr/src/pkg/Makefile @@ -180,7 +180,9 @@ PKGMOG_DEFINES= \ PKGVERS=$(PKGVERS) \ PERL_ARCH=$(PERL_ARCH) \ PERL_VERSION=$(PERL_VERSION) \ - PERL_PKGVERS=$(PERL_PKGVERS) + PERL_PKGVERS=$(PERL_PKGVERS) \ + PYTHON_VERSION=$(PYTHON_VERSION) \ + PYTHON_PKGVERS=$(PYTHON_PKGVERS) PKGDEP_TOKENS_i386= \ 'PLATFORM=i86hvm' \ diff --git a/usr/src/pkg/manifests/developer-build-onbld.mf b/usr/src/pkg/manifests/developer-build-onbld.mf index d6f449f2bc..5bc5cf97e6 100644 --- a/usr/src/pkg/manifests/developer-build-onbld.mf +++ b/usr/src/pkg/manifests/developer-build-onbld.mf @@ -52,11 +52,11 @@ dir path=opt/onbld/lib dir path=opt/onbld/lib/$(ARCH) dir path=opt/onbld/lib/$(ARCH)/64 dir path=opt/onbld/lib/perl -dir path=opt/onbld/lib/python2.6 -dir path=opt/onbld/lib/python2.6/onbld -dir path=opt/onbld/lib/python2.6/onbld/Checks -dir path=opt/onbld/lib/python2.6/onbld/Scm -dir path=opt/onbld/lib/python2.6/onbld/hgext +dir path=opt/onbld/lib/python$(PYTHON_VERSION) +dir path=opt/onbld/lib/python$(PYTHON_VERSION)/onbld +dir path=opt/onbld/lib/python$(PYTHON_VERSION)/onbld/Checks +dir path=opt/onbld/lib/python$(PYTHON_VERSION)/onbld/Scm +dir path=opt/onbld/lib/python$(PYTHON_VERSION)/onbld/hgext dir path=opt/onbld/man dir path=opt/onbld/man/man1onbld dir path=opt/onbld/share @@ -146,49 +146,83 @@ file path=opt/onbld/lib/$(ARCH)/libdwarf.so.1 file path=opt/onbld/lib/$(ARCH)/libmakestate.so.1 file path=opt/onbld/lib/perl/onbld_elfmod.pm file path=opt/onbld/lib/perl/onbld_elfmod_vertype.pm -file path=opt/onbld/lib/python2.6/onbld/Checks/CStyle.py mode=0444 -file path=opt/onbld/lib/python2.6/onbld/Checks/CStyle.pyc mode=0444 -file path=opt/onbld/lib/python2.6/onbld/Checks/Cddl.py mode=0444 -file path=opt/onbld/lib/python2.6/onbld/Checks/Cddl.pyc mode=0444 -file path=opt/onbld/lib/python2.6/onbld/Checks/CmtBlk.py mode=0444 -file path=opt/onbld/lib/python2.6/onbld/Checks/CmtBlk.pyc mode=0444 -file path=opt/onbld/lib/python2.6/onbld/Checks/Comments.py mode=0444 -file path=opt/onbld/lib/python2.6/onbld/Checks/Comments.pyc mode=0444 -file path=opt/onbld/lib/python2.6/onbld/Checks/Copyright.py mode=0444 -file path=opt/onbld/lib/python2.6/onbld/Checks/Copyright.pyc mode=0444 -file path=opt/onbld/lib/python2.6/onbld/Checks/DbLookups.py mode=0444 -file path=opt/onbld/lib/python2.6/onbld/Checks/DbLookups.pyc mode=0444 -file path=opt/onbld/lib/python2.6/onbld/Checks/HdrChk.py mode=0444 -file path=opt/onbld/lib/python2.6/onbld/Checks/HdrChk.pyc mode=0444 -file path=opt/onbld/lib/python2.6/onbld/Checks/JStyle.py mode=0444 -file path=opt/onbld/lib/python2.6/onbld/Checks/JStyle.pyc mode=0444 -file path=opt/onbld/lib/python2.6/onbld/Checks/Keywords.py mode=0444 -file path=opt/onbld/lib/python2.6/onbld/Checks/Keywords.pyc mode=0444 -file path=opt/onbld/lib/python2.6/onbld/Checks/ManLint.py mode=0444 -file path=opt/onbld/lib/python2.6/onbld/Checks/ManLint.pyc mode=0444 -file path=opt/onbld/lib/python2.6/onbld/Checks/Mapfile.py mode=0444 -file path=opt/onbld/lib/python2.6/onbld/Checks/Mapfile.pyc mode=0444 -file path=opt/onbld/lib/python2.6/onbld/Checks/ProcessCheck.py mode=0444 -file path=opt/onbld/lib/python2.6/onbld/Checks/ProcessCheck.pyc mode=0444 -file path=opt/onbld/lib/python2.6/onbld/Checks/SpellCheck.py mode=0444 -file path=opt/onbld/lib/python2.6/onbld/Checks/SpellCheck.pyc mode=0444 -file path=opt/onbld/lib/python2.6/onbld/Checks/__init__.py mode=0444 -file path=opt/onbld/lib/python2.6/onbld/Checks/__init__.pyc mode=0444 -file path=opt/onbld/lib/python2.6/onbld/Scm/Backup.py mode=0444 -file path=opt/onbld/lib/python2.6/onbld/Scm/Backup.pyc mode=0444 -file path=opt/onbld/lib/python2.6/onbld/Scm/Ignore.py mode=0444 -file path=opt/onbld/lib/python2.6/onbld/Scm/Ignore.pyc mode=0444 -file path=opt/onbld/lib/python2.6/onbld/Scm/Version.py mode=0444 -file path=opt/onbld/lib/python2.6/onbld/Scm/Version.pyc mode=0444 -file path=opt/onbld/lib/python2.6/onbld/Scm/WorkSpace.py mode=0444 -file path=opt/onbld/lib/python2.6/onbld/Scm/WorkSpace.pyc mode=0444 -file path=opt/onbld/lib/python2.6/onbld/Scm/__init__.py mode=0444 -file path=opt/onbld/lib/python2.6/onbld/Scm/__init__.pyc mode=0444 -file path=opt/onbld/lib/python2.6/onbld/__init__.py mode=0444 -file path=opt/onbld/lib/python2.6/onbld/__init__.pyc mode=0444 -file path=opt/onbld/lib/python2.6/onbld/hgext/__init__.py mode=0444 -file path=opt/onbld/lib/python2.6/onbld/hgext/__init__.pyc mode=0444 -file path=opt/onbld/lib/python2.6/onbld/hgext/cdm.py mode=0444 +file path=opt/onbld/lib/python$(PYTHON_VERSION)/onbld/Checks/CStyle.py \ + mode=0444 +file path=opt/onbld/lib/python$(PYTHON_VERSION)/onbld/Checks/CStyle.pyc \ + mode=0444 +file path=opt/onbld/lib/python$(PYTHON_VERSION)/onbld/Checks/Cddl.py mode=0444 +file path=opt/onbld/lib/python$(PYTHON_VERSION)/onbld/Checks/Cddl.pyc \ + mode=0444 +file path=opt/onbld/lib/python$(PYTHON_VERSION)/onbld/Checks/CmtBlk.py \ + mode=0444 +file path=opt/onbld/lib/python$(PYTHON_VERSION)/onbld/Checks/CmtBlk.pyc \ + mode=0444 +file path=opt/onbld/lib/python$(PYTHON_VERSION)/onbld/Checks/Comments.py \ + mode=0444 +file path=opt/onbld/lib/python$(PYTHON_VERSION)/onbld/Checks/Comments.pyc \ + mode=0444 +file path=opt/onbld/lib/python$(PYTHON_VERSION)/onbld/Checks/Copyright.py \ + mode=0444 +file path=opt/onbld/lib/python$(PYTHON_VERSION)/onbld/Checks/Copyright.pyc \ + mode=0444 +file path=opt/onbld/lib/python$(PYTHON_VERSION)/onbld/Checks/DbLookups.py \ + mode=0444 +file path=opt/onbld/lib/python$(PYTHON_VERSION)/onbld/Checks/DbLookups.pyc \ + mode=0444 +file path=opt/onbld/lib/python$(PYTHON_VERSION)/onbld/Checks/HdrChk.py \ + mode=0444 +file path=opt/onbld/lib/python$(PYTHON_VERSION)/onbld/Checks/HdrChk.pyc \ + mode=0444 +file path=opt/onbld/lib/python$(PYTHON_VERSION)/onbld/Checks/JStyle.py \ + mode=0444 +file path=opt/onbld/lib/python$(PYTHON_VERSION)/onbld/Checks/JStyle.pyc \ + mode=0444 +file path=opt/onbld/lib/python$(PYTHON_VERSION)/onbld/Checks/Keywords.py \ + mode=0444 +file path=opt/onbld/lib/python$(PYTHON_VERSION)/onbld/Checks/Keywords.pyc \ + mode=0444 +file path=opt/onbld/lib/python$(PYTHON_VERSION)/onbld/Checks/ManLint.py \ + mode=0444 +file path=opt/onbld/lib/python$(PYTHON_VERSION)/onbld/Checks/ManLint.pyc \ + mode=0444 +file path=opt/onbld/lib/python$(PYTHON_VERSION)/onbld/Checks/Mapfile.py \ + mode=0444 +file path=opt/onbld/lib/python$(PYTHON_VERSION)/onbld/Checks/Mapfile.pyc \ + mode=0444 +file path=opt/onbld/lib/python$(PYTHON_VERSION)/onbld/Checks/ProcessCheck.py \ + mode=0444 +file path=opt/onbld/lib/python$(PYTHON_VERSION)/onbld/Checks/ProcessCheck.pyc \ + mode=0444 +file path=opt/onbld/lib/python$(PYTHON_VERSION)/onbld/Checks/SpellCheck.py \ + mode=0444 +file path=opt/onbld/lib/python$(PYTHON_VERSION)/onbld/Checks/SpellCheck.pyc \ + mode=0444 +file path=opt/onbld/lib/python$(PYTHON_VERSION)/onbld/Checks/__init__.py \ + mode=0444 +file path=opt/onbld/lib/python$(PYTHON_VERSION)/onbld/Checks/__init__.pyc \ + mode=0444 +file path=opt/onbld/lib/python$(PYTHON_VERSION)/onbld/Scm/Backup.py mode=0444 +file path=opt/onbld/lib/python$(PYTHON_VERSION)/onbld/Scm/Backup.pyc mode=0444 +file path=opt/onbld/lib/python$(PYTHON_VERSION)/onbld/Scm/Ignore.py mode=0444 +file path=opt/onbld/lib/python$(PYTHON_VERSION)/onbld/Scm/Ignore.pyc mode=0444 +file path=opt/onbld/lib/python$(PYTHON_VERSION)/onbld/Scm/Version.py mode=0444 +file path=opt/onbld/lib/python$(PYTHON_VERSION)/onbld/Scm/Version.pyc \ + mode=0444 +file path=opt/onbld/lib/python$(PYTHON_VERSION)/onbld/Scm/WorkSpace.py \ + mode=0444 +file path=opt/onbld/lib/python$(PYTHON_VERSION)/onbld/Scm/WorkSpace.pyc \ + mode=0444 +file path=opt/onbld/lib/python$(PYTHON_VERSION)/onbld/Scm/__init__.py \ + mode=0444 +file path=opt/onbld/lib/python$(PYTHON_VERSION)/onbld/Scm/__init__.pyc \ + mode=0444 +file path=opt/onbld/lib/python$(PYTHON_VERSION)/onbld/__init__.py mode=0444 +file path=opt/onbld/lib/python$(PYTHON_VERSION)/onbld/__init__.pyc mode=0444 +file path=opt/onbld/lib/python$(PYTHON_VERSION)/onbld/hgext/__init__.py \ + mode=0444 +file path=opt/onbld/lib/python$(PYTHON_VERSION)/onbld/hgext/__init__.pyc \ + mode=0444 +file path=opt/onbld/lib/python$(PYTHON_VERSION)/onbld/hgext/cdm.py mode=0444 file path=opt/onbld/man/man1onbld/Install.1onbld file path=opt/onbld/man/man1onbld/bldenv.1onbld file path=opt/onbld/man/man1onbld/bringovercheck.1onbld @@ -234,7 +268,7 @@ license usr/src/tools/onbld/THIRDPARTYLICENSE \ license=usr/src/tools/onbld/THIRDPARTYLICENSE link path=opt/onbld/bin/$(ARCH)/dmake target=make link path=opt/onbld/bin/git-nits target=git-pbchk -link path=opt/onbld/lib/python target=python2.6 +link path=opt/onbld/lib/python target=python$(PYTHON_VERSION) link path=opt/onbld/man/man1onbld/git-nits.1onbld target=git-pbchk.1onbld # webrev(1) requires ps2pdf depend fmri=print/filter/ghostscript type=require diff --git a/usr/src/pkg/manifests/developer-debug-mdb.mf b/usr/src/pkg/manifests/developer-debug-mdb.mf index f1929da69b..2cd8a0299e 100644 --- a/usr/src/pkg/manifests/developer-debug-mdb.mf +++ b/usr/src/pkg/manifests/developer-debug-mdb.mf @@ -296,7 +296,8 @@ file path=usr/lib/mdb/proc/$(ARCH64)/libc.so group=sys mode=0555 file path=usr/lib/mdb/proc/$(ARCH64)/libcmdutils.so group=sys mode=0555 file path=usr/lib/mdb/proc/$(ARCH64)/libnvpair.so group=sys mode=0555 file path=usr/lib/mdb/proc/$(ARCH64)/libproc.so group=sys mode=0555 -file path=usr/lib/mdb/proc/$(ARCH64)/libpython2.6.so group=sys mode=0555 +file path=usr/lib/mdb/proc/$(ARCH64)/libpython$(PYTHON_VERSION).so group=sys \ + mode=0555 file path=usr/lib/mdb/proc/$(ARCH64)/libsysevent.so group=sys mode=0555 file path=usr/lib/mdb/proc/$(ARCH64)/libtopo.so group=sys mode=0555 file path=usr/lib/mdb/proc/$(ARCH64)/libumem.so group=sys mode=0555 @@ -310,7 +311,7 @@ file path=usr/lib/mdb/proc/libc.so group=sys mode=0555 file path=usr/lib/mdb/proc/libcmdutils.so group=sys mode=0555 file path=usr/lib/mdb/proc/libnvpair.so group=sys mode=0555 file path=usr/lib/mdb/proc/libproc.so group=sys mode=0555 -file path=usr/lib/mdb/proc/libpython2.6.so group=sys mode=0555 +file path=usr/lib/mdb/proc/libpython$(PYTHON_VERSION).so group=sys mode=0555 file path=usr/lib/mdb/proc/libsysevent.so group=sys mode=0555 file path=usr/lib/mdb/proc/libtopo.so group=sys mode=0555 file path=usr/lib/mdb/proc/libumem.so group=sys mode=0555 diff --git a/usr/src/pkg/manifests/install-beadm.mf b/usr/src/pkg/manifests/install-beadm.mf index 2a94f501ca..05f83de46b 100644 --- a/usr/src/pkg/manifests/install-beadm.mf +++ b/usr/src/pkg/manifests/install-beadm.mf @@ -34,10 +34,10 @@ dir path=usr group=sys dir path=usr/include dir path=usr/lib dir path=usr/lib/$(ARCH64) -dir path=usr/lib/python2.6 -dir path=usr/lib/python2.6/vendor-packages -dir path=usr/lib/python2.6/vendor-packages/64 -dir path=usr/lib/python2.6/vendor-packages/beadm +dir path=usr/lib/python$(PYTHON_VERSION) +dir path=usr/lib/python$(PYTHON_VERSION)/vendor-packages +dir path=usr/lib/python$(PYTHON_VERSION)/vendor-packages/64 +dir path=usr/lib/python$(PYTHON_VERSION)/vendor-packages/beadm dir path=usr/sbin dir path=usr/share dir path=usr/share/man @@ -49,8 +49,8 @@ file path=usr/lib/$(ARCH64)/llib-lbe.ln file path=usr/lib/libbe.so.1 file path=usr/lib/llib-lbe file path=usr/lib/llib-lbe.ln -file path=usr/lib/python2.6/vendor-packages/64/libbe_py.so -file path=usr/lib/python2.6/vendor-packages/libbe_py.so +file path=usr/lib/python$(PYTHON_VERSION)/vendor-packages/64/libbe_py.so +file path=usr/lib/python$(PYTHON_VERSION)/vendor-packages/libbe_py.so file path=usr/share/man/man1m/beadm.1m license cr_Sun license=cr_Sun license lic_CDDL license=lic_CDDL diff --git a/usr/src/pkg/manifests/system-file-system-zfs.mf b/usr/src/pkg/manifests/system-file-system-zfs.mf index 2b632296dd..e0474cd0d0 100644 --- a/usr/src/pkg/manifests/system-file-system-zfs.mf +++ b/usr/src/pkg/manifests/system-file-system-zfs.mf @@ -66,9 +66,9 @@ dir path=usr/lib/mdb/kvm/$(ARCH64) group=sys dir path=usr/lib/mdb/proc group=sys $(sparc_ONLY)dir path=usr/lib/mdb/proc/$(ARCH64) group=sys $(i386_ONLY)dir path=usr/lib/mdb/proc/$(ARCH64) -dir path=usr/lib/python2.6 -dir path=usr/lib/python2.6/vendor-packages -dir path=usr/lib/python2.6/vendor-packages/zfs +dir path=usr/lib/python$(PYTHON_VERSION) +dir path=usr/lib/python$(PYTHON_VERSION)/vendor-packages +dir path=usr/lib/python$(PYTHON_VERSION)/vendor-packages/zfs dir path=usr/lib/sysevent dir path=usr/lib/sysevent/modules dir path=usr/lib/zfs @@ -108,25 +108,25 @@ file path=usr/lib/mdb/kvm/$(ARCH64)/zfs.so group=sys mode=0555 $(i386_ONLY)file path=usr/lib/mdb/kvm/zfs.so group=sys mode=0555 file path=usr/lib/mdb/proc/$(ARCH64)/libzpool.so group=sys mode=0555 file path=usr/lib/mdb/proc/libzpool.so group=sys mode=0555 -file path=usr/lib/python2.6/vendor-packages/zfs/__init__.py -file path=usr/lib/python2.6/vendor-packages/zfs/__init__.pyc -file path=usr/lib/python2.6/vendor-packages/zfs/allow.py -file path=usr/lib/python2.6/vendor-packages/zfs/allow.pyc -file path=usr/lib/python2.6/vendor-packages/zfs/dataset.py -file path=usr/lib/python2.6/vendor-packages/zfs/dataset.pyc -file path=usr/lib/python2.6/vendor-packages/zfs/groupspace.py -file path=usr/lib/python2.6/vendor-packages/zfs/groupspace.pyc -file path=usr/lib/python2.6/vendor-packages/zfs/holds.py -file path=usr/lib/python2.6/vendor-packages/zfs/holds.pyc -file path=usr/lib/python2.6/vendor-packages/zfs/ioctl.so -file path=usr/lib/python2.6/vendor-packages/zfs/table.py -file path=usr/lib/python2.6/vendor-packages/zfs/table.pyc -file path=usr/lib/python2.6/vendor-packages/zfs/unallow.py -file path=usr/lib/python2.6/vendor-packages/zfs/unallow.pyc -file path=usr/lib/python2.6/vendor-packages/zfs/userspace.py -file path=usr/lib/python2.6/vendor-packages/zfs/userspace.pyc -file path=usr/lib/python2.6/vendor-packages/zfs/util.py -file path=usr/lib/python2.6/vendor-packages/zfs/util.pyc +file path=usr/lib/python$(PYTHON_VERSION)/vendor-packages/zfs/__init__.py +file path=usr/lib/python$(PYTHON_VERSION)/vendor-packages/zfs/__init__.pyc +file path=usr/lib/python$(PYTHON_VERSION)/vendor-packages/zfs/allow.py +file path=usr/lib/python$(PYTHON_VERSION)/vendor-packages/zfs/allow.pyc +file path=usr/lib/python$(PYTHON_VERSION)/vendor-packages/zfs/dataset.py +file path=usr/lib/python$(PYTHON_VERSION)/vendor-packages/zfs/dataset.pyc +file path=usr/lib/python$(PYTHON_VERSION)/vendor-packages/zfs/groupspace.py +file path=usr/lib/python$(PYTHON_VERSION)/vendor-packages/zfs/groupspace.pyc +file path=usr/lib/python$(PYTHON_VERSION)/vendor-packages/zfs/holds.py +file path=usr/lib/python$(PYTHON_VERSION)/vendor-packages/zfs/holds.pyc +file path=usr/lib/python$(PYTHON_VERSION)/vendor-packages/zfs/ioctl.so +file path=usr/lib/python$(PYTHON_VERSION)/vendor-packages/zfs/table.py +file path=usr/lib/python$(PYTHON_VERSION)/vendor-packages/zfs/table.pyc +file path=usr/lib/python$(PYTHON_VERSION)/vendor-packages/zfs/unallow.py +file path=usr/lib/python$(PYTHON_VERSION)/vendor-packages/zfs/unallow.pyc +file path=usr/lib/python$(PYTHON_VERSION)/vendor-packages/zfs/userspace.py +file path=usr/lib/python$(PYTHON_VERSION)/vendor-packages/zfs/userspace.pyc +file path=usr/lib/python$(PYTHON_VERSION)/vendor-packages/zfs/util.py +file path=usr/lib/python$(PYTHON_VERSION)/vendor-packages/zfs/util.pyc file path=usr/lib/sysevent/modules/zfs_mod.so group=sys file path=usr/lib/zfs/availdevs mode=0555 file path=usr/lib/zfs/pyzfs.py mode=0555 @@ -185,4 +185,4 @@ link path=usr/lib/llib-lzfs_core target=../../lib/llib-lzfs_core link path=usr/lib/llib-lzfs_core.ln target=../../lib/llib-lzfs_core.ln link path=usr/sbin/zfs target=../../sbin/zfs link path=usr/sbin/zpool target=../../sbin/zpool -depend fmri=runtime/python-26 type=require +depend fmri=runtime/python$(PYTHON_PKGVERS) type=require diff --git a/usr/src/pkg/manifests/system-library.mf b/usr/src/pkg/manifests/system-library.mf index 1842a62f06..000169ef55 100644 --- a/usr/src/pkg/manifests/system-library.mf +++ b/usr/src/pkg/manifests/system-library.mf @@ -99,9 +99,9 @@ dir path=usr/lib/iconv/$(ARCH64) $(i386_ONLY)dir path=usr/lib/libc dir path=usr/lib/lwp dir path=usr/lib/lwp/$(ARCH64) -dir path=usr/lib/python2.6 -dir path=usr/lib/python2.6/vendor-packages -dir path=usr/lib/python2.6/vendor-packages/solaris +dir path=usr/lib/python$(PYTHON_VERSION) +dir path=usr/lib/python$(PYTHON_VERSION)/vendor-packages +dir path=usr/lib/python$(PYTHON_VERSION)/vendor-packages/solaris dir path=usr/lib/raidcfg dir path=usr/lib/raidcfg/$(ARCH64) dir path=usr/lib/scsi @@ -452,9 +452,9 @@ file path=usr/lib/libzoneinfo.so.1 file path=usr/lib/nss_ad.so.1 file path=usr/lib/nss_ldap.so.1 file path=usr/lib/passwdutil.so.1 -file path=usr/lib/python2.6/vendor-packages/solaris/__init__.py -file path=usr/lib/python2.6/vendor-packages/solaris/__init__.pyc -file path=usr/lib/python2.6/vendor-packages/solaris/misc.so +file path=usr/lib/python$(PYTHON_VERSION)/vendor-packages/solaris/__init__.py +file path=usr/lib/python$(PYTHON_VERSION)/vendor-packages/solaris/__init__.pyc +file path=usr/lib/python$(PYTHON_VERSION)/vendor-packages/solaris/misc.so file path=usr/lib/raidcfg/$(ARCH64)/mpt.so.1 file path=usr/lib/raidcfg/mpt.so.1 file path=usr/lib/scsi/$(ARCH64)/libscsi.so.1 diff --git a/usr/src/test/test-runner/cmd/Makefile b/usr/src/test/test-runner/cmd/Makefile index 33e7a61275..68acf3b23c 100644 --- a/usr/src/test/test-runner/cmd/Makefile +++ b/usr/src/test/test-runner/cmd/Makefile @@ -33,5 +33,5 @@ $(CMDS): $(ROOTBIN) $(ROOTBIN): $(INS.dir) -$(ROOTBIN)/%: %.py - $(INS.rename) +$(ROOTBIN)/%: % + $(INS.pyfile) diff --git a/usr/src/test/test-runner/cmd/run b/usr/src/test/test-runner/cmd/run new file mode 100644 index 0000000000..ad0204d136 --- /dev/null +++ b/usr/src/test/test-runner/cmd/run @@ -0,0 +1,871 @@ +#!@PYTHON@ + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2012, 2015 by Delphix. All rights reserved. +# + +import ConfigParser +import os +import logging +from logging.handlers import WatchedFileHandler +from datetime import datetime +from optparse import OptionParser +from pwd import getpwnam +from pwd import getpwuid +from select import select +from subprocess import PIPE +from subprocess import Popen +from sys import argv +from sys import maxint +from threading import Timer +from time import time + +BASEDIR = '/var/tmp/test_results' +KILL = '/usr/bin/kill' +TRUE = '/usr/bin/true' +SUDO = '/usr/bin/sudo' + +# Custom class to reopen the log file in case it is forcibly closed by a test. +class WatchedFileHandlerClosed(WatchedFileHandler): + """Watch files, including closed files. + Similar to (and inherits from) logging.handler.WatchedFileHandler, + except that IOErrors are handled by reopening the stream and retrying. + This will be retried up to a configurable number of times before + giving up, default 5. + """ + + def __init__(self, filename, mode='a', encoding=None, delay=0, max_tries=5): + self.max_tries = max_tries + self.tries = 0 + WatchedFileHandler.__init__(self, filename, mode, encoding, delay) + + def emit(self, record): + while True: + try: + WatchedFileHandler.emit(self, record) + self.tries = 0 + return + except IOError as err: + if self.tries == self.max_tries: + raise + self.stream.close() + self.stream = self._open() + self.tries += 1 + +class Result(object): + total = 0 + runresults = {'PASS': 0, 'FAIL': 0, 'SKIP': 0, 'KILLED': 0} + + def __init__(self): + self.starttime = None + self.returncode = None + self.runtime = '' + self.stdout = [] + self.stderr = [] + self.result = '' + + def done(self, proc, killed): + """ + Finalize the results of this Cmd. + """ + Result.total += 1 + m, s = divmod(time() - self.starttime, 60) + self.runtime = '%02d:%02d' % (m, s) + self.returncode = proc.returncode + if killed: + self.result = 'KILLED' + Result.runresults['KILLED'] += 1 + elif self.returncode is 0: + self.result = 'PASS' + Result.runresults['PASS'] += 1 + elif self.returncode is not 0: + self.result = 'FAIL' + Result.runresults['FAIL'] += 1 + + +class Output(object): + """ + This class is a slightly modified version of the 'Stream' class found + here: http://goo.gl/aSGfv + """ + def __init__(self, stream): + self.stream = stream + self._buf = '' + self.lines = [] + + def fileno(self): + return self.stream.fileno() + + def read(self, drain=0): + """ + Read from the file descriptor. If 'drain' set, read until EOF. + """ + while self._read() is not None: + if not drain: + break + + def _read(self): + """ + Read up to 4k of data from this output stream. Collect the output + up to the last newline, and append it to any leftover data from a + previous call. The lines are stored as a (timestamp, data) tuple + for easy sorting/merging later. + """ + fd = self.fileno() + buf = os.read(fd, 4096) + if not buf: + return None + if '\n' not in buf: + self._buf += buf + return [] + + buf = self._buf + buf + tmp, rest = buf.rsplit('\n', 1) + self._buf = rest + now = datetime.now() + rows = tmp.split('\n') + self.lines += [(now, r) for r in rows] + + +class Cmd(object): + verified_users = [] + + def __init__(self, pathname, outputdir=None, timeout=None, user=None): + self.pathname = pathname + self.outputdir = outputdir or 'BASEDIR' + self.timeout = timeout + self.user = user or '' + self.killed = False + self.result = Result() + + if self.timeout is None: + self.timeout = 60 + + def __str__(self): + return "Pathname: %s\nOutputdir: %s\nTimeout: %d\nUser: %s\n" % \ + (self.pathname, self.outputdir, self.timeout, self.user) + + def kill_cmd(self, proc): + """ + Kill a running command due to timeout, or ^C from the keyboard. If + sudo is required, this user was verified previously. + """ + self.killed = True + do_sudo = len(self.user) != 0 + signal = '-TERM' + + cmd = [SUDO, KILL, signal, str(proc.pid)] + if not do_sudo: + del cmd[0] + + try: + kp = Popen(cmd) + kp.wait() + except: + pass + + def update_cmd_privs(self, cmd, user): + """ + If a user has been specified to run this Cmd and we're not already + running as that user, prepend the appropriate sudo command to run + as that user. + """ + me = getpwuid(os.getuid()) + + if not user or user is me: + return cmd + + ret = '%s -E -u %s %s' % (SUDO, user, cmd) + return ret.split(' ') + + def collect_output(self, proc): + """ + Read from stdout/stderr as data becomes available, until the + process is no longer running. Return the lines from the stdout and + stderr Output objects. + """ + out = Output(proc.stdout) + err = Output(proc.stderr) + res = [] + while proc.returncode is None: + proc.poll() + res = select([out, err], [], [], .1) + for fd in res[0]: + fd.read() + for fd in res[0]: + fd.read(drain=1) + + return out.lines, err.lines + + def run(self, options): + """ + This is the main function that runs each individual test. + Determine whether or not the command requires sudo, and modify it + if needed. Run the command, and update the result object. + """ + if options.dryrun is True: + print self + return + + privcmd = self.update_cmd_privs(self.pathname, self.user) + try: + old = os.umask(0) + if not os.path.isdir(self.outputdir): + os.makedirs(self.outputdir, mode=0777) + os.umask(old) + except OSError, e: + fail('%s' % e) + + try: + self.result.starttime = time() + proc = Popen(privcmd, stdout=PIPE, stderr=PIPE) + + # Allow a special timeout value of 0 to mean infinity + if int(self.timeout) == 0: + self.timeout = maxint + t = Timer(int(self.timeout), self.kill_cmd, [proc]) + t.start() + self.result.stdout, self.result.stderr = self.collect_output(proc) + except KeyboardInterrupt: + self.kill_cmd(proc) + fail('\nRun terminated at user request.') + finally: + t.cancel() + + self.result.done(proc, self.killed) + + def skip(self): + """ + Initialize enough of the test result that we can log a skipped + command. + """ + Result.total += 1 + Result.runresults['SKIP'] += 1 + self.result.stdout = self.result.stderr = [] + self.result.starttime = time() + m, s = divmod(time() - self.result.starttime, 60) + self.result.runtime = '%02d:%02d' % (m, s) + self.result.result = 'SKIP' + + def log(self, logger, options): + """ + This function is responsible for writing all output. This includes + the console output, the logfile of all results (with timestamped + merged stdout and stderr), and for each test, the unmodified + stdout/stderr/merged in it's own file. + """ + if logger is None: + return + + logname = getpwuid(os.getuid()).pw_name + user = ' (run as %s)' % (self.user if len(self.user) else logname) + msga = 'Test: %s%s ' % (self.pathname, user) + msgb = '[%s] [%s]' % (self.result.runtime, self.result.result) + pad = ' ' * (80 - (len(msga) + len(msgb))) + + # If -q is specified, only print a line for tests that didn't pass. + # This means passing tests need to be logged as DEBUG, or the one + # line summary will only be printed in the logfile for failures. + if not options.quiet: + logger.info('%s%s%s' % (msga, pad, msgb)) + elif self.result.result is not 'PASS': + logger.info('%s%s%s' % (msga, pad, msgb)) + else: + logger.debug('%s%s%s' % (msga, pad, msgb)) + + lines = sorted(self.result.stdout + self.result.stderr, + cmp=lambda x, y: cmp(x[0], y[0])) + + for dt, line in lines: + logger.debug('%s %s' % (dt.strftime("%H:%M:%S.%f ")[:11], line)) + + if len(self.result.stdout): + with open(os.path.join(self.outputdir, 'stdout'), 'w') as out: + for _, line in self.result.stdout: + os.write(out.fileno(), '%s\n' % line) + if len(self.result.stderr): + with open(os.path.join(self.outputdir, 'stderr'), 'w') as err: + for _, line in self.result.stderr: + os.write(err.fileno(), '%s\n' % line) + if len(self.result.stdout) and len(self.result.stderr): + with open(os.path.join(self.outputdir, 'merged'), 'w') as merged: + for _, line in lines: + os.write(merged.fileno(), '%s\n' % line) + + +class Test(Cmd): + props = ['outputdir', 'timeout', 'user', 'pre', 'pre_user', 'post', + 'post_user'] + + def __init__(self, pathname, outputdir=None, timeout=None, user=None, + pre=None, pre_user=None, post=None, post_user=None): + super(Test, self).__init__(pathname, outputdir, timeout, user) + self.pre = pre or '' + self.pre_user = pre_user or '' + self.post = post or '' + self.post_user = post_user or '' + + def __str__(self): + post_user = pre_user = '' + if len(self.pre_user): + pre_user = ' (as %s)' % (self.pre_user) + if len(self.post_user): + post_user = ' (as %s)' % (self.post_user) + return "Pathname: %s\nOutputdir: %s\nTimeout: %d\nPre: %s%s\nPost: " \ + "%s%s\nUser: %s\n" % \ + (self.pathname, self.outputdir, self.timeout, self.pre, + pre_user, self.post, post_user, self.user) + + def verify(self, logger): + """ + Check the pre/post scripts, user and Test. Omit the Test from this + run if there are any problems. + """ + files = [self.pre, self.pathname, self.post] + users = [self.pre_user, self.user, self.post_user] + + for f in [f for f in files if len(f)]: + if not verify_file(f): + logger.info("Warning: Test '%s' not added to this run because" + " it failed verification." % f) + return False + + for user in [user for user in users if len(user)]: + if not verify_user(user, logger): + logger.info("Not adding Test '%s' to this run." % + self.pathname) + return False + + return True + + def run(self, logger, options): + """ + Create Cmd instances for the pre/post scripts. If the pre script + doesn't pass, skip this Test. Run the post script regardless. + """ + odir = os.path.join(self.outputdir, os.path.basename(self.pre)) + pretest = Cmd(self.pre, outputdir=odir, timeout=self.timeout, + user=self.pre_user) + test = Cmd(self.pathname, outputdir=self.outputdir, + timeout=self.timeout, user=self.user) + odir = os.path.join(self.outputdir, os.path.basename(self.post)) + posttest = Cmd(self.post, outputdir=odir, timeout=self.timeout, + user=self.post_user) + + cont = True + if len(pretest.pathname): + pretest.run(options) + cont = pretest.result.result is 'PASS' + pretest.log(logger, options) + + if cont: + test.run(options) + else: + test.skip() + + test.log(logger, options) + + if len(posttest.pathname): + posttest.run(options) + posttest.log(logger, options) + + +class TestGroup(Test): + props = Test.props + ['tests'] + + def __init__(self, pathname, outputdir=None, timeout=None, user=None, + pre=None, pre_user=None, post=None, post_user=None, + tests=None): + super(TestGroup, self).__init__(pathname, outputdir, timeout, user, + pre, pre_user, post, post_user) + self.tests = tests or [] + + def __str__(self): + post_user = pre_user = '' + if len(self.pre_user): + pre_user = ' (as %s)' % (self.pre_user) + if len(self.post_user): + post_user = ' (as %s)' % (self.post_user) + return "Pathname: %s\nOutputdir: %s\nTests: %s\nTimeout: %d\n" \ + "Pre: %s%s\nPost: %s%s\nUser: %s\n" % \ + (self.pathname, self.outputdir, self.tests, self.timeout, + self.pre, pre_user, self.post, post_user, self.user) + + def verify(self, logger): + """ + Check the pre/post scripts, user and tests in this TestGroup. Omit + the TestGroup entirely, or simply delete the relevant tests in the + group, if that's all that's required. + """ + # If the pre or post scripts are relative pathnames, convert to + # absolute, so they stand a chance of passing verification. + if len(self.pre) and not os.path.isabs(self.pre): + self.pre = os.path.join(self.pathname, self.pre) + if len(self.post) and not os.path.isabs(self.post): + self.post = os.path.join(self.pathname, self.post) + + auxfiles = [self.pre, self.post] + users = [self.pre_user, self.user, self.post_user] + + for f in [f for f in auxfiles if len(f)]: + if self.pathname != os.path.dirname(f): + logger.info("Warning: TestGroup '%s' not added to this run. " + "Auxiliary script '%s' exists in a different " + "directory." % (self.pathname, f)) + return False + + if not verify_file(f): + logger.info("Warning: TestGroup '%s' not added to this run. " + "Auxiliary script '%s' failed verification." % + (self.pathname, f)) + return False + + for user in [user for user in users if len(user)]: + if not verify_user(user, logger): + logger.info("Not adding TestGroup '%s' to this run." % + self.pathname) + return False + + # If one of the tests is invalid, delete it, log it, and drive on. + for test in self.tests: + if not verify_file(os.path.join(self.pathname, test)): + del self.tests[self.tests.index(test)] + logger.info("Warning: Test '%s' removed from TestGroup '%s' " + "because it failed verification." % + (test, self.pathname)) + + return len(self.tests) is not 0 + + def run(self, logger, options): + """ + Create Cmd instances for the pre/post scripts. If the pre script + doesn't pass, skip all the tests in this TestGroup. Run the post + script regardless. + """ + odir = os.path.join(self.outputdir, os.path.basename(self.pre)) + pretest = Cmd(self.pre, outputdir=odir, timeout=self.timeout, + user=self.pre_user) + odir = os.path.join(self.outputdir, os.path.basename(self.post)) + posttest = Cmd(self.post, outputdir=odir, timeout=self.timeout, + user=self.post_user) + + cont = True + if len(pretest.pathname): + pretest.run(options) + cont = pretest.result.result is 'PASS' + pretest.log(logger, options) + + for fname in self.tests: + test = Cmd(os.path.join(self.pathname, fname), + outputdir=os.path.join(self.outputdir, fname), + timeout=self.timeout, user=self.user) + if cont: + test.run(options) + else: + test.skip() + + test.log(logger, options) + + if len(posttest.pathname): + posttest.run(options) + posttest.log(logger, options) + + +class TestRun(object): + props = ['quiet', 'outputdir'] + + def __init__(self, options): + self.tests = {} + self.testgroups = {} + self.starttime = time() + self.timestamp = datetime.now().strftime('%Y%m%dT%H%M%S') + self.outputdir = os.path.join(options.outputdir, self.timestamp) + self.logger = self.setup_logging(options) + self.defaults = [ + ('outputdir', BASEDIR), + ('quiet', False), + ('timeout', 60), + ('user', ''), + ('pre', ''), + ('pre_user', ''), + ('post', ''), + ('post_user', '') + ] + + def __str__(self): + s = 'TestRun:\n outputdir: %s\n' % self.outputdir + s += 'TESTS:\n' + for key in sorted(self.tests.keys()): + s += '%s%s' % (self.tests[key].__str__(), '\n') + s += 'TESTGROUPS:\n' + for key in sorted(self.testgroups.keys()): + s += '%s%s' % (self.testgroups[key].__str__(), '\n') + return s + + def addtest(self, pathname, options): + """ + Create a new Test, and apply any properties that were passed in + from the command line. If it passes verification, add it to the + TestRun. + """ + test = Test(pathname) + for prop in Test.props: + setattr(test, prop, getattr(options, prop)) + + if test.verify(self.logger): + self.tests[pathname] = test + + def addtestgroup(self, dirname, filenames, options): + """ + Create a new TestGroup, and apply any properties that were passed + in from the command line. If it passes verification, add it to the + TestRun. + """ + if dirname not in self.testgroups: + testgroup = TestGroup(dirname) + for prop in Test.props: + setattr(testgroup, prop, getattr(options, prop)) + + # Prevent pre/post scripts from running as regular tests + for f in [testgroup.pre, testgroup.post]: + if f in filenames: + del filenames[filenames.index(f)] + + self.testgroups[dirname] = testgroup + self.testgroups[dirname].tests = sorted(filenames) + + testgroup.verify(self.logger) + + def read(self, logger, options): + """ + Read in the specified runfile, and apply the TestRun properties + listed in the 'DEFAULT' section to our TestRun. Then read each + section, and apply the appropriate properties to the Test or + TestGroup. Properties from individual sections override those set + in the 'DEFAULT' section. If the Test or TestGroup passes + verification, add it to the TestRun. + """ + config = ConfigParser.RawConfigParser() + if not len(config.read(options.runfile)): + fail("Coulnd't read config file %s" % options.runfile) + + for opt in TestRun.props: + if config.has_option('DEFAULT', opt): + setattr(self, opt, config.get('DEFAULT', opt)) + self.outputdir = os.path.join(self.outputdir, self.timestamp) + + for section in config.sections(): + if 'tests' in config.options(section): + testgroup = TestGroup(section) + for prop in TestGroup.props: + for sect in ['DEFAULT', section]: + if config.has_option(sect, prop): + setattr(testgroup, prop, config.get(sect, prop)) + + # Repopulate tests using eval to convert the string to a list + testgroup.tests = eval(config.get(section, 'tests')) + + if testgroup.verify(logger): + self.testgroups[section] = testgroup + else: + test = Test(section) + for prop in Test.props: + for sect in ['DEFAULT', section]: + if config.has_option(sect, prop): + setattr(test, prop, config.get(sect, prop)) + + if test.verify(logger): + self.tests[section] = test + + def write(self, options): + """ + Create a configuration file for editing and later use. The + 'DEFAULT' section of the config file is created from the + properties that were specified on the command line. Tests are + simply added as sections that inherit everything from the + 'DEFAULT' section. TestGroups are the same, except they get an + option including all the tests to run in that directory. + """ + + defaults = dict([(prop, getattr(options, prop)) for prop, _ in + self.defaults]) + config = ConfigParser.RawConfigParser(defaults) + + for test in sorted(self.tests.keys()): + config.add_section(test) + + for testgroup in sorted(self.testgroups.keys()): + config.add_section(testgroup) + config.set(testgroup, 'tests', self.testgroups[testgroup].tests) + + try: + with open(options.template, 'w') as f: + return config.write(f) + except IOError: + fail('Could not open \'%s\' for writing.' % options.template) + + def complete_outputdirs(self): + """ + Collect all the pathnames for Tests, and TestGroups. Work + backwards one pathname component at a time, to create a unique + directory name in which to deposit test output. Tests will be able + to write output files directly in the newly modified outputdir. + TestGroups will be able to create one subdirectory per test in the + outputdir, and are guaranteed uniqueness because a group can only + contain files in one directory. Pre and post tests will create a + directory rooted at the outputdir of the Test or TestGroup in + question for their output. + """ + done = False + components = 0 + tmp_dict = dict(self.tests.items() + self.testgroups.items()) + total = len(tmp_dict) + base = self.outputdir + + while not done: + l = [] + components -= 1 + for testfile in tmp_dict.keys(): + uniq = '/'.join(testfile.split('/')[components:]).lstrip('/') + if uniq not in l: + l.append(uniq) + tmp_dict[testfile].outputdir = os.path.join(base, uniq) + else: + break + done = total == len(l) + + def setup_logging(self, options): + """ + Two loggers are set up here. The first is for the logfile which + will contain one line summarizing the test, including the test + name, result, and running time. This logger will also capture the + timestamped combined stdout and stderr of each run. The second + logger is optional console output, which will contain only the one + line summary. The loggers are initialized at two different levels + to facilitate segregating the output. + """ + if options.dryrun is True: + return + + testlogger = logging.getLogger(__name__) + testlogger.setLevel(logging.DEBUG) + + if options.cmd is not 'wrconfig': + try: + old = os.umask(0) + os.makedirs(self.outputdir, mode=0777) + os.umask(old) + except OSError, e: + fail('%s' % e) + filename = os.path.join(self.outputdir, 'log') + + logfile = WatchedFileHandlerClosed(filename) + logfile.setLevel(logging.DEBUG) + logfilefmt = logging.Formatter('%(message)s') + logfile.setFormatter(logfilefmt) + testlogger.addHandler(logfile) + + cons = logging.StreamHandler() + cons.setLevel(logging.INFO) + consfmt = logging.Formatter('%(message)s') + cons.setFormatter(consfmt) + testlogger.addHandler(cons) + + return testlogger + + def run(self, options): + """ + Walk through all the Tests and TestGroups, calling run(). + """ + try: + os.chdir(self.outputdir) + except OSError: + fail('Could not change to directory %s' % self.outputdir) + for test in sorted(self.tests.keys()): + self.tests[test].run(self.logger, options) + for testgroup in sorted(self.testgroups.keys()): + self.testgroups[testgroup].run(self.logger, options) + + def summary(self): + if Result.total is 0: + return + + print '\nResults Summary' + for key in Result.runresults.keys(): + if Result.runresults[key] is not 0: + print '%s\t% 4d' % (key, Result.runresults[key]) + + m, s = divmod(time() - self.starttime, 60) + h, m = divmod(m, 60) + print '\nRunning Time:\t%02d:%02d:%02d' % (h, m, s) + print 'Percent passed:\t%.1f%%' % ((float(Result.runresults['PASS']) / + float(Result.total)) * 100) + print 'Log directory:\t%s' % self.outputdir + + +def verify_file(pathname): + """ + Verify that the supplied pathname is an executable regular file. + """ + if os.path.isdir(pathname) or os.path.islink(pathname): + return False + + if os.path.isfile(pathname) and os.access(pathname, os.X_OK): + return True + + return False + + +def verify_user(user, logger): + """ + Verify that the specified user exists on this system, and can execute + sudo without being prompted for a password. + """ + testcmd = [SUDO, '-n', '-u', user, TRUE] + + if user in Cmd.verified_users: + return True + + try: + _ = getpwnam(user) + except KeyError: + logger.info("Warning: user '%s' does not exist.", user) + return False + + p = Popen(testcmd) + p.wait() + if p.returncode is not 0: + logger.info("Warning: user '%s' cannot use passwordless sudo.", user) + return False + else: + Cmd.verified_users.append(user) + + return True + + +def find_tests(testrun, options): + """ + For the given list of pathnames, add files as Tests. For directories, + if do_groups is True, add the directory as a TestGroup. If False, + recursively search for executable files. + """ + + for p in sorted(options.pathnames): + if os.path.isdir(p): + for dirname, _, filenames in os.walk(p): + if options.do_groups: + testrun.addtestgroup(dirname, filenames, options) + else: + for f in sorted(filenames): + testrun.addtest(os.path.join(dirname, f), options) + else: + testrun.addtest(p, options) + + +def fail(retstr, ret=1): + print '%s: %s' % (argv[0], retstr) + exit(ret) + + +def options_cb(option, opt_str, value, parser): + path_options = ['runfile', 'outputdir', 'template'] + + if option.dest is 'runfile' and '-w' in parser.rargs or \ + option.dest is 'template' and '-c' in parser.rargs: + fail('-c and -w are mutually exclusive.') + + if opt_str in parser.rargs: + fail('%s may only be specified once.' % opt_str) + + if option.dest is 'runfile': + parser.values.cmd = 'rdconfig' + if option.dest is 'template': + parser.values.cmd = 'wrconfig' + + setattr(parser.values, option.dest, value) + if option.dest in path_options: + setattr(parser.values, option.dest, os.path.abspath(value)) + + +def parse_args(): + parser = OptionParser() + parser.add_option('-c', action='callback', callback=options_cb, + type='string', dest='runfile', metavar='runfile', + help='Specify tests to run via config file.') + parser.add_option('-d', action='store_true', default=False, dest='dryrun', + help='Dry run. Print tests, but take no other action.') + parser.add_option('-g', action='store_true', default=False, + dest='do_groups', help='Make directories TestGroups.') + parser.add_option('-o', action='callback', callback=options_cb, + default=BASEDIR, dest='outputdir', type='string', + metavar='outputdir', help='Specify an output directory.') + parser.add_option('-p', action='callback', callback=options_cb, + default='', dest='pre', metavar='script', + type='string', help='Specify a pre script.') + parser.add_option('-P', action='callback', callback=options_cb, + default='', dest='post', metavar='script', + type='string', help='Specify a post script.') + parser.add_option('-q', action='store_true', default=False, dest='quiet', + help='Silence on the console during a test run.') + parser.add_option('-t', action='callback', callback=options_cb, default=60, + dest='timeout', metavar='seconds', type='int', + help='Timeout (in seconds) for an individual test.') + parser.add_option('-u', action='callback', callback=options_cb, + default='', dest='user', metavar='user', type='string', + help='Specify a different user name to run as.') + parser.add_option('-w', action='callback', callback=options_cb, + default=None, dest='template', metavar='template', + type='string', help='Create a new config file.') + parser.add_option('-x', action='callback', callback=options_cb, default='', + dest='pre_user', metavar='pre_user', type='string', + help='Specify a user to execute the pre script.') + parser.add_option('-X', action='callback', callback=options_cb, default='', + dest='post_user', metavar='post_user', type='string', + help='Specify a user to execute the post script.') + (options, pathnames) = parser.parse_args() + + if not options.runfile and not options.template: + options.cmd = 'runtests' + + if options.runfile and len(pathnames): + fail('Extraneous arguments.') + + options.pathnames = [os.path.abspath(path) for path in pathnames] + + return options + + +def main(): + options = parse_args() + testrun = TestRun(options) + + if options.cmd is 'runtests': + find_tests(testrun, options) + elif options.cmd is 'rdconfig': + testrun.read(testrun.logger, options) + elif options.cmd is 'wrconfig': + find_tests(testrun, options) + testrun.write(options) + exit(0) + else: + fail('Unknown command specified') + + testrun.complete_outputdirs() + testrun.run(options) + testrun.summary() + exit(0) + + +if __name__ == '__main__': + main() diff --git a/usr/src/test/test-runner/cmd/run.py b/usr/src/test/test-runner/cmd/run.py deleted file mode 100644 index 81e53b210f..0000000000 --- a/usr/src/test/test-runner/cmd/run.py +++ /dev/null @@ -1,871 +0,0 @@ -#!/usr/bin/python2.6 - -# -# This file and its contents are supplied under the terms of the -# Common Development and Distribution License ("CDDL"), version 1.0. -# You may only use this file in accordance with the terms of version -# 1.0 of the CDDL. -# -# A full copy of the text of the CDDL should have accompanied this -# source. A copy of the CDDL is also available via the Internet at -# http://www.illumos.org/license/CDDL. -# - -# -# Copyright (c) 2012, 2015 by Delphix. All rights reserved. -# - -import ConfigParser -import os -import logging -from logging.handlers import WatchedFileHandler -from datetime import datetime -from optparse import OptionParser -from pwd import getpwnam -from pwd import getpwuid -from select import select -from subprocess import PIPE -from subprocess import Popen -from sys import argv -from sys import maxint -from threading import Timer -from time import time - -BASEDIR = '/var/tmp/test_results' -KILL = '/usr/bin/kill' -TRUE = '/usr/bin/true' -SUDO = '/usr/bin/sudo' - -# Custom class to reopen the log file in case it is forcibly closed by a test. -class WatchedFileHandlerClosed(WatchedFileHandler): - """Watch files, including closed files. - Similar to (and inherits from) logging.handler.WatchedFileHandler, - except that IOErrors are handled by reopening the stream and retrying. - This will be retried up to a configurable number of times before - giving up, default 5. - """ - - def __init__(self, filename, mode='a', encoding=None, delay=0, max_tries=5): - self.max_tries = max_tries - self.tries = 0 - WatchedFileHandler.__init__(self, filename, mode, encoding, delay) - - def emit(self, record): - while True: - try: - WatchedFileHandler.emit(self, record) - self.tries = 0 - return - except IOError as err: - if self.tries == self.max_tries: - raise - self.stream.close() - self.stream = self._open() - self.tries += 1 - -class Result(object): - total = 0 - runresults = {'PASS': 0, 'FAIL': 0, 'SKIP': 0, 'KILLED': 0} - - def __init__(self): - self.starttime = None - self.returncode = None - self.runtime = '' - self.stdout = [] - self.stderr = [] - self.result = '' - - def done(self, proc, killed): - """ - Finalize the results of this Cmd. - """ - Result.total += 1 - m, s = divmod(time() - self.starttime, 60) - self.runtime = '%02d:%02d' % (m, s) - self.returncode = proc.returncode - if killed: - self.result = 'KILLED' - Result.runresults['KILLED'] += 1 - elif self.returncode is 0: - self.result = 'PASS' - Result.runresults['PASS'] += 1 - elif self.returncode is not 0: - self.result = 'FAIL' - Result.runresults['FAIL'] += 1 - - -class Output(object): - """ - This class is a slightly modified version of the 'Stream' class found - here: http://goo.gl/aSGfv - """ - def __init__(self, stream): - self.stream = stream - self._buf = '' - self.lines = [] - - def fileno(self): - return self.stream.fileno() - - def read(self, drain=0): - """ - Read from the file descriptor. If 'drain' set, read until EOF. - """ - while self._read() is not None: - if not drain: - break - - def _read(self): - """ - Read up to 4k of data from this output stream. Collect the output - up to the last newline, and append it to any leftover data from a - previous call. The lines are stored as a (timestamp, data) tuple - for easy sorting/merging later. - """ - fd = self.fileno() - buf = os.read(fd, 4096) - if not buf: - return None - if '\n' not in buf: - self._buf += buf - return [] - - buf = self._buf + buf - tmp, rest = buf.rsplit('\n', 1) - self._buf = rest - now = datetime.now() - rows = tmp.split('\n') - self.lines += [(now, r) for r in rows] - - -class Cmd(object): - verified_users = [] - - def __init__(self, pathname, outputdir=None, timeout=None, user=None): - self.pathname = pathname - self.outputdir = outputdir or 'BASEDIR' - self.timeout = timeout - self.user = user or '' - self.killed = False - self.result = Result() - - if self.timeout is None: - self.timeout = 60 - - def __str__(self): - return "Pathname: %s\nOutputdir: %s\nTimeout: %d\nUser: %s\n" % \ - (self.pathname, self.outputdir, self.timeout, self.user) - - def kill_cmd(self, proc): - """ - Kill a running command due to timeout, or ^C from the keyboard. If - sudo is required, this user was verified previously. - """ - self.killed = True - do_sudo = len(self.user) != 0 - signal = '-TERM' - - cmd = [SUDO, KILL, signal, str(proc.pid)] - if not do_sudo: - del cmd[0] - - try: - kp = Popen(cmd) - kp.wait() - except: - pass - - def update_cmd_privs(self, cmd, user): - """ - If a user has been specified to run this Cmd and we're not already - running as that user, prepend the appropriate sudo command to run - as that user. - """ - me = getpwuid(os.getuid()) - - if not user or user is me: - return cmd - - ret = '%s -E -u %s %s' % (SUDO, user, cmd) - return ret.split(' ') - - def collect_output(self, proc): - """ - Read from stdout/stderr as data becomes available, until the - process is no longer running. Return the lines from the stdout and - stderr Output objects. - """ - out = Output(proc.stdout) - err = Output(proc.stderr) - res = [] - while proc.returncode is None: - proc.poll() - res = select([out, err], [], [], .1) - for fd in res[0]: - fd.read() - for fd in res[0]: - fd.read(drain=1) - - return out.lines, err.lines - - def run(self, options): - """ - This is the main function that runs each individual test. - Determine whether or not the command requires sudo, and modify it - if needed. Run the command, and update the result object. - """ - if options.dryrun is True: - print self - return - - privcmd = self.update_cmd_privs(self.pathname, self.user) - try: - old = os.umask(0) - if not os.path.isdir(self.outputdir): - os.makedirs(self.outputdir, mode=0777) - os.umask(old) - except OSError, e: - fail('%s' % e) - - try: - self.result.starttime = time() - proc = Popen(privcmd, stdout=PIPE, stderr=PIPE) - - # Allow a special timeout value of 0 to mean infinity - if int(self.timeout) == 0: - self.timeout = maxint - t = Timer(int(self.timeout), self.kill_cmd, [proc]) - t.start() - self.result.stdout, self.result.stderr = self.collect_output(proc) - except KeyboardInterrupt: - self.kill_cmd(proc) - fail('\nRun terminated at user request.') - finally: - t.cancel() - - self.result.done(proc, self.killed) - - def skip(self): - """ - Initialize enough of the test result that we can log a skipped - command. - """ - Result.total += 1 - Result.runresults['SKIP'] += 1 - self.result.stdout = self.result.stderr = [] - self.result.starttime = time() - m, s = divmod(time() - self.result.starttime, 60) - self.result.runtime = '%02d:%02d' % (m, s) - self.result.result = 'SKIP' - - def log(self, logger, options): - """ - This function is responsible for writing all output. This includes - the console output, the logfile of all results (with timestamped - merged stdout and stderr), and for each test, the unmodified - stdout/stderr/merged in it's own file. - """ - if logger is None: - return - - logname = getpwuid(os.getuid()).pw_name - user = ' (run as %s)' % (self.user if len(self.user) else logname) - msga = 'Test: %s%s ' % (self.pathname, user) - msgb = '[%s] [%s]' % (self.result.runtime, self.result.result) - pad = ' ' * (80 - (len(msga) + len(msgb))) - - # If -q is specified, only print a line for tests that didn't pass. - # This means passing tests need to be logged as DEBUG, or the one - # line summary will only be printed in the logfile for failures. - if not options.quiet: - logger.info('%s%s%s' % (msga, pad, msgb)) - elif self.result.result is not 'PASS': - logger.info('%s%s%s' % (msga, pad, msgb)) - else: - logger.debug('%s%s%s' % (msga, pad, msgb)) - - lines = sorted(self.result.stdout + self.result.stderr, - cmp=lambda x, y: cmp(x[0], y[0])) - - for dt, line in lines: - logger.debug('%s %s' % (dt.strftime("%H:%M:%S.%f ")[:11], line)) - - if len(self.result.stdout): - with open(os.path.join(self.outputdir, 'stdout'), 'w') as out: - for _, line in self.result.stdout: - os.write(out.fileno(), '%s\n' % line) - if len(self.result.stderr): - with open(os.path.join(self.outputdir, 'stderr'), 'w') as err: - for _, line in self.result.stderr: - os.write(err.fileno(), '%s\n' % line) - if len(self.result.stdout) and len(self.result.stderr): - with open(os.path.join(self.outputdir, 'merged'), 'w') as merged: - for _, line in lines: - os.write(merged.fileno(), '%s\n' % line) - - -class Test(Cmd): - props = ['outputdir', 'timeout', 'user', 'pre', 'pre_user', 'post', - 'post_user'] - - def __init__(self, pathname, outputdir=None, timeout=None, user=None, - pre=None, pre_user=None, post=None, post_user=None): - super(Test, self).__init__(pathname, outputdir, timeout, user) - self.pre = pre or '' - self.pre_user = pre_user or '' - self.post = post or '' - self.post_user = post_user or '' - - def __str__(self): - post_user = pre_user = '' - if len(self.pre_user): - pre_user = ' (as %s)' % (self.pre_user) - if len(self.post_user): - post_user = ' (as %s)' % (self.post_user) - return "Pathname: %s\nOutputdir: %s\nTimeout: %d\nPre: %s%s\nPost: " \ - "%s%s\nUser: %s\n" % \ - (self.pathname, self.outputdir, self.timeout, self.pre, - pre_user, self.post, post_user, self.user) - - def verify(self, logger): - """ - Check the pre/post scripts, user and Test. Omit the Test from this - run if there are any problems. - """ - files = [self.pre, self.pathname, self.post] - users = [self.pre_user, self.user, self.post_user] - - for f in [f for f in files if len(f)]: - if not verify_file(f): - logger.info("Warning: Test '%s' not added to this run because" - " it failed verification." % f) - return False - - for user in [user for user in users if len(user)]: - if not verify_user(user, logger): - logger.info("Not adding Test '%s' to this run." % - self.pathname) - return False - - return True - - def run(self, logger, options): - """ - Create Cmd instances for the pre/post scripts. If the pre script - doesn't pass, skip this Test. Run the post script regardless. - """ - odir = os.path.join(self.outputdir, os.path.basename(self.pre)) - pretest = Cmd(self.pre, outputdir=odir, timeout=self.timeout, - user=self.pre_user) - test = Cmd(self.pathname, outputdir=self.outputdir, - timeout=self.timeout, user=self.user) - odir = os.path.join(self.outputdir, os.path.basename(self.post)) - posttest = Cmd(self.post, outputdir=odir, timeout=self.timeout, - user=self.post_user) - - cont = True - if len(pretest.pathname): - pretest.run(options) - cont = pretest.result.result is 'PASS' - pretest.log(logger, options) - - if cont: - test.run(options) - else: - test.skip() - - test.log(logger, options) - - if len(posttest.pathname): - posttest.run(options) - posttest.log(logger, options) - - -class TestGroup(Test): - props = Test.props + ['tests'] - - def __init__(self, pathname, outputdir=None, timeout=None, user=None, - pre=None, pre_user=None, post=None, post_user=None, - tests=None): - super(TestGroup, self).__init__(pathname, outputdir, timeout, user, - pre, pre_user, post, post_user) - self.tests = tests or [] - - def __str__(self): - post_user = pre_user = '' - if len(self.pre_user): - pre_user = ' (as %s)' % (self.pre_user) - if len(self.post_user): - post_user = ' (as %s)' % (self.post_user) - return "Pathname: %s\nOutputdir: %s\nTests: %s\nTimeout: %d\n" \ - "Pre: %s%s\nPost: %s%s\nUser: %s\n" % \ - (self.pathname, self.outputdir, self.tests, self.timeout, - self.pre, pre_user, self.post, post_user, self.user) - - def verify(self, logger): - """ - Check the pre/post scripts, user and tests in this TestGroup. Omit - the TestGroup entirely, or simply delete the relevant tests in the - group, if that's all that's required. - """ - # If the pre or post scripts are relative pathnames, convert to - # absolute, so they stand a chance of passing verification. - if len(self.pre) and not os.path.isabs(self.pre): - self.pre = os.path.join(self.pathname, self.pre) - if len(self.post) and not os.path.isabs(self.post): - self.post = os.path.join(self.pathname, self.post) - - auxfiles = [self.pre, self.post] - users = [self.pre_user, self.user, self.post_user] - - for f in [f for f in auxfiles if len(f)]: - if self.pathname != os.path.dirname(f): - logger.info("Warning: TestGroup '%s' not added to this run. " - "Auxiliary script '%s' exists in a different " - "directory." % (self.pathname, f)) - return False - - if not verify_file(f): - logger.info("Warning: TestGroup '%s' not added to this run. " - "Auxiliary script '%s' failed verification." % - (self.pathname, f)) - return False - - for user in [user for user in users if len(user)]: - if not verify_user(user, logger): - logger.info("Not adding TestGroup '%s' to this run." % - self.pathname) - return False - - # If one of the tests is invalid, delete it, log it, and drive on. - for test in self.tests: - if not verify_file(os.path.join(self.pathname, test)): - del self.tests[self.tests.index(test)] - logger.info("Warning: Test '%s' removed from TestGroup '%s' " - "because it failed verification." % - (test, self.pathname)) - - return len(self.tests) is not 0 - - def run(self, logger, options): - """ - Create Cmd instances for the pre/post scripts. If the pre script - doesn't pass, skip all the tests in this TestGroup. Run the post - script regardless. - """ - odir = os.path.join(self.outputdir, os.path.basename(self.pre)) - pretest = Cmd(self.pre, outputdir=odir, timeout=self.timeout, - user=self.pre_user) - odir = os.path.join(self.outputdir, os.path.basename(self.post)) - posttest = Cmd(self.post, outputdir=odir, timeout=self.timeout, - user=self.post_user) - - cont = True - if len(pretest.pathname): - pretest.run(options) - cont = pretest.result.result is 'PASS' - pretest.log(logger, options) - - for fname in self.tests: - test = Cmd(os.path.join(self.pathname, fname), - outputdir=os.path.join(self.outputdir, fname), - timeout=self.timeout, user=self.user) - if cont: - test.run(options) - else: - test.skip() - - test.log(logger, options) - - if len(posttest.pathname): - posttest.run(options) - posttest.log(logger, options) - - -class TestRun(object): - props = ['quiet', 'outputdir'] - - def __init__(self, options): - self.tests = {} - self.testgroups = {} - self.starttime = time() - self.timestamp = datetime.now().strftime('%Y%m%dT%H%M%S') - self.outputdir = os.path.join(options.outputdir, self.timestamp) - self.logger = self.setup_logging(options) - self.defaults = [ - ('outputdir', BASEDIR), - ('quiet', False), - ('timeout', 60), - ('user', ''), - ('pre', ''), - ('pre_user', ''), - ('post', ''), - ('post_user', '') - ] - - def __str__(self): - s = 'TestRun:\n outputdir: %s\n' % self.outputdir - s += 'TESTS:\n' - for key in sorted(self.tests.keys()): - s += '%s%s' % (self.tests[key].__str__(), '\n') - s += 'TESTGROUPS:\n' - for key in sorted(self.testgroups.keys()): - s += '%s%s' % (self.testgroups[key].__str__(), '\n') - return s - - def addtest(self, pathname, options): - """ - Create a new Test, and apply any properties that were passed in - from the command line. If it passes verification, add it to the - TestRun. - """ - test = Test(pathname) - for prop in Test.props: - setattr(test, prop, getattr(options, prop)) - - if test.verify(self.logger): - self.tests[pathname] = test - - def addtestgroup(self, dirname, filenames, options): - """ - Create a new TestGroup, and apply any properties that were passed - in from the command line. If it passes verification, add it to the - TestRun. - """ - if dirname not in self.testgroups: - testgroup = TestGroup(dirname) - for prop in Test.props: - setattr(testgroup, prop, getattr(options, prop)) - - # Prevent pre/post scripts from running as regular tests - for f in [testgroup.pre, testgroup.post]: - if f in filenames: - del filenames[filenames.index(f)] - - self.testgroups[dirname] = testgroup - self.testgroups[dirname].tests = sorted(filenames) - - testgroup.verify(self.logger) - - def read(self, logger, options): - """ - Read in the specified runfile, and apply the TestRun properties - listed in the 'DEFAULT' section to our TestRun. Then read each - section, and apply the appropriate properties to the Test or - TestGroup. Properties from individual sections override those set - in the 'DEFAULT' section. If the Test or TestGroup passes - verification, add it to the TestRun. - """ - config = ConfigParser.RawConfigParser() - if not len(config.read(options.runfile)): - fail("Coulnd't read config file %s" % options.runfile) - - for opt in TestRun.props: - if config.has_option('DEFAULT', opt): - setattr(self, opt, config.get('DEFAULT', opt)) - self.outputdir = os.path.join(self.outputdir, self.timestamp) - - for section in config.sections(): - if 'tests' in config.options(section): - testgroup = TestGroup(section) - for prop in TestGroup.props: - for sect in ['DEFAULT', section]: - if config.has_option(sect, prop): - setattr(testgroup, prop, config.get(sect, prop)) - - # Repopulate tests using eval to convert the string to a list - testgroup.tests = eval(config.get(section, 'tests')) - - if testgroup.verify(logger): - self.testgroups[section] = testgroup - else: - test = Test(section) - for prop in Test.props: - for sect in ['DEFAULT', section]: - if config.has_option(sect, prop): - setattr(test, prop, config.get(sect, prop)) - - if test.verify(logger): - self.tests[section] = test - - def write(self, options): - """ - Create a configuration file for editing and later use. The - 'DEFAULT' section of the config file is created from the - properties that were specified on the command line. Tests are - simply added as sections that inherit everything from the - 'DEFAULT' section. TestGroups are the same, except they get an - option including all the tests to run in that directory. - """ - - defaults = dict([(prop, getattr(options, prop)) for prop, _ in - self.defaults]) - config = ConfigParser.RawConfigParser(defaults) - - for test in sorted(self.tests.keys()): - config.add_section(test) - - for testgroup in sorted(self.testgroups.keys()): - config.add_section(testgroup) - config.set(testgroup, 'tests', self.testgroups[testgroup].tests) - - try: - with open(options.template, 'w') as f: - return config.write(f) - except IOError: - fail('Could not open \'%s\' for writing.' % options.template) - - def complete_outputdirs(self): - """ - Collect all the pathnames for Tests, and TestGroups. Work - backwards one pathname component at a time, to create a unique - directory name in which to deposit test output. Tests will be able - to write output files directly in the newly modified outputdir. - TestGroups will be able to create one subdirectory per test in the - outputdir, and are guaranteed uniqueness because a group can only - contain files in one directory. Pre and post tests will create a - directory rooted at the outputdir of the Test or TestGroup in - question for their output. - """ - done = False - components = 0 - tmp_dict = dict(self.tests.items() + self.testgroups.items()) - total = len(tmp_dict) - base = self.outputdir - - while not done: - l = [] - components -= 1 - for testfile in tmp_dict.keys(): - uniq = '/'.join(testfile.split('/')[components:]).lstrip('/') - if uniq not in l: - l.append(uniq) - tmp_dict[testfile].outputdir = os.path.join(base, uniq) - else: - break - done = total == len(l) - - def setup_logging(self, options): - """ - Two loggers are set up here. The first is for the logfile which - will contain one line summarizing the test, including the test - name, result, and running time. This logger will also capture the - timestamped combined stdout and stderr of each run. The second - logger is optional console output, which will contain only the one - line summary. The loggers are initialized at two different levels - to facilitate segregating the output. - """ - if options.dryrun is True: - return - - testlogger = logging.getLogger(__name__) - testlogger.setLevel(logging.DEBUG) - - if options.cmd is not 'wrconfig': - try: - old = os.umask(0) - os.makedirs(self.outputdir, mode=0777) - os.umask(old) - except OSError, e: - fail('%s' % e) - filename = os.path.join(self.outputdir, 'log') - - logfile = WatchedFileHandlerClosed(filename) - logfile.setLevel(logging.DEBUG) - logfilefmt = logging.Formatter('%(message)s') - logfile.setFormatter(logfilefmt) - testlogger.addHandler(logfile) - - cons = logging.StreamHandler() - cons.setLevel(logging.INFO) - consfmt = logging.Formatter('%(message)s') - cons.setFormatter(consfmt) - testlogger.addHandler(cons) - - return testlogger - - def run(self, options): - """ - Walk through all the Tests and TestGroups, calling run(). - """ - try: - os.chdir(self.outputdir) - except OSError: - fail('Could not change to directory %s' % self.outputdir) - for test in sorted(self.tests.keys()): - self.tests[test].run(self.logger, options) - for testgroup in sorted(self.testgroups.keys()): - self.testgroups[testgroup].run(self.logger, options) - - def summary(self): - if Result.total is 0: - return - - print '\nResults Summary' - for key in Result.runresults.keys(): - if Result.runresults[key] is not 0: - print '%s\t% 4d' % (key, Result.runresults[key]) - - m, s = divmod(time() - self.starttime, 60) - h, m = divmod(m, 60) - print '\nRunning Time:\t%02d:%02d:%02d' % (h, m, s) - print 'Percent passed:\t%.1f%%' % ((float(Result.runresults['PASS']) / - float(Result.total)) * 100) - print 'Log directory:\t%s' % self.outputdir - - -def verify_file(pathname): - """ - Verify that the supplied pathname is an executable regular file. - """ - if os.path.isdir(pathname) or os.path.islink(pathname): - return False - - if os.path.isfile(pathname) and os.access(pathname, os.X_OK): - return True - - return False - - -def verify_user(user, logger): - """ - Verify that the specified user exists on this system, and can execute - sudo without being prompted for a password. - """ - testcmd = [SUDO, '-n', '-u', user, TRUE] - - if user in Cmd.verified_users: - return True - - try: - _ = getpwnam(user) - except KeyError: - logger.info("Warning: user '%s' does not exist.", user) - return False - - p = Popen(testcmd) - p.wait() - if p.returncode is not 0: - logger.info("Warning: user '%s' cannot use passwordless sudo.", user) - return False - else: - Cmd.verified_users.append(user) - - return True - - -def find_tests(testrun, options): - """ - For the given list of pathnames, add files as Tests. For directories, - if do_groups is True, add the directory as a TestGroup. If False, - recursively search for executable files. - """ - - for p in sorted(options.pathnames): - if os.path.isdir(p): - for dirname, _, filenames in os.walk(p): - if options.do_groups: - testrun.addtestgroup(dirname, filenames, options) - else: - for f in sorted(filenames): - testrun.addtest(os.path.join(dirname, f), options) - else: - testrun.addtest(p, options) - - -def fail(retstr, ret=1): - print '%s: %s' % (argv[0], retstr) - exit(ret) - - -def options_cb(option, opt_str, value, parser): - path_options = ['runfile', 'outputdir', 'template'] - - if option.dest is 'runfile' and '-w' in parser.rargs or \ - option.dest is 'template' and '-c' in parser.rargs: - fail('-c and -w are mutually exclusive.') - - if opt_str in parser.rargs: - fail('%s may only be specified once.' % opt_str) - - if option.dest is 'runfile': - parser.values.cmd = 'rdconfig' - if option.dest is 'template': - parser.values.cmd = 'wrconfig' - - setattr(parser.values, option.dest, value) - if option.dest in path_options: - setattr(parser.values, option.dest, os.path.abspath(value)) - - -def parse_args(): - parser = OptionParser() - parser.add_option('-c', action='callback', callback=options_cb, - type='string', dest='runfile', metavar='runfile', - help='Specify tests to run via config file.') - parser.add_option('-d', action='store_true', default=False, dest='dryrun', - help='Dry run. Print tests, but take no other action.') - parser.add_option('-g', action='store_true', default=False, - dest='do_groups', help='Make directories TestGroups.') - parser.add_option('-o', action='callback', callback=options_cb, - default=BASEDIR, dest='outputdir', type='string', - metavar='outputdir', help='Specify an output directory.') - parser.add_option('-p', action='callback', callback=options_cb, - default='', dest='pre', metavar='script', - type='string', help='Specify a pre script.') - parser.add_option('-P', action='callback', callback=options_cb, - default='', dest='post', metavar='script', - type='string', help='Specify a post script.') - parser.add_option('-q', action='store_true', default=False, dest='quiet', - help='Silence on the console during a test run.') - parser.add_option('-t', action='callback', callback=options_cb, default=60, - dest='timeout', metavar='seconds', type='int', - help='Timeout (in seconds) for an individual test.') - parser.add_option('-u', action='callback', callback=options_cb, - default='', dest='user', metavar='user', type='string', - help='Specify a different user name to run as.') - parser.add_option('-w', action='callback', callback=options_cb, - default=None, dest='template', metavar='template', - type='string', help='Create a new config file.') - parser.add_option('-x', action='callback', callback=options_cb, default='', - dest='pre_user', metavar='pre_user', type='string', - help='Specify a user to execute the pre script.') - parser.add_option('-X', action='callback', callback=options_cb, default='', - dest='post_user', metavar='post_user', type='string', - help='Specify a user to execute the post script.') - (options, pathnames) = parser.parse_args() - - if not options.runfile and not options.template: - options.cmd = 'runtests' - - if options.runfile and len(pathnames): - fail('Extraneous arguments.') - - options.pathnames = [os.path.abspath(path) for path in pathnames] - - return options - - -def main(): - options = parse_args() - testrun = TestRun(options) - - if options.cmd is 'runtests': - find_tests(testrun, options) - elif options.cmd is 'rdconfig': - testrun.read(testrun.logger, options) - elif options.cmd is 'wrconfig': - find_tests(testrun, options) - testrun.write(options) - exit(0) - else: - fail('Unknown command specified') - - testrun.complete_outputdirs() - testrun.run(options) - testrun.summary() - exit(0) - - -if __name__ == '__main__': - main() diff --git a/usr/src/tools/Makefile b/usr/src/tools/Makefile index 22ff67aa3b..c8485b216a 100644 --- a/usr/src/tools/Makefile +++ b/usr/src/tools/Makefile @@ -96,11 +96,11 @@ ROOTDIRS= \ $(ROOTONBLD)/lib/$(MACH) \ $(ROOTONBLD)/lib/$(MACH)/64 \ $(ROOTONBLD)/lib/perl \ - $(ROOTONBLD)/lib/python2.6 \ - $(ROOTONBLD)/lib/python2.6/onbld \ - $(ROOTONBLD)/lib/python2.6/onbld/Checks \ - $(ROOTONBLD)/lib/python2.6/onbld/hgext \ - $(ROOTONBLD)/lib/python2.6/onbld/Scm \ + $(ROOTONBLD)/lib/python$(PYTHON_VERSION) \ + $(ROOTONBLD)/lib/python$(PYTHON_VERSION)/onbld \ + $(ROOTONBLD)/lib/python$(PYTHON_VERSION)/onbld/Checks \ + $(ROOTONBLD)/lib/python$(PYTHON_VERSION)/onbld/hgext \ + $(ROOTONBLD)/lib/python$(PYTHON_VERSION)/onbld/Scm \ $(ROOTONBLD)/env \ $(ROOTONBLD)/etc \ $(ROOTONBLD)/etc/exception_lists \ @@ -148,7 +148,7 @@ $(ROOTDIRS): $(INS.dir) $(ROOTONBLDLIBPY): $(ROOTDIRS) - $(RM) -r $@; $(SYMLINK) python2.6 $@ + $(RM) -r $@; $(SYMLINK) python$(PYTHON_VERSION) $@ make: ctf diff --git a/usr/src/tools/Makefile.python b/usr/src/tools/Makefile.python index dcf7c06360..147ef2b05c 100644 --- a/usr/src/tools/Makefile.python +++ b/usr/src/tools/Makefile.python @@ -63,34 +63,30 @@ PYFILES = $(PYSRCS) $(PYOBJS) -ROOTPYDIR_26 = $(PYTOPDIR)/python2.6/$(PYMODDIR) -ROOTPYFILES_26 = $(PYFILES:%=$(ROOTPYDIR_26)/%) +ROOTPYDIR = $(PYTOPDIR)/python$(PYTHON_VERSION)/$(PYMODDIR) +ROOTPYFILES = $(PYFILES:%=$(ROOTPYDIR)/%) -ROOTPYFILES = $(ROOTPYFILES_26) $(ROOTPYFILES) := FILEMODE = 0444 -PYVERSDIRS = python2.6 +PYVERSDIRS = python$(PYTHON_VERSION) -PY26OBJS = $(PYOBJS:%=python2.6/%) -$(PY26OBJS) := PYTHON = $(PYTHON_26) - -PYVERSOBJS = $(PY26OBJS) +PYVERSOBJS = $(PYOBJS:%=python$(PYTHON_VERSION)/%) CLOBBERFILES += $(PYVERSOBJS) CLOBBERDIRS += $(PYVERSDIRS) .KEEP_STATE: -python2.6/%.pyc: %.py +python$(PYTHON_VERSION)/%.pyc: %.py @[ -d $(@D) ] || mkdir $(@D) $(RM) $@ $(PYTHON) -mpy_compile $< $(MV) $(*).pyc $@ -$(ROOTPYDIR_26)/%.pyc: python2.6/%.pyc +$(ROOTPYDIR)/%.pyc: python$(PYTHON_VERSION)/%.pyc $(INS.pyfile) -$(ROOTPYDIR_26)/%.py: %.py +$(ROOTPYDIR)/%.py: %.py $(INS.pyfile) pyclobber: diff --git a/usr/src/tools/Makefile.tools b/usr/src/tools/Makefile.tools index 78fe5f1d14..2b6cf225ca 100644 --- a/usr/src/tools/Makefile.tools +++ b/usr/src/tools/Makefile.tools @@ -39,7 +39,6 @@ ROOTONBLDLIBMACH= $(ROOTONBLD)/lib/$(MACH) ROOTONBLDLIBMACH64= $(ROOTONBLD)/lib/$(MACH)/64 ROOTONBLDLIBPERL= $(ROOTONBLD)/lib/perl ROOTONBLDLIBPY= $(ROOTONBLD)/lib/python -ROOTONBLDLIBPY_26= $(ROOTONBLD)/lib/python2.6 ROOTONBLDENV= $(ROOTONBLD)/env ROOTONBLDGK= $(ROOTONBLD)/gk ROOTONBLDMAN= $(ROOTONBLD)/man diff --git a/usr/src/tools/scripts/Makefile b/usr/src/tools/scripts/Makefile index 717147d8ad..935ebcf80c 100644 --- a/usr/src/tools/scripts/Makefile +++ b/usr/src/tools/scripts/Makefile @@ -116,7 +116,10 @@ EXCEPTFILES= \ interface_check \ interface_cmp -CLEANFILES = $(SHFILES) $(PERLFILES) $(PYFILES) bldenv.1onbld +CLEANFILES = $(SHFILES) $(PERLFILES) $(PYFILES) bldenv.1onbld onu.sh + +onu.sh: onu.sh.in + $(SED) -e "s:@PYTHON_VERSION@:$(PYTHON_VERSION):g" < onu.sh.in > $@ include ../Makefile.tools diff --git a/usr/src/tools/scripts/cddlchk.py b/usr/src/tools/scripts/cddlchk.py index 816d2f33a7..d0b3a70474 100644 --- a/usr/src/tools/scripts/cddlchk.py +++ b/usr/src/tools/scripts/cddlchk.py @@ -1,4 +1,4 @@ -#!/usr/bin/python2.6 +#!@PYTHON@ # # CDDL HEADER START # diff --git a/usr/src/tools/scripts/copyrightchk.py b/usr/src/tools/scripts/copyrightchk.py index 210ef1b46c..8e6228f5f7 100644 --- a/usr/src/tools/scripts/copyrightchk.py +++ b/usr/src/tools/scripts/copyrightchk.py @@ -1,4 +1,4 @@ -#!/usr/bin/python2.6 +#!@PYTHON@ # # CDDL HEADER START # diff --git a/usr/src/tools/scripts/git-pbchk.py b/usr/src/tools/scripts/git-pbchk.py index 8dea5a8785..9420812a21 100644 --- a/usr/src/tools/scripts/git-pbchk.py +++ b/usr/src/tools/scripts/git-pbchk.py @@ -1,4 +1,4 @@ -#!/usr/bin/python2.6 +#!@PYTHON@ # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 diff --git a/usr/src/tools/scripts/hdrchk.py b/usr/src/tools/scripts/hdrchk.py index 84acbc8616..8c529b8c3d 100644 --- a/usr/src/tools/scripts/hdrchk.py +++ b/usr/src/tools/scripts/hdrchk.py @@ -1,4 +1,4 @@ -#!/usr/bin/python2.6 +#!@PYTHON@ # # CDDL HEADER START # diff --git a/usr/src/tools/scripts/hg-active.py b/usr/src/tools/scripts/hg-active.py index 88394e98e5..495cdfc0db 100644 --- a/usr/src/tools/scripts/hg-active.py +++ b/usr/src/tools/scripts/hg-active.py @@ -1,4 +1,4 @@ -#!/usr/bin/python2.6 +#!@PYTHON@ # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 diff --git a/usr/src/tools/scripts/mapfilechk.py b/usr/src/tools/scripts/mapfilechk.py index 9cf2000c7a..0e3ba283a5 100644 --- a/usr/src/tools/scripts/mapfilechk.py +++ b/usr/src/tools/scripts/mapfilechk.py @@ -1,4 +1,4 @@ -#!/usr/bin/python2.6 +#!@PYTHON@ # # CDDL HEADER START # diff --git a/usr/src/tools/scripts/onu.sh b/usr/src/tools/scripts/onu.sh deleted file mode 100644 index 53b0d85ce9..0000000000 --- a/usr/src/tools/scripts/onu.sh +++ /dev/null @@ -1,245 +0,0 @@ -#!/bin/ksh93 -p -# -# CDDL HEADER START -# -# The contents of this file are subject to the terms of the -# Common Development and Distribution License (the "License"). -# You may not use this file except in compliance with the License. -# -# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE -# or http://www.opensolaris.org/os/licensing. -# See the License for the specific language governing permissions -# and limitations under the License. -# -# When distributing Covered Code, include this CDDL HEADER in each -# file and include the License file at usr/src/OPENSOLARIS.LICENSE. -# If applicable, add the following below this CDDL HEADER, with the -# fields enclosed by brackets "[]" replaced with your own identifying -# information: Portions Copyright [yyyy] [name of copyright owner] -# -# CDDL HEADER END -# - -# -# Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved. -# Copyright 2010, Richard Lowe -# - -PATH=/usr/bin:/usr/sbin -export PATH - -DEFAULTONURI="http://ipkg.sfbay/on-nightly" -DEFAULTONPUB="on-nightly" - -usage() -{ - echo "usage: $0 [opts] [-s beName] -t beName" - echo "usage: $0 [opts] -r" - echo - echo "\t-c consolidation : consolidation being upgraded" - echo "\t-d repodir : directory for repositories" - echo "\t-r : configure publisher only" - echo "\t-s : source BE to clone" - echo "\t-t : new BE name" - echo "\t-u uri : origin URI for redist repository" - echo "\t-U prefix: prefix for redist repository" - echo "\t-v : verbose" - echo "\t-Z : skip updating zones" - echo - echo "Update to an ON build:" - echo "\tonu -t newbe -d /path/to/my/ws/packages/\`uname -p\`/nightly" - echo - echo "Update to the nightly build:" - echo "\tonu -t newbe" - echo - echo "Re-enable the publishers in the current BE:" - echo "\tonu -r -d /path/to/my/ws/packages/\`uname -p\`/nightly" - exit 1 -} - -exit_error() -{ - echo $* - exit 2 -} - -do_cmd() -{ - [ $verbose -gt 0 ] && echo $* - $* - exit_code=$? - [ $exit_code -eq 0 ] && return - # pkg(1) returns 4 if "nothing to do", which is safe to ignore - [ $1 = "pkg" -a $exit_code -eq 4 ] && return - exit_error "$*" failed: exit code $exit_code -} - -configure_publishers() -{ - root=$1 - - # - # Get the publisher name from the 'list -v' output. It may seem we - # could do this more tidily using 'info', but that is - # internationalized. - # - typeset on_publisher=$(pkg -R $root list -Hv \ - "${consolidation}-incorporation" | cut -d/ -f3) - - if [[ "$on_publisher" != "$redistpub" ]]; then - do_cmd pkg -R $root set-publisher --no-refresh \ - --non-sticky $on_publisher - fi - do_cmd pkg -R $root set-publisher -e --no-refresh -P -O $uri $redistpub - do_cmd pkg -R $root refresh --full -} - -update() -{ - root=$1 - - pkg -R $root list entire > /dev/null 2>&1 - [ $? -eq 0 ] && do_cmd pkg -R $root uninstall entire - - configure_publishers $root - - do_cmd pkg -R $root image-update -} - -update_zone() -{ - zone=$1 - - name=`echo $zone | cut -d: -f 2` - if [ $name = "global" ]; then - return - fi - - brand=`echo $zone | cut -d: -f 6` - if [ $brand != "ipkg" ]; then - return - fi - - if [ "$zone_warned" = 0 ]; then - echo "WARNING: Use of onu(1) will prevent use of zone attach in the new BE" >&2 - echo "See onu(1)" >&2 - zone_warned=1 - fi - - state=`echo $zone | cut -d: -f 3` - - case "$state" in - configured|incomplete) - return - ;; - esac - - zoneroot=`echo $zone | cut -d: -f 4` - - echo "Updating zone $name" - update $zoneroot/root -} - -sourcebe="" -targetbe="" -uri="" -repodir="" -consolidation="osnet" -verbose=0 -no_zones=0 -zone_warned=0 -reposonly=0 - -while getopts :c:d:Ors:t:U:u:vZ i ; do - case $i in - c) - consolidation=$OPTARG - ;; - d) - repodir=$OPTARG - ;; - O) # no-op, compatibility with recommended use - ;; - r) - reposonly=1 - ;; - s) - sourcebe=$OPTARG - ;; - t) - targetbe=$OPTARG - ;; - U) - redistpub=$OPTARG - ;; - u) - uri=$OPTARG - ;; - v) - verbose=1 - ;; - Z) - no_zones=1 - ;; - *) - usage - esac -done -shift `expr $OPTIND - 1` - -[ -n "$1" ] && usage - -if [ "$reposonly" -eq 1 ]; then - [ -n "$sourcebe" ] && usage - [ -n "$targetbe" ] && usage - [ "$no_zones" -eq 1 ] && usage -else - [ -z "$targetbe" ] && usage -fi -[ -z "$uri" ] && uri=$ONURI -[ -z "$uri" ] && uri=$DEFAULTONURI -[ -z "$redistpub" ] && redistpub=$ONPUB -[ -z "$redistpub" ] && redistpub=$DEFAULTONPUB - -if [ -n "$repodir" ]; then - redistdir=$repodir/repo.redist - [ -d $redistdir ] || exit_error "$redistdir not found" - typeset cfgfile=$redistdir/cfg_cache - [[ ! -e $cfgfile ]] && cfgfile=$redistdir/pkg5.repository - # need an absolute path - [[ $redistdir == /* ]] || redistdir=$PWD/$redistdir - redistpub=$(python2.6 <<# EOF - import ConfigParser - p = ConfigParser.SafeConfigParser() - p.read("$cfgfile") - pp = p.get("publisher", "prefix") - print "%s" % pp - EOF) || exit_error "Cannot determine publisher prefix" - [[ -n "$redistpub" ]] || exit_error "Repository has no publisher prefix" - uri="file://$redistdir" -fi - -if [ "$reposonly" -eq 1 ]; then - configure_publishers / - exit 0 -fi - -createargs="" -[ -n "$sourcebe" ] && createargs="-e $sourcebe" - -# ksh seems to have its own mktemp with slightly different semantics -tmpdir=`/usr/bin/mktemp -d /tmp/onu.XXXXXX` -[ -z "$tmpdir" ] && exit_error "mktemp failed" - -do_cmd beadm create $createargs $targetbe -do_cmd beadm mount $targetbe $tmpdir -update $tmpdir -do_cmd beadm activate $targetbe - -if [ "$no_zones" != 1 ]; then - for zone in `do_cmd zoneadm -R $tmpdir list -cip`; do - update_zone $zone - done -fi - -exit 0 diff --git a/usr/src/tools/scripts/onu.sh.in b/usr/src/tools/scripts/onu.sh.in new file mode 100644 index 0000000000..9a4e81e7a2 --- /dev/null +++ b/usr/src/tools/scripts/onu.sh.in @@ -0,0 +1,245 @@ +#!/bin/ksh93 -p +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# + +# +# Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved. +# Copyright 2010, Richard Lowe +# + +PATH=/usr/bin:/usr/sbin +export PATH + +DEFAULTONURI="http://ipkg.sfbay/on-nightly" +DEFAULTONPUB="on-nightly" + +usage() +{ + echo "usage: $0 [opts] [-s beName] -t beName" + echo "usage: $0 [opts] -r" + echo + echo "\t-c consolidation : consolidation being upgraded" + echo "\t-d repodir : directory for repositories" + echo "\t-r : configure publisher only" + echo "\t-s : source BE to clone" + echo "\t-t : new BE name" + echo "\t-u uri : origin URI for redist repository" + echo "\t-U prefix: prefix for redist repository" + echo "\t-v : verbose" + echo "\t-Z : skip updating zones" + echo + echo "Update to an ON build:" + echo "\tonu -t newbe -d /path/to/my/ws/packages/\`uname -p\`/nightly" + echo + echo "Update to the nightly build:" + echo "\tonu -t newbe" + echo + echo "Re-enable the publishers in the current BE:" + echo "\tonu -r -d /path/to/my/ws/packages/\`uname -p\`/nightly" + exit 1 +} + +exit_error() +{ + echo $* + exit 2 +} + +do_cmd() +{ + [ $verbose -gt 0 ] && echo $* + $* + exit_code=$? + [ $exit_code -eq 0 ] && return + # pkg(1) returns 4 if "nothing to do", which is safe to ignore + [ $1 = "pkg" -a $exit_code -eq 4 ] && return + exit_error "$*" failed: exit code $exit_code +} + +configure_publishers() +{ + root=$1 + + # + # Get the publisher name from the 'list -v' output. It may seem we + # could do this more tidily using 'info', but that is + # internationalized. + # + typeset on_publisher=$(pkg -R $root list -Hv \ + "${consolidation}-incorporation" | cut -d/ -f3) + + if [[ "$on_publisher" != "$redistpub" ]]; then + do_cmd pkg -R $root set-publisher --no-refresh \ + --non-sticky $on_publisher + fi + do_cmd pkg -R $root set-publisher -e --no-refresh -P -O $uri $redistpub + do_cmd pkg -R $root refresh --full +} + +update() +{ + root=$1 + + pkg -R $root list entire > /dev/null 2>&1 + [ $? -eq 0 ] && do_cmd pkg -R $root uninstall entire + + configure_publishers $root + + do_cmd pkg -R $root image-update +} + +update_zone() +{ + zone=$1 + + name=`echo $zone | cut -d: -f 2` + if [ $name = "global" ]; then + return + fi + + brand=`echo $zone | cut -d: -f 6` + if [ $brand != "ipkg" ]; then + return + fi + + if [ "$zone_warned" = 0 ]; then + echo "WARNING: Use of onu(1) will prevent use of zone attach in the new BE" >&2 + echo "See onu(1)" >&2 + zone_warned=1 + fi + + state=`echo $zone | cut -d: -f 3` + + case "$state" in + configured|incomplete) + return + ;; + esac + + zoneroot=`echo $zone | cut -d: -f 4` + + echo "Updating zone $name" + update $zoneroot/root +} + +sourcebe="" +targetbe="" +uri="" +repodir="" +consolidation="osnet" +verbose=0 +no_zones=0 +zone_warned=0 +reposonly=0 + +while getopts :c:d:Ors:t:U:u:vZ i ; do + case $i in + c) + consolidation=$OPTARG + ;; + d) + repodir=$OPTARG + ;; + O) # no-op, compatibility with recommended use + ;; + r) + reposonly=1 + ;; + s) + sourcebe=$OPTARG + ;; + t) + targetbe=$OPTARG + ;; + U) + redistpub=$OPTARG + ;; + u) + uri=$OPTARG + ;; + v) + verbose=1 + ;; + Z) + no_zones=1 + ;; + *) + usage + esac +done +shift `expr $OPTIND - 1` + +[ -n "$1" ] && usage + +if [ "$reposonly" -eq 1 ]; then + [ -n "$sourcebe" ] && usage + [ -n "$targetbe" ] && usage + [ "$no_zones" -eq 1 ] && usage +else + [ -z "$targetbe" ] && usage +fi +[ -z "$uri" ] && uri=$ONURI +[ -z "$uri" ] && uri=$DEFAULTONURI +[ -z "$redistpub" ] && redistpub=$ONPUB +[ -z "$redistpub" ] && redistpub=$DEFAULTONPUB + +if [ -n "$repodir" ]; then + redistdir=$repodir/repo.redist + [ -d $redistdir ] || exit_error "$redistdir not found" + typeset cfgfile=$redistdir/cfg_cache + [[ ! -e $cfgfile ]] && cfgfile=$redistdir/pkg5.repository + # need an absolute path + [[ $redistdir == /* ]] || redistdir=$PWD/$redistdir + redistpub=$(python@PYTHON_VERSION@ <<# EOF + import ConfigParser + p = ConfigParser.SafeConfigParser() + p.read("$cfgfile") + pp = p.get("publisher", "prefix") + print "%s" % pp + EOF) || exit_error "Cannot determine publisher prefix" + [[ -n "$redistpub" ]] || exit_error "Repository has no publisher prefix" + uri="file://$redistdir" +fi + +if [ "$reposonly" -eq 1 ]; then + configure_publishers / + exit 0 +fi + +createargs="" +[ -n "$sourcebe" ] && createargs="-e $sourcebe" + +# ksh seems to have its own mktemp with slightly different semantics +tmpdir=`/usr/bin/mktemp -d /tmp/onu.XXXXXX` +[ -z "$tmpdir" ] && exit_error "mktemp failed" + +do_cmd beadm create $createargs $targetbe +do_cmd beadm mount $targetbe $tmpdir +update $tmpdir +do_cmd beadm activate $targetbe + +if [ "$no_zones" != 1 ]; then + for zone in `do_cmd zoneadm -R $tmpdir list -cip`; do + update_zone $zone + done +fi + +exit 0 diff --git a/usr/src/tools/scripts/validate_pkg.py b/usr/src/tools/scripts/validate_pkg.py index 6678b211e8..6e5858a6ae 100644 --- a/usr/src/tools/scripts/validate_pkg.py +++ b/usr/src/tools/scripts/validate_pkg.py @@ -1,4 +1,4 @@ -#!/usr/bin/python2.6 +#!@PYTHON@ # # CDDL HEADER START # diff --git a/usr/src/tools/scripts/wsdiff.py b/usr/src/tools/scripts/wsdiff.py index 27458f43fc..367903a190 100644 --- a/usr/src/tools/scripts/wsdiff.py +++ b/usr/src/tools/scripts/wsdiff.py @@ -1,4 +1,4 @@ -#!/usr/bin/python2.6 +#!@PYTHON@ # # CDDL HEADER START # -- cgit v1.2.3