diff options
author | Sue Gleeson <Susan.Gleeson@Sun.COM> | 2009-05-21 06:50:10 -0400 |
---|---|---|
committer | Sue Gleeson <Susan.Gleeson@Sun.COM> | 2009-05-21 06:50:10 -0400 |
commit | 1bdd6c0e3710e91cb1f31aa78de33cb638494480 (patch) | |
tree | 0537482ce185b6bb91501dad007b4dd230dbfea5 | |
parent | d65b419ea7828ceaecc8f2ed7188237add6b14dc (diff) | |
download | illumos-gate-1bdd6c0e3710e91cb1f31aa78de33cb638494480.tar.gz |
6775678 Integrate SRP target
PSARC 2009/111 COMSTAR Infiniband SRP Target
54 files changed, 9978 insertions, 1 deletions
diff --git a/usr/src/cmd/Makefile b/usr/src/cmd/Makefile index 97c73a3e18..46de0433f5 100644 --- a/usr/src/cmd/Makefile +++ b/usr/src/cmd/Makefile @@ -378,6 +378,7 @@ COMMON_SUBDIRS= \ split \ sqlite \ srchtxt \ + srptsvc \ ssh \ stat \ stmfadm \ diff --git a/usr/src/cmd/mdb/Makefile.common b/usr/src/cmd/mdb/Makefile.common index 109b110645..ec17cc4334 100644 --- a/usr/src/cmd/mdb/Makefile.common +++ b/usr/src/cmd/mdb/Makefile.common @@ -89,6 +89,7 @@ COMMON_MODULES_KVM = \ sockfs \ specfs \ sppp \ + srpt \ stmf \ sv \ ufs \ diff --git a/usr/src/cmd/mdb/common/modules/srpt/srpt.c b/usr/src/cmd/mdb/common/modules/srpt/srpt.c new file mode 100644 index 0000000000..6da3af074c --- /dev/null +++ b/usr/src/cmd/mdb/common/modules/srpt/srpt.c @@ -0,0 +1,280 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright 2009 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +#include <sys/dditypes.h> +#include <sys/mdb_modapi.h> +#include <sys/modctl.h> +#include <sys/sunddi.h> +#include <sys/sysmacros.h> +#include <sys/lpif.h> +#include <srp.h> +#include <srpt_impl.h> + +/* + * byteswap macros since ntohl not available in kernel mdb + */ +#if defined(_LITTLE_ENDIAN) +#define SRPT_BSWAP_32(x) (((uint32_t)(x) << 24) | \ + (((uint32_t)(x) << 8) & 0xff0000) | \ + (((uint32_t)(x) >> 8) & 0xff00) | \ + ((uint32_t)(x) >> 24)) +#define SRPT_BSWAP_16(x) ((((x) & 0xff) << 8) | ((x) >> 8)) +#else +#define SRPT_BSWAP_32(x) (x) +#define SRPT_BSWAP_16(x) (x) +#endif /* _LITTLE_ENDIAN */ + +/* + * Walker to list the addresses of all the active I/O Controllers + */ +static int +srpt_ioc_walk_init(mdb_walk_state_t *wsp) +{ + srpt_ctxt_t *srpt; + uintptr_t srpt_global_addr, list_addr; + + if (mdb_readvar(&srpt, "srpt_ctxt") == -1) { + mdb_warn("failed to read srpt soft state"); + return (WALK_ERR); + } + + srpt_global_addr = (uintptr_t)srpt; + + list_addr = srpt_global_addr + offsetof(srpt_ctxt_t, sc_ioc_list); + + wsp->walk_addr = list_addr; + + if (mdb_layered_walk("list", wsp) == -1) { + mdb_warn("list walk failed"); + return (WALK_ERR); + } + return (WALK_NEXT); +} + +static int +srpt_list_walk_step(mdb_walk_state_t *wsp) +{ + if (wsp->walk_addr == NULL) { + return (WALK_DONE); + } + return (wsp->walk_callback(wsp->walk_addr, wsp->walk_layer, + wsp->walk_cbdata)); +} + +/* + * Walker to list the target services per I/O Controller. The I/O Controller is + * provided as input. + */ +static int +srpt_tgt_walk_init(mdb_walk_state_t *wsp) +{ + srpt_ioc_t srpt_ioc; + + /* + * Input should be a srpt_ioc_t, read it to get the + * srpt_target_port_t + */ + if (wsp->walk_addr == NULL) { + mdb_warn("<srpt_ioc_t addr>::walk srpt_target\n"); + return (WALK_ERR); + } + + if (mdb_vread(&srpt_ioc, sizeof (srpt_ioc_t), wsp->walk_addr) == -1) { + mdb_warn("failed to read in the srpt_ioc\n "); + return (WALK_ERR); + } + + wsp->walk_addr = (uintptr_t)srpt_ioc.ioc_tgt_port; + wsp->walk_data = mdb_alloc(sizeof (srpt_target_port_t), UM_SLEEP); + return (WALK_NEXT); +} + +static int +srpt_tgt_walk_step(mdb_walk_state_t *wsp) +{ + if (wsp->walk_addr == NULL) { + return (WALK_DONE); + } + + (void) wsp->walk_callback(wsp->walk_addr, wsp->walk_data, + wsp->walk_cbdata); + + /* Currently there is only one target per IOC */ + return (WALK_DONE); + +} + +static void +srpt_tgt_walk_fini(mdb_walk_state_t *wsp) +{ + mdb_free(wsp->walk_data, sizeof (srpt_target_port_t)); +} + +/* + * Walker to list the channels per SRP target service. The target port is + * provided as input. + */ +static int +srpt_channel_walk_init(mdb_walk_state_t *wsp) +{ + /* + * Input should be a srpt_target_port_t, read it to get the + * list of channels + */ + if (wsp->walk_addr == NULL) { + mdb_warn("<srpt_target_port_t addr>::walk srpt_channel\n"); + return (WALK_ERR); + } + + wsp->walk_addr += offsetof(srpt_target_port_t, tp_ch_list); + + if (mdb_layered_walk("list", wsp) == -1) { + mdb_warn("Could not walk tp_ch_list"); + return (WALK_ERR); + } + return (WALK_NEXT); +} + +/* + * Walker to list the SCSI sessions per target. The target is + * provided as input. + */ +static int +srpt_scsi_session_walk_init(mdb_walk_state_t *wsp) +{ + /* + * Input should be a srpt_target_port_t, read it to get the + * srpt_session_t + */ + if (wsp->walk_addr == NULL) { + mdb_warn("<srpt_target_port_t addr>::walk srpt_scsi_session\n"); + return (WALK_ERR); + } + + wsp->walk_addr += offsetof(srpt_target_port_t, tp_sess_list); + + if (mdb_layered_walk("list", wsp) == -1) { + mdb_warn("target session list walk failed"); + return (WALK_ERR); + } + return (WALK_NEXT); +} + +/* + * Walker to list the tasks in a session. The session is + * provided as input. + */ +static int +srpt_task_walk_init(mdb_walk_state_t *wsp) +{ + if (wsp->walk_addr == NULL) { + mdb_warn("<srpt_session_t addr>::walk srpt_tasks\n"); + return (WALK_ERR); + } + + wsp->walk_addr += offsetof(srpt_session_t, ss_task_list); + + if (mdb_layered_walk("list", wsp) == -1) { + mdb_warn("session task list walk failed"); + return (WALK_ERR); + } + return (WALK_NEXT); +} + +/* ARGSUSED */ +static int +srpt_print_ioc(uintptr_t addr, uint_t flags, int argc, const mdb_arg_t *argv) +{ + srpt_ioc_t ioc; + char mask[9]; + int i; + + if (addr == NULL) { + mdb_warn("address of srpt_ioc should be specified\n"); + return (DCMD_ERR); + } + + if (mdb_vread(&ioc, sizeof (srpt_ioc_t), addr) == -1) { + mdb_warn("failed to read srpt_ioc at %p", addr); + return (DCMD_ERR); + } + + mdb_printf("IOC %p\n", addr); + mdb_printf(" guid: %x\n", ioc.ioc_guid); + mdb_printf(" target port: %p\n", ioc.ioc_tgt_port); + mdb_printf(" srq handle: %p\n", ioc.ioc_srq_hdl); + mdb_printf(" current srq size: %u\n", ioc.ioc_num_iu_entries); + mdb_printf(" max srq size: %d\n", ioc.ioc_srq_attr.srq_wr_sz); + mdb_printf(" iu pool: %p\n", ioc.ioc_iu_pool); + mdb_printf(" profile send qdepth: %d\n", + SRPT_BSWAP_16(ioc.ioc_profile.ioc_send_msg_qdepth)); + mdb_printf(" profile rmda read qdepth: %d\n", + ioc.ioc_profile.ioc_rdma_read_qdepth); + mdb_printf(" profile send msg size: %d\n", + SRPT_BSWAP_32(ioc.ioc_profile.ioc_send_msg_sz)); + mdb_printf(" profile rmda xfer size: %d\n", + SRPT_BSWAP_32(ioc.ioc_profile.ioc_rdma_xfer_sz)); + for (i = 0; i < 8; i++) { + if (ioc.ioc_profile.ioc_ctrl_opcap_mask & 1<<i) { + mask[i] = 'x'; + } else { + mask[i] = '-'; + } + } + mask[i] = '\0'; + mdb_printf(" profile opcap mask: %s\n", mask); + + return (DCMD_OK); +} + +static const mdb_dcmd_t dcmds[] = { + { "srpt_print_ioc", ":", "Print information about an SRPT IOC", + srpt_print_ioc, NULL}, + { NULL } +}; + +static const mdb_walker_t walkers[] = { + { "srpt_ioc", "Walk active IO controllers", + srpt_ioc_walk_init, srpt_list_walk_step, NULL}, + { "srpt_tgt", "Walk the targets", + srpt_tgt_walk_init, srpt_tgt_walk_step, srpt_tgt_walk_fini}, + { "srpt_channel", "Walk the channels", + srpt_channel_walk_init, srpt_list_walk_step, NULL}, + { "srpt_scsi_session", "Walk the scsi sessions", + srpt_scsi_session_walk_init, srpt_list_walk_step, NULL}, + { "srpt_tasks", "Walk the tasks in a scsi session", + srpt_task_walk_init, srpt_list_walk_step, NULL}, + { NULL } +}; + +static const mdb_modinfo_t modinfo = { + MDB_API_VERSION, dcmds, walkers +}; + +const mdb_modinfo_t * +_mdb_init(void) +{ + return (&modinfo); +} diff --git a/usr/src/cmd/mdb/intel/amd64/srpt/Makefile b/usr/src/cmd/mdb/intel/amd64/srpt/Makefile new file mode 100644 index 0000000000..1c15d3abfc --- /dev/null +++ b/usr/src/cmd/mdb/intel/amd64/srpt/Makefile @@ -0,0 +1,43 @@ +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# +# +# Copyright 2009 Sun Microsystems, Inc. All rights reserved. +# Use is subject to license terms. +# + +MODULE = srpt.so +MDBTGT = kvm + +MODSRCS = srpt.c + +UTSBASE = ../../../../../uts +COMSTARBASE = $(UTSBASE)/common/io/comstar +SRPTBASE = $(COMSTARBASE)/port/srpt + +include ../../../../Makefile.cmd +include ../../../../Makefile.cmd.64 +include ../../Makefile.amd64 +include ../../../Makefile.module + +CPPFLAGS += -I$(UTSBASE)/common -I$(UTSBASE)/common/sys -I$(SRPTBASE) + +C99MODE= -xc99=%all +C99LMODE= -Xc99=%all diff --git a/usr/src/cmd/mdb/intel/ia32/srpt/Makefile b/usr/src/cmd/mdb/intel/ia32/srpt/Makefile new file mode 100644 index 0000000000..f222ec22fb --- /dev/null +++ b/usr/src/cmd/mdb/intel/ia32/srpt/Makefile @@ -0,0 +1,42 @@ +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# +# +# Copyright 2009 Sun Microsystems, Inc. All rights reserved. +# Use is subject to license terms. +# + +MODULE = srpt.so +MDBTGT = kvm + +MODSRCS = srpt.c + +UTSBASE = ../../../../../uts +COMSTARBASE = $(UTSBASE)/common/io/comstar +SRPTBASE = $(COMSTARBASE)/port/srpt + +include ../../../../Makefile.cmd +include ../../Makefile.ia32 +include ../../../Makefile.module + +CPPFLAGS += -I$(UTSBASE)/common -I$(UTSBASE)/common/sys -I$(SRPTBASE) + +C99MODE= -xc99=%all +C99LMODE= -Xc99=%all diff --git a/usr/src/cmd/mdb/sparc/v9/srpt/Makefile b/usr/src/cmd/mdb/sparc/v9/srpt/Makefile new file mode 100644 index 0000000000..7379a27cde --- /dev/null +++ b/usr/src/cmd/mdb/sparc/v9/srpt/Makefile @@ -0,0 +1,43 @@ +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# +# +# Copyright 2009 Sun Microsystems, Inc. All rights reserved. +# Use is subject to license terms. +# + +MODULE = srpt.so +MDBTGT = kvm + +MODSRCS = srpt.c + +UTSBASE = ../../../../../uts +COMSTARBASE = $(UTSBASE)/common/io/comstar +SRPTBASE = $(COMSTARBASE)/port/srpt + +include ../../../../Makefile.cmd +include ../../../../Makefile.cmd.64 +include ../../Makefile.sparcv9 +include ../../../Makefile.module + +CPPFLAGS += -I$(UTSBASE)/common -I$(UTSBASE)/common/sys -I$(SRPTBASE) + +C99MODE= -xc99=%all +C99LMODE= -Xc99=%all diff --git a/usr/src/cmd/srptsvc/Makefile b/usr/src/cmd/srptsvc/Makefile new file mode 100644 index 0000000000..70e243b34b --- /dev/null +++ b/usr/src/cmd/srptsvc/Makefile @@ -0,0 +1,64 @@ +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# + +# +# Copyright 2009 Sun Microsystems, Inc. All rights reserved. +# Use is subject to license terms. +# + +PROG = svc-srpt + +include ../Makefile.cmd + +SRPTBASE = ../../uts/common/io/comstar/port/srpt + +OBJS = srptsvc.o +SRCS = $(OBJS:%.o=%.c) + +CPPFLAGS += -I$(SRPTBASE) + +MANIFEST = target.xml +SVCMETHOD = svc-srpt + +ROOTMANIFESTDIR = $(ROOTSVCSYSTEM)/ibsrp +$(ROOTMANIFESTDIR)/$(MANIFEST) := OWNER = root +$(ROOTMANIFESTDIR)/$(MANIFEST) := GROUP = bin +$(ROOTMANIFESTDIR)/$(MANIFEST) := FILEMODE = 0444 + +.KEEP_STATE: + +all: $(PROG) + +$(PROG): $(OBJS) + $(LINK.c) -o $@ $(OBJS) $(LDLIBS) + $(POST_PROCESS) + +install: all $(ROOTMANIFEST) $(ROOTSVCMETHOD) + +check: $(CHKMANIFEST) + $(CSTYLE) -pPc $(SRCS:%=%) + +clean: + $(RM) $(OBJS) + +lint: lint_SRCS + +include ../Makefile.targ diff --git a/usr/src/cmd/srptsvc/srptsvc.c b/usr/src/cmd/srptsvc/srptsvc.c new file mode 100644 index 0000000000..a14d3c965b --- /dev/null +++ b/usr/src/cmd/srptsvc/srptsvc.c @@ -0,0 +1,105 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright 2009 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +#include <stdio.h> +#include <libintl.h> +#include <sys/types.h> +#include <sys/stat.h> +#include <fcntl.h> +#include <locale.h> +#include <errno.h> +#include <unistd.h> +#include <strings.h> + +#include <srpt_ioctl.h> + + +/* Globals */ +char cmdName[] = "svc-srpt"; + +int +main(int argc, char *argv[]) +{ + int ret; + boolean_t do_enable = B_FALSE; + int srpt_ioctl; + int fd = -1; + int saverr; + char *txt; + + (void) setlocale(LC_ALL, ""); + + if (argc < 2 || argc > 2) { + goto usage; + } + + if (strcasecmp(argv[1], "start") == 0) { + do_enable = B_TRUE; + srpt_ioctl = SRPT_IOC_ENABLE_SVC; + } else if (strcasecmp(argv[1], "stop") == 0) { + srpt_ioctl = SRPT_IOC_DISABLE_SVC; + } else { + goto usage; + } + + fd = open(SRPT_NODE, O_RDONLY); + if (fd < 0) { + saverr = errno; + + (void) fprintf(stderr, "%s: %s", cmdName, + gettext("Could not open SRP Target pseudodevice.")); + + if (saverr == ENOENT) { + (void) fprintf(stderr, + gettext(" Driver may not be loaded.")); + } else { + (void) fprintf(stderr, gettext(" error = %d"), saverr); + } + (void) fprintf(stderr, "\n"); + return (1); + } + + ret = ioctl(fd, srpt_ioctl, NULL); + if (ret != 0) { + if (do_enable) { + txt = gettext("Could not enable SRP Target"); + } else { + txt = gettext("Could not disable SRP Target"); + } + + (void) fprintf(stderr, "%s: %d", txt, ret); + ret = 1; + } + + (void) close(fd); + + return (ret); + +usage: + (void) fprintf(stderr, gettext("Usage: %s start|stop"), cmdName); + (void) fprintf(stderr, "\n"); + + return (1); +} diff --git a/usr/src/cmd/srptsvc/target.xml b/usr/src/cmd/srptsvc/target.xml new file mode 100644 index 0000000000..c21b4fe471 --- /dev/null +++ b/usr/src/cmd/srptsvc/target.xml @@ -0,0 +1,103 @@ +<?xml version='1.0'?> +<!DOCTYPE service_bundle SYSTEM '/usr/share/lib/xml/dtd/service_bundle.dtd.1'> + +<!-- + +CDDL HEADER START + +The contents of this file are subject to the terms of the +Common Development and Distribution License (the "License"). +You may not use this file except in compliance with the License. + +You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +or http://www.opensolaris.org/os/licensing. +See the License for the specific language governing permissions +and limitations under the License. + +When distributing Covered Code, include this CDDL HEADER in each +file and include the License file at usr/src/OPENSOLARIS.LICENSE. +If applicable, add the following below this CDDL HEADER, with the +fields enclosed by brackets "[]" replaced with your own identifying +information: Portions Copyright [yyyy] [name of copyright owner] + +CDDL HEADER END + +Copyright 2009 Sun Microsystems, Inc. All rights reserved. +Use is subject to license terms. + +Service manifests for SRP Target Port Provider +--> + +<!-- + system/ibsrp/target - Export SRP target services + +--> + +<service_bundle type='manifest' name='SUNWsrptr:target'> + +<service + name='system/ibsrp/target' + type='service' + version='1'> + + <create_default_instance enabled='false' /> + + <single_instance/> + + <dependency name = 'stmf' + grouping='require_all' + restart_on='none' + type='service'> + <service_fmri value='svc:/system/stmf:default'/> + </dependency> + + <exec_method + type='method' + name='start' + exec='/lib/svc/method/svc-srpt start' + timeout_seconds='600'> + <method_context> + <method_credential + user='daemon' + group='daemon' + privileges='basic,sys_devices,!file_link_any,!proc_session,!proc_info' + /> + </method_context> + </exec_method> + + <exec_method + type='method' + name='stop' + exec='/lib/svc/method/svc-srpt stop' + timeout_seconds='600'> + <method_context> + <method_credential + user='daemon' + group='daemon' + privileges='basic,sys_devices,!file_link_any,!proc_session,!proc_info' + /> + </method_context> + </exec_method> + + <property_group name='startd' type='framework'> + <propval name='duration' type='astring' + value='transient' /> + </property_group> + + <stability value='Evolving' /> + + <template> + <common_name> + <loctext xml:lang='C'> + SRP Target + </loctext> + </common_name> + <documentation> + <manpage title='srpt' section='7D' + manpath='/usr/share/man' /> + </documentation> + </template> + +</service> + +</service_bundle> diff --git a/usr/src/pkgdefs/Makefile b/usr/src/pkgdefs/Makefile index f38865052f..c07d27b25c 100644 --- a/usr/src/pkgdefs/Makefile +++ b/usr/src/pkgdefs/Makefile @@ -277,6 +277,7 @@ COMMON_SUBDIRS= \ SUNWhwdata \ SUNWhxge \ SUNWib \ + SUNWibdmar \ SUNWibsdpu \ SUNWibsdp \ SUNWiir \ @@ -470,6 +471,8 @@ COMMON_SUBDIRS= \ SUNWspsvu \ SUNWsra \ SUNWsrh \ + SUNWsrptr \ + SUNWsrptu \ SUNWsshcu \ SUNWsshr \ SUNWsshu \ diff --git a/usr/src/pkgdefs/SUNWibdmar/Makefile b/usr/src/pkgdefs/SUNWibdmar/Makefile new file mode 100644 index 0000000000..9e5b2d8335 --- /dev/null +++ b/usr/src/pkgdefs/SUNWibdmar/Makefile @@ -0,0 +1,35 @@ +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# +# +# Copyright 2009 Sun Microsystems, Inc. All rights reserved. +# Use is subject to license terms. +# + +include ../Makefile.com + +.KEEP_STATE: + +all: $(FILES) depend + +install: all pkg + +include ../Makefile.targ + diff --git a/usr/src/pkgdefs/SUNWibdmar/depend b/usr/src/pkgdefs/SUNWibdmar/depend new file mode 100644 index 0000000000..7985a9230a --- /dev/null +++ b/usr/src/pkgdefs/SUNWibdmar/depend @@ -0,0 +1,50 @@ +# +# Copyright 2009 Sun Microsystems, Inc. All rights reserved. +# Use is subject to license terms. +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# +# This package information file defines software dependencies associated +# with the pkg. You can define three types of pkg dependencies with this file: +# P indicates a prerequisite for installation +# I indicates an incompatible package +# R indicates a reverse dependency +# <pkg.abbr> see pkginfo(4), PKG parameter +# <name> see pkginfo(4), NAME parameter +# <version> see pkginfo(4), VERSION parameter +# <arch> see pkginfo(4), ARCH parameter +# <type> <pkg.abbr> <name> +# (<arch>)<version> +# (<arch>)<version> +# ... +# <type> <pkg.abbr> <name> +# ... +# + +P SUNWcar Core Architecture, (Root) +P SUNWcakr Core Solaris Kernel Architecture (Root) +P SUNWkvm Core Architecture, (Kvm) +P SUNWcsr Core Solaris, (Root) +P SUNWckr Core Solaris Kernel (Root) +P SUNWcnetr Core Solaris Network Infrastructure (Root) +P SUNWcsu Core Solaris, (Usr) +P SUNWcsd Core Solaris Devices +P SUNWcsl Core Solaris Libraries +P SUNWib Sun InfiniBand Framework diff --git a/usr/src/pkgdefs/SUNWibdmar/pkginfo.tmpl b/usr/src/pkgdefs/SUNWibdmar/pkginfo.tmpl new file mode 100644 index 0000000000..42c29079b9 --- /dev/null +++ b/usr/src/pkgdefs/SUNWibdmar/pkginfo.tmpl @@ -0,0 +1,49 @@ +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# +# +# Copyright 2009 Sun Microsystems, Inc. All rights reserved. +# Use is subject to license terms. +# +# +# +# This required package information file describes characteristics of the +# package, such as package abbreviation, full package name, package version, +# and package architecture. +# +PKG="SUNWibdmar" +NAME="Sun InfiniBand Device Management Agent (Root)" +ARCH="ISA" +CATEGORY="system" +BASEDIR=/ +SUNW_PKGVERS="1.0" +SUNW_PKGTYPE="root" +CLASSES="none" +DESC="Sun InfiniBand Device Management Agent (Root)" +SUNW_PRODNAME="SunOS" +SUNW_PRODVERS="RELEASE/VERSION" +VERSION="ONVERS,REV=0.0.0" +VENDOR="Sun Microsystems, Inc." +HOTLINE="Please contact your local service provider" +EMAIL="" +MAXINST="1000" +SUNW_PKG_ALLZONES="true" +SUNW_PKG_HOLLOW="true" +SUNW_PKG_THISZONE="false" diff --git a/usr/src/pkgdefs/SUNWibdmar/prototype_com b/usr/src/pkgdefs/SUNWibdmar/prototype_com new file mode 100644 index 0000000000..ba92b3898c --- /dev/null +++ b/usr/src/pkgdefs/SUNWibdmar/prototype_com @@ -0,0 +1,45 @@ +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# +# +# Copyright 2009 Sun Microsystems, Inc. All rights reserved. +# Use is subject to license terms. +# +# + +# This required package information file contains a list of package contents. +# The 'pkgmk' command uses this file to identify the contents of a package +# and their location on the development machine when building the package. +# Can be created via a text editor or through use of the 'pkgproto' command. + +#!search <pathname pathname ...> # where to find pkg objects +#!include <filename> # include another 'prototype' file +#!default <mode> <owner> <group> # default used if not specified on entry +#!<param>=<value> # puts parameter in pkg environment + +# +i copyright +i pkginfo +i depend +# +# SUNWibdmar files +# +d none kernel 0755 root sys +d none kernel/misc 0755 root sys diff --git a/usr/src/pkgdefs/SUNWibdmar/prototype_i386 b/usr/src/pkgdefs/SUNWibdmar/prototype_i386 new file mode 100644 index 0000000000..557a0f4e30 --- /dev/null +++ b/usr/src/pkgdefs/SUNWibdmar/prototype_i386 @@ -0,0 +1,51 @@ +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# +# +# Copyright 2009 Sun Microsystems, Inc. All rights reserved. +# Use is subject to license terms. +# +# +# This required package information file contains a list of package contents. +# The 'pkgmk' command uses this file to identify the contents of a package +# and their location on the development machine when building the package. +# Can be created via a text editor or through use of the 'pkgproto' command. + +#!search <pathname pathname ...> # where to find pkg objects +#!include <filename> # include another 'prototype' file +#!default <mode> <owner> <group> # default used if not specified on entry +#!<param>=<value> # puts parameter in pkg environment + +# +# Include ISA independent files (prototype_com) +# +!include prototype_com +# +# +# List files which are Intel specific here +# +# source locations relative to the prototype file +# +# +# SUNWibdmar +# +f none kernel/misc/ibdma 0755 root sys +d none kernel/misc/amd64 0755 root sys +f none kernel/misc/amd64/ibdma 0755 root sys diff --git a/usr/src/pkgdefs/SUNWibdmar/prototype_sparc b/usr/src/pkgdefs/SUNWibdmar/prototype_sparc new file mode 100644 index 0000000000..87b0865fed --- /dev/null +++ b/usr/src/pkgdefs/SUNWibdmar/prototype_sparc @@ -0,0 +1,49 @@ +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# +# +# Copyright 2009 Sun Microsystems, Inc. All rights reserved. +# Use is subject to license terms. +# +# +# This required package information file contains a list of package contents. +# The 'pkgmk' command uses this file to identify the contents of a package +# and their location on the development machine when building the package. +# Can be created via a text editor or through use of the 'pkgproto' command. + +#!search <pathname pathname ...> # where to find pkg objects +#!include <filename> # include another 'prototype' file +#!default <mode> <owner> <group> # default used if not specified on entry +#!<param>=<value> # puts parameter in pkg environment +# +# Include ISA independent files (prototype_com) +# +!include prototype_com +# +# +# List files which are SPARC specific here +# +# source locations relative to the prototype file +# +# +# SUNWibdmar +# +d none kernel/misc/sparcv9 0755 root sys +f none kernel/misc/sparcv9/ibdma 0755 root sys diff --git a/usr/src/pkgdefs/SUNWsrptr/Makefile b/usr/src/pkgdefs/SUNWsrptr/Makefile new file mode 100644 index 0000000000..948948735d --- /dev/null +++ b/usr/src/pkgdefs/SUNWsrptr/Makefile @@ -0,0 +1,37 @@ +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# +# +# Copyright 2009 Sun Microsystems, Inc. All rights reserved. +# Use is subject to license terms. +# + +include ../Makefile.com + +DATAFILES += i.manifest r.manifest + +.KEEP_STATE: + +all: $(FILES) depend i.manifest r.manifest + +install: all pkg + +include ../Makefile.targ + diff --git a/usr/src/pkgdefs/SUNWsrptr/depend b/usr/src/pkgdefs/SUNWsrptr/depend new file mode 100644 index 0000000000..55d64e36b5 --- /dev/null +++ b/usr/src/pkgdefs/SUNWsrptr/depend @@ -0,0 +1,51 @@ +# +# Copyright 2009 Sun Microsystems, Inc. All rights reserved. +# Use is subject to license terms. +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# +# This package information file defines software dependencies associated +# with the pkg. You can define three types of pkg dependencies with this file: +# P indicates a prerequisite for installation +# I indicates an incompatible package +# R indicates a reverse dependency +# <pkg.abbr> see pkginfo(4), PKG parameter +# <name> see pkginfo(4), NAME parameter +# <version> see pkginfo(4), VERSION parameter +# <arch> see pkginfo(4), ARCH parameter +# <type> <pkg.abbr> <name> +# (<arch>)<version> +# (<arch>)<version> +# ... +# <type> <pkg.abbr> <name> +# ... +# + +P SUNWcar Core Architecture, (Root) +P SUNWcakr Core Solaris Kernel Architecture (Root) +P SUNWkvm Core Architecture, (Kvm) +P SUNWcsr Core Solaris, (Root) +P SUNWckr Core Solaris Kernel (Root) +P SUNWcnetr Core Solaris Network Infrastructure (Root) +P SUNWcsu Core Solaris, (Usr) +P SUNWcsd Core Solaris Devices +P SUNWcsl Core Solaris Libraries +P SUNWstmf Sun Common Multiprotocol SCSI Target +P SUNWibdmar Sun InfiniBand Device Management Agent (Root) diff --git a/usr/src/pkgdefs/SUNWsrptr/pkginfo.tmpl b/usr/src/pkgdefs/SUNWsrptr/pkginfo.tmpl new file mode 100644 index 0000000000..c861ff2de3 --- /dev/null +++ b/usr/src/pkgdefs/SUNWsrptr/pkginfo.tmpl @@ -0,0 +1,48 @@ +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# +# +# Copyright 2009 Sun Microsystems, Inc. All rights reserved. +# Use is subject to license terms. +# +# +# This required package information file describes characteristics of the +# package, such as package abbreviation, full package name, package version, +# and package architecture. +# +PKG="SUNWsrptr" +NAME="Sun SRP COMSTAR Port Provider (Root)" +ARCH="ISA" +CATEGORY="system" +BASEDIR=/ +SUNW_PKGVERS="1.0" +SUNW_PKGTYPE="root" +CLASSES="manifest none" +DESC="Sun SRP COMSTAR Port Provider (Root)" +SUNW_PRODNAME="SunOS" +SUNW_PRODVERS="RELEASE/VERSION" +VERSION="ONVERS,REV=0.0.0" +VENDOR="Sun Microsystems, Inc." +HOTLINE="Please contact your local service provider" +EMAIL="" +MAXINST="1000" +SUNW_PKG_ALLZONES="true" +SUNW_PKG_HOLLOW="true" +SUNW_PKG_THISZONE="false" diff --git a/usr/src/pkgdefs/SUNWsrptr/postinstall b/usr/src/pkgdefs/SUNWsrptr/postinstall new file mode 100644 index 0000000000..6d9052f036 --- /dev/null +++ b/usr/src/pkgdefs/SUNWsrptr/postinstall @@ -0,0 +1,42 @@ +#!/bin/sh +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# +# +# Copyright 2009 Sun Microsystems, Inc. All rights reserved. +# Use is subject to license terms. +# +# + +PATH="/usr/bin:/usr/sbin:${PATH}"; export PATH + +# Driver definitions +DRVR_NAME=srpt; export DRVR_NAME +DRVR_PERM='* 0644 root sys'; export DRVR_PERM +BASEDIR_OPT= + +if [ "${BASEDIR:=/}" != "/" ] +then + BASEDIR_OPT="-n -b $BASEDIR" +fi + +/usr/sbin/add_drv $BASEDIR_OPT -m "${DRVR_PERM}" ${DRVR_NAME} + +exit 0 diff --git a/usr/src/pkgdefs/SUNWsrptr/postremove b/usr/src/pkgdefs/SUNWsrptr/postremove new file mode 100644 index 0000000000..c476b26afd --- /dev/null +++ b/usr/src/pkgdefs/SUNWsrptr/postremove @@ -0,0 +1,43 @@ +#!/bin/sh +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# +# +# Copyright 2009 Sun Microsystems, Inc. All rights reserved. +# Use is subject to license terms. +# +# + +PATH="/usr/bin:/usr/sbin:${PATH}" +export PATH + +DRVR_NAME=srpt +BASEDIR_OPT= + +# Unload and remove the driver +if [ "${BASEDIR:=/}" != "/" ] +then + BASEDIR_OPT="-b $BASEDIR" +fi + + +/usr/sbin/rem_drv ${BASEDIR_OPT} ${DRVR_NAME} + +exit 0 diff --git a/usr/src/pkgdefs/SUNWsrptr/prototype_com b/usr/src/pkgdefs/SUNWsrptr/prototype_com new file mode 100644 index 0000000000..4a3583bf86 --- /dev/null +++ b/usr/src/pkgdefs/SUNWsrptr/prototype_com @@ -0,0 +1,62 @@ +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# +# +# Copyright 2009 Sun Microsystems, Inc. All rights reserved. +# Use is subject to license terms. +# +# +# This required package information file contains a list of package contents. +# The 'pkgmk' command uses this file to identify the contents of a package +# and their location on the development machine when building the package. +# Can be created via a text editor or through use of the 'pkgproto' command. + +#!search <pathname pathname ...> # where to find pkg objects +#!include <filename> # include another 'prototype' file +#!default <mode> <owner> <group> # default used if not specified on entry +#!<param>=<value> # puts parameter in pkg environment + +# +# +i copyright +i pkginfo +i depend +i postinstall +i postremove +i i.manifest +i r.manifest + +# +# SUNWsrptr files +# +d none kernel 0755 root sys +d none kernel/drv 0755 root sys +f none kernel/drv/srpt.conf 0644 root sys +d none kernel/kmdb 755 root sys +d none lib 0755 root bin +d none lib/svc 0755 root bin +d none lib/svc/method 0755 root bin +f none lib/svc/method/svc-srpt 0555 root bin +d none var 0755 root sys +d none var/svc 0755 root sys +d none var/svc/manifest 0755 root sys +d none var/svc/manifest/system 0755 root sys +d none var/svc/manifest/system/ibsrp 0755 root sys +f manifest var/svc/manifest/system/ibsrp/target.xml 0444 root sys diff --git a/usr/src/pkgdefs/SUNWsrptr/prototype_i386 b/usr/src/pkgdefs/SUNWsrptr/prototype_i386 new file mode 100644 index 0000000000..2b1afbb77b --- /dev/null +++ b/usr/src/pkgdefs/SUNWsrptr/prototype_i386 @@ -0,0 +1,54 @@ +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# +# +# Copyright 2009 Sun Microsystems, Inc. All rights reserved. +# Use is subject to license terms. +# +# +# This required package information file contains a list of package contents. +# The 'pkgmk' command uses this file to identify the contents of a package +# and their location on the development machine when building the package. +# Can be created via a text editor or through use of the 'pkgproto' command. + +#!search <pathname pathname ...> # where to find pkg objects +#!include <filename> # include another 'prototype' file +#!default <mode> <owner> <group> # default used if not specified on entry +#!<param>=<value> # puts parameter in pkg environment + +# +# Include ISA independent files (prototype_com) +# +!include prototype_com +# +# +# List files which are Intel specific here +# +# source locations relative to the prototype file +# +# +# SUNWsrptr +# +f none kernel/drv/srpt 0755 root sys +d none kernel/drv/amd64 0755 root sys +f none kernel/drv/amd64/srpt 0755 root sys +d none kernel/kmdb/amd64 755 root sys +f none kernel/kmdb/amd64/srpt 555 root sys +f none kernel/kmdb/srpt 555 root sys diff --git a/usr/src/pkgdefs/SUNWsrptr/prototype_sparc b/usr/src/pkgdefs/SUNWsrptr/prototype_sparc new file mode 100644 index 0000000000..735ba00711 --- /dev/null +++ b/usr/src/pkgdefs/SUNWsrptr/prototype_sparc @@ -0,0 +1,51 @@ +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# +# +# Copyright 2009 Sun Microsystems, Inc. All rights reserved. +# Use is subject to license terms. +# +# +# This required package information file contains a list of package contents. +# The 'pkgmk' command uses this file to identify the contents of a package +# and their location on the development machine when building the package. +# Can be created via a text editor or through use of the 'pkgproto' command. + +#!search <pathname pathname ...> # where to find pkg objects +#!include <filename> # include another 'prototype' file +#!default <mode> <owner> <group> # default used if not specified on entry +#!<param>=<value> # puts parameter in pkg environment +# +# Include ISA independent files (prototype_com) +# +!include prototype_com +# +# +# List files which are SPARC specific here +# +# source locations relative to the prototype file +# +# +# SUNWsrptr +# +d none kernel/drv/sparcv9 0755 root sys +f none kernel/drv/sparcv9/srpt 0755 root sys +d none kernel/kmdb/sparcv9 0755 root sys +f none kernel/kmdb/sparcv9/srpt 0555 root sys diff --git a/usr/src/pkgdefs/SUNWsrptu/Makefile b/usr/src/pkgdefs/SUNWsrptu/Makefile new file mode 100644 index 0000000000..9e5b2d8335 --- /dev/null +++ b/usr/src/pkgdefs/SUNWsrptu/Makefile @@ -0,0 +1,35 @@ +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# +# +# Copyright 2009 Sun Microsystems, Inc. All rights reserved. +# Use is subject to license terms. +# + +include ../Makefile.com + +.KEEP_STATE: + +all: $(FILES) depend + +install: all pkg + +include ../Makefile.targ + diff --git a/usr/src/pkgdefs/SUNWsrptu/depend b/usr/src/pkgdefs/SUNWsrptu/depend new file mode 100644 index 0000000000..5df204fe4a --- /dev/null +++ b/usr/src/pkgdefs/SUNWsrptu/depend @@ -0,0 +1,48 @@ +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# +# Copyright 2009 Sun Microsystems, Inc. All rights reserved. +# Use is subject to license terms. +# +# This package information file defines software dependencies associated +# with the pkg. You can define three types of pkg dependencies with this file: +# P indicates a prerequisite for installation +# I indicates an incompatible package +# R indicates a reverse dependency +# <pkg.abbr> see pkginfo(4), PKG parameter +# <name> see pkginfo(4), NAME parameter +# <version> see pkginfo(4), VERSION parameter +# <arch> see pkginfo(4), ARCH parameter +# <type> <pkg.abbr> <name> +# (<arch>)<version> +# (<arch>)<version> +# ... +# <type> <pkg.abbr> <name> +# ... +# + +P SUNWcar Core Architecture, (Root) +P SUNWcakr Core Solaris Kernel Architecture (Root) +P SUNWkvm Core Architecture, (Kvm) +P SUNWcsr Core Solaris, (Root) +P SUNWckr Core Solaris Kernel (Root) +P SUNWcnetr Core Solaris Network Infrastructure (Root) +P SUNWcsu Core Solaris, (Usr) +P SUNWcsd Core Solaris Devices +P SUNWcsl Core Solaris Libraries diff --git a/usr/src/pkgdefs/SUNWsrptu/pkginfo.tmpl b/usr/src/pkgdefs/SUNWsrptu/pkginfo.tmpl new file mode 100644 index 0000000000..662b92bbae --- /dev/null +++ b/usr/src/pkgdefs/SUNWsrptu/pkginfo.tmpl @@ -0,0 +1,48 @@ +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# +# +# Copyright 2009 Sun Microsystems, Inc. All rights reserved. +# Use is subject to license terms. +# +# +# This required package information file describes characteristics of the +# package, such as package abbreviation, full package name, package version, +# and package architecture. +# +PKG="SUNWsrptu" +NAME="Sun SRP COMSTAR Port Provider utilities (Usr)" +ARCH="ISA" +CATEGORY="system" +BASEDIR=/ +SUNW_PKGVERS="1.0" +SUNW_PKGTYPE="usr" +CLASSES="none" +DESC="Sun SRP COMSTAR Port Provider utilities (Usr)" +SUNW_PRODNAME="SunOS" +SUNW_PRODVERS="RELEASE/VERSION" +VERSION="ONVERS,REV=0.0.0" +VENDOR="Sun Microsystems, Inc." +HOTLINE="Please contact your local service provider" +EMAIL="" +MAXINST="1000" +SUNW_PKG_ALLZONES="true" +SUNW_PKG_HOLLOW="false" +SUNW_PKG_THISZONE="false" diff --git a/usr/src/pkgdefs/SUNWsrptu/prototype_com b/usr/src/pkgdefs/SUNWsrptu/prototype_com new file mode 100644 index 0000000000..3b1247f851 --- /dev/null +++ b/usr/src/pkgdefs/SUNWsrptu/prototype_com @@ -0,0 +1,43 @@ +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# +# +# Copyright 2009 Sun Microsystems, Inc. All rights reserved. +# Use is subject to license terms. +# +# +# This required package information file contains a list of package contents. +# The 'pkgmk' command uses this file to identify the contents of a package +# and their location on the development machine when building the package. +# Can be created via a text editor or through use of the 'pkgproto' command. + +#!search <pathname pathname ...> # where to find pkg objects +#!include <filename> # include another 'prototype' file +#!default <mode> <owner> <group> # default used if not specified on entry +#!<param>=<value> # puts parameter in pkg environment + +i copyright +i pkginfo +i depend + +d none usr 755 root sys +d none usr/lib 755 root bin +d none usr/lib/mdb 755 root sys +d none usr/lib/mdb/kvm 755 root sys diff --git a/usr/src/pkgdefs/SUNWsrptu/prototype_i386 b/usr/src/pkgdefs/SUNWsrptu/prototype_i386 new file mode 100644 index 0000000000..8d14da762f --- /dev/null +++ b/usr/src/pkgdefs/SUNWsrptu/prototype_i386 @@ -0,0 +1,51 @@ +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# +# +# Copyright 2009 Sun Microsystems, Inc. All rights reserved. +# Use is subject to license terms. +# +# +# This required package information file contains a list of package contents. +# The 'pkgmk' command uses this file to identify the contents of a package +# and their location on the development machine when building the package. +# Can be created via a text editor or through use of the 'pkgproto' command. + +#!search <pathname pathname ...> # where to find pkg objects +#!include <filename> # include another 'prototype' file +#!default <mode> <owner> <group> # default used if not specified on entry +#!<param>=<value> # puts parameter in pkg environment + +# +# Include ISA independent files (prototype_com) +# +!include prototype_com +# +# +# List files which are Intel specific here +# +# source locations relative to the prototype file +# +# SUNWsrptu +# +d none usr/lib/mdb/kvm/amd64 755 root sys +f none usr/lib/mdb/kvm/srpt.so 555 root sys +f none usr/lib/mdb/kvm/amd64/srpt.so 555 root sys + diff --git a/usr/src/pkgdefs/SUNWsrptu/prototype_sparc b/usr/src/pkgdefs/SUNWsrptu/prototype_sparc new file mode 100644 index 0000000000..98bdfbd172 --- /dev/null +++ b/usr/src/pkgdefs/SUNWsrptu/prototype_sparc @@ -0,0 +1,50 @@ +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# +# +# Copyright 2009 Sun Microsystems, Inc. All rights reserved. +# Use is subject to license terms. +# +# +# This required package information file contains a list of package contents. +# The 'pkgmk' command uses this file to identify the contents of a package +# and their location on the development machine when building the package. +# Can be created via a text editor or through use of the 'pkgproto' command. + +#!search <pathname pathname ...> # where to find pkg objects +#!include <filename> # include another 'prototype' file +#!default <mode> <owner> <group> # default used if not specified on entry +#!<param>=<value> # puts parameter in pkg environment +# +# Include ISA independent files (prototype_com) +# +!include prototype_com +# +# +# List files which are SPARC specific here +# +# source locations relative to the prototype file +# +# +# SUNWsrptu +# +d none usr/lib/mdb/kvm/sparcv9 755 root sys +f none usr/lib/mdb/kvm/sparcv9/srpt.so 555 root sys + diff --git a/usr/src/uts/common/Makefile.files b/usr/src/uts/common/Makefile.files index 6f9dc7d5a0..b4462e3118 100644 --- a/usr/src/uts/common/Makefile.files +++ b/usr/src/uts/common/Makefile.files @@ -631,6 +631,8 @@ IBCM_OBJS += ibcm_impl.o ibcm_sm.o ibcm_ti.o ibcm_utils.o ibcm_path.o \ IBDM_OBJS += ibdm.o +IBDMA_OBJS += ibdma.o + IBMF_OBJS += ibmf.o ibmf_impl.o ibmf_dr.o ibmf_wqe.o ibmf_ud_dest.o ibmf_mod.o \ ibmf_send.o ibmf_recv.o ibmf_handlers.o ibmf_trans.o \ ibmf_timers.o ibmf_msg.o ibmf_utils.o ibmf_rmpp.o \ @@ -876,6 +878,8 @@ FCT_OBJS += discovery.o fct.o QLT_OBJS += 2400.o 2500.o qlt.o qlt_dma.o +SRPT_OBJS += srpt_mod.o srpt_ch.o srpt_cm.o srpt_ioc.o srpt_stp.o + FCOE_OBJS += fcoe.o fcoe_eth.o fcoe_fc.o FCOET_OBJS += fcoet.o fcoet_eth.o fcoet_fc.o diff --git a/usr/src/uts/common/Makefile.rules b/usr/src/uts/common/Makefile.rules index 2de822fa57..5158c543b7 100644 --- a/usr/src/uts/common/Makefile.rules +++ b/usr/src/uts/common/Makefile.rules @@ -592,10 +592,13 @@ $(OBJS_DIR)/%.o: $(UTSBASE)/common/io/comstar/port/qlt/%.c $(COMPILE.c) -o $@ $< $(CTFCONVERT_O) -$(OBJS_DIR)/%.o: $(UTSBASE)/common/io/comstar/port/fcoet/%.c +$(OBJS_DIR)/%.o: $(UTSBASE)/common/io/comstar/port/srpt/%.c $(COMPILE.c) -o $@ $< $(CTFCONVERT_O) +$(OBJS_DIR)/%.o: $(UTSBASE)/common/io/comstar/port/fcoet/%.c + $(COMPILE.c) -o $@ $< + $(CTFCONVERT_O) $(OBJS_DIR)/%.o: $(COMMONBASE)/iscsit/%.c $(COMPILE.c) -o $@ $< @@ -673,6 +676,10 @@ $(OBJS_DIR)/%.o: $(UTSBASE)/common/io/ib/mgt/ibdm/%.c $(COMPILE.c) -o $@ $< $(CTFCONVERT_O) +$(OBJS_DIR)/%.o: $(UTSBASE)/common/io/ib/mgt/ibdma/%.c + $(COMPILE.c) -o $@ $< + $(CTFCONVERT_O) + $(OBJS_DIR)/%.o: $(UTSBASE)/common/io/ib/mgt/ibmf/%.c $(COMPILE.c) -o $@ $< $(CTFCONVERT_O) @@ -1776,6 +1783,9 @@ $(LINTS_DIR)/%.ln: $(UTSBASE)/common/io/comstar/port/fct/%.c $(LINTS_DIR)/%.ln: $(UTSBASE)/common/io/comstar/port/qlt/%.c @($(LHEAD) $(LINT.c) $< $(LTAIL)) +$(LINTS_DIR)/%.ln: $(UTSBASE)/common/io/comstar/port/srpt/%.c + @($(LHEAD) $(LINT.c) $< $(LTAIL)) + $(LINTS_DIR)/%.ln: $(COMMONBASE)/iscsit/%.c @($(LHEAD) $(LINT.c) $< $(LTAIL)) @@ -1836,6 +1846,9 @@ $(LINTS_DIR)/%.ln: $(UTSBASE)/common/io/ib/mgt/ibcm/%.c $(LINTS_DIR)/%.ln: $(UTSBASE)/common/io/ib/mgt/ibdm/%.c @($(LHEAD) $(LINT.c) $< $(LTAIL)) +$(LINTS_DIR)/%.ln: $(UTSBASE)/common/io/ib/mgt/ibdma/%.c + @($(LHEAD) $(LINT.c) $< $(LTAIL)) + $(LINTS_DIR)/%.ln: $(UTSBASE)/common/io/ib/mgt/ibmf/%.c @($(LHEAD) $(LINT.c) $< $(LTAIL)) diff --git a/usr/src/uts/common/io/comstar/port/srpt/srp.h b/usr/src/uts/common/io/comstar/port/srpt/srp.h new file mode 100644 index 0000000000..d6183a2299 --- /dev/null +++ b/usr/src/uts/common/io/comstar/port/srpt/srp.h @@ -0,0 +1,298 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ + +/* + * Copyright 2009 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +#ifndef _SRP_H +#define _SRP_H + +/* + * General SCSI RDMA Protocol generic defines + */ + +#ifdef __cplusplus +extern "C" { +#endif + +/* + * The following defines and structures are based on revision 16A of + * the T10 Project 1415-D SRP Protocol specification. + */ + +/* Protocol revsion information */ +enum { + SRP_PROTOCOL = 0x0108, + SRP_PROTOCOL_VERSION = 0x0001, + SRP_REV_16A_IO_CLASS = 0x0100, + SRP_REV_10_IO_CLASS = 0xFF00, /* Old targets */ + SRP_IO_SUBCLASS = 0x690E +}; + +/* SRP memory descriptors; direct and indirect formats */ +typedef struct srp_direct_desc_s { + uint64_t dd_vaddr; + uint32_t dd_hdl; + uint32_t dd_len; +} srp_direct_desc_t; + +#pragma pack(1) +typedef struct srp_indirect_desc_s { + srp_direct_desc_t id_table; + uint32_t id_len; + srp_direct_desc_t id_desc[1]; +} srp_indirect_desc_t; +#pragma pack() +enum { + SRP_DIRECT_BUFR_DESC = 1 << 1, + SRP_INDIRECT_BUFR_DESC = 1 << 2 +}; + +/* General constants */ +enum { + SRP_CDB_SIZE = 16, + SRP_LUN_SIZE = 8, + SRP_PORT_ID_LEN = 16, + SRP_MIN_IU_SIZE = 64 +}; + +/* SRP IU types */ +enum { + SRP_IU_LOGIN_REQ = 0x00, + SRP_IU_TASK_MGMT = 0x01, + SRP_IU_CMD = 0x02, + SRP_IU_I_LOGOUT = 0x03, + SRP_IU_LOGIN_RSP = 0xC0, + SRP_IU_RSP = 0xC1, + SRP_IU_LOGIN_REJ = 0xC2, + SRP_IU_T_LOGOUT = 0x80, + SRP_IU_CRED_REQ = 0x81, + SRP_IU_AER_REQ = 0x82, + SRP_IU_CRED_RSP = 0x41, + SRP_IU_AER_RSP = 0x42 +}; + +/* SRP Initiator Login IU, 64 bytes */ +enum { + SRP_LOGIN_MULTI_CH_SINGLE = 0, + SRP_LOGIN_MULTI_CH_MULTIPLE = 1, + SRP_LOGIN_MULTI_CH_MASK = 0x03, + SRP_LOGIN_AESOL_NOTIFICATION = 1 << 6, + SRP_LOGIN_CRSOL_NOTIFICATION = 1 << 5, + SRP_LOGIN_LOSOL_NOTIFICATION = 1 << 4 +}; + +typedef struct srp_login_req_s { + uint8_t lreq_type; + uint8_t lreq_rsvd[7]; + uint64_t lreq_tag; + uint32_t lreq_req_it_iu_len; + uint8_t lreq_rsvd2[4]; + uint16_t lreq_buf_format; + uint8_t lreq_req_flags; + uint8_t lreq_rsvd3[5]; + uint8_t lreq_initiator_port_id[SRP_PORT_ID_LEN]; + uint8_t lreq_target_port_id[SRP_PORT_ID_LEN]; +} srp_login_req_t; + +/* SRP Task Management IU, 64 bytes. */ +enum { + SRP_TSK_MGMT_SUCCESSFUL_COMP_SOLNT = 1 << 1, + SRP_TSK_MGMT_UNSUCCESSFUL_COMP_SOLNT = 1 << 2 +}; + +enum { + SRP_TSK_ATTR_QTYPE_SIMPLE = 0, + SRP_TSK_ATTR_QTYPE_HEAD_OF_Q = 1, + SRP_TSK_ATTR_QTYPE_ORDERED = 2, + SRP_TSK_ATTR_QTYPE_ACA_Q_TAG = 4 +}; + +enum { + SRP_TSK_MGMT_ABORT_TASK = 1, + SRP_TSK_MGMT_ABORT_TASK_SET = 2, + SRP_TSK_MGMT_CLEAR_TASK_SET = 4, + SRP_TSK_MGMT_LUN_RESET = 8, + SRP_TSK_MGMT_CLEAR_ACA = 0x40 +}; + +typedef struct srp_tsk_mgmt_s { + uint8_t tm_type; + uint8_t tm_not_flags; + uint8_t tm_rsvd[6]; + uint64_t tm_tag; + uint8_t tm_rsvd2[4]; + uint8_t tm_lun[8]; + uint8_t tm_rsvd3[2]; + uint8_t tm_function; + uint8_t tm_rsvd4; + uint64_t tm_task_tag; + uint8_t tm_rsvd5[8]; +} srp_tsk_mgmt_t; + +/* SRP Command Request IU, 48 bytes minimum */ +enum { + SRP_DATA_DESC_NONE = 0, + SRP_DATA_DESC_DIRECT = 1, + SRP_DATA_DESC_INDIRECT = 2 +}; + +#pragma pack(1) +typedef struct srp_cmd_req_s { + uint8_t cr_type; + uint8_t cr_not_flags; + uint8_t cr_rsvd[3]; + uint8_t cr_buf_fmt; + uint8_t cr_docnt; + uint8_t cr_dicnt; + uint64_t cr_tag; + uint8_t cr_rsvd2[4]; + uint8_t cr_lun[8]; + uint8_t cr_rsvd3; + uint8_t cr_task_attr; + uint8_t cr_rsvd4; + uint8_t cr_add_cdb_len; + uint8_t cr_cdb[SRP_CDB_SIZE]; + uint8_t cr_add_data; +} srp_cmd_req_t; +#pragma pack() + +/* SRP Initiator Logout IU, 16 bytes */ +typedef struct srp_i_logout_s { + uint8_t il_type; + uint8_t il_rsvd[7]; + uint64_t il_tag; +} srp_i_logout_t; + +/* SRP Login Response IU, 52 bytes */ +enum { + SRP_MULTI_CH_RESULT_NO_EXISTING = 0, + SRP_MULTI_CH_RESULT_TERM_EXISTING = 1, + SRP_MULTI_CH_RESULT_EXISTING_EXISTS = 1 << 1, + SRP_SOLNT_SUPPORTED = 1 << 4 +}; + +#define SRP_LOGIN_RSP_SIZE 52 + +typedef struct srp_login_rsp_s { + uint8_t lrsp_type; + uint8_t lrsp_rsvd[3]; + uint32_t lrsp_req_limit_delta; + uint64_t lrsp_tag; + uint32_t lrsp_max_it_iu_len; + uint32_t lrsp_max_ti_iu_len; + uint16_t lrsp_sup_buf_format; + uint8_t lrsp_rsp_flags; + uint8_t lrsp_rsvd2[25]; +} srp_login_rsp_t; + +/* SRP Response IU, 36 byte minimum */ +enum { + SRP_RSP_SOLICITED_NOTIFICATION = 1 +}; + +enum { + SRP_RSP_VALID = 1, + SRP_RSP_SNS_VALID = 1 << 1, + SRP_RSP_DO_OVER = 1 << 2, + SRP_RSP_DO_UNDER = 1 << 3, + SRP_RSP_DI_OVER = 1 << 4, + SRP_RSP_DI_UNDER = 1 << 5 +}; + +/* Additional response data used for task mgmt responses */ +enum { + SRP_TM_SUCCESS = 0, + SRP_TM_REQ_INVALID = 2, + SRP_TM_NOT_SUPPORTED = 4, + SRP_TM_FAILED = 5 +}; + +typedef struct srp_rsp_data_s { + uint8_t rd_rsvd[3]; + uint8_t rd_rsp_status; +} srp_rsp_data_t; + +#define SRP_RSP_SIZE 36 + +typedef struct srp_rsp_s { + uint8_t rsp_type; + uint8_t rsp_sol_not; + uint8_t rsp_rsvd[2]; + uint32_t rsp_req_limit_delta; + uint64_t rsp_tag; + uint8_t rsp_rsvd2[2]; + uint8_t rsp_flags; + uint8_t rsp_status; + uint32_t rsp_do_resid_cnt; + uint32_t rsp_di_resid_cnt; + uint32_t rsp_sense_data_len; + uint32_t rsp_data_len; +} srp_rsp_t; + +/* SRP Login Reject IU, 32 bytes */ +enum { + SRP_LOGIN_REJ_NO_REASON = 0x00010000, + SRP_LOGIN_REJ_INSUFFICIENT_CH_RESOURCES = 0x00010001, + SRP_LOGIN_REJ_REQ_IT_IU_LENGTH_TOO_LARGE = 0x00010002, + SRP_LOGIN_REJ_UNABLE_TO_ASSOCIATE_I_T_NEXUS = 0x00010003, + SRP_LOGIN_REJ_REQ_BUF_FORMAT_NOT_SUPPORTED = 0x00010004, + SRP_LOGIN_REJ_MULTI_CH_NOT_SUPPORTED = 0x00010005, + SRP_LOGIN_REJ_INIT_CH_LIMIT = 0x00010006 +}; + +typedef struct srp_login_rej_s { + uint8_t lrej_type; + uint8_t lrej_rsvd[3]; + uint32_t lrej_reason; + uint64_t lrej_tag; + uint8_t lrej_rsvd2[8]; + uint16_t lrej_sup_buf_format; + uint8_t lrej_rsvd3[6]; +} srp_login_rej_t; + +/* SRP Target Logout IU, 16 bytes */ +enum { + SRP_T_LOGOUT_NO_REASON = 0, + SRP_T_LOGOUT_INACTIVE = 1, + SRP_T_LOGOUT_INVALID_IU_TYPE = 2, + SRP_T_LOGOUT_UNEXPECTED_INITIATOR_RSP = 3, + SRP_T_LOGOUT_MULTI_CHANNEL_ACTION = 4, + SRP_T_LOGOUT_UNSUPPORTED_DO_FORMAT = 6, + SRP_T_LOGOUT_UNSUPPORTED_DI_FORMAT = 7, + SRP_T_LOGOUT_INVALID_IU_LENGTH = 8 +}; + +typedef struct srp_t_logout_s { + uint8_t tl_type; + uint8_t tl_sol_not; + uint8_t tl_rsvd[2]; + uint32_t tl_reason; + uint64_t tl_tag; +} srp_t_logout_t; + +#ifdef __cplusplus +} +#endif + +#endif /* _SRP_H */ diff --git a/usr/src/uts/common/io/comstar/port/srpt/srpt.conf b/usr/src/uts/common/io/comstar/port/srpt/srpt.conf new file mode 100644 index 0000000000..b5f327a9fd --- /dev/null +++ b/usr/src/uts/common/io/comstar/port/srpt/srpt.conf @@ -0,0 +1,26 @@ +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# +# +# Copyright 2009 Sun Microsystems, Inc. All rights reserved. +# Use is subject to license terms. +# + +name="srpt" parent="ib" unit-address="0"; diff --git a/usr/src/uts/common/io/comstar/port/srpt/srpt_ch.c b/usr/src/uts/common/io/comstar/port/srpt/srpt_ch.c new file mode 100644 index 0000000000..d8b569bec8 --- /dev/null +++ b/usr/src/uts/common/io/comstar/port/srpt/srpt_ch.c @@ -0,0 +1,1446 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ + +/* + * Copyright 2009 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +/* + * RDMA channel interface for Solaris SCSI RDMA Protocol Target (SRP) + * transport port provider module for the COMSTAR framework. + */ + +#include <sys/cpuvar.h> +#include <sys/types.h> +#include <sys/conf.h> +#include <sys/stat.h> +#include <sys/file.h> +#include <sys/ddi.h> +#include <sys/sunddi.h> +#include <sys/modctl.h> +#include <sys/sysmacros.h> +#include <sys/sdt.h> +#include <sys/taskq.h> +#include <sys/scsi/scsi.h> +#include <sys/ib/ibtl/ibti.h> + +#include <stmf.h> +#include <stmf_ioctl.h> +#include <portif.h> + +#include "srp.h" +#include "srpt_impl.h" +#include "srpt_ioc.h" +#include "srpt_stp.h" +#include "srpt_ch.h" + +extern srpt_ctxt_t *srpt_ctxt; +extern uint16_t srpt_send_msg_depth; + +/* + * Prototypes. + */ +static void srpt_ch_scq_hdlr(ibt_cq_hdl_t cq_dhl, void *arg); +static void srpt_ch_rcq_hdlr(ibt_cq_hdl_t cq_dhl, void *arg); +static void srpt_ch_process_iu(srpt_channel_t *ch, srpt_iu_t *iu); + +/* + * srpt_ch_alloc() + */ +srpt_channel_t * +srpt_ch_alloc(srpt_target_port_t *tgt, uint8_t port) +{ + ibt_status_t status; + srpt_channel_t *ch; + ibt_cq_attr_t cq_attr; + ibt_rc_chan_alloc_args_t ch_args; + uint32_t cq_real_size; + srpt_ioc_t *ioc; + + ASSERT(tgt != NULL); + ioc = tgt->tp_ioc; + ASSERT(ioc != NULL); + + ch = kmem_zalloc(sizeof (*ch), KM_SLEEP); + rw_init(&ch->ch_rwlock, NULL, RW_DRIVER, NULL); + mutex_init(&ch->ch_reflock, NULL, MUTEX_DRIVER, NULL); + cv_init(&ch->ch_cv_complete, NULL, CV_DRIVER, NULL); + ch->ch_refcnt = 1; + ch->ch_cv_waiters = 0; + + ch->ch_state = SRPT_CHANNEL_CONNECTING; + ch->ch_tgt = tgt; + ch->ch_req_lim_delta = 0; + ch->ch_ti_iu_len = 0; + + cq_attr.cq_size = srpt_send_msg_depth * 2; + cq_attr.cq_sched = 0; + cq_attr.cq_flags = IBT_CQ_NO_FLAGS; + + status = ibt_alloc_cq(ioc->ioc_ibt_hdl, &cq_attr, &ch->ch_scq_hdl, + &cq_real_size); + if (status != IBT_SUCCESS) { + SRPT_DPRINTF_L1("ch_alloc, send CQ alloc error (%d)", + status); + goto scq_alloc_err; + } + + cq_attr.cq_size = srpt_send_msg_depth + 1; + cq_attr.cq_sched = 0; + cq_attr.cq_flags = IBT_CQ_NO_FLAGS; + + status = ibt_alloc_cq(ioc->ioc_ibt_hdl, &cq_attr, &ch->ch_rcq_hdl, + &cq_real_size); + if (status != IBT_SUCCESS) { + SRPT_DPRINTF_L2("ch_alloc, receive CQ alloc error (%d)", + status); + goto rcq_alloc_err; + } + + ibt_set_cq_handler(ch->ch_scq_hdl, srpt_ch_scq_hdlr, ch); + ibt_set_cq_handler(ch->ch_rcq_hdl, srpt_ch_rcq_hdlr, ch); + ibt_enable_cq_notify(ch->ch_scq_hdl, IBT_NEXT_COMPLETION); + ibt_enable_cq_notify(ch->ch_rcq_hdl, IBT_NEXT_COMPLETION); + + ch_args.rc_flags = IBT_WR_SIGNALED; + + /* Maker certain initiator can not read/write our memory */ + ch_args.rc_control = 0; + + ch_args.rc_hca_port_num = port; + + /* + * Any SRP IU can result in a number of STMF data buffer transfers + * and those transfers themselves could span multiple initiator + * buffers. Therefore, the number of send WQE's actually required + * can vary. Here we assume that on average an I/O will require + * no more than SRPT_MAX_OUT_IO_PER_CMD send WQE's. In practice + * this will prevent send work queue overrun, but we will also + * inform STMF to throttle I/O should the work queue become full. + * + * If the HCA tells us the max outstanding WRs for a channel is + * lower than our default, use the HCA value. + */ + ch_args.rc_sizes.cs_sq = min(ioc->ioc_attr.hca_max_chan_sz, + (srpt_send_msg_depth * SRPT_MAX_OUT_IO_PER_CMD)); + ch_args.rc_sizes.cs_rq = 0; + ch_args.rc_sizes.cs_sq_sgl = 2; + ch_args.rc_sizes.cs_rq_sgl = 0; + + ch_args.rc_scq = ch->ch_scq_hdl; + ch_args.rc_rcq = ch->ch_rcq_hdl; + ch_args.rc_pd = ioc->ioc_pd_hdl; + ch_args.rc_clone_chan = NULL; + ch_args.rc_srq = ioc->ioc_srq_hdl; + + status = ibt_alloc_rc_channel(ioc->ioc_ibt_hdl, IBT_ACHAN_USES_SRQ, + &ch_args, &ch->ch_chan_hdl, &ch->ch_sizes); + if (status != IBT_SUCCESS) { + SRPT_DPRINTF_L2("ch_alloc, IBT channel alloc error (%d)", + status); + goto qp_alloc_err; + } + + /* + * Create pool of send WQE entries to map send wqe work IDs + * to various types (specifically in error cases where OP + * is not known). + */ + ch->ch_num_swqe = ch->ch_sizes.cs_sq; + SRPT_DPRINTF_L2("ch_alloc, number of SWQEs = %u", ch->ch_num_swqe); + ch->ch_swqe = kmem_zalloc(sizeof (srpt_swqe_t) * ch->ch_num_swqe, + KM_SLEEP); + if (ch->ch_swqe == NULL) { + SRPT_DPRINTF_L2("ch_alloc, SWQE alloc error"); + ibt_free_channel(ch->ch_chan_hdl); + goto qp_alloc_err; + } + mutex_init(&ch->ch_swqe_lock, NULL, MUTEX_DRIVER, NULL); + ch->ch_head = 1; + for (ch->ch_tail = 1; ch->ch_tail < ch->ch_num_swqe -1; ch->ch_tail++) { + ch->ch_swqe[ch->ch_tail].sw_next = ch->ch_tail + 1; + } + ch->ch_swqe[ch->ch_tail].sw_next = 0; + + ibt_set_chan_private(ch->ch_chan_hdl, ch); + return (ch); + +qp_alloc_err: + ibt_free_cq(ch->ch_rcq_hdl); + +rcq_alloc_err: + ibt_free_cq(ch->ch_scq_hdl); + +scq_alloc_err: + cv_destroy(&ch->ch_cv_complete); + mutex_destroy(&ch->ch_reflock); + rw_destroy(&ch->ch_rwlock); + kmem_free(ch, sizeof (*ch)); + + return (NULL); +} + +/* + * srpt_ch_add_ref() + */ +void +srpt_ch_add_ref(srpt_channel_t *ch) +{ + mutex_enter(&ch->ch_reflock); + ch->ch_refcnt++; + SRPT_DPRINTF_L4("ch_add_ref, ch (%p), refcnt (%d)", + (void *)ch, ch->ch_refcnt); + ASSERT(ch->ch_refcnt != 0); + mutex_exit(&ch->ch_reflock); +} + +/* + * srpt_ch_release_ref() + * + * A non-zero value for wait causes thread to block until all references + * to channel are released. + */ +void +srpt_ch_release_ref(srpt_channel_t *ch, uint_t wait) +{ + mutex_enter(&ch->ch_reflock); + + SRPT_DPRINTF_L4("ch_release_ref, ch (%p), refcnt (%d), wait (%d)", + (void *)ch, ch->ch_refcnt, wait); + + ASSERT(ch->ch_refcnt != 0); + + ch->ch_refcnt--; + + if (ch->ch_refcnt != 0) { + if (wait) { + ch->ch_cv_waiters++; + while (ch->ch_refcnt != 0) { + cv_wait(&ch->ch_cv_complete, &ch->ch_reflock); + } + ch->ch_cv_waiters--; + } else { + mutex_exit(&ch->ch_reflock); + return; + } + } + + /* + * Last thread out frees the IB resources, locks/conditions and memory + */ + if (ch->ch_cv_waiters > 0) { + /* we're not last, wake someone else up */ + cv_signal(&ch->ch_cv_complete); + mutex_exit(&ch->ch_reflock); + return; + } + + SRPT_DPRINTF_L3("ch_release_ref - release resources"); + if (ch->ch_chan_hdl) { + SRPT_DPRINTF_L3("ch_release_ref - free channel"); + ibt_free_channel(ch->ch_chan_hdl); + } + + if (ch->ch_scq_hdl) { + ibt_free_cq(ch->ch_scq_hdl); + } + + if (ch->ch_rcq_hdl) { + ibt_free_cq(ch->ch_rcq_hdl); + } + + /* + * There should be no IU's associated with this + * channel on the SCSI session. + */ + if (ch->ch_session != NULL) { + ASSERT(list_is_empty(&ch->ch_session->ss_task_list)); + + /* + * Currently only have one channel per session, we will + * need to release a reference when support is added + * for multi-channel target login. + */ + srpt_stp_free_session(ch->ch_session); + ch->ch_session = NULL; + } + + kmem_free(ch->ch_swqe, sizeof (srpt_swqe_t) * ch->ch_num_swqe); + mutex_destroy(&ch->ch_swqe_lock); + mutex_exit(&ch->ch_reflock); + mutex_destroy(&ch->ch_reflock); + rw_destroy(&ch->ch_rwlock); + kmem_free(ch, sizeof (srpt_channel_t)); +} + +/* + * srpt_ch_disconnect() + */ +void +srpt_ch_disconnect(srpt_channel_t *ch) +{ + ibt_status_t status; + + SRPT_DPRINTF_L3("ch_disconnect, invoked for ch (%p)", + (void *)ch); + + rw_enter(&ch->ch_rwlock, RW_WRITER); + + /* + * If we are already in the process of disconnecting then + * nothing need be done, CM will call-back into us when done. + */ + if (ch->ch_state == SRPT_CHANNEL_DISCONNECTING) { + SRPT_DPRINTF_L2("ch_disconnect, called when" + " disconnect in progress"); + rw_exit(&ch->ch_rwlock); + return; + } + ch->ch_state = SRPT_CHANNEL_DISCONNECTING; + rw_exit(&ch->ch_rwlock); + + /* + * Initiate the sending of the CM DREQ message, the private data + * should be the SRP Target logout IU. We don't really care about + * the remote CM DREP message returned. We issue this in an + * asynchronous manner and will cleanup when called back by CM. + */ + status = ibt_close_rc_channel(ch->ch_chan_hdl, IBT_NONBLOCKING, + NULL, 0, NULL, NULL, 0); + + if (status != IBT_SUCCESS) { + SRPT_DPRINTF_L2("ch_disconnect, close RC channel" + " err(%d)", status); + } +} + +/* + * srpt_ch_cleanup() + */ +void +srpt_ch_cleanup(srpt_channel_t *ch) +{ + srpt_iu_t *iu; + srpt_iu_t *next; + ibt_wc_t wc; + srpt_target_port_t *tgt; + srpt_channel_t *tgt_ch; + scsi_task_t *iutask; + + SRPT_DPRINTF_L3("ch_cleanup, invoked for ch(%p), state(%d)", + (void *)ch, ch->ch_state); + + /* add a ref for the channel until we're done */ + srpt_ch_add_ref(ch); + + tgt = ch->ch_tgt; + ASSERT(tgt != NULL); + + /* + * Make certain the channel is in the target ports list of + * known channels and remove it (releasing the target + * ports reference to the channel). + */ + mutex_enter(&tgt->tp_ch_list_lock); + tgt_ch = list_head(&tgt->tp_ch_list); + while (tgt_ch != NULL) { + if (tgt_ch == ch) { + list_remove(&tgt->tp_ch_list, tgt_ch); + srpt_ch_release_ref(tgt_ch, 0); + break; + } + tgt_ch = list_next(&tgt->tp_ch_list, tgt_ch); + } + mutex_exit(&tgt->tp_ch_list_lock); + + if (tgt_ch == NULL) { + SRPT_DPRINTF_L2("ch_cleanup, target channel no" + "longer known to target"); + srpt_ch_release_ref(ch, 0); + return; + } + + rw_enter(&ch->ch_rwlock, RW_WRITER); + ch->ch_state = SRPT_CHANNEL_DISCONNECTING; + rw_exit(&ch->ch_rwlock); + + + /* + * Generally the IB CQ's will have been drained prior to + * getting to this call; but we check here to make certain. + */ + if (ch->ch_scq_hdl) { + SRPT_DPRINTF_L4("ch_cleanup, start drain (%d)", + ch->ch_swqe_posted); + while ((int)ch->ch_swqe_posted > 0) { + delay(drv_usectohz(1000)); + } + ibt_set_cq_handler(ch->ch_scq_hdl, NULL, NULL); + } + + if (ch->ch_rcq_hdl) { + ibt_set_cq_handler(ch->ch_rcq_hdl, NULL, NULL); + + while (ibt_poll_cq(ch->ch_rcq_hdl, &wc, 1, NULL) == + IBT_SUCCESS) { + iu = (srpt_iu_t *)(uintptr_t)wc.wc_id; + SRPT_DPRINTF_L4("ch_cleanup, recovering" + " outstanding RX iu(%p)", (void *)iu); + mutex_enter(&iu->iu_lock); + srpt_ioc_repost_recv_iu(iu->iu_ioc, iu); + /* + * Channel reference has not yet been added for this + * IU, so do not decrement. + */ + mutex_exit(&iu->iu_lock); + } + } + + /* + * Go through the list of outstanding IU for the channel's SCSI + * session and for each either abort or complete an abort. + */ + rw_enter(&ch->ch_rwlock, RW_READER); + if (ch->ch_session != NULL) { + rw_enter(&ch->ch_session->ss_rwlock, RW_READER); + iu = list_head(&ch->ch_session->ss_task_list); + while (iu != NULL) { + next = list_next(&ch->ch_session->ss_task_list, iu); + + mutex_enter(&iu->iu_lock); + if (ch == iu->iu_ch) { + if (iu->iu_stmf_task == NULL) { + cmn_err(CE_NOTE, + "ch_cleanup, NULL stmf task"); + ASSERT(0); + } + iutask = iu->iu_stmf_task; + } else { + iutask = NULL; + } + mutex_exit(&iu->iu_lock); + + if (iutask != NULL) { + SRPT_DPRINTF_L4("ch_cleanup, aborting " + "task(%p)", (void *)iutask); + stmf_abort(STMF_QUEUE_TASK_ABORT, iutask, + STMF_ABORTED, NULL); + } + iu = next; + } + rw_exit(&ch->ch_session->ss_rwlock); + } + rw_exit(&ch->ch_rwlock); + + srpt_ch_release_ref(ch, 0); +} + +/* + * srpt_ch_rsp_comp() + * + * Process a completion for an IB SEND message. A SEND completion + * is for a SRP response packet sent back to the initiator. It + * will not have a STMF SCSI task associated with it if it was + * sent for a rejected IU, or was a task management abort response. + */ +static void +srpt_ch_rsp_comp(srpt_channel_t *ch, srpt_iu_t *iu, + ibt_wc_status_t wc_status) +{ + ASSERT(iu->iu_ch == ch); + + /* + * If work completion indicates failure, decrement the + * send posted count. If it is a flush error, we are + * done; for all other errors start a channel disconnect. + */ + if (wc_status != IBT_SUCCESS) { + SRPT_DPRINTF_L2("ch_rsp_comp, WC status err(%d)", + wc_status); + atomic_dec_32(&iu->iu_sq_posted_cnt); + + if (wc_status != IBT_WC_WR_FLUSHED_ERR) { + srpt_ch_disconnect(ch); + } + + mutex_enter(&iu->iu_lock); + if (iu->iu_stmf_task == NULL) { + srpt_ioc_repost_recv_iu(iu->iu_ioc, iu); + mutex_exit(&iu->iu_lock); + srpt_ch_release_ref(ch, 0); + } else { + /* cleanup handled in task_free */ + mutex_exit(&iu->iu_lock); + } + return; + } + + /* + * If the IU response completion is not associated with + * with a SCSI task, release the IU to return the resource + * and the reference to the channel it holds. + */ + mutex_enter(&iu->iu_lock); + atomic_dec_32(&iu->iu_sq_posted_cnt); + + if (iu->iu_stmf_task == NULL) { + srpt_ioc_repost_recv_iu(iu->iu_ioc, iu); + mutex_exit(&iu->iu_lock); + srpt_ch_release_ref(ch, 0); + return; + } + + /* + * If STMF has requested the IU task be aborted, then notify STMF + * the command is now aborted. + */ + if ((iu->iu_flags & SRPT_IU_STMF_ABORTING) != 0) { + scsi_task_t *abort_task = iu->iu_stmf_task; + + mutex_exit(&iu->iu_lock); + stmf_abort(STMF_REQUEUE_TASK_ABORT_LPORT, abort_task, + STMF_ABORTED, NULL); + return; + } + + /* + * We should not get a SEND completion where the task has already + * completed aborting and STMF has been informed. + */ + ASSERT((iu->iu_flags & SRPT_IU_ABORTED) == 0); + + /* + * Successful status response completion for SCSI task. + * Let STMF know we are done. + */ + mutex_exit(&iu->iu_lock); + + stmf_send_status_done(iu->iu_stmf_task, STMF_SUCCESS, + STMF_IOF_LPORT_DONE); +} + +/* + * srpt_ch_data_comp() + * + * Process an IB completion for a RDMA operation. This completion + * should be associated with the last RDMA operation for any + * data buffer transfer. + */ +static void +srpt_ch_data_comp(srpt_channel_t *ch, stmf_data_buf_t *stmf_dbuf, + ibt_wc_status_t wc_status) +{ + srpt_ds_dbuf_t *dbuf; + srpt_iu_t *iu; + stmf_status_t status; + + ASSERT(stmf_dbuf != NULL); + + dbuf = (srpt_ds_dbuf_t *)stmf_dbuf->db_port_private; + + ASSERT(dbuf != NULL); + + iu = dbuf->db_iu; + + ASSERT(iu != NULL); + ASSERT(iu->iu_ch == ch); + + /* + * If work completion indicates non-flush failure, then + * start a channel disconnect (asynchronous) and release + * the reference to the IU. The task will be cleaned + * up with STMF during channel shutdown processing. + */ + if (wc_status != IBT_SUCCESS) { + SRPT_DPRINTF_L2("ch_data_comp, WC status err(%d)", + wc_status); + if (wc_status != IBT_WC_WR_FLUSHED_ERR) { + srpt_ch_disconnect(ch); + } + atomic_dec_32(&iu->iu_sq_posted_cnt); + return; + } + + /* + * If STMF has requested this task be aborted, then if this is the + * last I/O operation outstanding, notify STMF the task has been + * aborted and ignore the completion. + */ + mutex_enter(&iu->iu_lock); + atomic_dec_32(&iu->iu_sq_posted_cnt); + + if ((iu->iu_flags & SRPT_IU_STMF_ABORTING) != 0) { + scsi_task_t *abort_task = iu->iu_stmf_task; + + mutex_exit(&iu->iu_lock); + stmf_abort(STMF_REQUEUE_TASK_ABORT_LPORT, abort_task, + STMF_ABORTED, NULL); + return; + } + + /* + * We should not get an RDMA completion where the task has already + * completed aborting and STMF has been informed. + */ + ASSERT((iu->iu_flags & SRPT_IU_ABORTED) == 0); + + /* + * Good completion for last RDMA op associated with a data buffer + * I/O, if specified initiate status otherwise let STMF know we are + * done. + */ + stmf_dbuf->db_xfer_status = STMF_SUCCESS; + mutex_exit(&iu->iu_lock); + if ((stmf_dbuf->db_flags & DB_SEND_STATUS_GOOD) != 0) { + status = srpt_stp_send_status(dbuf->db_iu->iu_stmf_task, 0); + if (status == STMF_SUCCESS) { + return; + } + stmf_dbuf->db_xfer_status = STMF_FAILURE; + } + + stmf_data_xfer_done(dbuf->db_iu->iu_stmf_task, stmf_dbuf, 0); +} + +/* + * srpt_ch_scq_hdlr() + */ +static void +srpt_ch_scq_hdlr(ibt_cq_hdl_t cq_hdl, void *arg) +{ + ibt_status_t status; + srpt_channel_t *ch = arg; + ibt_wc_t wc[SRPT_SEND_WC_POLL_SIZE]; + ibt_wc_t *wcp; + int i; + uint32_t cq_rearmed = 0; + uint32_t entries; + srpt_swqe_t *swqe; + + ASSERT(ch != NULL); + + /* Reference channel for the duration of this call */ + srpt_ch_add_ref(ch); + + for (;;) { + status = ibt_poll_cq(cq_hdl, &wc[0], SRPT_SEND_WC_POLL_SIZE, + &entries); + if (status == IBT_CQ_EMPTY) { + /* + * CQ drained, if we have not rearmed the CQ + * do so and poll to eliminate race; otherwise + * we are done. + */ + if (cq_rearmed == 0) { + ibt_enable_cq_notify(ch->ch_scq_hdl, + IBT_NEXT_COMPLETION); + cq_rearmed = 1; + continue; + } else { + break; + } + } else if (status != IBT_SUCCESS) { + /* + * This error should not happen, it indicates something + * abnormal has gone wrong and represents either a + * hardware or programming logic coding error. + */ + SRPT_DPRINTF_L2("ch_scq_hdlr, unexpected CQ err(%d)", + status); + srpt_ch_disconnect(ch); + break; + } + + for (wcp = wc, i = 0; i < entries; i++, wcp++) { + + /* + * A zero work ID indicates this CQE is associated + * with an intermediate post of a RDMA data transfer + * operation. Since intermediate data requests are + * unsignaled, we should only get these if there was + * an error. No action is required. + */ + if (wcp->wc_id == 0) { + continue; + } + swqe = ch->ch_swqe + wcp->wc_id; + + switch (swqe->sw_type) { + case SRPT_SWQE_TYPE_RESP: + srpt_ch_rsp_comp(ch, (srpt_iu_t *) + swqe->sw_addr, wcp->wc_status); + break; + + case SRPT_SWQE_TYPE_DATA: + srpt_ch_data_comp(ch, (stmf_data_buf_t *) + swqe->sw_addr, wcp->wc_status); + break; + + default: + SRPT_DPRINTF_L2("ch_scq_hdlr, bad type(%d)", + swqe->sw_type); + ASSERT(0); + } + + srpt_ch_free_swqe_wrid(ch, wcp->wc_id); + } + } + + srpt_ch_release_ref(ch, 0); +} + +/* + * srpt_ch_rcq_hdlr() + */ +static void +srpt_ch_rcq_hdlr(ibt_cq_hdl_t cq_hdl, void *arg) +{ + ibt_status_t status; + srpt_channel_t *ch = arg; + ibt_wc_t wc[SRPT_RECV_WC_POLL_SIZE]; + ibt_wc_t *wcp; + int i; + uint32_t entries; + srpt_iu_t *iu; + uint_t cq_rearmed = 0; + + /* + * The channel object will exists while the CQ handler call-back + * is installed. + */ + ASSERT(ch != NULL); + srpt_ch_add_ref(ch); + + /* + * If we know a channel disconnect has started do nothing + * and let channel cleanup code recover resources from the CQ. + * We are not concerned about races with the state transition + * since the code will do the correct thing either way. This + * is simply to circumvent rearming the CQ, and it will + * catch the state next time. + */ + rw_enter(&ch->ch_rwlock, RW_READER); + if (ch->ch_state == SRPT_CHANNEL_DISCONNECTING) { + SRPT_DPRINTF_L2("ch_rcq_hdlr, channel disconnecting"); + rw_exit(&ch->ch_rwlock); + srpt_ch_release_ref(ch, 0); + return; + } + rw_exit(&ch->ch_rwlock); + + for (;;) { + status = ibt_poll_cq(cq_hdl, &wc[0], SRPT_RECV_WC_POLL_SIZE, + &entries); + if (status == IBT_CQ_EMPTY) { + /* + * OK, empty, if we have not rearmed the CQ + * do so, and poll to eliminate race; otherwise + * we are done. + */ + if (cq_rearmed == 0) { + ibt_enable_cq_notify(ch->ch_rcq_hdl, + IBT_NEXT_COMPLETION); + cq_rearmed = 1; + continue; + } else { + break; + } + } else if (status != IBT_SUCCESS) { + /* + * This error should not happen, it indicates something + * abnormal has gone wrong and represents either a + * hardware or programming logic coding error. + */ + SRPT_DPRINTF_L2("ch_rcq_hdlr, unexpected CQ err(%d)", + status); + srpt_ch_disconnect(ch); + break; + } + + for (wcp = wc, i = 0; i < entries; i++, wcp++) { + + /* + * Check wc_status before proceeding. If the + * status indicates a channel problem, stop processing. + */ + if (wcp->wc_status != IBT_WC_SUCCESS) { + if (wcp->wc_status == IBT_WC_WR_FLUSHED_ERR) { + SRPT_DPRINTF_L2( + "ch_rcq, unexpected" + " wc_status err(%d)", + wcp->wc_status); + srpt_ch_disconnect(ch); + /* XXX - verify not leaking IUs */ + goto done; + } else { + /* skip IUs with errors */ + SRPT_DPRINTF_L2( + "ch_rcq, ERROR comp(%d)", + wcp->wc_status); + /* XXX - verify not leaking IUs */ + continue; + } + } + + iu = (srpt_iu_t *)(uintptr_t)wcp->wc_id; + ASSERT(iu != NULL); + + /* + * Process the IU. + */ + ASSERT(wcp->wc_type == IBT_WRC_RECV); + srpt_ch_process_iu(ch, iu); + } + } + +done: + srpt_ch_release_ref(ch, 0); +} + +/* + * srpt_ch_srp_cmd() + */ +static int +srpt_ch_srp_cmd(srpt_channel_t *ch, srpt_iu_t *iu) +{ + srp_cmd_req_t *cmd = (srp_cmd_req_t *)iu->iu_buf; + srp_indirect_desc_t *i_desc; + uint_t i_di_cnt; + uint_t i_do_cnt; + uint8_t do_fmt; + uint8_t di_fmt; + uint32_t *cur_desc_off; + int i; + ibt_status_t status; + uint8_t addlen; + + iu->iu_ch = ch; + iu->iu_tag = cmd->cr_tag; + + /* + * The SRP specification and SAM require support for bi-directional + * data transfer, so we create a single buffer descriptor list that + * in the IU buffer that covers the data-in and data-out buffers. + * In practice we will just see unidirectional transfers with either + * data-in or data out descriptors. If we were to take that as fact, + * we could reduce overhead slightly. + */ + + /* + * additional length is a 6-bit number in 4-byte words, so multiply by 4 + * to get bytes. + */ + addlen = cmd->cr_add_cdb_len & 0x3f; /* mask off 6 bits */ + + cur_desc_off = (uint32_t *)(void *)&cmd->cr_add_data; + cur_desc_off += addlen; /* 32-bit arithmetic */ + iu->iu_num_rdescs = 0; + iu->iu_rdescs = (srp_direct_desc_t *)(void *)cur_desc_off; + + /* + * Examine buffer description for Data In (i.e. data flows + * to the initiator). + */ + i_do_cnt = i_di_cnt = 0; + di_fmt = cmd->cr_buf_fmt >> 4; + if (di_fmt == SRP_DATA_DESC_DIRECT) { + iu->iu_num_rdescs = 1; + cur_desc_off = (uint32_t *)(void *)&iu->iu_rdescs[1]; + } else if (di_fmt == SRP_DATA_DESC_INDIRECT) { + i_desc = (srp_indirect_desc_t *)iu->iu_rdescs; + i_di_cnt = b2h32(i_desc->id_table.dd_len) / + sizeof (srp_direct_desc_t); + + /* + * Some initiators like OFED occasionally use the wrong counts, + * so check total to allow for this. NOTE: we do not support + * reading of the descriptor table from the initiator, so if + * not all descriptors are in the IU we drop the task. + */ + if (i_di_cnt > (cmd->cr_dicnt + cmd->cr_docnt)) { + SRPT_DPRINTF_L2("ch_srp_cmd, remote RDMA of" + " descriptors not supported"); + SRPT_DPRINTF_L2("ch_srp_cmd, sizeof entry (%d)," + " i_di_cnt(%d), cr_dicnt(%d)", + (uint_t)sizeof (srp_direct_desc_t), + i_di_cnt, cmd->cr_dicnt); + iu->iu_rdescs = NULL; + return (1); + } + bcopy(&i_desc->id_desc[0], iu->iu_rdescs, + sizeof (srp_direct_desc_t) * i_di_cnt); + iu->iu_num_rdescs += i_di_cnt; + cur_desc_off = (uint32_t *)(void *)&i_desc->id_desc[i_di_cnt]; + } + + /* + * Examine buffer description for Data Out (i.e. data flows + * from the initiator). + */ + do_fmt = cmd->cr_buf_fmt & 0x0F; + if (do_fmt == SRP_DATA_DESC_DIRECT) { + if (di_fmt == SRP_DATA_DESC_DIRECT) { + bcopy(cur_desc_off, &iu->iu_rdescs[iu->iu_num_rdescs], + sizeof (srp_direct_desc_t)); + } + iu->iu_num_rdescs++; + } else if (do_fmt == SRP_DATA_DESC_INDIRECT) { + i_desc = (srp_indirect_desc_t *)cur_desc_off; + i_do_cnt = b2h32(i_desc->id_table.dd_len) / + sizeof (srp_direct_desc_t); + + /* + * Some initiators like OFED occasionally use the wrong counts, + * so check total to allow for this. NOTE: we do not support + * reading of the descriptor table from the initiator, so if + * not all descriptors are in the IU we drop the task. + */ + if ((i_di_cnt + i_do_cnt) > (cmd->cr_dicnt + cmd->cr_docnt)) { + SRPT_DPRINTF_L2("ch_srp_cmd, remote RDMA of" + " descriptors not supported"); + SRPT_DPRINTF_L2("ch_srp_cmd, sizeof entry (%d)," + " i_do_cnt(%d), cr_docnt(%d)", + (uint_t)sizeof (srp_direct_desc_t), + i_do_cnt, cmd->cr_docnt); + iu->iu_rdescs = 0; + return (1); + } + bcopy(&i_desc->id_desc[0], &iu->iu_rdescs[iu->iu_num_rdescs], + sizeof (srp_direct_desc_t) * i_do_cnt); + iu->iu_num_rdescs += i_do_cnt; + } + + iu->iu_tot_xfer_len = 0; + for (i = 0; i < iu->iu_num_rdescs; i++) { + iu->iu_rdescs[i].dd_vaddr = b2h64(iu->iu_rdescs[i].dd_vaddr); + iu->iu_rdescs[i].dd_hdl = b2h32(iu->iu_rdescs[i].dd_hdl); + iu->iu_rdescs[i].dd_len = b2h32(iu->iu_rdescs[i].dd_len); + iu->iu_tot_xfer_len += iu->iu_rdescs[i].dd_len; + } + +#ifdef DEBUG + if (srpt_errlevel >= SRPT_LOG_L4) { + SRPT_DPRINTF_L4("ch_srp_cmd, iu->iu_tot_xfer_len (%d)", + iu->iu_tot_xfer_len); + for (i = 0; i < iu->iu_num_rdescs; i++) { + SRPT_DPRINTF_L4("ch_srp_cmd, rdescs[%d].dd_vaddr" + " (0x%08llx)", + i, (u_longlong_t)iu->iu_rdescs[i].dd_vaddr); + SRPT_DPRINTF_L4("ch_srp_cmd, rdescs[%d].dd_hdl" + " (0x%08x)", i, iu->iu_rdescs[i].dd_hdl); + SRPT_DPRINTF_L4("ch_srp_cmd, rdescs[%d].dd_len (%d)", + i, iu->iu_rdescs[i].dd_len); + } + SRPT_DPRINTF_L4("ch_srp_cmd, LUN (0x%08lx)", + (unsigned long int) *((uint64_t *)(void *) cmd->cr_lun)); + } +#endif + rw_enter(&ch->ch_rwlock, RW_READER); + + if (ch->ch_state == SRPT_CHANNEL_DISCONNECTING) { + /* + * The channel has begun disconnecting, so ignore the + * the command returning the IU resources. + */ + rw_exit(&ch->ch_rwlock); + return (1); + } + + /* + * Once a SCSI task is allocated and assigned to the IU, it + * owns those IU resources, which will be held until STMF + * is notified the task is done (from a lport perspective). + */ + iu->iu_stmf_task = stmf_task_alloc(ch->ch_tgt->tp_lport, + ch->ch_session->ss_ss, cmd->cr_lun, + SRP_CDB_SIZE + (addlen * 4), 0); + if (iu->iu_stmf_task == NULL) { + /* + * Could not allocate, return status to the initiator + * indicating that we are temporarily unable to process + * commands. If unable to send, immediately return IU + * resource. + */ + SRPT_DPRINTF_L2("ch_srp_cmd, SCSI task allocation failure"); + rw_exit(&ch->ch_rwlock); + mutex_enter(&iu->iu_lock); + status = srpt_stp_send_response(iu, STATUS_BUSY, 0, 0, 0, + NULL, SRPT_NO_FENCE_SEND); + mutex_exit(&iu->iu_lock); + if (status != IBT_SUCCESS) { + SRPT_DPRINTF_L2("ch_srp_cmd, error(%d) posting error" + " response", status); + return (1); + } else { + return (0); + } + } + + iu->iu_stmf_task->task_port_private = iu; + iu->iu_stmf_task->task_flags = 0; + + if (di_fmt != 0) { + iu->iu_stmf_task->task_flags |= TF_WRITE_DATA; + } + if (do_fmt != 0) { + iu->iu_stmf_task->task_flags |= TF_READ_DATA; + } + + switch (cmd->cr_task_attr) { + case SRP_TSK_ATTR_QTYPE_SIMPLE: + iu->iu_stmf_task->task_flags |= TF_ATTR_SIMPLE_QUEUE; + break; + + case SRP_TSK_ATTR_QTYPE_HEAD_OF_Q: + iu->iu_stmf_task->task_flags |= TF_ATTR_HEAD_OF_QUEUE; + break; + + case SRP_TSK_ATTR_QTYPE_ORDERED: + iu->iu_stmf_task->task_flags |= TF_ATTR_ORDERED_QUEUE; + break; + + case SRP_TSK_ATTR_QTYPE_ACA_Q_TAG: + iu->iu_stmf_task->task_flags |= TF_ATTR_ACA; + break; + + default: + SRPT_DPRINTF_L2("ch_srp_cmd, reserved task attr (%d)", + cmd->cr_task_attr); + iu->iu_stmf_task->task_flags |= TF_ATTR_ORDERED_QUEUE; + break; + } + iu->iu_stmf_task->task_additional_flags = 0; + iu->iu_stmf_task->task_priority = 0; + iu->iu_stmf_task->task_mgmt_function = TM_NONE; + iu->iu_stmf_task->task_max_nbufs = STMF_BUFS_MAX; + iu->iu_stmf_task->task_expected_xfer_length = iu->iu_tot_xfer_len; + iu->iu_stmf_task->task_csn_size = 0; + + bcopy(cmd->cr_cdb, iu->iu_stmf_task->task_cdb, + SRP_CDB_SIZE); + if (addlen != 0) { + bcopy(&cmd->cr_add_data, + iu->iu_stmf_task->task_cdb + SRP_CDB_SIZE, + addlen * 4); + } + + /* + * Add the IU/task to the session and post to STMF. The task will + * remain in the session's list until STMF is informed by SRP that + * it is done with the task. + */ + srpt_stp_add_task(ch->ch_session, iu); + + SRPT_DPRINTF_L3("ch_srp_cmd, new task (%p) posted", + (void *)iu->iu_stmf_task); + stmf_post_task(iu->iu_stmf_task, NULL); + rw_exit(&ch->ch_rwlock); + + return (0); +} + +/* + * srpt_ch_task_mgmt_abort() + * + * Returns 0 on success, indicating we've sent a management response. + * Returns !0 to indicate failure; the IU should be reposted. + */ +static ibt_status_t +srpt_ch_task_mgmt_abort(srpt_channel_t *ch, srpt_iu_t *iu, + uint64_t tag_to_abort) +{ + srpt_session_t *session = ch->ch_session; + srpt_iu_t *ss_iu; + ibt_status_t status; + + /* + * Locate the associated task (tag_to_abort) in the + * session's active task list. + */ + rw_enter(&session->ss_rwlock, RW_READER); + ss_iu = list_head(&session->ss_task_list); + while (ss_iu != NULL) { + mutex_enter(&ss_iu->iu_lock); + if ((tag_to_abort == ss_iu->iu_tag)) { + mutex_exit(&ss_iu->iu_lock); + break; + } + mutex_exit(&ss_iu->iu_lock); + ss_iu = list_next(&session->ss_task_list, ss_iu); + } + rw_exit(&session->ss_rwlock); + + /* + * Take appropriate action based on state of task + * to be aborted: + * 1) No longer exists - do nothing. + * 2) Previously aborted or status queued - do nothing. + * 3) Otherwise - initiate abort. + */ + if (ss_iu == NULL) { + goto send_mgmt_resp; + } + + mutex_enter(&ss_iu->iu_lock); + if ((ss_iu->iu_flags & (SRPT_IU_STMF_ABORTING | + SRPT_IU_ABORTED | SRPT_IU_RESP_SENT)) != 0) { + mutex_exit(&ss_iu->iu_lock); + goto send_mgmt_resp; + } + + /* + * Set aborting flag and notify STMF of abort request. No + * additional I/O will be queued for this IU. + */ + SRPT_DPRINTF_L3("ch_task_mgmt_abort, task found"); + ss_iu->iu_flags |= SRPT_IU_SRP_ABORTING; + mutex_exit(&ss_iu->iu_lock); + stmf_abort(STMF_QUEUE_TASK_ABORT, + ss_iu->iu_stmf_task, STMF_ABORTED, NULL); + +send_mgmt_resp: + mutex_enter(&iu->iu_lock); + status = srpt_stp_send_mgmt_response(iu, SRP_TM_SUCCESS, + SRPT_FENCE_SEND); + mutex_exit(&iu->iu_lock); + + if (status != IBT_SUCCESS) { + SRPT_DPRINTF_L2("ch_task_mgmt_abort, err(%d)" + " posting abort response", status); + } + + return (status); +} + +/* + * srpt_ch_srp_task_mgmt() + */ +static int +srpt_ch_srp_task_mgmt(srpt_channel_t *ch, srpt_iu_t *iu) +{ + srp_tsk_mgmt_t *tsk = (srp_tsk_mgmt_t *)iu->iu_buf; + uint8_t tm_fn; + ibt_status_t status; + + SRPT_DPRINTF_L3("ch_srp_task_mgmt, SRP TASK MGMT func(%d)", + tsk->tm_function); + + iu->iu_ch = ch; + iu->iu_tag = tsk->tm_tag; + + /* + * Task management aborts are processed directly by the SRP driver; + * all other task management requests are handed off to STMF. + */ + switch (tsk->tm_function) { + case SRP_TSK_MGMT_ABORT_TASK: + /* + * Initiate SCSI transport protocol specific task abort + * logic. + */ + status = srpt_ch_task_mgmt_abort(ch, iu, tsk->tm_task_tag); + if (status != IBT_SUCCESS) { + /* repost this IU */ + return (1); + } else { + return (0); + } + + case SRP_TSK_MGMT_ABORT_TASK_SET: + tm_fn = TM_ABORT_TASK_SET; + break; + + case SRP_TSK_MGMT_CLEAR_TASK_SET: + tm_fn = TM_CLEAR_TASK_SET; + break; + + case SRP_TSK_MGMT_LUN_RESET: + tm_fn = TM_LUN_RESET; + break; + + case SRP_TSK_MGMT_CLEAR_ACA: + tm_fn = TM_CLEAR_ACA; + break; + + default: + /* + * SRP does not support the requested task management + * function; return a not supported status in the response. + */ + SRPT_DPRINTF_L2("ch_srp_task_mgmt, SRP task mgmt fn(%d)" + " not supported", tsk->tm_function); + mutex_enter(&iu->iu_lock); + status = srpt_stp_send_mgmt_response(iu, + SRP_TM_NOT_SUPPORTED, SRPT_NO_FENCE_SEND); + mutex_exit(&iu->iu_lock); + if (status != IBT_SUCCESS) { + SRPT_DPRINTF_L2("ch_srp_task_mgmt, err(%d) posting" + " response", status); + return (1); + } + return (0); + } + + rw_enter(&ch->ch_rwlock, RW_READER); + if (ch->ch_state == SRPT_CHANNEL_DISCONNECTING) { + /* + * The channel has begun disconnecting, so ignore the + * the command returning the IU resources. + */ + rw_exit(&ch->ch_rwlock); + return (1); + } + + /* + * Once a SCSI mgmt task is allocated and assigned to the IU, it + * owns those IU resources, which will be held until we inform + * STMF that we are done with the task (from an lports perspective). + */ + iu->iu_stmf_task = stmf_task_alloc(ch->ch_tgt->tp_lport, + ch->ch_session->ss_ss, tsk->tm_lun, 0, STMF_TASK_EXT_NONE); + if (iu->iu_stmf_task == NULL) { + /* + * Could not allocate, return status to the initiator + * indicating that we are temporarily unable to process + * commands. If unable to send, immediately return IU + * resource. + */ + SRPT_DPRINTF_L2("ch_srp_task_mgmt, SCSI task allocation" + " failure"); + rw_exit(&ch->ch_rwlock); + mutex_enter(&iu->iu_lock); + status = srpt_stp_send_response(iu, STATUS_BUSY, 0, 0, 0, + NULL, SRPT_NO_FENCE_SEND); + mutex_exit(&iu->iu_lock); + if (status != IBT_SUCCESS) { + SRPT_DPRINTF_L2("ch_srp_task_mgmt, err(%d) posting" + "busy response", status); + /* repost the IU */ + return (1); + } + return (0); + } + + iu->iu_stmf_task->task_port_private = iu; + iu->iu_stmf_task->task_flags = 0; + iu->iu_stmf_task->task_additional_flags = + TASK_AF_NO_EXPECTED_XFER_LENGTH; + iu->iu_stmf_task->task_priority = 0; + iu->iu_stmf_task->task_mgmt_function = tm_fn; + iu->iu_stmf_task->task_max_nbufs = STMF_BUFS_MAX; + iu->iu_stmf_task->task_expected_xfer_length = 0; + iu->iu_stmf_task->task_csn_size = 0; + + /* + * Add the IU/task to the session and post to STMF. The task will + * remain in the session's list until STMF is informed by SRP that + * it is done with the task. + */ + srpt_stp_add_task(ch->ch_session, iu); + + SRPT_DPRINTF_L3("ch_srp_task_mgmt, new mgmt task(%p) posted", + (void *)iu->iu_stmf_task); + stmf_post_task(iu->iu_stmf_task, NULL); + rw_exit(&ch->ch_rwlock); + + return (0); +} + +/* + * srpt_ch_process_iu() + */ +static void +srpt_ch_process_iu(srpt_channel_t *ch, srpt_iu_t *iu) +{ + srpt_iu_data_t *iud; + int status = 1; + + /* + * IU adds reference to channel which will represent a + * a reference by STMF. If for whatever reason the IU + * is not handed off to STMF, then this reference will be + * released. Otherwise, the reference will be released when + * SRP informs STMF that the associated SCSI task is done. + */ + srpt_ch_add_ref(ch); + + /* + * Validate login RC channel state. Normally active, if + * not active then we need to handle a possible race between the + * receipt of a implied RTU and CM calling back to notify of the + * state transition. + */ + rw_enter(&ch->ch_rwlock, RW_READER); + if (ch->ch_state == SRPT_CHANNEL_DISCONNECTING) { + rw_exit(&ch->ch_rwlock); + goto repost_iu; + } + rw_exit(&ch->ch_rwlock); + + iud = iu->iu_buf; + + switch (iud->rx_iu.srp_op) { + case SRP_IU_CMD: + status = srpt_ch_srp_cmd(ch, iu); + break; + + case SRP_IU_TASK_MGMT: + status = srpt_ch_srp_task_mgmt(ch, iu); + return; + + case SRP_IU_I_LOGOUT: + SRPT_DPRINTF_L3("ch_process_iu, SRP INITIATOR LOGOUT"); + /* + * Initiators should logout by issuing a CM disconnect + * request (DREQ) with the logout IU in the private data; + * however some initiators have been known to send the + * IU in-band, if this happens just initiate the logout. + * Note that we do not return a response as per the + * specification. + */ + srpt_stp_logout(ch); + break; + + case SRP_IU_AER_RSP: + case SRP_IU_CRED_RSP: + default: + /* + * We don't send asynchronous events or ask for credit + * adjustments, so nothing need be done. Log we got an + * unexpected IU but then just repost the IU to the SRQ. + */ + SRPT_DPRINTF_L2("ch_process_iu, invalid IU from initiator," + " IU opcode(%d)", iud->rx_iu.srp_op); + break; + } + + if (status == 0) { + return; + } + +repost_iu: + SRPT_DPRINTF_L4("process_iu: reposting iu %p", (void *)iu); + mutex_enter(&iu->iu_lock); + srpt_ioc_repost_recv_iu(iu->iu_ioc, iu); + mutex_exit(&iu->iu_lock); + srpt_ch_release_ref(ch, 0); +} + +/* + * srpt_ch_post_send + */ +ibt_status_t +srpt_ch_post_send(srpt_channel_t *ch, srpt_iu_t *iu, uint32_t len, + uint_t fence) +{ + ibt_status_t status; + ibt_send_wr_t wr; + ibt_wr_ds_t ds; + uint_t posted; + + ASSERT(ch != NULL); + ASSERT(iu != NULL); + ASSERT(mutex_owned(&iu->iu_lock)); + + rw_enter(&ch->ch_rwlock, RW_READER); + if (ch->ch_state == SRPT_CHANNEL_DISCONNECTING) { + rw_exit(&ch->ch_rwlock); + SRPT_DPRINTF_L2("ch_post_send, bad ch state (%d)", + ch->ch_state); + return (IBT_FAILURE); + } + rw_exit(&ch->ch_rwlock); + + wr.wr_id = srpt_ch_alloc_swqe_wrid(ch, SRPT_SWQE_TYPE_RESP, + (void *)iu); + if (wr.wr_id == 0) { + SRPT_DPRINTF_L2("ch_post_send, queue full"); + return (IBT_FAILURE); + } + + atomic_inc_32(&iu->iu_sq_posted_cnt); + + wr.wr_flags = IBT_WR_SEND_SIGNAL; + if (fence == SRPT_FENCE_SEND) { + wr.wr_flags |= IBT_WR_SEND_FENCE; + } + wr.wr_opcode = IBT_WRC_SEND; + wr.wr_trans = IBT_RC_SRV; + wr.wr_nds = 1; + wr.wr_sgl = &ds; + + ds.ds_va = iu->iu_sge.ds_va; + ds.ds_key = iu->iu_sge.ds_key; + ds.ds_len = len; + + SRPT_DPRINTF_L4("ch_post_send, posting SRP response to channel" + " ds.ds_va (0x%16llx), ds.ds_key (0x%08x), " + " ds.ds_len (%d)", + (u_longlong_t)ds.ds_va, ds.ds_key, ds.ds_len); + + status = ibt_post_send(ch->ch_chan_hdl, &wr, 1, &posted); + if (status != IBT_SUCCESS) { + SRPT_DPRINTF_L2("ch_post_send, post_send failed (%d)", + status); + atomic_dec_32(&iu->iu_sq_posted_cnt); + srpt_ch_free_swqe_wrid(ch, wr.wr_id); + return (status); + } + + return (IBT_SUCCESS); +} + +/* + * srpt_ch_alloc_swqe_wrid() + */ +ibt_wrid_t +srpt_ch_alloc_swqe_wrid(srpt_channel_t *ch, + srpt_swqe_type_t wqe_type, void *addr) +{ + ibt_wrid_t wrid; + + mutex_enter(&ch->ch_swqe_lock); + if (ch->ch_head == ch->ch_tail) { + mutex_exit(&ch->ch_swqe_lock); + return ((ibt_wrid_t)0); + } + wrid = (ibt_wrid_t)ch->ch_head; + ch->ch_swqe[ch->ch_head].sw_type = wqe_type; + ch->ch_swqe[ch->ch_head].sw_addr = addr; + ch->ch_head = ch->ch_swqe[ch->ch_head].sw_next; + ch->ch_swqe_posted++; + mutex_exit(&ch->ch_swqe_lock); + return (wrid); +} + +/* + * srpt_ch_free_swqe_wrid() + */ +void +srpt_ch_free_swqe_wrid(srpt_channel_t *ch, ibt_wrid_t id) +{ + mutex_enter(&ch->ch_swqe_lock); + ch->ch_swqe[ch->ch_tail].sw_next = id; + ch->ch_tail = (uint32_t)id; + ch->ch_swqe_posted--; + mutex_exit(&ch->ch_swqe_lock); +} diff --git a/usr/src/uts/common/io/comstar/port/srpt/srpt_ch.h b/usr/src/uts/common/io/comstar/port/srpt/srpt_ch.h new file mode 100644 index 0000000000..4e85a58ab9 --- /dev/null +++ b/usr/src/uts/common/io/comstar/port/srpt/srpt_ch.h @@ -0,0 +1,59 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ + +/* + * Copyright 2009 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +#ifndef _SRPT_CH_H +#define _SRPT_CH_H + +/* + * Prototypes and data structures specific to SCSI Session + * interface. + */ + +#ifdef __cplusplus +extern "C" { +#endif + +/* + * Prototypes + */ +srpt_channel_t *srpt_ch_alloc(srpt_target_port_t *tgt, uint8_t port); +void srpt_ch_add_ref(srpt_channel_t *ch); +void srpt_ch_release_ref(srpt_channel_t *ch, uint_t wait); +void srpt_ch_disconnect(srpt_channel_t *ch); +void srpt_ch_cleanup(srpt_channel_t *ch); + +ibt_wrid_t srpt_ch_alloc_swqe_wrid(srpt_channel_t *ch, + srpt_swqe_type_t wqe_type, void *addr); +void srpt_ch_free_swqe_wrid(srpt_channel_t *ch, ibt_wrid_t id); + +ibt_status_t srpt_ch_post_send(srpt_channel_t *ch, srpt_iu_t *iu, + uint32_t len, uint_t fence); + +#ifdef __cplusplus +} +#endif + +#endif /* _SRPT_CH_H */ diff --git a/usr/src/uts/common/io/comstar/port/srpt/srpt_cm.c b/usr/src/uts/common/io/comstar/port/srpt/srpt_cm.c new file mode 100644 index 0000000000..2c1b4d0c12 --- /dev/null +++ b/usr/src/uts/common/io/comstar/port/srpt/srpt_cm.c @@ -0,0 +1,339 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ + +/* + * Copyright 2009 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +/* + * IB CM handlers for s Solaris SCSI RDMA Protocol Target (SRP) + * transport port provider module for the COMSTAR framework. + */ + +#include <sys/cpuvar.h> +#include <sys/types.h> +#include <sys/conf.h> +#include <sys/stat.h> +#include <sys/file.h> +#include <sys/ddi.h> +#include <sys/sunddi.h> +#include <sys/modctl.h> +#include <sys/sysmacros.h> +#include <sys/sdt.h> +#include <sys/taskq.h> +#include <sys/ib/ibtl/ibti.h> + +#include <stmf.h> +#include <stmf_ioctl.h> +#include <portif.h> + +#include "srp.h" +#include "srpt_impl.h" +#include "srpt_cm.h" +#include "srpt_stp.h" +#include "srpt_ch.h" + +extern uint16_t srpt_send_msg_depth; +extern srpt_ctxt_t *srpt_ctxt; + +/* + * srpt_cm_req_hdlr() - Login request + * + * CM has called back with a CM REQ message associated with an + * SRP initiator login request. + */ +static ibt_cm_status_t +srpt_cm_req_hdlr(srpt_target_port_t *tgt, ibt_cm_event_t *event, + ibt_cm_return_args_t *ret_args, void *ret_priv_data, + ibt_priv_data_len_t ret_priv_data_len) +{ + ibt_cm_status_t status; + ibt_cm_req_rcv_t *req; + srp_login_req_t login; + srp_login_rej_t login_rej; + srp_login_rsp_t login_rsp; + srpt_channel_t *ch = NULL; + + ASSERT(tgt != NULL); + req = &event->cm_event.req; + + if (event->cm_priv_data_len < sizeof (srp_login_req_t)) { + SRPT_DPRINTF_L2("cm_req_hdlr, IU size expected (>= %d)," + " received size (%d)", (uint_t)sizeof (srp_login_req_t), + event->cm_priv_data_len); + return (IBT_CM_REJECT); + } + + if (event->cm_priv_data == NULL) { + SRPT_DPRINTF_L2("cm_req_hdlr, NULL ULP private data pointer"); + return (IBT_CM_REJECT); + } + + if (ret_priv_data_len < sizeof (srp_login_rej_t)) { + SRPT_DPRINTF_L2("cm_req_hdlr, return private len too" + " small (%d)", ret_priv_data_len); + return (IBT_CM_REJECT); + } + + if (ret_priv_data == NULL) { + SRPT_DPRINTF_L2("cm_req_hdlr, NULL ULP return private data" + " pointer"); + return (IBT_CM_REJECT); + } + + /* + * Copy to avoid potential alignment problems, process login + * creating a new channel and possibly session. + */ + bcopy(event->cm_priv_data, &login, sizeof (login)); + + ch = srpt_stp_login(tgt, &login, &login_rsp, + &login_rej, req->req_prim_hca_port); + if (ch != NULL) { + bcopy(&login_rsp, ret_priv_data, SRP_LOGIN_RSP_SIZE); + ret_args->cm_ret_len = SRP_LOGIN_RSP_SIZE; + + SRPT_DPRINTF_L3("cm_req_hdlr, rsp priv len(%d)" + " ch created on port(%d)" + ", cm_req_hdlr, req ra_out(%d), ra_in(%d)" + ", retry(%d)", + ret_args->cm_ret_len, req->req_prim_hca_port, + req->req_rdma_ra_out, req->req_rdma_ra_in, + req->req_retry_cnt); + + ret_args->cm_ret.rep.cm_channel = ch->ch_chan_hdl; + ret_args->cm_ret.rep.cm_rdma_ra_out = + min(tgt->tp_ioc->ioc_attr.hca_max_rdma_out_chan, + req->req_rdma_ra_in); + ret_args->cm_ret.rep.cm_rdma_ra_in = + min(tgt->tp_ioc->ioc_attr.hca_max_rdma_in_chan, + req->req_rdma_ra_out); + ret_args->cm_ret.rep.cm_rnr_retry_cnt = req->req_retry_cnt; + + SRPT_DPRINTF_L3("cm_req_hdlr, hca_max_rdma_in_chan (%d)" + ", hca_max_rdma_out_chan (%d)" + ", updated ra_out(%d), ra_in(%d), retry(%d)", + tgt->tp_ioc->ioc_attr.hca_max_rdma_in_chan, + tgt->tp_ioc->ioc_attr.hca_max_rdma_out_chan, + ret_args->cm_ret.rep.cm_rdma_ra_out, + ret_args->cm_ret.rep.cm_rdma_ra_in, + ret_args->cm_ret.rep.cm_rnr_retry_cnt); + status = IBT_CM_ACCEPT; + + } else { + bcopy(&login_rej, ret_priv_data, sizeof (login_rej)); + ret_args->cm_ret_len = sizeof (login_rej); + status = IBT_CM_REJECT; + } + + return (status); +} + +/* + * srpt_cm_conn_est_hdlr() - Connection established + * + * CM has called back to inform us that a connection attempt has + * completed (explicit or implicit) and may now be used. + */ +/* ARGSUSED */ +static ibt_cm_status_t +srpt_cm_conn_est_hdlr(srpt_target_port_t *tgt, ibt_cm_event_t *event) +{ + srpt_channel_t *ch; + + ASSERT(tgt != NULL); + ASSERT(event != NULL); + + ch = (srpt_channel_t *)ibt_get_chan_private(event->cm_channel); + ASSERT(ch != NULL); + + SRPT_DPRINTF_L3("cm_conn_est_hdlr, invoked for ch(%p)", + (void *)ch); + + rw_enter(&ch->ch_rwlock, RW_WRITER); + if (ch->ch_state != SRPT_CHANNEL_CONNECTING && + ch->ch_state != SRPT_CHANNEL_CONNECTED) { + SRPT_DPRINTF_L2("cm_conn_est_hdlr, invalid ch state (%d)", + ch->ch_state); + rw_exit(&ch->ch_rwlock); + return (IBT_CM_REJECT); + } + + ch->ch_state = SRPT_CHANNEL_CONNECTED; + + rw_exit(&ch->ch_rwlock); + return (IBT_CM_ACCEPT); +} + +/* + * srpt_cm_conn_closed_hdlr() - Channel closed + * + * CM callback indicating a channel has been completely closed. + */ +/* ARGSUSED */ +static ibt_cm_status_t +srpt_cm_conn_closed_hdlr(srpt_target_port_t *tgt, ibt_cm_event_t *event) +{ + ibt_cm_status_t status = IBT_CM_ACCEPT; + srpt_channel_t *ch; + + ASSERT(tgt != NULL); + ASSERT(event != NULL); + + ch = (srpt_channel_t *)ibt_get_chan_private(event->cm_channel); + ASSERT(ch != NULL); + + SRPT_DPRINTF_L3("cm_conn_closed_hdlr, invoked for chan_hdl(%p)," + " event(%d)", (void *)ch->ch_chan_hdl, + event->cm_event.closed); + + switch (event->cm_event.closed) { + + case IBT_CM_CLOSED_DREP_RCVD: + case IBT_CM_CLOSED_DREQ_TIMEOUT: + case IBT_CM_CLOSED_DUP: + case IBT_CM_CLOSED_ABORT: + case IBT_CM_CLOSED_ALREADY: + /* + * These cases indicate the SRP target initiated + * the closing of the channel and it is now closed. + * Cleanup the channel (which will remove the targets + * reference) and then release CM's reference. + */ + SRPT_DPRINTF_L3("cm_conn_closed_hdlr, local close call-back"); + srpt_ch_cleanup(ch); + srpt_ch_release_ref(ch, 1); + break; + + case IBT_CM_CLOSED_DREQ_RCVD: + case IBT_CM_CLOSED_REJ_RCVD: + case IBT_CM_CLOSED_STALE: + /* + * These cases indicate that the SRP initiator is closing + * the channel. CM will have already closed the RC channel, + * so simply initiate cleanup which will remove the target + * ports reference to the channel and then release the + * reference held by the CM. + */ + SRPT_DPRINTF_L3("cm_conn_closed_hdlr, remote close," + " free channel"); + if (ch != NULL) { + srpt_ch_cleanup(ch); + srpt_ch_release_ref(ch, 1); + } else { + SRPT_DPRINTF_L2("cm_conn_closed_hdlr, NULL channel"); + } + break; + + default: + SRPT_DPRINTF_L2("cm_conn_closed_hdlr, unknown close type (%d)", + event->cm_event.closed); + status = IBT_CM_DEFAULT; + break; + } + return (status); +} + +/* + * srpt_cm_failure_hdlr() - Called when the channel is in error. Cleanup + * and release the channel. + */ +static ibt_cm_status_t +srpt_cm_failure_hdlr(ibt_cm_event_t *event) +{ + srpt_channel_t *ch; + + ASSERT(event != NULL); + + ch = (srpt_channel_t *)ibt_get_chan_private(event->cm_channel); + ASSERT(ch != NULL); + + SRPT_DPRINTF_L3("cm_failure_hdlr, chan_hdl: 0x%p, code: %d" + "msg: %d reason: %d", (void *)event->cm_channel, + event->cm_event.failed.cf_code, + event->cm_event.failed.cf_msg, + event->cm_event.failed.cf_reason); + + srpt_ch_cleanup(ch); + srpt_ch_release_ref(ch, 1); + + return (IBT_CM_ACCEPT); +} + +/* + * srpt_cm_hdlr() - CM call-back handler. + */ +ibt_cm_status_t +srpt_cm_hdlr(void *cm_private, ibt_cm_event_t *event, + ibt_cm_return_args_t *ret_args, void *ret_priv_data, + ibt_priv_data_len_t ret_len_max) +{ + ibt_cm_status_t status = IBT_CM_ACCEPT; + + switch (event->cm_type) { + + case IBT_CM_EVENT_REQ_RCV: + SRPT_DPRINTF_L3("cm_hdlr, REQ received"); + status = srpt_cm_req_hdlr((srpt_target_port_t *)cm_private, + event, ret_args, ret_priv_data, ret_len_max); + break; + + case IBT_CM_EVENT_REP_RCV: + SRPT_DPRINTF_L3("cm_hdlr, REP received"); + break; + + case IBT_CM_EVENT_MRA_RCV: + SRPT_DPRINTF_L3("cm_hdlr, MRA received"); + break; + + case IBT_CM_EVENT_CONN_EST: + SRPT_DPRINTF_L3("cm_hdlr, Connection established"); + status = srpt_cm_conn_est_hdlr( + (srpt_target_port_t *)cm_private, event); + break; + + case IBT_CM_EVENT_CONN_CLOSED: + SRPT_DPRINTF_L3("cm_hdlr, Connection closed"); + status = srpt_cm_conn_closed_hdlr( + (srpt_target_port_t *)cm_private, event); + break; + + case IBT_CM_EVENT_FAILURE: + SRPT_DPRINTF_L3("cm_hdlr, Event failure"); + status = srpt_cm_failure_hdlr(event); + break; + + case IBT_CM_EVENT_LAP_RCV: + SRPT_DPRINTF_L3("cm_hdlr, LAP received"); + break; + + case IBT_CM_EVENT_APR_RCV: + SRPT_DPRINTF_L3("cm_hdlr, APR received"); + break; + + default: + SRPT_DPRINTF_L3("cm_hdlr, unknown event received"); + break; + } + + return (status); +} diff --git a/usr/src/uts/common/io/comstar/port/srpt/srpt_cm.h b/usr/src/uts/common/io/comstar/port/srpt/srpt_cm.h new file mode 100644 index 0000000000..5fbb2c55bf --- /dev/null +++ b/usr/src/uts/common/io/comstar/port/srpt/srpt_cm.h @@ -0,0 +1,49 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ + +/* + * Copyright 2009 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +#ifndef _SRPT_CM_H +#define _SRPT_CM_H + +/* + * Prototypes and data structures specific to Infiniband CM + * interface. + */ + +#ifdef __cplusplus +extern "C" { +#endif + +/* Prototypes */ + +ibt_cm_status_t srpt_cm_hdlr(void *cm_private, ibt_cm_event_t *eventp, + ibt_cm_return_args_t *ret_args, void *ret_priv_data, + ibt_priv_data_len_t ret_len_max); + +#ifdef __cplusplus +} +#endif + +#endif /* _SRPT_CM_H */ diff --git a/usr/src/uts/common/io/comstar/port/srpt/srpt_impl.h b/usr/src/uts/common/io/comstar/port/srpt/srpt_impl.h new file mode 100644 index 0000000000..2f18098b3b --- /dev/null +++ b/usr/src/uts/common/io/comstar/port/srpt/srpt_impl.h @@ -0,0 +1,497 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ + +/* + * Copyright 2009 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +#ifndef _SRPT_IMPL_H_ +#define _SRPT_IMPL_H_ + +/* + * Prototypes and data structures for the SRP Target Port Provider. + */ + +#include <sys/types.h> +#include <sys/ddi.h> +#include <sys/ib/ibtl/ibti.h> +#include <sys/modctl.h> + +#include <stmf.h> +#include <stmf_ioctl.h> +#include <portif.h> + +#include <sys/ib/mgt/ibdma/ibdma.h> + +#ifdef __cplusplus +extern "C" { +#endif + +/* + * We should/could consider making some of these values tunables. + * Specifically, SEND_MSG_SIZE and SEND_MSG_DEPTH. + */ +enum { + SRPT_DEFAULT_IOC_SRQ_SIZE = 4096, + SRPT_DEFAULT_SEND_MSG_DEPTH = 128, + SRPT_DEFAULT_SEND_MSG_SIZE = 960, /* must be multiple of 64 */ + SRPT_DEFAULT_MAX_RDMA_SIZE = 65536, + SRPT_MIN_T_I_IU_LEN = 52, + SRPT_EUI_ID_LEN = 20, + SRPT_RECV_WC_POLL_SIZE = 16, + SRPT_SEND_WC_POLL_SIZE = 16, + SRPT_MAX_OUT_IO_PER_CMD = 16, + SRPT_FENCE_SEND = 1, + SRPT_NO_FENCE_SEND = 0 +}; + +struct srpt_target_port_s; + +/* + * SRP Session - represents a SCSI I_T_Nexus. + * + * Sessions map 1 or more initiator logins to a specific I/O + * Controller SCSI Target Port. Targets create sessions + * at initiator login and release when no longer referenced + * by a login. + */ +typedef struct srpt_session_s { + krwlock_t ss_rwlock; + list_node_t ss_node; + + /* + * ADVANCED FEATURE, NOT YET SUPPORTED. + * In multi-channel mode, multiple RDMA communication + * channels may reference the same SCSI session. When + * a channel releases its reference to the SCSI session, + * it should have no tasks associated with the session. + * + * If multi-channel is implemented, add a channel list + * to this object instead of tracking it on the target. + * + * Will also need a session state & mode. Mode is to + * track if the session is MULTI or SINGLE channel. + */ + + stmf_scsi_session_t *ss_ss; + struct srpt_target_port_s *ss_tgt; + list_t ss_task_list; + + /* + * SRP Initiator and target identifiers are 128-bit. + * + * The specification defines the initiator to be 64-bits of + * ID extension and 64 bits of GUID, but these are really + * just a recommendation. Generally the extension is used + * to create unique I_T_Nexus from the same initiator and + * target. Initiators are inconsistent on the GUID they + * use, some use the HCA Node, some the HCA port. + * + * The specification defines the target to be 64-bits of + * service ID followed by 64-bits of I/O Controller GUID. + * In the case where there is a single default target + * service, they will be the same (our default). + */ + uint8_t ss_i_id[SRP_PORT_ID_LEN]; + uint8_t ss_t_id[SRP_PORT_ID_LEN]; + + /* So we can see the full 128-bit initiator login from stmfadm */ + char ss_alias[SRP_PORT_ID_LEN * 2 + 2]; + uint8_t ss_hw_port; +} srpt_session_t; + +/* + * Send work request types. + */ +typedef enum srpt_swqe_type_e { + SRPT_SWQE_TYPE_DATA = 1, + SRPT_SWQE_TYPE_RESP +} srpt_swqe_type_t; + +typedef struct srpt_swqe_s { + srpt_swqe_type_t sw_type; + void *sw_addr; + ibt_wrid_t sw_next; +} srpt_swqe_t; + +/* + * SRP Channel - the RDMA communications channel associated with + * a specific SRP login. + */ +typedef enum srpt_channel_state_e { + SRPT_CHANNEL_CONNECTING = 0, + SRPT_CHANNEL_CONNECTED, + SRPT_CHANNEL_DISCONNECTING +} srpt_channel_state_t; + +typedef struct srpt_channel_s { + krwlock_t ch_rwlock; + + kmutex_t ch_reflock; + uint_t ch_refcnt; + kcondvar_t ch_cv_complete; + uint_t ch_cv_waiters; + + list_node_t ch_stp_node; + srpt_channel_state_t ch_state; + ibt_cq_hdl_t ch_scq_hdl; + ibt_cq_hdl_t ch_rcq_hdl; + ibt_channel_hdl_t ch_chan_hdl; + ibt_chan_sizes_t ch_sizes; + + uint32_t ch_req_lim_delta; + uint32_t ch_ti_iu_len; + struct srpt_target_port_s *ch_tgt; + srpt_session_t *ch_session; + + /* + * Map IB send WQE request IDs to the + * apporpriate operation type (for errors). + */ + kmutex_t ch_swqe_lock; + srpt_swqe_t *ch_swqe; + uint32_t ch_num_swqe; + uint32_t ch_head; + uint32_t ch_tail; + uint32_t ch_swqe_posted; +} srpt_channel_t; + +/* + * SRP Information Unit (IU). Each IU structure contains + * the buffer for the IU itself (received over the RC + * channel), and all of the context required by the target + * to process this request represented by the IU. + * Available IU structures are managed on the I/O Controller + * shared receive queue. + */ +enum { + SRPT_IU_STMF_ABORTING = 1 << 0, /* STMF called abort */ + SRPT_IU_SRP_ABORTING = 1 << 1, /* SRP initiator aborting */ + SRPT_IU_ABORTED = 1 << 2, /* Task has been aborted */ + SRPT_IU_RESP_SENT = 1 << 3 /* Response queued */ +}; + +typedef struct srpt_iu_s { + /* + * The buffer for the IU itself. When unused (a + * reference count of zero), this buffer is posted + * on the I/O Controllers SRPT SRQ. + */ + void *iu_buf; + ibt_wr_ds_t iu_sge; + struct srpt_ioc_s *iu_ioc; + uint_t iu_pool_ndx; + kmutex_t iu_lock; + + /* + * The following are reset for each IU request + * processed by this buffer. + */ + list_node_t iu_ss_task_node; + srpt_channel_t *iu_ch; + + uint_t iu_num_rdescs; + srp_direct_desc_t *iu_rdescs; + uint_t iu_tot_xfer_len; + + uint64_t iu_tag; + uint_t iu_flags; + uint32_t iu_sq_posted_cnt; + scsi_task_t *iu_stmf_task; +} srpt_iu_t; + +/* + * SRP SCSI Target Port. By default each HCA creates a single + * SCSI Target Port based on the associated I/O Controller + * (HCA) node GUID and made available through each physical + * hardware port of the I/O Controller. + */ +typedef enum srpt_target_state_e { + SRPT_TGT_STATE_OFFLINE = 0, + SRPT_TGT_STATE_ONLINING, + SRPT_TGT_STATE_ONLINE, + SRPT_TGT_STATE_OFFLINING +} srpt_target_state_t; + +typedef struct srpt_hw_port_s { + ibt_sbind_hdl_t hwp_bind_hdl; + ib_gid_t hwp_gid; +} srpt_hw_port_t; + +typedef struct srpt_target_port_s { + stmf_local_port_t *tp_lport; + struct srpt_ioc_s *tp_ioc; + + kmutex_t tp_lock; + srpt_target_state_t tp_state; + kcondvar_t tp_offline_complete; + uint_t tp_drv_disabled; + + /* + * We are using a simple list for channels right now, we + * probably should switch this over to the AVL + * implementation eventually (but lookups are not done + * in the data path so this is not urgent). + */ + kmutex_t tp_ch_list_lock; + list_t tp_ch_list; + + /* + * A list of active sessions. Session lifetime is + * determined by having active channels, but track + * them here for easier determination to when a + * target can truly be offlined, and as a step toward + * being session-focused rather than channel-focused. + * If we ever truly support multi-channel, move the + * channels to be part of the session object. + * + * Sessions should remain on this list until they + * are deregistered from STMF. This allows the target + * to properly track when it can consider itself 'offline'. + */ + kmutex_t tp_sess_list_lock; + kcondvar_t tp_sess_complete; + list_t tp_sess_list; + + uint_t tp_srp_enabled; + ibt_srv_hdl_t tp_ibt_svc_hdl; + ibt_srv_desc_t tp_ibt_svc_desc; + ib_svc_id_t tp_ibt_svc_id; + scsi_devid_desc_t *tp_scsi_devid; + uint8_t tp_srp_port_id[SRP_PORT_ID_LEN]; + + uint_t tp_nports; + srpt_hw_port_t *tp_hw_port; +} srpt_target_port_t; + +/* + * SRP Target hardware device. A SRP Target hardware device + * is an IB HCA. All ports of the HCA comprise a single + * I/O Controller that is registered with the IB Device + * Managment Agent. + */ +typedef struct srpt_ioc_s { + list_node_t ioc_node; + + krwlock_t ioc_rwlock; + ibt_hca_hdl_t ioc_ibt_hdl; + ibt_hca_attr_t ioc_attr; + ib_guid_t ioc_guid; + + /* + * By default each HCA is a single SRP.T10 service based on + * the HCA GUID. We have implemented the target here as a + * pointer to facilitate moving to a list of targets if + * appropriate down the road. + */ + srpt_target_port_t *ioc_tgt_port; + + + /* + * Each HCA registers a single I/O Controller with the + * IB Device Management Agent. + */ + ibdma_hdl_t ioc_ibdma_hdl; + ib_dm_ioc_ctrl_profile_t ioc_profile; + ib_dm_srv_t ioc_svc; + + ibt_pd_hdl_t ioc_pd_hdl; + ibt_srq_sizes_t ioc_srq_attr; + ibt_srq_hdl_t ioc_srq_hdl; + + /* + * The I/O Controller pool of IU resources allocated + * at controller creation. + */ + uint32_t ioc_num_iu_entries; + srpt_iu_t *ioc_iu_pool; + ibt_mr_hdl_t ioc_iu_mr_hdl; + void *ioc_iu_bufs; /* iu buffer space */ + + /* + * Each I/O Controller has it's own data buffer + * vmem arena. Pool is created at controller creation, + * and expanded as required. This keeps IB memory + * registrations to a minimum in the data path. + */ + struct srpt_vmem_pool_s *ioc_dbuf_pool; + stmf_dbuf_store_t *ioc_stmf_ds; +} srpt_ioc_t; + +/* + * Memory regions + */ +typedef struct srpt_mr_s { + ibt_mr_hdl_t mr_hdl; + ib_vaddr_t mr_va; + ib_memlen_t mr_len; + ibt_lkey_t mr_lkey; + ibt_rkey_t mr_rkey; + avl_node_t mr_avl; +} srpt_mr_t; + +/* + * SRP Target vmem arena definition + */ +typedef struct srpt_vmem_pool_s { + srpt_ioc_t *svp_ioc; + ib_memlen_t svp_chunksize; + vmem_t *svp_vmem; + uint64_t svp_total_size; + uint64_t svp_max_size; + avl_tree_t svp_mr_list; + krwlock_t svp_lock; + ibt_mr_flags_t svp_flags; +} srpt_vmem_pool_t; + +/* + * SRP port provider data buffer, allocated and freed + * via calls to the IOC datastore. + */ +typedef struct srpt_ds_dbuf_s { + stmf_data_buf_t *db_stmf_buf; + srpt_ioc_t *db_ioc; + ibt_mr_hdl_t db_mr_hdl; + ibt_wr_ds_t db_sge; + srpt_iu_t *db_iu; +} srpt_ds_dbuf_t; + +/* + * SRP Target service state + */ +typedef enum { + SRPT_SVC_DISABLED, + SRPT_SVC_ENABLED +} srpt_svc_state_t; + +typedef struct { + ddi_modhandle_t ibdmah; + ibdma_hdl_t (*ibdma_register)(ib_guid_t, + ib_dm_ioc_ctrl_profile_t *, ib_dm_srv_t *); + ibdma_status_t (*ibdma_unregister)(ibdma_hdl_t); + ibdma_status_t (*ibdma_update)(ibdma_hdl_t, + ib_dm_ioc_ctrl_profile_t *, ib_dm_srv_t *); +} srpt_ibdma_ops_t; + +/* + * SRP Target protocol driver context data structure, maintaining + * the global state of the protocol. + */ +typedef struct srpt_ctxt_s { + dev_info_t *sc_dip; + krwlock_t sc_rwlock; + srpt_svc_state_t sc_svc_state; + + ibt_clnt_hdl_t sc_ibt_hdl; + + /* + * SRP Target I/O Controllers. Each IBT HCA represents an + * I/O Controller. Must hold rwlock as a writer to update. + */ + list_t sc_ioc_list; + uint_t sc_num_iocs; + + /* Back-end COMSTAR port provider interface. */ + stmf_port_provider_t *sc_pp; + + /* IBDMA entry points */ + srpt_ibdma_ops_t sc_ibdma_ops; +} srpt_ctxt_t; + +typedef struct srpt_iu_data_s { + union { + uint8_t srp_op; + srp_cmd_req_t srp_cmd; + srp_tsk_mgmt_t srp_tsk_mgmt; + srp_i_logout_t srp_i_logout; + srp_rsp_t srp_rsp; + } rx_iu; +} srpt_iu_data_t; + +extern srpt_ctxt_t *srpt_ctxt; + +/* + * For Non recoverable or Major Errors + */ +#define SRPT_LOG_L0 0 + +/* + * For additional information on Non recoverable errors and + * warnings/informational message for sys-admin types. + */ +#define SRPT_LOG_L1 1 + +/* + * debug only + * for more verbose trace than L1, for e.g. recoverable errors, + * or intersting trace + */ +#define SRPT_LOG_L2 2 + +/* + * debug only + * for more verbose trace than L2, for e.g. printing function entries.... + */ +#define SRPT_LOG_L3 3 + +/* + * debug only + * for more verbose trace than L3, for e.g. printing minor function entries... + */ +#define SRPT_LOG_L4 4 + +/* + * srpt_errlevel can be set in the debugger to enable additional logging. + * You can also add set srpt:srpt_errlevel={0,1,2,3,4} in /etc/system. + * The default log level is L2 for debug builds, otherwise L1. + */ +#ifdef DEBUG +#define SRPT_LOG_DEFAULT_LEVEL SRPT_LOG_L2 +#else +#define SRPT_LOG_DEFAULT_LEVEL SRPT_LOG_L1 +#endif + +extern uint_t srpt_errlevel; + + +#define SRPT_DPRINTF_L0(...) cmn_err(CE_WARN, __VA_ARGS__) +#define SRPT_DPRINTF_L1(...) cmn_err(CE_NOTE, __VA_ARGS__) +#define SRPT_DPRINTF_L2(...) if (srpt_errlevel >= SRPT_LOG_L2) { \ + cmn_err(CE_NOTE, __VA_ARGS__);\ + } +#ifdef DEBUG +#define SRPT_DPRINTF_L3(...) if (srpt_errlevel >= SRPT_LOG_L3) { \ + cmn_err(CE_NOTE, __VA_ARGS__);\ + } +#define SRPT_DPRINTF_L4(...) if (srpt_errlevel >= SRPT_LOG_L4) { \ + cmn_err(CE_NOTE, __VA_ARGS__);\ + } +#else +#define SRPT_DPRINTF_L3 0 && +#define SRPT_DPRINTF_L4 0 && +#endif + +#ifdef __cplusplus +} +#endif + +#endif /* _SRPT_IMPL_H_ */ diff --git a/usr/src/uts/common/io/comstar/port/srpt/srpt_ioc.c b/usr/src/uts/common/io/comstar/port/srpt/srpt_ioc.c new file mode 100644 index 0000000000..a1654ce592 --- /dev/null +++ b/usr/src/uts/common/io/comstar/port/srpt/srpt_ioc.c @@ -0,0 +1,1368 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ + +/* + * Copyright 2009 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +/* + * I/O Controller functions for the Solaris COMSTAR SCSI RDMA Protocol + * Target (SRPT) port provider. + */ + +#include <sys/types.h> +#include <sys/ddi.h> +#include <sys/types.h> +#include <sys/sunddi.h> +#include <sys/atomic.h> +#include <sys/sysmacros.h> +#include <sys/ib/ibtl/ibti.h> + +#include "srp.h" +#include "srpt_impl.h" +#include "srpt_ioc.h" +#include "srpt_stp.h" +#include "srpt_ch.h" + +/* + * srpt_ioc_srq_size - Tunable parameter that specifies the number + * of receive WQ entries that can be posted to the IOC shared + * receive queue. + */ +uint32_t srpt_ioc_srq_size = SRPT_DEFAULT_IOC_SRQ_SIZE; +extern uint16_t srpt_send_msg_depth; + +/* IOC profile capabilities mask must be big-endian */ +typedef struct srpt_ioc_opcap_bits_s { +#if defined(_BIT_FIELDS_LTOH) + uint8_t af:1, + at:1, + wf:1, + wt:1, + rf:1, + rt:1, + sf:1, + st:1; +#elif defined(_BIT_FIELDS_HTOL) + uint8_t st:1, + sf:1, + rt:1, + rf:1, + wt:1, + wf:1, + at:1, + af:1; +#else +#error One of _BIT_FIELDS_LTOH or _BIT_FIELDS_HTOL must be defined +#endif +} srpt_ioc_opcap_bits_t; + +typedef union { + srpt_ioc_opcap_bits_t bits; + uint8_t mask; +} srpt_ioc_opcap_mask_t; + +/* + * vmem arena variables - values derived from iSER + */ +#define SRPT_MR_QUANTSIZE 0x400 /* 1K */ +#define SRPT_MIN_CHUNKSIZE 0x100000 /* 1MB */ + +/* use less memory on 32-bit kernels as it's much more constrained */ +#ifdef _LP64 +#define SRPT_BUF_MR_CHUNKSIZE 0x1000000 /* 16MB */ +#define SRPT_BUF_POOL_MAX 0x40000000 /* 1GB */ +#else +#define SRPT_BUF_MR_CHUNKSIZE 0x400000 /* 4MB */ +#define SRPT_BUF_POOL_MAX 0x4000000 /* 64MB */ +#endif + +static ibt_mr_flags_t srpt_dbuf_mr_flags = + IBT_MR_ENABLE_LOCAL_WRITE | IBT_MR_ENABLE_REMOTE_WRITE | + IBT_MR_ENABLE_REMOTE_READ; + +void srpt_ioc_ib_async_hdlr(void *clnt, ibt_hca_hdl_t hdl, + ibt_async_code_t code, ibt_async_event_t *event); + +static struct ibt_clnt_modinfo_s srpt_ibt_modinfo = { + IBTI_V_CURR, + IBT_STORAGE_DEV, + srpt_ioc_ib_async_hdlr, + NULL, + "srpt" +}; + +static srpt_ioc_t *srpt_ioc_init(ib_guid_t guid); +static void srpt_ioc_fini(srpt_ioc_t *ioc); + +static srpt_vmem_pool_t *srpt_vmem_create(const char *name, srpt_ioc_t *ioc, + ib_memlen_t chunksize, uint64_t maxsize, ibt_mr_flags_t flags); +static void *srpt_vmem_alloc(srpt_vmem_pool_t *vm_pool, size_t size); +static int srpt_vmem_mr_compare(const void *a, const void *b); +static srpt_mr_t *srpt_vmem_chunk_alloc(srpt_vmem_pool_t *ioc, + ib_memlen_t chunksize); +static void srpt_vmem_destroy(srpt_vmem_pool_t *vm_pool); +static void srpt_vmem_free(srpt_vmem_pool_t *vm_pool, void *vaddr, size_t size); +static srpt_mr_t *srpt_reg_mem(srpt_vmem_pool_t *vm_pool, ib_vaddr_t vaddr, + ib_memlen_t len); +static void srpt_vmem_chunk_free(srpt_vmem_pool_t *vm_pool, srpt_mr_t *mr); +static void srpt_dereg_mem(srpt_ioc_t *ioc, srpt_mr_t *mr); +static int srpt_vmem_mr(srpt_vmem_pool_t *vm_pool, void *vaddr, size_t size, + srpt_mr_t *mr); + +/* + * srpt_ioc_attach() - I/O Controller attach + * + * Attach to IBTF and initialize I/O controllers. The srpt_ctxt->sc_rwlock + * should be held outside of this call. + */ +int +srpt_ioc_attach() +{ + int status; + int hca_cnt; + int hca_ndx; + ib_guid_t *guid; + srpt_ioc_t *ioc; + + ASSERT(srpt_ctxt != NULL); + + /* + * Attach to IBTF and initialize a list of IB devices. Each + * HCA will be represented by an I/O Controller. + */ + status = ibt_attach(&srpt_ibt_modinfo, srpt_ctxt->sc_dip, + srpt_ctxt, &srpt_ctxt->sc_ibt_hdl); + if (status != DDI_SUCCESS) { + SRPT_DPRINTF_L1("ioc_attach, ibt_attach failed (0x%x)", + status); + return (DDI_FAILURE); + } + + hca_cnt = ibt_get_hca_list(&guid); + if (hca_cnt < 1) { + SRPT_DPRINTF_L2("ioc_attach, no HCA found"); + ibt_detach(srpt_ctxt->sc_ibt_hdl); + srpt_ctxt->sc_ibt_hdl = NULL; + return (DDI_FAILURE); + } + + list_create(&srpt_ctxt->sc_ioc_list, sizeof (srpt_ioc_t), + offsetof(srpt_ioc_t, ioc_node)); + + for (hca_ndx = 0; hca_ndx < hca_cnt; hca_ndx++) { + SRPT_DPRINTF_L2("ioc_attach, adding I/O" + " Controller (%016llx)", (u_longlong_t)guid[hca_ndx]); + + ioc = srpt_ioc_init(guid[hca_ndx]); + if (ioc == NULL) { + SRPT_DPRINTF_L1("ioc_attach, ioc_init GUID(%016llx)" + " failed", (u_longlong_t)guid[hca_ndx]); + continue; + } + list_insert_tail(&srpt_ctxt->sc_ioc_list, ioc); + SRPT_DPRINTF_L2("ioc_attach, I/O Controller ibt HCA hdl (%p)", + (void *)ioc->ioc_ibt_hdl); + srpt_ctxt->sc_num_iocs++; + } + + ibt_free_hca_list(guid, hca_cnt); + SRPT_DPRINTF_L3("ioc_attach, added %d I/O Controller(s)", + srpt_ctxt->sc_num_iocs); + return (DDI_SUCCESS); +} + +/* + * srpt_ioc_detach() - I/O Controller detach + * + * srpt_ctxt->sc_rwlock should be held outside of this call. + */ +void +srpt_ioc_detach() +{ + srpt_ioc_t *ioc; + + ASSERT(srpt_ctxt != NULL); + + while ((ioc = list_head(&srpt_ctxt->sc_ioc_list)) != NULL) { + list_remove(&srpt_ctxt->sc_ioc_list, ioc); + SRPT_DPRINTF_L2("ioc_detach, removing I/O Controller(%p)" + " (%016llx), ibt_hdl(%p)", + (void *)ioc, + ioc ? (u_longlong_t)ioc->ioc_guid : 0x0ll, + (void *)ioc->ioc_ibt_hdl); + srpt_ioc_fini(ioc); + } + + list_destroy(&srpt_ctxt->sc_ioc_list); + + ibt_detach(srpt_ctxt->sc_ibt_hdl); + srpt_ctxt->sc_ibt_hdl = NULL; +} + +/* + * srpt_ioc_init() - I/O Controller initialization + * + * Requires srpt_ctxt->rw_lock be held outside of call. + */ +static srpt_ioc_t * +srpt_ioc_init(ib_guid_t guid) +{ + ibt_status_t status; + srpt_ioc_t *ioc; + ibt_hca_attr_t hca_attr; + uint_t iu_ndx; + uint_t err_ndx; + ibt_mr_attr_t mr_attr; + ibt_mr_desc_t mr_desc; + srpt_iu_t *iu; + ibt_srq_sizes_t srq_attr; + char namebuf[32]; + size_t iu_offset; + + status = ibt_query_hca_byguid(guid, &hca_attr); + if (status != IBT_SUCCESS) { + SRPT_DPRINTF_L1("ioc_init, HCA query error (%d)", + status); + return (NULL); + } + + ioc = srpt_ioc_get_locked(guid); + if (ioc != NULL) { + SRPT_DPRINTF_L1("ioc_init, HCA already exists"); + return (NULL); + } + + ioc = kmem_zalloc(sizeof (srpt_ioc_t), KM_SLEEP); + + rw_init(&ioc->ioc_rwlock, NULL, RW_DRIVER, NULL); + rw_enter(&ioc->ioc_rwlock, RW_WRITER); + + bcopy(&hca_attr, &ioc->ioc_attr, sizeof (ibt_hca_attr_t)); + + SRPT_DPRINTF_L2("ioc_init, HCA max mr=%d, mrlen=%lld", + hca_attr.hca_max_memr, (u_longlong_t)hca_attr.hca_max_memr_len); + ioc->ioc_guid = guid; + + status = ibt_open_hca(srpt_ctxt->sc_ibt_hdl, guid, &ioc->ioc_ibt_hdl); + if (status != IBT_SUCCESS) { + SRPT_DPRINTF_L1("ioc_init, IBT open failed (%d)", status); + goto hca_open_err; + } + + status = ibt_alloc_pd(ioc->ioc_ibt_hdl, IBT_PD_NO_FLAGS, + &ioc->ioc_pd_hdl); + if (status != IBT_SUCCESS) { + SRPT_DPRINTF_L1("ioc_init, IBT create PD failed (%d)", status); + goto pd_alloc_err; + } + + /* + * We require hardware support for SRQs. We use a common SRQ to + * reduce channel memory consumption. + */ + if ((ioc->ioc_attr.hca_flags & IBT_HCA_SRQ) == 0) { + SRPT_DPRINTF_L0("ioc_init, no SRQ capability, not supported"); + goto srq_alloc_err; + } + + SRPT_DPRINTF_L3("ioc_init, Using shared receive queues, max srq work" + " queue size(%d), def size = %d", ioc->ioc_attr.hca_max_srqs_sz, + srpt_ioc_srq_size); + srq_attr.srq_wr_sz = min(srpt_ioc_srq_size, + ioc->ioc_attr.hca_max_srqs_sz); + srq_attr.srq_sgl_sz = 1; + + status = ibt_alloc_srq(ioc->ioc_ibt_hdl, IBT_SRQ_NO_FLAGS, + ioc->ioc_pd_hdl, &srq_attr, &ioc->ioc_srq_hdl, + &ioc->ioc_srq_attr); + if (status != IBT_SUCCESS) { + SRPT_DPRINTF_L1("ioc_init, IBT create SRQ failed(%d)", status); + goto srq_alloc_err; + } + + SRPT_DPRINTF_L2("ioc_init, SRQ WR size(%d), SG size(%d)", + ioc->ioc_srq_attr.srq_wr_sz, ioc->ioc_srq_attr.srq_sgl_sz); + + ibt_set_srq_private(ioc->ioc_srq_hdl, ioc); + + /* + * Allocate a pool of SRP IU message buffers and post them to + * the I/O Controller SRQ. We let the SRQ manage the free IU + * messages. + */ + ioc->ioc_num_iu_entries = + min(srq_attr.srq_wr_sz, srpt_ioc_srq_size) - 1; + + ioc->ioc_iu_pool = kmem_zalloc(sizeof (srpt_iu_t) * + ioc->ioc_num_iu_entries, KM_SLEEP); + + ioc->ioc_iu_bufs = kmem_alloc(SRPT_DEFAULT_SEND_MSG_SIZE * + ioc->ioc_num_iu_entries, KM_SLEEP); + + if ((ioc->ioc_iu_pool == NULL) || (ioc->ioc_iu_bufs == NULL)) { + SRPT_DPRINTF_L1("ioc_init, failed to allocate SRQ IUs"); + goto srq_iu_alloc_err; + } + + mr_attr.mr_vaddr = (ib_vaddr_t)(uintptr_t)ioc->ioc_iu_bufs; + mr_attr.mr_len = SRPT_DEFAULT_SEND_MSG_SIZE * ioc->ioc_num_iu_entries; + mr_attr.mr_as = NULL; + mr_attr.mr_flags = IBT_MR_ENABLE_LOCAL_WRITE; + + status = ibt_register_mr(ioc->ioc_ibt_hdl, ioc->ioc_pd_hdl, + &mr_attr, &ioc->ioc_iu_mr_hdl, &mr_desc); + if (status != IBT_SUCCESS) { + SRPT_DPRINTF_L1("ioc_init, IU buffer pool MR err(%d)", + status); + goto srq_iu_alloc_err; + } + + for (iu_ndx = 0, iu = ioc->ioc_iu_pool; iu_ndx < + ioc->ioc_num_iu_entries; iu_ndx++, iu++) { + + iu_offset = (iu_ndx * SRPT_DEFAULT_SEND_MSG_SIZE); + iu->iu_buf = (void *)((uintptr_t)ioc->ioc_iu_bufs + iu_offset); + + mutex_init(&iu->iu_lock, NULL, MUTEX_DRIVER, NULL); + + iu->iu_sge.ds_va = mr_desc.md_vaddr + iu_offset; + iu->iu_sge.ds_key = mr_desc.md_lkey; + iu->iu_sge.ds_len = SRPT_DEFAULT_SEND_MSG_SIZE; + iu->iu_ioc = ioc; + iu->iu_pool_ndx = iu_ndx; + + status = srpt_ioc_post_recv_iu(ioc, &ioc->ioc_iu_pool[iu_ndx]); + if (status != IBT_SUCCESS) { + SRPT_DPRINTF_L1("ioc_init, SRQ IU post err(%d)", + status); + goto srq_iu_post_err; + } + } + + /* + * Initialize the dbuf vmem arena + */ + (void) snprintf(namebuf, sizeof (namebuf), + "srpt_buf_pool_%16llX", (u_longlong_t)guid); + ioc->ioc_dbuf_pool = srpt_vmem_create(namebuf, ioc, + SRPT_BUF_MR_CHUNKSIZE, SRPT_BUF_POOL_MAX, srpt_dbuf_mr_flags); + + if (ioc->ioc_dbuf_pool == NULL) { + goto stmf_db_alloc_err; + } + + /* + * Allocate the I/O Controller STMF data buffer allocator. The + * data store will span all targets associated with this IOC. + */ + ioc->ioc_stmf_ds = stmf_alloc(STMF_STRUCT_DBUF_STORE, 0, 0); + if (ioc->ioc_stmf_ds == NULL) { + SRPT_DPRINTF_L1("ioc_attach, STMF DBUF alloc failure for IOC"); + goto stmf_db_alloc_err; + } + ioc->ioc_stmf_ds->ds_alloc_data_buf = &srpt_ioc_ds_alloc_dbuf; + ioc->ioc_stmf_ds->ds_free_data_buf = &srpt_ioc_ds_free_dbuf; + ioc->ioc_stmf_ds->ds_port_private = ioc; + + rw_exit(&ioc->ioc_rwlock); + return (ioc); + +stmf_db_alloc_err: + if (ioc->ioc_dbuf_pool != NULL) { + srpt_vmem_destroy(ioc->ioc_dbuf_pool); + } + +srq_iu_post_err: + if (ioc->ioc_iu_mr_hdl != NULL) { + status = ibt_deregister_mr(ioc->ioc_ibt_hdl, + ioc->ioc_iu_mr_hdl); + if (status != IBT_SUCCESS) { + SRPT_DPRINTF_L1("ioc_init, error deregistering" + " memory region (%d)", status); + } + } + for (err_ndx = 0, iu = ioc->ioc_iu_pool; err_ndx < iu_ndx; + err_ndx++, iu++) { + mutex_destroy(&iu->iu_lock); + } + +srq_iu_alloc_err: + if (ioc->ioc_iu_bufs != NULL) { + kmem_free(ioc->ioc_iu_bufs, SRPT_DEFAULT_SEND_MSG_SIZE * + ioc->ioc_num_iu_entries); + } + if (ioc->ioc_iu_pool != NULL) { + kmem_free(ioc->ioc_iu_pool, + sizeof (srpt_iu_t) * ioc->ioc_num_iu_entries); + } + if (ioc->ioc_srq_hdl != NULL) { + status = ibt_free_srq(ioc->ioc_srq_hdl); + if (status != IBT_SUCCESS) { + SRPT_DPRINTF_L1("ioc_init, error freeing SRQ (%d)", + status); + } + + } + +srq_alloc_err: + status = ibt_free_pd(ioc->ioc_ibt_hdl, ioc->ioc_pd_hdl); + if (status != IBT_SUCCESS) { + SRPT_DPRINTF_L1("ioc_init, free PD error (%d)", status); + } + +pd_alloc_err: + status = ibt_close_hca(ioc->ioc_ibt_hdl); + if (status != IBT_SUCCESS) { + SRPT_DPRINTF_L1("ioc_init, close ioc error (%d)", status); + } + +hca_open_err: + rw_exit(&ioc->ioc_rwlock); + rw_destroy(&ioc->ioc_rwlock); + kmem_free(ioc, sizeof (*ioc)); + return (NULL); +} + +/* + * srpt_ioc_fini() - I/O Controller Cleanup + * + * Requires srpt_ctxt->sc_rwlock be held outside of call. + */ +static void +srpt_ioc_fini(srpt_ioc_t *ioc) +{ + int status; + int ndx; + + /* + * Note driver flows will have already taken all SRP + * services running on the I/O Controller off-line. + */ + rw_enter(&ioc->ioc_rwlock, RW_WRITER); + if (ioc->ioc_ibt_hdl != NULL) { + if (ioc->ioc_stmf_ds != NULL) { + stmf_free(ioc->ioc_stmf_ds); + } + + if (ioc->ioc_srq_hdl != NULL) { + SRPT_DPRINTF_L4("ioc_fini, freeing SRQ"); + status = ibt_free_srq(ioc->ioc_srq_hdl); + if (status != IBT_SUCCESS) { + SRPT_DPRINTF_L1("ioc_fini, free SRQ" + " error (%d)", status); + } + } + + if (ioc->ioc_iu_mr_hdl != NULL) { + status = ibt_deregister_mr( + ioc->ioc_ibt_hdl, ioc->ioc_iu_mr_hdl); + if (status != IBT_SUCCESS) { + SRPT_DPRINTF_L1("ioc_fini, error deregistering" + " memory region (%d)", status); + } + } + + if (ioc->ioc_iu_bufs != NULL) { + kmem_free(ioc->ioc_iu_bufs, SRPT_DEFAULT_SEND_MSG_SIZE * + ioc->ioc_num_iu_entries); + } + + if (ioc->ioc_iu_pool != NULL) { + SRPT_DPRINTF_L4("ioc_fini, freeing IU entries"); + for (ndx = 0; ndx < ioc->ioc_num_iu_entries; ndx++) { + mutex_destroy(&ioc->ioc_iu_pool[ndx].iu_lock); + } + + SRPT_DPRINTF_L4("ioc_fini, free IU pool struct"); + kmem_free(ioc->ioc_iu_pool, + sizeof (srpt_iu_t) * (ioc->ioc_num_iu_entries)); + ioc->ioc_iu_pool = NULL; + ioc->ioc_num_iu_entries = 0; + } + + if (ioc->ioc_dbuf_pool != NULL) { + srpt_vmem_destroy(ioc->ioc_dbuf_pool); + } + + if (ioc->ioc_pd_hdl != NULL) { + status = ibt_free_pd(ioc->ioc_ibt_hdl, + ioc->ioc_pd_hdl); + if (status != IBT_SUCCESS) { + SRPT_DPRINTF_L1("ioc_fini, free PD" + " error (%d)", status); + } + } + + status = ibt_close_hca(ioc->ioc_ibt_hdl); + if (status != IBT_SUCCESS) { + SRPT_DPRINTF_L1( + "ioc_fini, close ioc error (%d)", status); + } + } + rw_exit(&ioc->ioc_rwlock); + rw_destroy(&ioc->ioc_rwlock); + kmem_free(ioc, sizeof (srpt_ioc_t)); +} + +/* + * srpt_ioc_port_active() - I/O Controller port active + */ +static void +srpt_ioc_port_active(ibt_async_event_t *event) +{ + ibt_status_t status; + srpt_ioc_t *ioc; + + ASSERT(event != NULL); + + SRPT_DPRINTF_L3("ioc_port_active event handler, invoked"); + + /* + * Find the HCA in question and if the HCA has completed + * initialization, and the SRP Target service for the + * the I/O Controller exists, then bind this port. + */ + ioc = srpt_ioc_get(event->ev_hca_guid); + + if (ioc == NULL) { + SRPT_DPRINTF_L2("ioc_port_active, I/O Controller not" + " active"); + return; + } + + if (ioc->ioc_tgt_port == NULL) { + SRPT_DPRINTF_L2("ioc_port_active, no I/O Controller target" + " undefined"); + return; + } + + + /* + * We take the target lock here to serialize this operation + * with any STMF initiated target state transitions. If + * SRP is off-line then the service handle is NULL. + */ + mutex_enter(&ioc->ioc_tgt_port->tp_lock); + + if (ioc->ioc_tgt_port->tp_ibt_svc_hdl != NULL) { + status = srpt_ioc_svc_bind(ioc->ioc_tgt_port, event->ev_port); + if (status != IBT_SUCCESS && + status != IBT_HCA_PORT_NOT_ACTIVE) { + SRPT_DPRINTF_L1("ioc_port_active, bind failed (%d)", + status); + } + } + mutex_exit(&ioc->ioc_tgt_port->tp_lock); +} + +/* + * srpt_ioc_port_down() + */ +static void +srpt_ioc_port_down(ibt_async_event_t *event) +{ + srpt_ioc_t *ioc; + srpt_target_port_t *tgt; + srpt_channel_t *ch; + srpt_channel_t *next_ch; + + SRPT_DPRINTF_L3("ioc_port_down event handler, invoked"); + + /* + * Find the HCA in question and if the HCA has completed + * initialization, and the SRP Target service for the + * the I/O Controller exists, then logout initiators + * through this port. + */ + ioc = srpt_ioc_get(event->ev_hca_guid); + + if (ioc == NULL) { + SRPT_DPRINTF_L2("ioc_port_down, I/O Controller not" + " active"); + return; + } + + /* + * We only have one target now, but we could go through all + * SCSI target ports if more are added. + */ + tgt = ioc->ioc_tgt_port; + if (tgt == NULL) { + SRPT_DPRINTF_L2("ioc_port_down, no I/O Controller target" + " undefined"); + return; + } + mutex_enter(&tgt->tp_lock); + + /* + * For all channel's logged in through this port, initiate a + * disconnect. + */ + mutex_enter(&tgt->tp_ch_list_lock); + ch = list_head(&tgt->tp_ch_list); + while (ch != NULL) { + next_ch = list_next(&tgt->tp_ch_list, ch); + if (ch->ch_session && (ch->ch_session->ss_hw_port == + event->ev_port)) { + srpt_ch_disconnect(ch); + } + ch = next_ch; + } + mutex_exit(&tgt->tp_ch_list_lock); + + mutex_exit(&tgt->tp_lock); +} + +/* + * srpt_ioc_ib_async_hdlr - I/O Controller IB asynchronous events + */ +/* ARGSUSED */ +void +srpt_ioc_ib_async_hdlr(void *clnt, ibt_hca_hdl_t hdl, + ibt_async_code_t code, ibt_async_event_t *event) +{ + srpt_ioc_t *ioc; + srpt_channel_t *ch; + + switch (code) { + case IBT_EVENT_PORT_UP: + srpt_ioc_port_active(event); + break; + + case IBT_ERROR_PORT_DOWN: + srpt_ioc_port_down(event); + break; + + case IBT_HCA_ATTACH_EVENT: + rw_enter(&srpt_ctxt->sc_rwlock, RW_WRITER); + ioc = srpt_ioc_init(event->ev_hca_guid); + + if (ioc == NULL) { + rw_exit(&srpt_ctxt->sc_rwlock); + SRPT_DPRINTF_L1("ib_async_hdlr, HCA_ATTACH" + " event failed to initialize HCA (0x%016llx)", + (u_longlong_t)event->ev_hca_guid); + return; + } + SRPT_DPRINTF_L2("HCA_ATTACH_EVENT: I/O Controller" + " ibt hdl (%p)", + (void *)ioc->ioc_ibt_hdl); + + rw_enter(&ioc->ioc_rwlock, RW_WRITER); + ioc->ioc_tgt_port = srpt_stp_alloc_port(ioc, ioc->ioc_guid); + if (ioc->ioc_tgt_port == NULL) { + SRPT_DPRINTF_L1("ioc_ib_async_hdlr, alloc SCSI " + "target port error for HCA (0x%016llx)", + (u_longlong_t)event->ev_hca_guid); + rw_exit(&ioc->ioc_rwlock); + srpt_ioc_fini(ioc); + rw_exit(&srpt_ctxt->sc_rwlock); + return; + } + + /* + * New HCA added with default SCSI Target Port, SRP service + * will be started when SCSI Target Port is brought + * on-line by STMF. + */ + srpt_ctxt->sc_num_iocs++; + list_insert_tail(&srpt_ctxt->sc_ioc_list, ioc); + + rw_exit(&ioc->ioc_rwlock); + rw_exit(&srpt_ctxt->sc_rwlock); + break; + + case IBT_HCA_DETACH_EVENT: + SRPT_DPRINTF_L1( + "ioc_iob_async_hdlr, HCA_DETACH_EVENT received."); + break; + + case IBT_EVENT_EMPTY_CHAN: + /* Channel in ERROR state is now empty */ + ch = (srpt_channel_t *)ibt_get_chan_private(event->ev_chan_hdl); + SRPT_DPRINTF_L3( + "ioc_iob_async_hdlr, received empty channel error on %p", + (void *)ch); + break; + + default: + SRPT_DPRINTF_L2("ioc_ib_async_hdlr, event not " + "handled (%d)", code); + break; + } +} + +/* + * srpt_ioc_svc_bind() + */ +ibt_status_t +srpt_ioc_svc_bind(srpt_target_port_t *tgt, uint_t portnum) +{ + ibt_status_t status; + srpt_hw_port_t *port; + ibt_hca_portinfo_t *portinfo; + uint_t qportinfo_sz; + uint_t qportnum; + ib_gid_t new_gid; + srpt_ioc_t *ioc; + + ASSERT(tgt != NULL); + ASSERT(tgt->tp_ioc != NULL); + ioc = tgt->tp_ioc; + + if (tgt->tp_ibt_svc_hdl == NULL) { + SRPT_DPRINTF_L2("ioc_svc_bind, NULL SCSI target port" + " service"); + return (IBT_INVALID_PARAM); + } + + if (portnum == 0 || portnum > tgt->tp_nports) { + SRPT_DPRINTF_L2("ioc_svc_bind, bad port (%d)", portnum); + return (IBT_INVALID_PARAM); + } + status = ibt_query_hca_ports(ioc->ioc_ibt_hdl, portnum, + &portinfo, &qportnum, &qportinfo_sz); + if (status != IBT_SUCCESS) { + SRPT_DPRINTF_L1("ioc_svc_bind, query port error (%d)", + portnum); + return (IBT_INVALID_PARAM); + } + + ASSERT(portinfo != NULL); + + /* + * If port is not active do nothing, caller should attempt to bind + * after the port goes active. + */ + if (portinfo->p_linkstate != IBT_PORT_ACTIVE) { + SRPT_DPRINTF_L2("ioc_svc_bind, port %d not in active state", + portnum); + ibt_free_portinfo(portinfo, qportinfo_sz); + return (IBT_HCA_PORT_NOT_ACTIVE); + } + + port = &tgt->tp_hw_port[portnum-1]; + new_gid = portinfo->p_sgid_tbl[0]; + ibt_free_portinfo(portinfo, qportinfo_sz); + + /* + * If previously bound and the port GID has changed, + * rebind to the new GID. + */ + if (port->hwp_bind_hdl != NULL) { + if (new_gid.gid_guid != port->hwp_gid.gid_guid || + new_gid.gid_prefix != port->hwp_gid.gid_prefix) { + SRPT_DPRINTF_L2("ioc_svc_bind, unregister current" + " bind"); + ibt_unbind_service(tgt->tp_ibt_svc_hdl, + port->hwp_bind_hdl); + port->hwp_bind_hdl = NULL; + } + } + SRPT_DPRINTF_L2("ioc_svc_bind, bind service, %016llx:%016llx", + (u_longlong_t)new_gid.gid_prefix, + (u_longlong_t)new_gid.gid_guid); + + /* + * Pass SCSI Target Port as CM private data, the target will always + * exist while this service is bound. + */ + status = ibt_bind_service(tgt->tp_ibt_svc_hdl, new_gid, NULL, tgt, + &port->hwp_bind_hdl); + if (status != IBT_SUCCESS && status != IBT_CM_SERVICE_EXISTS) { + SRPT_DPRINTF_L1("ioc_svc_bind, bind error (%d)", status); + return (status); + } + + return (IBT_SUCCESS); +} + +/* + * srpt_ioc_svc_unbind() + */ +void +srpt_ioc_svc_unbind(srpt_target_port_t *tgt, uint_t portnum) +{ + srpt_hw_port_t *port; + + if (tgt == NULL) { + SRPT_DPRINTF_L2("ioc_svc_unbind, SCSI target does not exist"); + return; + } + + if (portnum == 0 || portnum > tgt->tp_nports) { + SRPT_DPRINTF_L2("ioc_svc_unbind, bad port (%d)", portnum); + return; + } + port = &tgt->tp_hw_port[portnum-1]; + + if (tgt->tp_ibt_svc_hdl != NULL && port->hwp_bind_hdl != NULL) { + SRPT_DPRINTF_L2("ioc_svc_unbind, unregister current bind"); + ibt_unbind_service(tgt->tp_ibt_svc_hdl, port->hwp_bind_hdl); + } + port->hwp_bind_hdl = NULL; +} + +/* + * srpt_ioc_svc_unbind_all() + */ +void +srpt_ioc_svc_unbind_all(srpt_target_port_t *tgt) +{ + uint_t portnum; + + if (tgt == NULL) { + SRPT_DPRINTF_L2("ioc_svc_unbind_all, NULL SCSI target port" + " specified"); + return; + } + for (portnum = 1; portnum <= tgt->tp_nports; portnum++) { + srpt_ioc_svc_unbind(tgt, portnum); + } +} + +/* + * srpt_ioc_get_locked() + * + * Requires srpt_ctxt->rw_lock be held outside of call. + */ +srpt_ioc_t * +srpt_ioc_get_locked(ib_guid_t guid) +{ + srpt_ioc_t *ioc; + + ioc = list_head(&srpt_ctxt->sc_ioc_list); + while (ioc != NULL) { + if (ioc->ioc_guid == guid) { + break; + } + ioc = list_next(&srpt_ctxt->sc_ioc_list, ioc); + } + return (ioc); +} + +/* + * srpt_ioc_get() + */ +srpt_ioc_t * +srpt_ioc_get(ib_guid_t guid) +{ + srpt_ioc_t *ioc; + + rw_enter(&srpt_ctxt->sc_rwlock, RW_READER); + ioc = srpt_ioc_get_locked(guid); + rw_exit(&srpt_ctxt->sc_rwlock); + return (ioc); +} + +/* + * srpt_ioc_post_recv_iu() + */ +ibt_status_t +srpt_ioc_post_recv_iu(srpt_ioc_t *ioc, srpt_iu_t *iu) +{ + ibt_status_t status; + ibt_recv_wr_t wr; + uint_t posted; + + ASSERT(ioc != NULL); + ASSERT(iu != NULL); + + wr.wr_id = (ibt_wrid_t)(uintptr_t)iu; + wr.wr_nds = 1; + wr.wr_sgl = &iu->iu_sge; + posted = 0; + + status = ibt_post_srq(ioc->ioc_srq_hdl, &wr, 1, &posted); + if (status != IBT_SUCCESS) { + SRPT_DPRINTF_L2("ioc_post_recv_iu, post error (%d)", + status); + } + return (status); +} + +/* + * srpt_ioc_repost_recv_iu() + */ +void +srpt_ioc_repost_recv_iu(srpt_ioc_t *ioc, srpt_iu_t *iu) +{ + srpt_channel_t *ch; + ibt_status_t status; + + ASSERT(iu != NULL); + ASSERT(mutex_owned(&iu->iu_lock)); + + /* + * Some additional sanity checks while in debug state, all STMF + * related task activities should be complete prior to returning + * this IU to the available pool. + */ + ASSERT(iu->iu_stmf_task == NULL); + ASSERT(iu->iu_sq_posted_cnt == 0); + + ch = iu->iu_ch; + iu->iu_ch = NULL; + iu->iu_num_rdescs = 0; + iu->iu_rdescs = NULL; + iu->iu_tot_xfer_len = 0; + iu->iu_tag = 0; + iu->iu_flags = 0; + iu->iu_sq_posted_cnt = 0; + + status = srpt_ioc_post_recv_iu(ioc, iu); + + if (status != IBT_SUCCESS) { + /* + * Very bad, we should initiate a shutdown of the I/O + * Controller here, off-lining any targets associated + * with this I/O Controller (and therefore disconnecting + * any logins that remain). + * + * In practice this should never happen so we put + * the code near the bottom of the implementation list. + */ + SRPT_DPRINTF_L0("ioc_repost_recv_iu, error RX IU (%d)", + status); + ASSERT(0); + } else if (ch != NULL) { + atomic_inc_32(&ch->ch_req_lim_delta); + } +} + +/* + * srpt_ioc_init_profile() + * + * SRP I/O Controller serialization lock must be held when this + * routine is invoked. + */ +void +srpt_ioc_init_profile(srpt_ioc_t *ioc) +{ + srpt_ioc_opcap_mask_t capmask = {0}; + + ASSERT(ioc != NULL); + + ioc->ioc_profile.ioc_guid = h2b64(ioc->ioc_guid); + (void) memcpy(ioc->ioc_profile.ioc_id_string, + "Solaris SRP Target 0.9a", 23); + + /* + * Note vendor ID and subsystem ID are 24 bit values. Low order + * 8 bits in vendor ID field is slot and is initialized to zero. + * Low order 8 bits of subsystem ID is a reserved field and + * initialized to zero. + */ + ioc->ioc_profile.ioc_vendorid = + h2b32((uint32_t)(ioc->ioc_attr.hca_vendor_id << 8)); + ioc->ioc_profile.ioc_deviceid = + h2b32((uint32_t)ioc->ioc_attr.hca_device_id); + ioc->ioc_profile.ioc_device_ver = + h2b16((uint16_t)ioc->ioc_attr.hca_version_id); + ioc->ioc_profile.ioc_subsys_vendorid = + h2b32((uint32_t)(ioc->ioc_attr.hca_vendor_id << 8)); + ioc->ioc_profile.ioc_subsys_id = h2b32(0); + ioc->ioc_profile.ioc_io_class = h2b16(SRP_REV_16A_IO_CLASS); + ioc->ioc_profile.ioc_io_subclass = h2b16(SRP_IO_SUBCLASS); + ioc->ioc_profile.ioc_protocol = h2b16(SRP_PROTOCOL); + ioc->ioc_profile.ioc_protocol_ver = h2b16(SRP_PROTOCOL_VERSION); + ioc->ioc_profile.ioc_send_msg_qdepth = h2b16(srpt_send_msg_depth); + ioc->ioc_profile.ioc_rdma_read_qdepth = + ioc->ioc_attr.hca_max_rdma_out_chan; + ioc->ioc_profile.ioc_send_msg_sz = h2b32(SRPT_DEFAULT_SEND_MSG_SIZE); + ioc->ioc_profile.ioc_rdma_xfer_sz = h2b32(SRPT_DEFAULT_MAX_RDMA_SIZE); + + capmask.bits.st = 1; /* Messages can be sent to IOC */ + capmask.bits.sf = 1; /* Messages can be sent from IOC */ + capmask.bits.rf = 1; /* RDMA Reads can be sent from IOC */ + capmask.bits.wf = 1; /* RDMA Writes can be sent from IOC */ + ioc->ioc_profile.ioc_ctrl_opcap_mask = capmask.mask; + + /* + * We currently only have one target, but if we had a list we would + * go through that list and only count those that are ONLINE when + * setting the services count and entries. + */ + if (ioc->ioc_tgt_port->tp_srp_enabled) { + ioc->ioc_profile.ioc_service_entries = 1; + ioc->ioc_svc.srv_id = h2b64(ioc->ioc_guid); + (void) snprintf((char *)ioc->ioc_svc.srv_name, + IB_DM_MAX_SVC_NAME_LEN, "SRP.T10:%016llx", + (u_longlong_t)ioc->ioc_guid); + } else { + ioc->ioc_profile.ioc_service_entries = 0; + ioc->ioc_svc.srv_id = 0; + } +} + +/* + * srpt_ioc_ds_alloc_dbuf() + */ +/* ARGSUSED */ +stmf_data_buf_t * +srpt_ioc_ds_alloc_dbuf(struct scsi_task *task, uint32_t size, + uint32_t *pminsize, uint32_t flags) +{ + srpt_iu_t *iu; + srpt_ioc_t *ioc; + srpt_ds_dbuf_t *dbuf; + stmf_data_buf_t *stmf_dbuf; + void *buf; + srpt_mr_t mr; + + ASSERT(task != NULL); + iu = task->task_port_private; + ioc = iu->iu_ioc; + + SRPT_DPRINTF_L4("ioc_ds_alloc_dbuf, invoked ioc(%p)" + " size(%d), flags(%x)", + (void *)ioc, size, flags); + + buf = srpt_vmem_alloc(ioc->ioc_dbuf_pool, size); + if (buf == NULL) { + return (NULL); + } + + if (srpt_vmem_mr(ioc->ioc_dbuf_pool, buf, size, &mr) != 0) { + goto stmf_alloc_err; + } + + stmf_dbuf = stmf_alloc(STMF_STRUCT_DATA_BUF, sizeof (srpt_ds_dbuf_t), + 0); + if (stmf_dbuf == NULL) { + SRPT_DPRINTF_L2("ioc_ds_alloc_dbuf, stmf_alloc failed"); + goto stmf_alloc_err; + } + + dbuf = stmf_dbuf->db_port_private; + dbuf->db_stmf_buf = stmf_dbuf; + dbuf->db_mr_hdl = mr.mr_hdl; + dbuf->db_ioc = ioc; + dbuf->db_sge.ds_va = mr.mr_va; + dbuf->db_sge.ds_key = mr.mr_lkey; + dbuf->db_sge.ds_len = size; + + stmf_dbuf->db_buf_size = size; + stmf_dbuf->db_data_size = size; + stmf_dbuf->db_relative_offset = 0; + stmf_dbuf->db_flags = 0; + stmf_dbuf->db_xfer_status = 0; + stmf_dbuf->db_sglist_length = 1; + stmf_dbuf->db_sglist[0].seg_addr = buf; + stmf_dbuf->db_sglist[0].seg_length = size; + + return (stmf_dbuf); + +buf_mr_err: + stmf_free(stmf_dbuf); + +stmf_alloc_err: + srpt_vmem_free(ioc->ioc_dbuf_pool, buf, size); + + return (NULL); +} + +void +srpt_ioc_ds_free_dbuf(struct stmf_dbuf_store *ds, + stmf_data_buf_t *dbuf) +{ + srpt_ioc_t *ioc; + + SRPT_DPRINTF_L4("ioc_ds_free_dbuf, invoked buf (%p)", + (void *)dbuf); + ioc = ds->ds_port_private; + + srpt_vmem_free(ioc->ioc_dbuf_pool, dbuf->db_sglist[0].seg_addr, + dbuf->db_buf_size); + stmf_free(dbuf); +} + +/* Memory arena routines */ + +static srpt_vmem_pool_t * +srpt_vmem_create(const char *name, srpt_ioc_t *ioc, ib_memlen_t chunksize, + uint64_t maxsize, ibt_mr_flags_t flags) +{ + srpt_mr_t *chunk; + srpt_vmem_pool_t *result; + + ASSERT(chunksize <= maxsize); + + result = kmem_zalloc(sizeof (srpt_vmem_pool_t), KM_SLEEP); + + result->svp_ioc = ioc; + result->svp_chunksize = chunksize; + result->svp_max_size = maxsize; + result->svp_flags = flags; + + rw_init(&result->svp_lock, NULL, RW_DRIVER, NULL); + avl_create(&result->svp_mr_list, srpt_vmem_mr_compare, + sizeof (srpt_mr_t), offsetof(srpt_mr_t, mr_avl)); + + chunk = srpt_vmem_chunk_alloc(result, chunksize); + + avl_add(&result->svp_mr_list, chunk); + result->svp_total_size = chunksize; + + result->svp_vmem = vmem_create(name, + (void*)(uintptr_t)chunk->mr_va, + (size_t)chunk->mr_len, SRPT_MR_QUANTSIZE, + NULL, NULL, NULL, 0, VM_SLEEP); + + return (result); +} + +static void +srpt_vmem_destroy(srpt_vmem_pool_t *vm_pool) +{ + srpt_mr_t *chunk; + srpt_mr_t *next; + + rw_enter(&vm_pool->svp_lock, RW_WRITER); + vmem_destroy(vm_pool->svp_vmem); + + chunk = avl_first(&vm_pool->svp_mr_list); + + while (chunk != NULL) { + next = AVL_NEXT(&vm_pool->svp_mr_list, chunk); + avl_remove(&vm_pool->svp_mr_list, chunk); + srpt_vmem_chunk_free(vm_pool, chunk); + chunk = next; + } + + avl_destroy(&vm_pool->svp_mr_list); + + rw_exit(&vm_pool->svp_lock); + rw_destroy(&vm_pool->svp_lock); + + kmem_free(vm_pool, sizeof (srpt_vmem_pool_t)); +} + +static void * +srpt_vmem_alloc(srpt_vmem_pool_t *vm_pool, size_t size) +{ + void *result; + srpt_mr_t *next; + ib_memlen_t chunklen; + + ASSERT(vm_pool != NULL); + + result = vmem_alloc(vm_pool->svp_vmem, size, + VM_NOSLEEP | VM_FIRSTFIT); + + if (result != NULL) { + /* memory successfully allocated */ + return (result); + } + + /* need more vmem */ + rw_enter(&vm_pool->svp_lock, RW_WRITER); + chunklen = vm_pool->svp_chunksize; + + if (vm_pool->svp_total_size >= vm_pool->svp_max_size) { + /* no more room to alloc */ + rw_exit(&vm_pool->svp_lock); + return (NULL); + } + + if ((vm_pool->svp_total_size + chunklen) > vm_pool->svp_max_size) { + chunklen = vm_pool->svp_max_size - vm_pool->svp_total_size; + } + + next = srpt_vmem_chunk_alloc(vm_pool, chunklen); + if (next != NULL) { + /* + * Note that the size of the chunk we got + * may not be the size we requested. Use the + * length returned in the chunk itself. + */ + if (vmem_add(vm_pool->svp_vmem, (void*)(uintptr_t)next->mr_va, + next->mr_len, VM_NOSLEEP) == NULL) { + srpt_vmem_chunk_free(vm_pool, next); + SRPT_DPRINTF_L2("vmem_add failed"); + } else { + vm_pool->svp_total_size += next->mr_len; + avl_add(&vm_pool->svp_mr_list, next); + } + } + + rw_exit(&vm_pool->svp_lock); + + result = vmem_alloc(vm_pool->svp_vmem, size, VM_NOSLEEP | VM_FIRSTFIT); + + return (result); +} + +static void +srpt_vmem_free(srpt_vmem_pool_t *vm_pool, void *vaddr, size_t size) +{ + vmem_free(vm_pool->svp_vmem, vaddr, size); +} + +static int +srpt_vmem_mr(srpt_vmem_pool_t *vm_pool, void *vaddr, size_t size, + srpt_mr_t *mr) +{ + avl_index_t where; + ib_vaddr_t mrva = (ib_vaddr_t)(uintptr_t)vaddr; + srpt_mr_t chunk; + srpt_mr_t *nearest; + ib_vaddr_t chunk_end; + int status = DDI_FAILURE; + + rw_enter(&vm_pool->svp_lock, RW_READER); + + chunk.mr_va = mrva; + nearest = avl_find(&vm_pool->svp_mr_list, &chunk, &where); + + if (nearest == NULL) { + nearest = avl_nearest(&vm_pool->svp_mr_list, where, + AVL_BEFORE); + } + + if (nearest != NULL) { + /* Verify this chunk contains the specified address range */ + ASSERT(nearest->mr_va <= mrva); + + chunk_end = nearest->mr_va + nearest->mr_len; + if (chunk_end >= mrva + size) { + mr->mr_hdl = nearest->mr_hdl; + mr->mr_va = mrva; + mr->mr_len = size; + mr->mr_lkey = nearest->mr_lkey; + mr->mr_rkey = nearest->mr_rkey; + status = DDI_SUCCESS; + } + } + + rw_exit(&vm_pool->svp_lock); + return (status); +} + +static srpt_mr_t * +srpt_vmem_chunk_alloc(srpt_vmem_pool_t *vm_pool, ib_memlen_t chunksize) +{ + void *chunk = NULL; + srpt_mr_t *result = NULL; + + while ((chunk == NULL) && (chunksize >= SRPT_MIN_CHUNKSIZE)) { + chunk = kmem_alloc(chunksize, KM_NOSLEEP); + if (chunk == NULL) { + SRPT_DPRINTF_L2("srpt_vmem_chunk_alloc: " + "failed to alloc chunk of %d, trying %d", + (int)chunksize, (int)chunksize/2); + chunksize /= 2; + } + } + + if (chunk != NULL) { + result = srpt_reg_mem(vm_pool, (ib_vaddr_t)(uintptr_t)chunk, + chunksize); + if (result == NULL) { + SRPT_DPRINTF_L2("srpt_vmem_chunk_alloc: " + "chunk registration failed"); + kmem_free(chunk, chunksize); + } + } + + return (result); +} + +static void +srpt_vmem_chunk_free(srpt_vmem_pool_t *vm_pool, srpt_mr_t *mr) +{ + void *chunk = (void *)(uintptr_t)mr->mr_va; + ib_memlen_t chunksize = mr->mr_len; + + srpt_dereg_mem(vm_pool->svp_ioc, mr); + kmem_free(chunk, chunksize); +} + +static srpt_mr_t * +srpt_reg_mem(srpt_vmem_pool_t *vm_pool, ib_vaddr_t vaddr, ib_memlen_t len) +{ + srpt_mr_t *result = NULL; + ibt_mr_attr_t mr_attr; + ibt_mr_desc_t mr_desc; + ibt_status_t status; + srpt_ioc_t *ioc = vm_pool->svp_ioc; + + result = kmem_zalloc(sizeof (srpt_mr_t), KM_NOSLEEP); + if (result == NULL) { + SRPT_DPRINTF_L2("srpt_reg_mem: failed to allocate"); + return (NULL); + } + + bzero(&mr_attr, sizeof (ibt_mr_attr_t)); + bzero(&mr_desc, sizeof (ibt_mr_desc_t)); + + mr_attr.mr_vaddr = vaddr; + mr_attr.mr_len = len; + mr_attr.mr_as = NULL; + mr_attr.mr_flags = vm_pool->svp_flags; + + status = ibt_register_mr(ioc->ioc_ibt_hdl, ioc->ioc_pd_hdl, + &mr_attr, &result->mr_hdl, &mr_desc); + if (status != IBT_SUCCESS) { + SRPT_DPRINTF_L2("srpt_reg_mem: ibt_register_mr " + "failed %d", status); + kmem_free(result, sizeof (srpt_mr_t)); + return (NULL); + } + + result->mr_va = mr_attr.mr_vaddr; + result->mr_len = mr_attr.mr_len; + result->mr_lkey = mr_desc.md_lkey; + result->mr_rkey = mr_desc.md_rkey; + + return (result); +} + +static void +srpt_dereg_mem(srpt_ioc_t *ioc, srpt_mr_t *mr) +{ + ibt_status_t status; + + status = ibt_deregister_mr(ioc->ioc_ibt_hdl, mr->mr_hdl); + if (status != IBT_SUCCESS) { + SRPT_DPRINTF_L1("ioc_fini, error deregistering MR (%d)", + status); + } + kmem_free(mr, sizeof (srpt_mr_t)); +} + +static int +srpt_vmem_mr_compare(const void *a, const void *b) +{ + srpt_mr_t *mr1 = (srpt_mr_t *)a; + srpt_mr_t *mr2 = (srpt_mr_t *)b; + + /* sort and match by virtual address */ + if (mr1->mr_va < mr2->mr_va) { + return (-1); + } else if (mr1->mr_va > mr2->mr_va) { + return (1); + } + + return (0); +} diff --git a/usr/src/uts/common/io/comstar/port/srpt/srpt_ioc.h b/usr/src/uts/common/io/comstar/port/srpt/srpt_ioc.h new file mode 100644 index 0000000000..f9c6fab614 --- /dev/null +++ b/usr/src/uts/common/io/comstar/port/srpt/srpt_ioc.h @@ -0,0 +1,62 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ + +/* + * Copyright 2009 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +#ifndef _SRPT_IOC_H +#define _SRPT_IOC_H + +/* + * Prototypes and data structures specific to I/O Controller + * operation. + */ + +#ifdef __cplusplus +extern "C" { +#endif +#include "srpt_impl.h" + +int srpt_ioc_attach(); +void srpt_ioc_detach(); +void srpt_ioc_init_profile(srpt_ioc_t *ioc); +ibt_status_t srpt_ioc_svc_bind(srpt_target_port_t *tgt, uint_t portnum); +void srpt_ioc_svc_unbind(srpt_target_port_t *tgt, uint_t portnum); +void srpt_ioc_svc_unbind_all(srpt_target_port_t *tgt); + +srpt_ioc_t *srpt_ioc_get_locked(ib_guid_t guid); +srpt_ioc_t *srpt_ioc_get(ib_guid_t guid); + +ibt_status_t srpt_ioc_post_recv_iu(srpt_ioc_t *ioc, srpt_iu_t *iu); +void srpt_ioc_repost_recv_iu(srpt_ioc_t *ioc, srpt_iu_t *iu); + +stmf_data_buf_t *srpt_ioc_ds_alloc_dbuf(struct scsi_task *task, + uint32_t size, uint32_t *pminsize, uint32_t flags); +void srpt_ioc_ds_free_dbuf(struct stmf_dbuf_store *ds, + stmf_data_buf_t *dbuf); + +#ifdef __cplusplus +} +#endif + +#endif /* _SRPT_IOC_H */ diff --git a/usr/src/uts/common/io/comstar/port/srpt/srpt_ioctl.h b/usr/src/uts/common/io/comstar/port/srpt/srpt_ioctl.h new file mode 100644 index 0000000000..52a5419345 --- /dev/null +++ b/usr/src/uts/common/io/comstar/port/srpt/srpt_ioctl.h @@ -0,0 +1,50 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ + +/* + * Copyright 2009 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +#ifndef _SRPT_IOCTL_H_ +#define _SRPT_IOCTL_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +/* + * SRP Target Port Provider ioctls. + */ + +#define SRPT_IOC_ENABLE_SVC 1 +#define SRPT_IOC_DISABLE_SVC 2 + +/* + * SRP Target pseudo device + */ +#define SRPT_NODE "/devices/ib/srpt@0:srpt" + +#ifdef __cplusplus +} +#endif + +#endif /* _SRPT_IOCTL_H_ */ diff --git a/usr/src/uts/common/io/comstar/port/srpt/srpt_mod.c b/usr/src/uts/common/io/comstar/port/srpt/srpt_mod.c new file mode 100644 index 0000000000..1c9fe24a91 --- /dev/null +++ b/usr/src/uts/common/io/comstar/port/srpt/srpt_mod.c @@ -0,0 +1,642 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ + +/* + * Copyright 2009 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +/* + * Solaris SCSI RDMA Protocol Target (SRP) transport port provider + * module for the COMSTAR framework. + */ + +#include <sys/cpuvar.h> +#include <sys/types.h> +#include <sys/conf.h> +#include <sys/stat.h> +#include <sys/file.h> +#include <sys/ddi.h> +#include <sys/sunddi.h> +#include <sys/modctl.h> +#include <sys/sysmacros.h> +#include <sys/sdt.h> +#include <sys/taskq.h> + +#include <stmf.h> +#include <stmf_ioctl.h> +#include <portif.h> + +#include "srp.h" +#include "srpt_impl.h" +#include "srpt_ioc.h" +#include "srpt_stp.h" +#include "srpt_cm.h" +#include "srpt_ioctl.h" + +#define SRPT_NAME_VERSION "COMSTAR SRP Target" + +/* + * srpt_send_msg_depth - Tunable parameter that specifies the + * maximum messages that could be in flight for a channel. + */ +uint16_t srpt_send_msg_depth = SRPT_DEFAULT_SEND_MSG_DEPTH; + +/* + * srpt_errlevel -- determine which error conditions are logged + */ +uint_t srpt_errlevel = SRPT_LOG_DEFAULT_LEVEL; + +srpt_ctxt_t *srpt_ctxt; + +/* + * DDI entry points. + */ +static int srpt_drv_attach(dev_info_t *, ddi_attach_cmd_t); +static int srpt_drv_detach(dev_info_t *, ddi_detach_cmd_t); +static int srpt_drv_getinfo(dev_info_t *, ddi_info_cmd_t, void *, void **); +static int srpt_drv_open(dev_t *, int, int, cred_t *); +static int srpt_drv_close(dev_t, int, int, cred_t *); +static int srpt_drv_ioctl(dev_t, int, intptr_t, int, cred_t *, int *); + +/* helper functions */ +static int srpt_disable_srp_services(void); +static int srpt_enable_srp_services(void); +static int srpt_ibdma_ops_load(srpt_ibdma_ops_t *); +static void srpt_ibdma_ops_unload(srpt_ibdma_ops_t *); + +extern struct mod_ops mod_miscops; + +static struct cb_ops srpt_cb_ops = { + srpt_drv_open, /* cb_open */ + srpt_drv_close, /* cb_close */ + nodev, /* cb_strategy */ + nodev, /* cb_print */ + nodev, /* cb_dump */ + nodev, /* cb_read */ + nodev, /* cb_write */ + srpt_drv_ioctl, /* cb_ioctl */ + nodev, /* cb_devmap */ + nodev, /* cb_mmap */ + nodev, /* cb_segmap */ + nochpoll, /* cb_chpoll */ + ddi_prop_op, /* cb_prop_op */ + NULL, /* cb_streamtab */ + D_MP, /* cb_flag */ + CB_REV, /* cb_rev */ + nodev, /* cb_aread */ + nodev, /* cb_awrite */ +}; + +static struct dev_ops srpt_dev_ops = { + DEVO_REV, /* devo_rev */ + 0, /* devo_refcnt */ + srpt_drv_getinfo, /* devo_getinfo */ + nulldev, /* devo_identify */ + nulldev, /* devo_probe */ + srpt_drv_attach, /* devo_attach */ + srpt_drv_detach, /* devo_detach */ + nodev, /* devo_reset */ + &srpt_cb_ops, /* devo_cb_ops */ + NULL, /* devo_bus_ops */ + NULL, /* devo_power */ + ddi_quiesce_not_needed, /* quiesce */ +}; + +static struct modldrv modldrv = { + &mod_driverops, + SRPT_NAME_VERSION, + &srpt_dev_ops, +}; + +static struct modlinkage srpt_modlinkage = { + MODREV_1, + &modldrv, + NULL, +}; + +static char srpt_pp_name[] = "srpt"; + +/* + * Prototypes + */ +static void srpt_pp_cb(stmf_port_provider_t *, int, void *, uint32_t); + +/* + * _init() + */ +int +_init(void) +{ + int status; + + /* + * Global one time initialization. + */ + srpt_ctxt = kmem_zalloc(sizeof (srpt_ctxt_t), KM_SLEEP); + ASSERT(srpt_ctxt != NULL); + rw_init(&srpt_ctxt->sc_rwlock, NULL, RW_DRIVER, NULL); + + /* Start-up state is DISABLED. SMF will tell us if we should enable. */ + srpt_ctxt->sc_svc_state = SRPT_SVC_DISABLED; + + status = mod_install(&srpt_modlinkage); + if (status != DDI_SUCCESS) { + cmn_err(CE_CONT, "_init, failed mod_install %d", status); + rw_destroy(&srpt_ctxt->sc_rwlock); + kmem_free(srpt_ctxt, sizeof (srpt_ctxt_t)); + srpt_ctxt = NULL; + } + + return (status); +} + +/* + * _info() + */ +int +_info(struct modinfo *modinfop) +{ + return (mod_info(&srpt_modlinkage, modinfop)); +} + +/* + * _fini() + */ +int +_fini(void) +{ + int status; + + status = mod_remove(&srpt_modlinkage); + if (status != DDI_SUCCESS) { + return (status); + } + + rw_destroy(&srpt_ctxt->sc_rwlock); + kmem_free(srpt_ctxt, sizeof (srpt_ctxt_t)); + srpt_ctxt = NULL; + + return (status); +} + +/* + * DDI entry points. + */ + +/* + * srpt_getinfo() + */ +/* ARGSUSED */ +static int +srpt_drv_getinfo(dev_info_t *dip, ddi_info_cmd_t cmd, void *arg, void **result) +{ + + switch (cmd) { + case DDI_INFO_DEVT2DEVINFO: + *result = srpt_ctxt->sc_dip; + return (DDI_SUCCESS); + + case DDI_INFO_DEVT2INSTANCE: + *result = NULL; + return (DDI_SUCCESS); + + default: + break; + } + return (DDI_FAILURE); +} + +/* + * srpt_drv_attach() + */ +static int +srpt_drv_attach(dev_info_t *dip, ddi_attach_cmd_t cmd) +{ + int status; + + switch (cmd) { + case DDI_ATTACH: + break; + + case DDI_RESUME: + return (DDI_SUCCESS); + + default: + return (DDI_FAILURE); + } + + /* + * We only allow a single instance. + */ + if (ddi_get_instance(dip) != 0) { + SRPT_DPRINTF_L1("drv_attach, error non-zero instance"); + return (DDI_FAILURE); + } + + /* + * Create minor node that might ultimately be used to create + * targets outside of srpt. + */ + status = ddi_create_minor_node(dip, ddi_get_name(dip), + S_IFCHR, 0, DDI_PSEUDO, 0); + if (status != DDI_SUCCESS) { + SRPT_DPRINTF_L1("drv_attach, minor node creation error (%d)", + status); + return (DDI_FAILURE); + } + + rw_enter(&srpt_ctxt->sc_rwlock, RW_WRITER); + srpt_ctxt->sc_dip = dip; + rw_exit(&srpt_ctxt->sc_rwlock); + + return (DDI_SUCCESS); +} + +/* + * srpt_enable_srp_services() + * + * Caller must be holding the sc_rwlock as RW_WRITER. + */ +static int +srpt_enable_srp_services(void) +{ + int status; + srpt_ioc_t *ioc; + + ASSERT((rw_read_locked(&srpt_ctxt->sc_rwlock)) == 0); + + SRPT_DPRINTF_L3("srpt_enable_srp_services"); + + /* Register the port provider */ + srpt_ctxt->sc_pp = (stmf_port_provider_t *) + stmf_alloc(STMF_STRUCT_PORT_PROVIDER, 0, 0); + srpt_ctxt->sc_pp->pp_portif_rev = PORTIF_REV_1; + srpt_ctxt->sc_pp->pp_name = srpt_pp_name; + srpt_ctxt->sc_pp->pp_cb = srpt_pp_cb; + status = stmf_register_port_provider(srpt_ctxt->sc_pp); + if (status != STMF_SUCCESS) { + SRPT_DPRINTF_L1("enable_srp: SRP port_provider registration" + " failed(%d)", status); + goto err_exit_1; + } + + /* + * Initialize IB resources, creating a list of SRP I/O Controllers. + */ + status = srpt_ioc_attach(); + if (status != DDI_SUCCESS) { + SRPT_DPRINTF_L1("enable_srp: error attach I/O" + " Controllers (%d)", status); + goto err_exit_2; + } + + if (srpt_ctxt->sc_num_iocs == 0) { + SRPT_DPRINTF_L2("enable_srp: no IB I/O Controllers found"); + status = DDI_FAILURE; + goto err_exit_3; + } + + /* + * For each I/O Controller register the default SCSI Target Port + * with STMF, and prepare profile and services. SRP will not + * start until the associated LPORT is brought on-line. + */ + ioc = list_head(&srpt_ctxt->sc_ioc_list); + + while (ioc != NULL) { + rw_enter(&ioc->ioc_rwlock, RW_WRITER); + ioc->ioc_tgt_port = srpt_stp_alloc_port(ioc, ioc->ioc_guid); + if (ioc->ioc_tgt_port == NULL) { + SRPT_DPRINTF_L1("enable_srp: alloc SCSI" + " Target Port error on GUID(%016llx)", + (u_longlong_t)ioc->ioc_guid); + } + + rw_exit(&ioc->ioc_rwlock); + ioc = list_next(&srpt_ctxt->sc_ioc_list, ioc); + } + + return (DDI_SUCCESS); + +err_exit_3: + srpt_ioc_detach(); + +err_exit_2: + stmf_deregister_port_provider(srpt_ctxt->sc_pp); + +err_exit_1: + stmf_free(srpt_ctxt->sc_pp); + srpt_ctxt->sc_pp = NULL; + + return (status); +} + +/* + * srpt_drv_detach() + * + * Refuse the detach request if we have channels open on + * any IOC. Users should use 'svcadm disable' to shutdown + * active targets. + */ +/*ARGSUSED*/ +static int +srpt_drv_detach(dev_info_t *dip, ddi_detach_cmd_t cmd) +{ + switch (cmd) { + case DDI_DETACH: + rw_enter(&srpt_ctxt->sc_rwlock, RW_WRITER); + if (srpt_ctxt->sc_svc_state != SRPT_SVC_DISABLED) { + rw_exit(&srpt_ctxt->sc_rwlock); + return (DDI_FAILURE); + } + + ddi_remove_minor_node(dip, NULL); + srpt_ctxt->sc_dip = NULL; + + rw_exit(&srpt_ctxt->sc_rwlock); + + break; + + case DDI_SUSPEND: + return (DDI_FAILURE); + + default: + return (DDI_FAILURE); + } + + return (DDI_SUCCESS); +} + +/* + * srpt_disable_srp_services() + * + * Offlines all targets, deregisters all IOCs. Caller must hold + * the srpt_ctxt->sc_rwlock as RW_WRITER. + */ +static int +srpt_disable_srp_services(void) +{ + stmf_status_t stmf_status; + stmf_change_status_t cstatus; + srpt_ioc_t *ioc; + srpt_target_port_t *tgt; + int ret_status = 0; + + ASSERT((rw_read_locked(&srpt_ctxt->sc_rwlock)) == 0); + + /* + * For each I/O Controller remove all SRP services and de-register + * with the associated I/O Unit's IB Device Management Agent. + */ + ioc = list_head(&srpt_ctxt->sc_ioc_list); + + while (ioc != NULL) { + /* + * Notify STMF to take the I/O Controller SCSI Target Port(s) + * off-line after we mark them as disabled so that they will + * stay off-line. + */ + rw_enter(&ioc->ioc_rwlock, RW_WRITER); + + tgt = ioc->ioc_tgt_port; + if (tgt != NULL) { + mutex_enter(&tgt->tp_lock); + tgt->tp_drv_disabled = 1; + mutex_exit(&tgt->tp_lock); + + SRPT_DPRINTF_L2("disable_srp: unbind and de-register" + " services for GUID(%016llx)", + (u_longlong_t)ioc->ioc_guid); + + cstatus.st_completion_status = STMF_SUCCESS; + cstatus.st_additional_info = NULL; + + stmf_status = stmf_ctl(STMF_CMD_LPORT_OFFLINE, + tgt->tp_lport, &cstatus); + + /* + * Wait for asynchronous target off-line operation + * to complete and then deregister the target + * port. + */ + mutex_enter(&tgt->tp_lock); + while (tgt->tp_state != SRPT_TGT_STATE_OFFLINE) { + cv_wait(&tgt->tp_offline_complete, + &tgt->tp_lock); + } + mutex_exit(&tgt->tp_lock); + ioc->ioc_tgt_port = NULL; + + SRPT_DPRINTF_L3("disable_srp: IOC (0x%016llx) Target" + " SRP off-line complete", + (u_longlong_t)ioc->ioc_guid); + + stmf_status = srpt_stp_deregister_port(tgt); + if (stmf_status != STMF_SUCCESS) { + /* Fails if I/O is pending */ + if (ret_status == 0) { + ret_status = EBUSY; + } + SRPT_DPRINTF_L1("disable_srp: could not" + " de-register LPORT, err(0x%llx)", + (u_longlong_t)stmf_status); + } else { + (void) srpt_stp_free_port(tgt); + } + } + + rw_exit(&ioc->ioc_rwlock); + ioc = list_next(&srpt_ctxt->sc_ioc_list, ioc); + } + + /* don't release IOCs until all ports are deregistered */ + if (ret_status != 0) { + return (ret_status); + } + + /* + * Release I/O Controller(s) resources and detach. + */ + srpt_ioc_detach(); + + /* De-register ourselves as an STMF port provider */ + stmf_deregister_port_provider(srpt_ctxt->sc_pp); + stmf_free(srpt_ctxt->sc_pp); + srpt_ctxt->sc_pp = NULL; + + return (0); +} + +/* + * srpt_drv_open() + */ +/* ARGSUSED */ +static int +srpt_drv_open(dev_t *devp, int flag, int otyp, cred_t *credp) +{ + SRPT_DPRINTF_L3("drv_open, invoked"); + return (0); +} + +/* + * srpt_drv_close() + */ +/* ARGSUSED */ +static int +srpt_drv_close(dev_t dev, int flag, int otyp, cred_t *credp) +{ + SRPT_DPRINTF_L3("drv_close, invoked"); + return (0); +} + +/* + * srpt_drv_ioctl() + */ +/* ARGSUSED */ +static int +srpt_drv_ioctl(dev_t drv, int cmd, intptr_t argp, int flag, cred_t *cred, + int *retval) +{ + int ret = 0; + + SRPT_DPRINTF_L3("drv_ioctl, invoked, cmd = %d", cmd); + + if (drv_priv(cred) != 0) { + return (EPERM); + } + + rw_enter(&srpt_ctxt->sc_rwlock, RW_WRITER); + + switch (cmd) { + case SRPT_IOC_ENABLE_SVC: + if (srpt_ctxt->sc_svc_state != SRPT_SVC_DISABLED) { + break; + } + + ret = srpt_ibdma_ops_load(&srpt_ctxt->sc_ibdma_ops); + if (ret != 0) { + break; + } + + ret = srpt_enable_srp_services(); + if (ret == 0) { + srpt_ctxt->sc_svc_state = SRPT_SVC_ENABLED; + } + + break; + + case SRPT_IOC_DISABLE_SVC: + if (srpt_ctxt->sc_svc_state != SRPT_SVC_ENABLED) { + break; + } + + ret = srpt_disable_srp_services(); + if (ret == 0) { + srpt_ctxt->sc_svc_state = SRPT_SVC_DISABLED; + } + + srpt_ibdma_ops_unload(&srpt_ctxt->sc_ibdma_ops); + + break; + + default: + ret = EFAULT; + break; + } + + rw_exit(&srpt_ctxt->sc_rwlock); + + return (ret); +} + +/* + * srpt_pp_cb() + */ +/* ARGSUSED */ +static void +srpt_pp_cb(stmf_port_provider_t *pp, int cmd, void *arg, uint32_t flags) +{ + SRPT_DPRINTF_L3("srpt_pp_cb, invoked (%d)", cmd); + /* + * We don't currently utilize the port provider call-back, in the + * future we might use it to synchronize provider data via STMF. + */ +} + +static int +srpt_ibdma_ops_load(srpt_ibdma_ops_t *ops) +{ + int ibdma_err = 0; + + ASSERT(ops != NULL); + + ops->ibdmah = ddi_modopen("ibdma", KRTLD_MODE_FIRST, &ibdma_err); + if (ops->ibdmah == NULL) { + SRPT_DPRINTF_L0("failed to open ibdma driver, error = %d", + ibdma_err); + return (ibdma_err); + } + + ops->ibdma_register = (ibdma_hdl_t (*)())ddi_modsym(ops->ibdmah, + "ibdma_ioc_register", &ibdma_err); + if (ops->ibdma_register == NULL) { + SRPT_DPRINTF_L0( + "failed to modsym ibdma_ioc_register, error = %d", + ibdma_err); + goto done; + } + + ops->ibdma_unregister = (ibdma_status_t (*)())ddi_modsym(ops->ibdmah, + "ibdma_ioc_unregister", &ibdma_err); + if (ops->ibdma_unregister == NULL) { + SRPT_DPRINTF_L0( + "failed to modsym ibdma_ioc_unregister, error = %d", + ibdma_err); + goto done; + } + + ops->ibdma_update = (ibdma_status_t (*)())ddi_modsym(ops->ibdmah, + "ibdma_ioc_update", &ibdma_err); + if (ops->ibdma_update == NULL) { + SRPT_DPRINTF_L0( + "failed to modsym ibdma_ioc_update, error = %d", + ibdma_err); + } + +done: + if (ibdma_err != 0) { + srpt_ibdma_ops_unload(ops); + } + + return (ibdma_err); +} + +static void +srpt_ibdma_ops_unload(srpt_ibdma_ops_t *ops) +{ + if (ops != NULL) { + if (ops->ibdmah != NULL) { + (void) ddi_modclose(ops->ibdmah); + } + bzero(ops, sizeof (srpt_ibdma_ops_t)); + } +} diff --git a/usr/src/uts/common/io/comstar/port/srpt/srpt_stp.c b/usr/src/uts/common/io/comstar/port/srpt/srpt_stp.c new file mode 100644 index 0000000000..5eff03d994 --- /dev/null +++ b/usr/src/uts/common/io/comstar/port/srpt/srpt_stp.c @@ -0,0 +1,1527 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ + +/* + * Copyright 2009 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +/* + * SCSI Target Port I/F for Solaris SCSI RDMA Protocol Target (SRP) + * port provider module for the COMSTAR framework. + */ + +#include <sys/cpuvar.h> +#include <sys/types.h> +#include <sys/conf.h> +#include <sys/stat.h> +#include <sys/file.h> +#include <sys/ddi.h> +#include <sys/sunddi.h> +#include <sys/modctl.h> +#include <sys/sysmacros.h> +#include <sys/sdt.h> +#include <sys/taskq.h> +#include <sys/atomic.h> + +#include <stmf.h> +#include <stmf_ioctl.h> +#include <portif.h> + +#include <sys/ib/mgt/ibdma/ibdma.h> + +#include "srp.h" +#include "srpt_impl.h" +#include "srpt_cm.h" +#include "srpt_ioc.h" +#include "srpt_ch.h" +#include "srpt_stp.h" + +extern srpt_ctxt_t *srpt_ctxt; + +/* + * STMF LPort Interface Prototypes + */ +static stmf_status_t srpt_stp_xfer_data(struct scsi_task *task, + struct stmf_data_buf *dbuf, uint32_t ioflags); +stmf_status_t srpt_stp_send_status(struct scsi_task *task, + uint32_t ioflags); +static void srpt_stp_task_free(struct scsi_task *task); +static stmf_status_t srpt_stp_abort(struct stmf_local_port *lport, + int abort_cmd, void *arg, uint32_t flags); +static void srpt_stp_task_poll(struct scsi_task *task); +static void srpt_stp_ctl(struct stmf_local_port *lport, + int cmd, void *arg); +static stmf_status_t srpt_stp_info(uint32_t cmd, + struct stmf_local_port *lport, void *arg, uint8_t *buf, + uint32_t *bufsizep); +static void srpt_stp_event_handler(struct stmf_local_port *lport, + int eventid, void *arg, uint32_t flags); + +static void srpt_format_login_rsp(srp_login_req_t *req, + srp_login_rsp_t *rsp, uint8_t flags); +static void srpt_format_login_rej(srp_login_req_t *req, + srp_login_rej_t *rej, uint32_t reason); + +static scsi_devid_desc_t *srpt_stp_alloc_scsi_devid_desc(uint64_t guid); +static void srpt_stp_free_scsi_devid_desc(scsi_devid_desc_t *sdd); + +extern uint16_t srpt_send_msg_depth; + +/* + * srpt_stp_start_srp() - Start SRP service + * + * Enable the SRP service for the specified SCSI Target Port. + */ +int +srpt_stp_start_srp(srpt_target_port_t *tgt) +{ + ibt_status_t status; + ibdma_status_t dma_status; + int port; + srpt_ioc_t *ioc; + + if (tgt == NULL) { + SRPT_DPRINTF_L1("stp_start_srp, NULL SCSI target port"); + return (IBT_FAILURE); + } + + if (tgt->tp_ioc == NULL) { + SRPT_DPRINTF_L1("stp_start_srp, SCSI target port NULL" + " IOC pointer"); + return (IBT_FAILURE); + } + ioc = tgt->tp_ioc; + + SRPT_DPRINTF_L2("stp_start_srp, register SRP service for" + " svc_id (%016llx)", (u_longlong_t)tgt->tp_ibt_svc_id); + status = ibt_register_service(srpt_ctxt->sc_ibt_hdl, + &tgt->tp_ibt_svc_desc, tgt->tp_ibt_svc_id, 1, + &tgt->tp_ibt_svc_hdl, NULL); + if (status != IBT_SUCCESS) { + tgt->tp_ibt_svc_hdl = NULL; + SRPT_DPRINTF_L1("stp_start_srp, SRP service creation err (%d)", + status); + return (status); + } + + /* + * Bind the service associated with the SCSI target port to + * each active port of the I/O Controller. + */ + for (port = 0; port < ioc->ioc_attr.hca_nports; port++) { + status = srpt_ioc_svc_bind(tgt, port+1); + if (status != IBT_SUCCESS && + status != IBT_HCA_PORT_NOT_ACTIVE) { + SRPT_DPRINTF_L1("start_srp, Unable to bind" + " service (%d)", status); + goto srp_start_err; + } + } + tgt->tp_srp_enabled = 1; + + /* + * Calculate the new I/O Controller profile and either update the + * profile if previously registered or register it with the IB + * Device Management Agent. + */ + SRPT_DPRINTF_L3("start_srp, update I/O Controller profile (%016llx)", + (u_longlong_t)ioc->ioc_guid); + + srpt_ioc_init_profile(ioc); + if (ioc->ioc_ibdma_hdl == NULL) { + ioc->ioc_ibdma_hdl = + srpt_ctxt->sc_ibdma_ops.ibdma_register(ioc->ioc_guid, + &ioc->ioc_profile, &ioc->ioc_svc); + if (ioc->ioc_ibdma_hdl == NULL) { + SRPT_DPRINTF_L1("start_srp, Unable to register" + " I/O Profile (%d)", status); + goto srp_start_err; + } + } else { + dma_status = + srpt_ctxt->sc_ibdma_ops.ibdma_update(ioc->ioc_ibdma_hdl, + &ioc->ioc_profile, &ioc->ioc_svc); + if (dma_status != IBDMA_SUCCESS) { + SRPT_DPRINTF_L1("start_srp, Unable to update I/O" + " Profile (%d)", dma_status); + goto srp_start_err; + } + } + + return (IBT_SUCCESS); + +srp_start_err: + tgt->tp_srp_enabled = 0; + srpt_ioc_svc_unbind_all(tgt); + if (tgt->tp_ibt_svc_hdl != NULL) { + ibt_deregister_service(srpt_ctxt->sc_ibt_hdl, + tgt->tp_ibt_svc_hdl); + tgt->tp_ibt_svc_hdl = NULL; + } + return (status); +} + +/* + * srpt_stp_stop_srp() - Stop SRP service. + * + * Disable the SRP service on the specified SCSI Target Port. + */ +void +srpt_stp_stop_srp(srpt_target_port_t *tgt) +{ + ibt_status_t status; + ibdma_status_t dma_status; + srpt_ioc_t *ioc; + srpt_channel_t *ch; + + if (tgt == NULL) { + SRPT_DPRINTF_L2("stp_stop_srp, NULL SCSI Target Port" + " specified"); + return; + } + + if (tgt->tp_ioc == NULL) { + SRPT_DPRINTF_L2("stp_stop_srp, bad Target, IOC NULL"); + return; + } + ioc = tgt->tp_ioc; + + /* + * Update the I/O Controller profile to remove the SRP service + * for this SCSI target port. + */ + tgt->tp_srp_enabled = 0; + + if (ioc->ioc_ibdma_hdl != NULL) { + SRPT_DPRINTF_L3("stp_stop_srp, update I/O Controller" + " profile (%016llx)", (u_longlong_t)ioc->ioc_guid); + srpt_ioc_init_profile(ioc); + + if (ioc->ioc_profile.ioc_service_entries == 0) { + SRPT_DPRINTF_L3("stp_stop_srp, no services active" + " unregister IOC profile"); + srpt_ctxt->sc_ibdma_ops.ibdma_unregister( + ioc->ioc_ibdma_hdl); + ioc->ioc_ibdma_hdl = NULL; + } else { + dma_status = srpt_ctxt->sc_ibdma_ops.ibdma_update( + ioc->ioc_ibdma_hdl, &ioc->ioc_profile, + &ioc->ioc_svc); + if (dma_status != IBDMA_SUCCESS) { + SRPT_DPRINTF_L1("stp_stop_srp, Unable to" + " update I/O Profile (%d)", dma_status); + return; + } + } + } + + /* + * Unbind the SRP service associated with the SCSI target port + * from all of the I/O Controller physical ports. + */ + SRPT_DPRINTF_L2("stp_stop_srp, unbind and de-register service" + "(%016llx)", (u_longlong_t)tgt->tp_ibt_svc_id); + if (tgt->tp_ibt_svc_hdl != NULL) { + srpt_ioc_svc_unbind_all(tgt); + } + + if (tgt->tp_ibt_svc_hdl != NULL) { + status = ibt_deregister_service(srpt_ctxt->sc_ibt_hdl, + tgt->tp_ibt_svc_hdl); + if (status != IBT_SUCCESS) { + SRPT_DPRINTF_L1("stp_stop_srp, de-register service" + " error(%d)", status); + } + tgt->tp_ibt_svc_hdl = NULL; + } + + /* + * SRP service is now off-line for this SCSI Target Port. + * We force a disconnect (i.e. SRP Target Logout) for any + * active SRP logins. + */ + mutex_enter(&tgt->tp_ch_list_lock); + ch = list_head(&tgt->tp_ch_list); + while (ch != NULL) { + SRPT_DPRINTF_L3("stp_stop_srp, disconnect ch(%p)", + (void *)ch); + srpt_ch_disconnect(ch); + ch = list_next(&tgt->tp_ch_list, ch); + } + mutex_exit(&tgt->tp_ch_list_lock); + + /* + * wait for all sessions to terminate before returning + */ + mutex_enter(&tgt->tp_sess_list_lock); + while (!list_is_empty(&tgt->tp_sess_list)) { + cv_wait(&tgt->tp_sess_complete, &tgt->tp_sess_list_lock); + } + mutex_exit(&tgt->tp_sess_list_lock); +} + +/* + * srpt_stp_alloc_port() - Allocate SCSI Target Port + */ +srpt_target_port_t * +srpt_stp_alloc_port(srpt_ioc_t *ioc, ib_guid_t guid) +{ + stmf_status_t status; + srpt_target_port_t *tgt; + stmf_local_port_t *lport; + uint64_t temp; + + if (ioc == NULL) { + SRPT_DPRINTF_L1("stp_alloc_port, NULL I/O Controller"); + return (NULL); + } + + SRPT_DPRINTF_L3("stp_alloc_port, allocate STMF local port"); + lport = stmf_alloc(STMF_STRUCT_STMF_LOCAL_PORT, sizeof (*tgt), 0); + if (lport == NULL) { + SRPT_DPRINTF_L1("tgt_alloc_port, stmf_alloc failed"); + return (NULL); + } + + tgt = lport->lport_port_private; + ASSERT(tgt != NULL); + + mutex_init(&tgt->tp_lock, NULL, MUTEX_DRIVER, NULL); + + mutex_init(&tgt->tp_ch_list_lock, NULL, MUTEX_DRIVER, NULL); + cv_init(&tgt->tp_offline_complete, NULL, CV_DRIVER, NULL); + list_create(&tgt->tp_ch_list, sizeof (srpt_channel_t), + offsetof(srpt_channel_t, ch_stp_node)); + + mutex_init(&tgt->tp_sess_list_lock, NULL, MUTEX_DRIVER, NULL); + cv_init(&tgt->tp_sess_complete, NULL, CV_DRIVER, NULL); + list_create(&tgt->tp_sess_list, sizeof (srpt_session_t), + offsetof(srpt_session_t, ss_node)); + + tgt->tp_state = SRPT_TGT_STATE_OFFLINE; + tgt->tp_drv_disabled = 0; + tgt->tp_srp_enabled = 0; + tgt->tp_lport = lport; + tgt->tp_ioc = ioc; + tgt->tp_ibt_svc_id = guid; + tgt->tp_ibt_svc_desc.sd_handler = srpt_cm_hdlr; + tgt->tp_ibt_svc_desc.sd_flags = IBT_SRV_NO_FLAGS; + temp = h2b64(tgt->tp_ibt_svc_id); + bcopy(&temp, &tgt->tp_srp_port_id[0], 8); + temp = h2b64(tgt->tp_ioc->ioc_guid); + bcopy(&temp, &tgt->tp_srp_port_id[8], 8); + + tgt->tp_nports = ioc->ioc_attr.hca_nports; + tgt->tp_hw_port = + kmem_zalloc(sizeof (srpt_hw_port_t) * tgt->tp_nports, KM_SLEEP); + + tgt->tp_scsi_devid = srpt_stp_alloc_scsi_devid_desc(tgt->tp_ibt_svc_id); + lport->lport_id = tgt->tp_scsi_devid; + lport->lport_pp = srpt_ctxt->sc_pp; + lport->lport_ds = ioc->ioc_stmf_ds; + lport->lport_xfer_data = &srpt_stp_xfer_data; + lport->lport_send_status = &srpt_stp_send_status; + lport->lport_task_free = &srpt_stp_task_free; + lport->lport_abort = &srpt_stp_abort; + lport->lport_task_poll = &srpt_stp_task_poll; + lport->lport_ctl = &srpt_stp_ctl; + lport->lport_info = &srpt_stp_info; + lport->lport_event_handler = &srpt_stp_event_handler; + + SRPT_DPRINTF_L3("stp_alloc_port, register STMF LPORT"); + +retry_registration: + status = stmf_register_local_port(lport); + if (status == STMF_SUCCESS) { + SRPT_DPRINTF_L3("stp_alloc_port, LPORT successfully" + " registered"); + return (tgt); + } + + if (status == STMF_BUSY) { + /* + * This is only done on an administrative thread of + * execution so it is ok to take a while. + */ + SRPT_DPRINTF_L3("stp_alloc_port, delaying"); + delay(2 * drv_usectohz(1000000)); + goto retry_registration; + } + SRPT_DPRINTF_L1("stp_alloc_port, STMF register local port err(0x%llx)", + (u_longlong_t)status); + + SRPT_DPRINTF_L3("stp_alloc_port, free STMF local port"); + cv_destroy(&tgt->tp_offline_complete); + mutex_destroy(&tgt->tp_ch_list_lock); + mutex_destroy(&tgt->tp_lock); + if (tgt->tp_hw_port) { + kmem_free(tgt->tp_hw_port, + sizeof (srpt_hw_port_t) * tgt->tp_nports); + } + if (tgt->tp_scsi_devid) { + srpt_stp_free_scsi_devid_desc(tgt->tp_scsi_devid); + } + + stmf_free(lport); + + return (NULL); +} + +/* + * srpt_stp_free_port() - Free SCSI Target Port + */ +stmf_status_t +srpt_stp_free_port(srpt_target_port_t *tgt) +{ + ASSERT(tgt != NULL); + ASSERT(list_is_empty(&tgt->tp_sess_list)); + ASSERT(list_is_empty(&tgt->tp_ch_list)); + + list_destroy(&tgt->tp_ch_list); + list_destroy(&tgt->tp_sess_list); + + cv_destroy(&tgt->tp_sess_complete); + cv_destroy(&tgt->tp_offline_complete); + + mutex_destroy(&tgt->tp_sess_list_lock); + mutex_destroy(&tgt->tp_ch_list_lock); + mutex_destroy(&tgt->tp_lock); + + + SRPT_DPRINTF_L3("stp_free_port, free STMF local port"); + if (tgt->tp_hw_port) { + kmem_free(tgt->tp_hw_port, + sizeof (srpt_hw_port_t) * tgt->tp_nports); + } + + if (tgt->tp_scsi_devid) { + srpt_stp_free_scsi_devid_desc(tgt->tp_scsi_devid); + } + + stmf_free(tgt->tp_lport); + + return (STMF_SUCCESS); +} + +/* + * srpt_stp_deregister_port() + */ +stmf_status_t +srpt_stp_deregister_port(srpt_target_port_t *tgt) +{ + stmf_status_t status; + + ASSERT(tgt != NULL); + ASSERT(tgt->tp_lport != NULL); + + SRPT_DPRINTF_L3("stp_deregister_port, de-register STMF LPORT"); + +retry_deregistration: + status = stmf_deregister_local_port(tgt->tp_lport); + if (status == STMF_SUCCESS) { + SRPT_DPRINTF_L3("stp_deregister_port, LPORT de-register" + " complete"); + return (status); + } + /* + * This is only done on an administrative thread of + * execution so it is ok to take a while. + */ + if (status == STMF_BUSY) { + delay(drv_usectohz(1000000)); + goto retry_deregistration; + } + + /* + * Something other than a BUSY error, this should not happen. + */ + SRPT_DPRINTF_L1("stp_deregister_port, de-register STMF error(0x%llx)", + (u_longlong_t)status); + return (status); +} + +/* + * srpt_stp_xfer_data() + */ +/* ARGSUSED */ +static stmf_status_t +srpt_stp_xfer_data(struct scsi_task *task, struct stmf_data_buf *dbuf, + uint32_t ioflags) +{ + srpt_iu_t *iu; + srpt_channel_t *ch; + srpt_ds_dbuf_t *db; + ibt_send_wr_t wr; + ibt_wr_ds_t ds; + ibt_status_t status; + uint32_t xfer_len; + uint32_t xferred_len; + uint32_t rdma_len; + uint32_t base_offset; + uint32_t desc_offset; + srp_direct_desc_t *desc; + + SRPT_DPRINTF_L3("stp_xfer_data, invoked task (%p), dbuf (%p)", + (void *)task, (void *)dbuf); + iu = task->task_port_private; + ASSERT(iu != NULL); + ASSERT(iu->iu_ch != NULL); + + /* + * We should use iu->iu_ch->ch_swqe_posted to throttle + * send wqe posting. This is very unlikely because we limit + * the maximum number of initiator descriptors per IU (impact + * of fragmentation of intiator buffer space) but it could occur + * if the back-end (STMF) were to use too many small buffers. In + * that case we would want to return STMF_BUSY. + */ + + SRPT_DPRINTF_L4("stp_xfer_data, dbuf->db_flags (0x%x)", + dbuf->db_flags); + SRPT_DPRINTF_L4("stp_xfer_data, dbuf->db_data_size (%d)", + dbuf->db_data_size); + SRPT_DPRINTF_L4("stp_xfer_data, dbuf->db_relative_offset (%d)", + dbuf->db_relative_offset); + + ASSERT((dbuf->db_flags & (DB_DIRECTION_TO_RPORT | + DB_DIRECTION_FROM_RPORT)) != (DB_DIRECTION_TO_RPORT | + DB_DIRECTION_FROM_RPORT)); + + db = dbuf->db_port_private; + + /* + * Check to see if request will overflow the remote buffer; if so + * return a bad status and let STMF abort the task. + */ + if ((dbuf->db_relative_offset + dbuf->db_data_size) > + iu->iu_tot_xfer_len) { + SRPT_DPRINTF_L2("stp_xfer_data, overflow of remote buffer"); + return (STMF_FAILURE); + } + + db->db_iu = iu; + wr.wr_trans = IBT_RC_SRV; + wr.wr_opcode = (dbuf->db_flags & DB_DIRECTION_TO_RPORT) ? + IBT_WRC_RDMAW : IBT_WRC_RDMAR; + wr.wr_nds = 1; + wr.wr_sgl = &ds; + + /* + * We know that the data transfer is within the bounds described + * by our list of remote buffer descriptors. Find the starting + * point based on the offset for the transfer, then perform the + * RDMA operations required of this transfer. + */ + base_offset = 0; + desc = iu->iu_rdescs; + + while ((base_offset + desc->dd_len) < dbuf->db_relative_offset) { + base_offset += desc->dd_len; + desc++; + } + + xfer_len = dbuf->db_data_size; + xferred_len = 0; + desc_offset = dbuf->db_relative_offset - base_offset; + + ch = iu->iu_ch; + + /* + * If the channel is no longer connected then return an + * error and do not initiate I/O. STMF should abort the + * task. + */ + rw_enter(&ch->ch_rwlock, RW_READER); + + if (iu->iu_ch->ch_state == SRPT_CHANNEL_DISCONNECTING) { + rw_exit(&iu->iu_ch->ch_rwlock); + return (STMF_FAILURE); + } + + while (xfer_len > 0) { + rdma_len = desc->dd_len - desc_offset; + + /* + * We only generate completion entries on the last IB + * operation associated with any STMF buffer. + */ + if (rdma_len >= xfer_len) { + rdma_len = xfer_len; + wr.wr_flags = IBT_WR_SEND_SIGNAL; + } else { + wr.wr_flags = IBT_WR_NO_FLAGS; + } + + wr.wr.rc.rcwr.rdma.rdma_raddr = desc->dd_vaddr + desc_offset; + wr.wr.rc.rcwr.rdma.rdma_rkey = desc->dd_hdl; + ds.ds_va = db->db_sge.ds_va + xferred_len; + ds.ds_key = db->db_sge.ds_key; + ds.ds_len = rdma_len; + + SRPT_DPRINTF_L4("stp_xfer_data, post RDMA operation"); + + /* + * If this task is being aborted or has been aborted, + * do not post additional I/O. + */ + mutex_enter(&iu->iu_lock); + if ((iu->iu_flags & (SRPT_IU_SRP_ABORTING | + SRPT_IU_STMF_ABORTING | SRPT_IU_ABORTED)) != 0) { + mutex_exit(&iu->iu_lock); + rw_exit(&iu->iu_ch->ch_rwlock); + return (STMF_SUCCESS); + } + + /* + * If a non-error CQE will be requested, add a reference to + * the IU and initialize the work request appropriately. + */ + if ((wr.wr_flags & IBT_WR_SEND_SIGNAL) != 0) { + wr.wr_id = srpt_ch_alloc_swqe_wrid(ch, + SRPT_SWQE_TYPE_DATA, (void *)dbuf); + if (wr.wr_id == 0) { + rw_exit(&iu->iu_ch->ch_rwlock); + mutex_exit(&iu->iu_lock); + return (STMF_BUSY); + } + atomic_inc_32(&iu->iu_sq_posted_cnt); + } else { + wr.wr_id = 0; + } + + status = ibt_post_send(iu->iu_ch->ch_chan_hdl, &wr, 1, NULL); + mutex_exit(&iu->iu_lock); + + if (status != IBT_SUCCESS) { + /* + * Could not post to IB transport, report to STMF and + * and let it initiate an abort of the task. + */ + SRPT_DPRINTF_L2("stp_xfer_data, post RDMA" + " error (%d)", status); + + if ((wr.wr_flags & IBT_WR_SEND_SIGNAL) != 0) { + srpt_ch_free_swqe_wrid(ch, wr.wr_id); + atomic_dec_32(&iu->iu_sq_posted_cnt); + } + rw_exit(&iu->iu_ch->ch_rwlock); + return (STMF_FAILURE); + } + + xferred_len += rdma_len; + xfer_len -= rdma_len; + desc_offset = 0; + desc++; + } + + rw_exit(&ch->ch_rwlock); + return (STMF_SUCCESS); +} + +/* + * srpt_stp_send_mgmt_response() - Return SRP task managment response IU + */ +ibt_status_t +srpt_stp_send_mgmt_response(srpt_iu_t *iu, uint8_t srp_rsp, + uint_t fence) +{ + srp_rsp_t *rsp; + srp_rsp_data_t *data; + uint32_t rsp_length; + ibt_status_t status; + uint8_t *bufp; + + ASSERT(mutex_owned(&iu->iu_lock)); + rsp = iu->iu_buf; + bufp = (uint8_t *)iu->iu_buf + SRP_RSP_SIZE; + bzero(rsp, SRP_RSP_SIZE + sizeof (srp_rsp_data_t)); + rsp->rsp_type = SRP_IU_RSP; + + /* + * Report ULP credits we have added since last response sent + * over this channel. + */ + rsp->rsp_req_limit_delta = + h2b32(atomic_swap_32(&iu->iu_ch->ch_req_lim_delta, 0)); + rsp->rsp_tag = iu->iu_tag; + + /* srp_rsp_t is padded out, so use explicit size here */ + rsp_length = SRP_RSP_SIZE; + if (srp_rsp != SRP_TM_SUCCESS) { + rsp->rsp_flags |= SRP_RSP_VALID; + data = (srp_rsp_data_t *)bufp; + data->rd_rsp_status = srp_rsp; + rsp->rsp_data_len = h2b32(sizeof (srp_rsp_data_t)); + rsp_length += sizeof (srp_rsp_data_t); + } + + SRPT_DPRINTF_L4("stp_send_mgmt_response, sending on ch(%p)," + " iu(%p), mgmt status(%d)", (void *)iu->iu_ch, + (void *)iu, srp_rsp); + + status = srpt_ch_post_send(iu->iu_ch, iu, rsp_length, fence); + if (status != IBT_SUCCESS) { + SRPT_DPRINTF_L2("stp_send_mgmt_response, post " + "response err(%d)", status); + } + return (status); +} + +/* + * srpt_stp_send_response() - Send SRP command response IU + */ +ibt_status_t +srpt_stp_send_response(srpt_iu_t *iu, uint8_t scsi_status, + uint8_t flags, uint32_t resid, uint16_t sense_length, + uint8_t *sense_data, uint_t fence) +{ + srp_rsp_t *rsp; + uint32_t rsp_length; + uint8_t *bufp; + ibt_status_t status; + + ASSERT(mutex_owned(&iu->iu_lock)); + rsp = iu->iu_buf; + bufp = (uint8_t *)iu->iu_buf + SRP_RSP_SIZE; + bzero(rsp, SRP_RSP_SIZE); + rsp->rsp_type = SRP_IU_RSP; + + /* + * Report ULP credits we have added since last response sent + * over this channel. + */ + rsp->rsp_req_limit_delta = + h2b32(atomic_swap_32(&iu->iu_ch->ch_req_lim_delta, 0)); + rsp->rsp_tag = iu->iu_tag; + rsp->rsp_status = scsi_status; + + rsp_length = SRP_RSP_SIZE; + + if (resid != 0) { + rsp->rsp_flags |= flags; + + if ((flags & SRP_RSP_DO_OVER) || + (flags & SRP_RSP_DO_UNDER)) { + rsp->rsp_do_resid_cnt = h2b32(resid); + } else if ((flags & SRP_RSP_DI_OVER) || + (flags & SRP_RSP_DI_UNDER)) { + rsp->rsp_di_resid_cnt = h2b32(resid); + } + } + + if (sense_length != 0) { + rsp->rsp_flags |= SRP_RSP_SNS_VALID; + if (SRP_RSP_SIZE + sense_length > + iu->iu_ch->ch_ti_iu_len) { + sense_length = iu->iu_ch->ch_ti_iu_len - + SRP_RSP_SIZE; + } + bcopy(sense_data, bufp, sense_length); + rsp->rsp_sense_data_len = h2b32(sense_length); + rsp_length += sense_length; + } + + SRPT_DPRINTF_L4("stp_send_reponse, sending on ch(%p)," + " iu(%p), length(%d)", (void *)iu->iu_ch, + (void *)iu, rsp_length); + + status = srpt_ch_post_send(iu->iu_ch, iu, rsp_length, fence); + if (status != IBT_SUCCESS) { + SRPT_DPRINTF_L2("stp_send_response, post response err(%d)", + status); + } + return (status); +} + +/* + * srpt_stp_send_status() + */ +/* ARGSUSED */ +stmf_status_t +srpt_stp_send_status(struct scsi_task *task, uint32_t ioflags) +{ + srpt_iu_t *iu; + ibt_status_t status; + + ASSERT(task != NULL); + iu = task->task_port_private; + + ASSERT(iu != NULL); + ASSERT(iu->iu_ch != NULL); + + SRPT_DPRINTF_L3("stp_send_status, invoked task (%p)" + ", task_completion_status (%d)" + ", task_resid (%d)" + ", task_status_ctrl (%d)" + ", task_scsi_status (%d)" + ", task_sense_length (%d)" + ", task_sense_data (%p)", + (void *)task, + (int)task->task_completion_status, + task->task_resid, + task->task_status_ctrl, + task->task_scsi_status, + task->task_sense_length, + (void *)task->task_sense_data); + + /* + * Indicate future aborts can not be initiated (although + * we will handle any that have been requested since the + * last I/O completed and before we are sending status). + */ + mutex_enter(&iu->iu_lock); + iu->iu_flags |= SRPT_IU_RESP_SENT; + + if ((iu->iu_flags & (SRPT_IU_STMF_ABORTING | + SRPT_IU_SRP_ABORTING | SRPT_IU_ABORTED)) != 0) { + mutex_exit(&iu->iu_lock); + return (STMF_FAILURE); + } + + /* + * Send SRP command response or SRP task mgmt response. + */ + if (task->task_mgmt_function == 0) { + uint8_t rsp_flags = 0; + uint32_t resbytes = 0; + + if (task->task_status_ctrl == TASK_SCTRL_OVER) { + resbytes = task->task_resid; + + if (task->task_flags & TF_READ_DATA) { + SRPT_DPRINTF_L3( + "stp_send_status, data out overrun"); + rsp_flags |= SRP_RSP_DO_OVER; + } else if (task->task_flags & TF_WRITE_DATA) { + SRPT_DPRINTF_L3( + "stp_send_status, data in overrun"); + rsp_flags |= SRP_RSP_DI_OVER; + } + } else if (task->task_status_ctrl == TASK_SCTRL_UNDER) { + resbytes = task->task_resid; + + if (task->task_flags & TF_READ_DATA) { + SRPT_DPRINTF_L3( + "stp_send_status, data out underrun"); + rsp_flags |= SRP_RSP_DO_UNDER; + } else if (task->task_flags & TF_WRITE_DATA) { + SRPT_DPRINTF_L3( + "stp_send_status, data in underrun"); + rsp_flags |= SRP_RSP_DI_UNDER; + } + } + + status = srpt_stp_send_response(iu, + task->task_scsi_status, rsp_flags, resbytes, + task->task_sense_length, task->task_sense_data, 0); + } else { + status = srpt_stp_send_mgmt_response(iu, + (task->task_scsi_status ? + SRP_TM_FAILED : SRP_TM_SUCCESS), + SRPT_FENCE_SEND); + } + + /* + * If we have an error posting the response return bad status + * to STMF and let it initiate an abort for the task. + */ + if (status != IBT_SUCCESS) { + SRPT_DPRINTF_L2("stp_send_status, post response err(%d)", + status); + mutex_exit(&iu->iu_lock); + return (STMF_FAILURE); + } + mutex_exit(&iu->iu_lock); + return (STMF_SUCCESS); +} + +/* + * srpt_stp_task_free() - STMF call-back. + */ +static void +srpt_stp_task_free(struct scsi_task *task) +{ + srpt_iu_t *iu; + srpt_channel_t *ch; + + SRPT_DPRINTF_L3("stp_task_free, invoked task (%p)", + (void *)task); + + iu = task->task_port_private; + ASSERT(iu != NULL); + + mutex_enter(&iu->iu_lock); + ch = iu->iu_ch; + mutex_exit(&iu->iu_lock); + + ASSERT(ch != NULL); + ASSERT(ch->ch_session != NULL); + + /* + * Do not hold IU lock while task is being removed from + * the session list - possible deadlock if cleaning up + * channel when this is called. + */ + srpt_stp_remove_task(ch->ch_session, iu); + + mutex_enter(&iu->iu_lock); + iu->iu_stmf_task = NULL; + + srpt_ioc_repost_recv_iu(iu->iu_ioc, iu); + + mutex_exit(&iu->iu_lock); + + srpt_ch_release_ref(ch, 0); +} + +/* + * srpt_stp_abort() - STMF call-back. + */ +/* ARGSUSED */ +static stmf_status_t +srpt_stp_abort(struct stmf_local_port *lport, int abort_cmd, + void *arg, uint32_t flags) +{ + struct scsi_task *task; + srpt_iu_t *iu; + stmf_status_t status; + + SRPT_DPRINTF_L3("stp_abort, invoked lport (%p), arg (%p)", + (void *)lport, (void *)arg); + + task = (struct scsi_task *)arg; + ASSERT(task != NULL); + + iu = (srpt_iu_t *)task->task_port_private; + ASSERT(iu != NULL); + + mutex_enter(&iu->iu_lock); + + /* + * If no I/O is outstanding then immediately transition to + * aborted state. If I/O are in progress, then indicate that an + * STMF abort has been requested and tell STMF we will complete + * it asynchronously. + */ + if (iu->iu_sq_posted_cnt == 0) { + SRPT_DPRINTF_L3("stp_abort, no outstanding I/O for %p", + (void *)iu); + iu->iu_flags |= SRPT_IU_ABORTED; + mutex_exit(&iu->iu_lock); + /* Synchronous abort - STMF will call task_free */ + status = STMF_ABORT_SUCCESS; + } else { + SRPT_DPRINTF_L3("stp_abort, %d outstanding I/O for %p", + iu->iu_sq_posted_cnt, (void *)iu); + iu->iu_flags |= SRPT_IU_STMF_ABORTING; + mutex_exit(&iu->iu_lock); + status = STMF_SUCCESS; + } + + return (status); +} + +/* + * srpt_stp_task_poll() - STMF call-back + */ +static void +srpt_stp_task_poll(struct scsi_task *task) +{ + SRPT_DPRINTF_L3("stp_task_poll, invoked, task (%p)", + (void *)task); +} + +/* + * srpt_stp_ctl() - STMF call-back + */ +static void +srpt_stp_ctl(struct stmf_local_port *lport, int cmd, void *arg) +{ + stmf_state_change_info_t *sc_info = arg; + stmf_change_status_t cstatus; + stmf_status_t status; + srpt_target_port_t *tgt; + + ASSERT(sc_info != NULL); + ASSERT(lport != NULL); + + tgt = lport->lport_port_private; + ASSERT(tgt->tp_ioc != NULL); + + SRPT_DPRINTF_L2("stp_ctl, invoked for LPORT (0x%016llx), cmd (%d)", + (u_longlong_t)tgt->tp_ibt_svc_id, cmd); + + cstatus.st_completion_status = STMF_SUCCESS; + cstatus.st_additional_info = NULL; + + switch (cmd) { + case STMF_CMD_LPORT_ONLINE: + SRPT_DPRINTF_L2("stp_ctl, LPORT_ONLINE command," + " st_rflags(0x%llx)", (u_longlong_t)sc_info->st_rflags); + /* + * If the SCSI Target Port is not enabled by the driver, + * don't start and instead return busy. This is a + * creation/destruction transitional state and the will + * either go away or become enabled. + */ + mutex_enter(&tgt->tp_lock); + if (tgt->tp_drv_disabled != 0) { + SRPT_DPRINTF_L1("stp_ctl, set LPORT_ONLINE failed - " + "LPORT (0x%016llx) BUSY", + (u_longlong_t)tgt->tp_ibt_svc_id); + cstatus.st_completion_status = STMF_BUSY; + } else if (tgt->tp_state == SRPT_TGT_STATE_ONLINE) { + cstatus.st_completion_status = STMF_ALREADY; + } else if (tgt->tp_state != SRPT_TGT_STATE_OFFLINE) { + cstatus.st_completion_status = STMF_INVALID_ARG; + } else { + tgt->tp_state = SRPT_TGT_STATE_ONLINING; + status = srpt_stp_start_srp(tgt); + if (status != STMF_SUCCESS) { + tgt->tp_state = SRPT_TGT_STATE_OFFLINE; + cstatus.st_completion_status = STMF_INVALID_ARG; + } + } + mutex_exit(&tgt->tp_lock); + SRPT_DPRINTF_L3("stp_ctl, (0x%016llx) LPORT_ONLINE command" + " status (0x%llx)", (u_longlong_t)tgt->tp_ibt_svc_id, + (u_longlong_t)cstatus.st_completion_status); + status = stmf_ctl(STMF_CMD_LPORT_ONLINE_COMPLETE, lport, + &cstatus); + if (status != STMF_SUCCESS) { + SRPT_DPRINTF_L1("stp_ctl, ONLINE_COMPLETE returned" + " error(0x%llx)", (u_longlong_t)status); + } + break; + + case STMF_CMD_LPORT_OFFLINE: + SRPT_DPRINTF_L2("stp_ctl, LPORT_OFFLINE"); + mutex_enter(&tgt->tp_lock); + if (tgt->tp_state == SRPT_TGT_STATE_OFFLINE) { + cstatus.st_completion_status = STMF_ALREADY; + } else if (tgt->tp_state != SRPT_TGT_STATE_ONLINE) { + cstatus.st_completion_status = STMF_INVALID_ARG; + } else { + tgt->tp_state = SRPT_TGT_STATE_OFFLINING; + srpt_stp_stop_srp(tgt); + } + mutex_exit(&tgt->tp_lock); + SRPT_DPRINTF_L3("stp_ctl, notify STMF OFFLINE complete" + " (0x%016llx)", (u_longlong_t)tgt->tp_ibt_svc_id); + status = stmf_ctl(STMF_CMD_LPORT_OFFLINE_COMPLETE, + lport, &cstatus); + if (status != STMF_SUCCESS) { + SRPT_DPRINTF_L1("stp_ctl, OFFLINE_COMPLETE returned" + " error(0x%llx)", (u_longlong_t)status); + } + break; + + case STMF_ACK_LPORT_ONLINE_COMPLETE: + SRPT_DPRINTF_L2("stp_ctl, LPORT_ONLINE_COMPLETE ACK from" + " STMF"); + mutex_enter(&tgt->tp_lock); + if (tgt->tp_state == SRPT_TGT_STATE_ONLINING) { + SRPT_DPRINTF_L2("stp_ctl, LPORT is ONLINE"); + tgt->tp_state = SRPT_TGT_STATE_ONLINE; + } else { + SRPT_DPRINTF_L2("stp_ctl, LPORT not on-lining"); + } + mutex_exit(&tgt->tp_lock); + break; + + case STMF_ACK_LPORT_OFFLINE_COMPLETE: + SRPT_DPRINTF_L2("stp_ctl, LPORT_OFFLINE_COMPLETE ACK from" + " STMF"); + mutex_enter(&tgt->tp_lock); + if (tgt->tp_state == SRPT_TGT_STATE_OFFLINING) { + SRPT_DPRINTF_L2("stp_ctl, LPORT is OFFLINE"); + tgt->tp_state = SRPT_TGT_STATE_OFFLINE; + cv_broadcast(&tgt->tp_offline_complete); + } else { + SRPT_DPRINTF_L2("stp_ctl, LPORT not off-lining"); + } + mutex_exit(&tgt->tp_lock); + break; + + default: + SRPT_DPRINTF_L2("stp_ctl, cmd (%d) not handled", + cmd); + break; + } +} + +/* + * srpt_stp_info() - STMF call-back + */ +/* ARGSUSED */ +static stmf_status_t +srpt_stp_info(uint32_t cmd, struct stmf_local_port *lport, + void *arg, uint8_t *buf, uint32_t *bufsizep) +{ + SRPT_DPRINTF_L3("stp_info, invoked"); + return (STMF_SUCCESS); +} + +/* + * srpt_stp_event_handler() - STMF call-back + */ +/* ARGSUSED */ +static void +srpt_stp_event_handler(struct stmf_local_port *lport, int eventid, + void *arg, uint32_t flags) +{ + SRPT_DPRINTF_L3("stp_event_handler, invoked"); +} + +/* + * srpt_stp_alloc_scsi_devid_desc() + * + * Allocate and initialize a SCSI device ID descriptor for + * the SRP protocol. Names are eui.GUID format. + * + * Both extension and guid are passed in host order. + */ +static scsi_devid_desc_t * +srpt_stp_alloc_scsi_devid_desc(uint64_t guid) +{ + scsi_devid_desc_t *sdd; + + sdd = kmem_zalloc(sizeof (*sdd) + SRPT_EUI_ID_LEN + 1, KM_SLEEP); + sdd->protocol_id = PROTOCOL_SRP; + sdd->piv = 1; + sdd->code_set = CODE_SET_ASCII; + sdd->association = ID_IS_TARGET_PORT; + sdd->ident_length = SRPT_EUI_ID_LEN; + (void) sprintf((char *)sdd->ident, "eui.%016llX", (u_longlong_t)guid); + return (sdd); +} + +/* + * srpt_stp_free_scsi_devid_desc() + * + * Free a SRPT SCSI device ID descriptor previously allocated via + * srpt_stp_alloc_scsi_devid_desc(). + */ +static void +srpt_stp_free_scsi_devid_desc(scsi_devid_desc_t *sdd) +{ + kmem_free(sdd, sizeof (*sdd) + SRPT_EUI_ID_LEN + 1); +} + +/* + * srpt_stp_alloc_session() + */ +srpt_session_t * +srpt_stp_alloc_session(srpt_target_port_t *tgt, + uint8_t *i_id, uint8_t *t_id, uint8_t port) +{ + stmf_status_t status; + srpt_session_t *ss; + stmf_scsi_session_t *stmf_ss; + uint64_t i_guid; + + ASSERT(tgt != NULL); + SRPT_DPRINTF_L3("stp_alloc_session, invoked"); + + mutex_enter(&tgt->tp_sess_list_lock); + + i_guid = BE_IN64(&i_id[8]); + + stmf_ss = stmf_alloc(STMF_STRUCT_SCSI_SESSION, + sizeof (srpt_session_t), 0); + if (stmf_ss == NULL) { + SRPT_DPRINTF_L2("stp_alloc_session, stmf_alloc" + " returned NULL"); + mutex_exit(&tgt->tp_sess_list_lock); + return (NULL); + } + ss = stmf_ss->ss_port_private; + ASSERT(ss != NULL); + + + rw_init(&ss->ss_rwlock, NULL, RW_DRIVER, NULL); + list_create(&ss->ss_task_list, sizeof (srpt_iu_t), + offsetof(srpt_iu_t, iu_ss_task_node)); + + stmf_ss->ss_rport_id = srpt_stp_alloc_scsi_devid_desc(i_guid); + stmf_ss->ss_lport = tgt->tp_lport; + + ss->ss_ss = stmf_ss; + ss->ss_hw_port = port; + ss->ss_tgt = tgt; + bcopy(i_id, ss->ss_i_id, SRP_PORT_ID_LEN); + bcopy(t_id, ss->ss_t_id, SRP_PORT_ID_LEN); + + /* + * Set the alias to include the initiator extension, this will enable + * the administrator to identify multiple unique sessions originating + * from the same initiator. + */ + (void) sprintf(ss->ss_alias, "%016llx:%016llx", + (u_longlong_t)BE_IN64(&ss->ss_i_id[0]), + (u_longlong_t)BE_IN64(&ss->ss_i_id[8])); + + stmf_ss->ss_rport_alias = ss->ss_alias; + + status = stmf_register_scsi_session(tgt->tp_lport, stmf_ss); + if (status != STMF_SUCCESS) { + SRPT_DPRINTF_L1("stp_alloc_session, STMF register session" + " err(0x%llx)", (u_longlong_t)status); + list_destroy(&ss->ss_task_list); + rw_destroy(&ss->ss_rwlock); + stmf_free(stmf_ss); + mutex_exit(&tgt->tp_sess_list_lock); + return (NULL); + } + + list_insert_tail(&tgt->tp_sess_list, ss); + mutex_exit(&tgt->tp_sess_list_lock); + return (ss); +} + +/* + * srpt_stp_free_session() + */ +void +srpt_stp_free_session(srpt_session_t *session) +{ + stmf_scsi_session_t *stmf_ss; + srpt_target_port_t *tgt; + + ASSERT(session != NULL); + + tgt = session->ss_tgt; + + ASSERT(tgt != NULL); + + SRPT_DPRINTF_L3("stp_free_session, invoked"); + + mutex_enter(&tgt->tp_sess_list_lock); + + stmf_ss = session->ss_ss; + + list_destroy(&session->ss_task_list); + rw_destroy(&session->ss_rwlock); + + stmf_deregister_scsi_session(tgt->tp_lport, stmf_ss); + srpt_stp_free_scsi_devid_desc(stmf_ss->ss_rport_id); + + list_remove(&tgt->tp_sess_list, session); + cv_signal(&tgt->tp_sess_complete); + mutex_exit(&tgt->tp_sess_list_lock); + stmf_free(stmf_ss); +} + +/* + * srpt_stp_login() - SRP SCSI Target port login + */ +srpt_channel_t * +srpt_stp_login(srpt_target_port_t *tgt, srp_login_req_t *login, + srp_login_rsp_t *login_rsp, srp_login_rej_t *login_rej, + uint8_t login_port) +{ + uint32_t reason; + uint32_t req_it_ui_len; + uint8_t rsp_flags; + srpt_ioc_t *ioc; + srpt_channel_t *ch = NULL; + srpt_channel_t *next_ch = NULL; + srpt_session_t *session = NULL; + + ASSERT(tgt != NULL); + ASSERT(login != NULL); + ASSERT(login_rsp != NULL); + ASSERT(login_rej != NULL); + + /* + * The target lock taken here serializes logins to this target + * and prevents an STMF target port from starting a control + * operation to transition the target state while a login is + * being processed. + */ + mutex_enter(&tgt->tp_lock); + ioc = tgt->tp_ioc; + if (ioc == NULL) { + SRPT_DPRINTF_L1("stp_login, NULL I/O Controller"); + reason = SRP_LOGIN_REJ_UNABLE_TO_ASSOCIATE_I_T_NEXUS; + goto reject_login; + } + + /* + * Validate that the SRP Target ID in the login request specifies + * this I/O Controller SCSI Target Port. + */ + if (memcmp(login->lreq_target_port_id, tgt->tp_srp_port_id, + SRP_PORT_ID_LEN) != 0) { + SRPT_DPRINTF_L2("stp_login, SRP CM SVC target ID mismatch." + " Incoming TgtID 0x%016llx:0x%016llx", + (u_longlong_t)BE_IN64(&login->lreq_target_port_id[0]), + (u_longlong_t)BE_IN64(&login->lreq_target_port_id[8])); + + reason = SRP_LOGIN_REJ_UNABLE_TO_ASSOCIATE_I_T_NEXUS; + goto reject_login; + } + + if (tgt->tp_state != SRPT_TGT_STATE_ONLINE) { + SRPT_DPRINTF_L2("stp_login, SRP Login target not on-line"); + reason = SRP_LOGIN_REJ_UNABLE_TO_ASSOCIATE_I_T_NEXUS; + goto reject_login; + } + + /* + * Initiator requested IU size must be as large as the specification + * minimum and no greater than what we chose to support. + */ + req_it_ui_len = b2h32(login->lreq_req_it_iu_len); + SRPT_DPRINTF_L2("stp_login, requested iu size = %d", req_it_ui_len); + if (req_it_ui_len > SRPT_DEFAULT_SEND_MSG_SIZE) { + SRPT_DPRINTF_L2("stp_login, SRP Login IU size (%d) too large", + req_it_ui_len); + reason = SRP_LOGIN_REJ_REQ_IT_IU_LENGTH_TOO_LARGE; + goto reject_login; + } + if (req_it_ui_len < SRP_MIN_IU_SIZE) { + SRPT_DPRINTF_L2("stp_login, SRP Login IU size (%d) too small", + req_it_ui_len); + reason = SRP_LOGIN_REJ_NO_REASON; + goto reject_login; + } + + SRPT_DPRINTF_L2("stp_login, login req InitID 0x%016llx:0x%016llx", + (u_longlong_t)BE_IN64(&login->lreq_initiator_port_id[0]), + (u_longlong_t)BE_IN64(&login->lreq_initiator_port_id[8])); + SRPT_DPRINTF_L2("stp_login, login req TgtID 0x%016llx:0x%016llx", + (u_longlong_t)BE_IN64(&login->lreq_target_port_id[0]), + (u_longlong_t)BE_IN64(&login->lreq_target_port_id[8])); + + /* + * Processing is based on either single channel or multi-channel + * operation. In single channel, all current logins for this + * same I_T_Nexus should be logged out. In multi-channel + * mode we would add an additional channel to an existing + * I_T_Nexus if one currently exists (i.e. reference the + * same SCSI session). + */ + rsp_flags = SRP_MULTI_CH_RESULT_NO_EXISTING; + + switch (login->lreq_req_flags & SRP_LOGIN_MULTI_CH_MASK) { + + case SRP_LOGIN_MULTI_CH_SINGLE: + /* + * Only a single channel may be associated with a I_T_Nexus. + * Disconnect any channel with the same SRP Initiator and + * SRP target IDs. + */ + mutex_enter(&tgt->tp_ch_list_lock); + ch = list_head(&tgt->tp_ch_list); + while (ch != NULL) { + SRPT_DPRINTF_L3("stp_login, compare session," + " ch_state(%d)", ch->ch_state); + next_ch = list_next(&tgt->tp_ch_list, ch); + + if (ch->ch_state != SRPT_CHANNEL_CONNECTING && + ch->ch_state != SRPT_CHANNEL_CONNECTED) { + SRPT_DPRINTF_L3("stp_login, compare session," + " channel not active"); + ch = next_ch; + continue; + } + + ASSERT(ch->ch_session != NULL); + SRPT_DPRINTF_L3("stp_login, compare session" + " I_ID 0x%016llx:0x%016llx", + (u_longlong_t)b2h64(*((uint64_t *)(void *) + &ch->ch_session->ss_i_id[0])), + (u_longlong_t)b2h64(*((uint64_t *)(void *) + &ch->ch_session->ss_i_id[8]))); + SRPT_DPRINTF_L3("stp_login, compare session" + " T_ID 0x%016llx:0x%016llx", + (u_longlong_t)b2h64(*((uint64_t *)(void *) + &ch->ch_session->ss_t_id[0])), + (u_longlong_t)b2h64(*((uint64_t *)(void *) + &ch->ch_session->ss_t_id[8]))); + if ((bcmp(login->lreq_initiator_port_id, + ch->ch_session->ss_i_id, + SRP_PORT_ID_LEN) == 0) && + (bcmp(login->lreq_target_port_id, + ch->ch_session->ss_t_id, + SRP_PORT_ID_LEN) == 0)) { + /* + * if a session is in the process of connecting, + * reject subsequent equivalent requests. + */ + if (ch->ch_state == SRPT_CHANNEL_CONNECTING) { + reason = SRP_LOGIN_REJ_INIT_CH_LIMIT; + mutex_exit(&tgt->tp_ch_list_lock); + goto reject_login; + } + + SRPT_DPRINTF_L2("stp_login, terminate" + " existing login"); + rsp_flags = + SRP_MULTI_CH_RESULT_TERM_EXISTING; + srpt_ch_disconnect(ch); + } + + ch = next_ch; + } + mutex_exit(&tgt->tp_ch_list_lock); + + /* Create the new session for this SRP login */ + session = srpt_stp_alloc_session(tgt, + login->lreq_initiator_port_id, + login->lreq_target_port_id, login_port); + if (session == NULL) { + SRPT_DPRINTF_L2("stp_login, session allocation" + " failed"); + reason = SRP_LOGIN_REJ_UNABLE_TO_ASSOCIATE_I_T_NEXUS; + goto reject_login; + } + break; + + case SRP_LOGIN_MULTI_CH_MULTIPLE: + SRPT_DPRINTF_L2("stp_login, multichannel not supported yet"); + reason = SRP_LOGIN_REJ_MULTI_CH_NOT_SUPPORTED; + goto reject_login; + /* break via goto */ + + default: + SRPT_DPRINTF_L2("stp_login, invalid multichannel field (%d)", + login->lreq_req_flags & SRP_LOGIN_MULTI_CH_MASK); + reason = SRP_LOGIN_REJ_NO_REASON; + goto reject_login; + /* break via goto */ + } + + /* + * Create new RDMA channel for this SRP login request. + * The channel is returned with a single reference which + * represents the reference held by the CM. + */ + ch = srpt_ch_alloc(tgt, login_port); + if (ch == NULL) { + SRPT_DPRINTF_L2("stp_login, unable to alloc RDMA channel"); + reason = SRP_LOGIN_REJ_INSUFFICIENT_CH_RESOURCES; + srpt_stp_free_session(session); + goto reject_login; + } + ch->ch_session = session; + ch->ch_ti_iu_len = b2h32(login->lreq_req_it_iu_len); + + /* + * Add another reference to the channel which represents + * a reference placed by the target port and add it to + * the store of channels logged in for this target port. + */ + srpt_ch_add_ref(ch); + mutex_enter(&tgt->tp_ch_list_lock); + list_insert_tail(&tgt->tp_ch_list, ch); + mutex_exit(&tgt->tp_ch_list_lock); + + srpt_format_login_rsp(login, login_rsp, rsp_flags); + mutex_exit(&tgt->tp_lock); + SRPT_DPRINTF_L2("stp_login, login successful"); + + return (ch); + +reject_login: + srpt_format_login_rej(login, login_rej, reason); + mutex_exit(&tgt->tp_lock); + return (NULL); +} + +/* + * srpt_stp_logout() - SRP logout + * + * Logout is not normally initiated in-band, but is so, just + * initiate a disconnect. + */ +void +srpt_stp_logout(srpt_channel_t *ch) +{ + SRPT_DPRINTF_L2("stp_logout, invoked for ch (%p)", + (void *)ch); + srpt_ch_disconnect(ch); +} + +/* + * srpt_format_login_rej() - Format login reject IU + */ +static void +srpt_format_login_rej(srp_login_req_t *req, srp_login_rej_t *rej, + uint32_t reason) +{ + bzero(rej, sizeof (srp_login_rej_t)); + + rej->lrej_type = SRP_IU_LOGIN_REJ; + rej->lrej_reason = h2b32(reason); + rej->lrej_tag = req->lreq_tag; + rej->lrej_sup_buf_format = + h2b16(SRP_DIRECT_BUFR_DESC | SRP_INDIRECT_BUFR_DESC); +} + +/* + * srpt_format_login_rsp() - Format login response IU + */ +static void +srpt_format_login_rsp(srp_login_req_t *req, srp_login_rsp_t *rsp, + uint8_t flags) +{ + bzero(rsp, sizeof (srp_login_rsp_t)); + + rsp->lrsp_type = SRP_IU_LOGIN_RSP; + rsp->lrsp_req_limit_delta = h2b32((uint32_t)srpt_send_msg_depth); + rsp->lrsp_tag = req->lreq_tag; + + rsp->lrsp_max_it_iu_len = req->lreq_req_it_iu_len; + /* by def. > min T_IU_LEN */ + rsp->lrsp_max_ti_iu_len = req->lreq_req_it_iu_len; + + rsp->lrsp_sup_buf_format = + h2b16(SRP_DIRECT_BUFR_DESC | SRP_INDIRECT_BUFR_DESC); + rsp->lrsp_rsp_flags = flags; +} + +/* + * srpt_stp_add_task() + */ +void +srpt_stp_add_task(srpt_session_t *session, srpt_iu_t *iu) +{ + rw_enter(&session->ss_rwlock, RW_WRITER); + list_insert_tail(&session->ss_task_list, iu); + rw_exit(&session->ss_rwlock); +} + +/* + * srpt_stp_remove_task() + */ +void +srpt_stp_remove_task(srpt_session_t *session, srpt_iu_t *iu) +{ + rw_enter(&session->ss_rwlock, RW_WRITER); + + ASSERT(!list_is_empty(&session->ss_task_list)); + + list_remove(&session->ss_task_list, iu); + rw_exit(&session->ss_rwlock); +} diff --git a/usr/src/uts/common/io/comstar/port/srpt/srpt_stp.h b/usr/src/uts/common/io/comstar/port/srpt/srpt_stp.h new file mode 100644 index 0000000000..043422d662 --- /dev/null +++ b/usr/src/uts/common/io/comstar/port/srpt/srpt_stp.h @@ -0,0 +1,75 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ + +/* + * Copyright 2009 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +#ifndef _SRPT_STP_H +#define _SRPT_STP_H + +/* + * Prototypes and data structures providing the SRP SCSI + * target port COMSTAR port provider function. + */ + +#ifdef __cplusplus +extern "C" { +#endif + +/* + * Prototypes + */ +int srpt_stp_start_srp(srpt_target_port_t *tgt); +void srpt_stp_stop_srp(srpt_target_port_t *tgt); +srpt_target_port_t *srpt_stp_alloc_port(srpt_ioc_t *ioc, ib_guid_t guid); +stmf_status_t srpt_stp_free_port(srpt_target_port_t *tgt); +stmf_status_t srpt_stp_deregister_port(srpt_target_port_t *tgt); + +srpt_session_t *srpt_stp_alloc_session(srpt_target_port_t *tgt, + uint8_t *i_id, uint8_t *t_id, uint8_t port); +void srpt_stp_free_session(srpt_session_t *session); + +srpt_channel_t *srpt_stp_login(srpt_target_port_t *tgt, + srp_login_req_t *login, srp_login_rsp_t *login_rsp, + srp_login_rej_t *login_rej, uint8_t login_port); + +void srpt_stp_logout(srpt_channel_t *ch); + +stmf_status_t srpt_stp_send_status(struct scsi_task *task, + uint32_t ioflags); + +ibt_status_t srpt_stp_send_response(srpt_iu_t *iu, uint8_t scsi_status, + uint8_t flags, uint32_t resid, uint16_t sense_length, + uint8_t *sense_data, uint_t fence); +ibt_status_t srpt_stp_send_mgmt_response(srpt_iu_t *iu, uint8_t srp_rsp, + uint_t fence); +void srpt_stp_add_task(srpt_session_t *session, srpt_iu_t *iu); +void srpt_stp_remove_task(srpt_session_t *session, srpt_iu_t *iu); + +uint64_t srpt_stp_u8array2u64(uint8_t *array); + +#ifdef __cplusplus +} +#endif + +#endif /* _SRPT_STP_H */ diff --git a/usr/src/uts/common/io/ib/mgt/ibdma/ibdma.c b/usr/src/uts/common/io/ib/mgt/ibdma/ibdma.c new file mode 100644 index 0000000000..d4fb5f3d9f --- /dev/null +++ b/usr/src/uts/common/io/ib/mgt/ibdma/ibdma.c @@ -0,0 +1,1198 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ + +/* + * Copyright 2009 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +/* + * Infiniband Device Management Agent for IB storage. + */ + +#include <sys/conf.h> +#include <sys/file.h> +#include <sys/ddi.h> +#include <sys/sunddi.h> +#include <sys/modctl.h> +#include <sys/priv.h> +#include <sys/sysmacros.h> + +#include <sys/ib/ibtl/ibti.h> /* IB public interfaces */ + +#include <sys/ib/mgt/ibdma/ibdma.h> +#include <sys/ib/mgt/ibdma/ibdma_impl.h> + +/* + * NOTE: The IB Device Management Agent function, like other IB + * managers and agents is best implemented as a kernel misc. + * module. + * Eventually we could modify IBT_DM_AGENT so that we don't need to + * open each HCA to receive asynchronous events. + */ + +#define IBDMA_NAME_VERSION "IB Device Management Agent" + +extern struct mod_ops mod_miscops; + +static void ibdma_ibt_async_handler(void *clnt, ibt_hca_hdl_t hdl, + ibt_async_code_t code, ibt_async_event_t *event); + +static void ibdma_mad_recv_cb(ibmf_handle_t ibmf_hdl, + ibmf_msg_t *msgp, void *args); +static void ibdma_create_resp_mad(ibmf_msg_t *msgp); + +/* + * Misc. kernel module for now. + */ +static struct modlmisc modlmisc = { + &mod_miscops, + IBDMA_NAME_VERSION +}; + +static struct modlinkage modlinkage = { + MODREV_1, (void *)&modlmisc, NULL +}; + +static ibt_clnt_modinfo_t ibdma_ibt_modinfo = { + IBTI_V_CURR, + IBT_DM_AGENT, + ibdma_ibt_async_handler, + NULL, + "ibdma" +}; + +/* + * Module global state allocated at init(). + */ +static ibdma_mod_state_t *ibdma = NULL; + +/* + * Init/Fini handlers and IBTL HCA management prototypes. + */ +static int ibdma_init(); +static int ibdma_fini(); +static int ibdma_ibt_init(); +static void ibdma_ibt_fini(); +static ibdma_hca_t *ibdma_hca_init(ib_guid_t guid); +static void ibdma_hca_fini(ibdma_hca_t *hca); +static ibdma_hca_t *ibdma_find_hca(ib_guid_t guid); + +/* + * DevMgmt Agent MAD attribute handlers prototypes. + */ +static void ibdma_get_class_portinfo(ibmf_msg_t *msg); +static void ibdma_get_io_unitinfo(ibdma_hca_t *hca, ibmf_msg_t *msg); +static void ibdma_get_ioc_profile(ibdma_hca_t *hca, ibmf_msg_t *msg); +static void ibdma_get_ioc_services(ibdma_hca_t *hca, ibmf_msg_t *msg); + +/* + * _init() + */ +int +_init(void) +{ + int status; + + ASSERT(ibdma == NULL); + + ibdma = kmem_zalloc(sizeof (*ibdma), KM_SLEEP); + ASSERT(ibdma != NULL); + + status = ibdma_init(); + if (status != DDI_SUCCESS) { + kmem_free(ibdma, sizeof (*ibdma)); + ibdma = NULL; + return (status); + } + + status = mod_install(&modlinkage); + if (status != DDI_SUCCESS) { + cmn_err(CE_NOTE, "_init, mod_install error (%d)", status); + (void) ibdma_fini(); + kmem_free(ibdma, sizeof (*ibdma)); + ibdma = NULL; + } + return (status); +} + +/* + * _info() + */ +int +_info(struct modinfo *modinfop) +{ + return (mod_info(&modlinkage, modinfop)); +} + +/* + * _fini() + */ +int +_fini(void) +{ + int status; + int slot; + ibdma_hca_t *hca; + + status = mod_remove(&modlinkage); + if (status != DDI_SUCCESS) { + cmn_err(CE_NOTE, "_fini, mod_remove error (%d)", status); + return (status); + } + + /* + * Sanity check to see if anyone is not cleaning + * up appropriately. + */ + mutex_enter(&ibdma->ms_hca_list_lock); + hca = list_head(&ibdma->ms_hca_list); + while (hca != NULL) { + for (slot = 0; slot < IBDMA_MAX_IOC; slot++) { + if (hca->ih_ioc[slot].ii_inuse) { + cmn_err(CE_NOTE, "_fini, IOC %d still attached" + " for (0x%0llx)", slot+1, + (u_longlong_t)hca->ih_iou_guid); + } + } + hca = list_next(&ibdma->ms_hca_list, hca); + } + mutex_exit(&ibdma->ms_hca_list_lock); + + (void) ibdma_fini(); + kmem_free(ibdma, sizeof (*ibdma)); + return (status); +} + +/* + * ibdma_init() + * + * Initialize I/O Unit structure, generate initial HCA list and register + * it port with the IBMF. + */ +static int +ibdma_init() +{ + int status; + + /* + * Global lock and I/O Unit initialization. + */ + mutex_init(&ibdma->ms_hca_list_lock, NULL, MUTEX_DRIVER, NULL); + + /* + * Discover IB hardware and setup for device management agent + * support. + */ + status = ibdma_ibt_init(); + if (status != DDI_SUCCESS) { + cmn_err(CE_NOTE, "ibdma_init, ibt_attach failed (%d)", + status); + mutex_destroy(&ibdma->ms_hca_list_lock); + return (status); + } + + return (status); +} + +/* + * ibdma_fini() + * + * Release resource if we are no longer in use. + */ +static int +ibdma_fini() +{ + ibdma_ibt_fini(); + mutex_destroy(&ibdma->ms_hca_list_lock); + return (DDI_SUCCESS); +} + +/* + * ibdma_ibt_async_handler() + */ +/* ARGSUSED */ +static void +ibdma_ibt_async_handler(void *clnt, ibt_hca_hdl_t hdl, + ibt_async_code_t code, ibt_async_event_t *event) +{ + ibdma_hca_t *hca; + + switch (code) { + + case IBT_EVENT_PORT_UP: + case IBT_ERROR_PORT_DOWN: + break; + + case IBT_HCA_ATTACH_EVENT: + mutex_enter(&ibdma->ms_hca_list_lock); + hca = ibdma_hca_init(event->ev_hca_guid); + if (hca != NULL) { + list_insert_tail(&ibdma->ms_hca_list, hca); + cmn_err(CE_NOTE, "hca ibt hdl (%p)", + (void *)hca->ih_ibt_hdl); + ibdma->ms_num_hcas++; + } + mutex_exit(&ibdma->ms_hca_list_lock); + break; + + case IBT_HCA_DETACH_EVENT: + mutex_enter(&ibdma->ms_hca_list_lock); + hca = ibdma_find_hca(event->ev_hca_guid); + if (hca != NULL) { + list_remove(&ibdma->ms_hca_list, hca); + cmn_err(CE_NOTE, "removing hca (%p) (0x%llx)", + (void *)hca, hca ? + (u_longlong_t)hca->ih_iou_guid : 0x0ll); + ibdma_hca_fini(hca); + } + mutex_exit(&ibdma->ms_hca_list_lock); + break; + + default: + cmn_err(CE_NOTE, "ibt_async_handler, unhandled event(%d)", + code); + break; + } + +} + +/* + * ibdma_ibt_init() + */ +static int +ibdma_ibt_init() +{ + int status; + int hca_cnt; + int hca_ndx; + ib_guid_t *guid; + ibdma_hca_t *hca; + + /* + * Attach to IBTF and get HCA list. + */ + status = ibt_attach(&ibdma_ibt_modinfo, NULL, + ibdma, &ibdma->ms_ibt_hdl); + if (status != DDI_SUCCESS) { + cmn_err(CE_NOTE, "ibt_init, ibt_attach failed (%d)", + status); + return (status); + } + + hca_cnt = ibt_get_hca_list(&guid); + if (hca_cnt < 1) { +#ifdef DEBUG_IBDMA + cmn_err(CE_NOTE, "ibt_init, no HCA(s) found"); +#endif + ibt_detach(ibdma->ms_ibt_hdl); + return (DDI_FAILURE); + } + + list_create(&ibdma->ms_hca_list, sizeof (ibdma_hca_t), + offsetof(ibdma_hca_t, ih_node)); + + mutex_enter(&ibdma->ms_hca_list_lock); + + for (hca_ndx = 0; hca_ndx < hca_cnt; hca_ndx++) { +#ifdef DEBUG_IBDMA + cmn_err(CE_NOTE, "adding hca GUID(0x%llx)", + (u_longlong_t)guid[hca_ndx]); +#endif + + hca = ibdma_hca_init(guid[hca_ndx]); + if (hca == NULL) { + cmn_err(CE_NOTE, "ibt_init, hca_init GUID(0x%llx)" + " failed", (u_longlong_t)guid[hca_ndx]); + continue; + } + list_insert_tail(&ibdma->ms_hca_list, hca); + ibdma->ms_num_hcas++; + } + + mutex_exit(&ibdma->ms_hca_list_lock); + + ibt_free_hca_list(guid, hca_cnt); +#ifdef DEBUG_IBDMA + cmn_err(CE_NOTE, "Added %d HCA(s)", + ibdma->ms_num_hcas); +#endif + return (DDI_SUCCESS); +} + +/* + * ibdma_ibt_fini() + */ +static void +ibdma_ibt_fini() +{ + ibdma_hca_t *hca; + ibdma_hca_t *next; + + mutex_enter(&ibdma->ms_hca_list_lock); + hca = list_head(&ibdma->ms_hca_list); + while (hca != NULL) { + next = list_next(&ibdma->ms_hca_list, hca); + list_remove(&ibdma->ms_hca_list, hca); +#ifdef DEBUG_IBDMA + cmn_err(CE_NOTE, "removing hca (%p) (0x%llx)", + (void *)hca, hca ? + (u_longlong_t)hca->ih_iou_guid : 0x0ll); + cmn_err(CE_NOTE, "hca ibt hdl (%p)", + (void *)hca->ih_ibt_hdl); +#endif + ibdma_hca_fini(hca); + hca = next; + } + list_destroy(&ibdma->ms_hca_list); + + ibt_detach(ibdma->ms_ibt_hdl); + ibdma->ms_ibt_hdl = NULL; + ibdma->ms_num_hcas = 0; + mutex_exit(&ibdma->ms_hca_list_lock); +} + +/* + * ibdma_find_hca() + */ +static ibdma_hca_t * +ibdma_find_hca(ib_guid_t guid) +{ + ibdma_hca_t *hca; + + ASSERT(mutex_owned(&ibdma->ms_hca_list_lock)); + + hca = list_head(&ibdma->ms_hca_list); + while (hca != NULL) { + if (hca->ih_iou_guid == guid) { + break; + } + hca = list_next(&ibdma->ms_hca_list, hca); + } + return (hca); +} + +/* + * ibdma_hca_init() + */ +static ibdma_hca_t * +ibdma_hca_init(ib_guid_t guid) +{ + ibt_status_t status; + ibdma_hca_t *hca; + ibdma_port_t *port; + ibt_hca_attr_t hca_attr; + int ndx; + + ASSERT(mutex_owned(&ibdma->ms_hca_list_lock)); + + status = ibt_query_hca_byguid(guid, &hca_attr); + if (status != IBT_SUCCESS) { + cmn_err(CE_NOTE, "hca_init HCA query error (%d)", + status); + return (NULL); + } + + if (ibdma_find_hca(guid) != NULL) { +#ifdef DEBUG_IBDMA + cmn_err(CE_NOTE, "hca_init HCA already exists"); +#endif + return (NULL); + } + + hca = kmem_zalloc(sizeof (ibdma_hca_t) + + (hca_attr.hca_nports-1)*sizeof (ibdma_port_t), KM_SLEEP); + ASSERT(hca != NULL); + + hca->ih_nports = hca_attr.hca_nports; + + rw_init(&hca->ih_iou_rwlock, NULL, RW_DRIVER, NULL); + rw_enter(&hca->ih_iou_rwlock, RW_WRITER); + hca->ih_iou_guid = guid; + hca->ih_iou.iou_changeid = h2b16(1); + hca->ih_iou.iou_num_ctrl_slots = IBDMA_MAX_IOC; + hca->ih_iou.iou_flag = IB_DM_IOU_OPTIONROM_ABSENT; + + list_create(&hca->ih_hdl_list, sizeof (ibdma_hdl_impl_t), + offsetof(ibdma_hdl_impl_t, ih_node)); + rw_exit(&hca->ih_iou_rwlock); + + /* + * It would be better to not open, but IBTL is setup to only allow + * certain managers to get async call backs if not open. + */ + status = ibt_open_hca(ibdma->ms_ibt_hdl, guid, &hca->ih_ibt_hdl); + if (status != IBT_SUCCESS) { + cmn_err(CE_NOTE, "hca_init() IBT open failed (%d)", + status); + + list_destroy(&hca->ih_hdl_list); + rw_destroy(&hca->ih_iou_rwlock); + kmem_free(hca, sizeof (ibdma_hca_t) + + (hca_attr.hca_nports-1)*sizeof (ibdma_port_t)); + return (NULL); + } + + /* + * Register with the IB Management Framework and setup MAD call-back. + */ + for (ndx = 0; ndx < hca->ih_nports; ndx++) { + port = &hca->ih_port[ndx]; + port->ip_hcap = hca; + port->ip_ibmf_reg.ir_ci_guid = hca->ih_iou_guid; + port->ip_ibmf_reg.ir_port_num = ndx + 1; + port->ip_ibmf_reg.ir_client_class = DEV_MGT_AGENT; + + status = ibmf_register(&port->ip_ibmf_reg, IBMF_VERSION, + 0, NULL, NULL, &port->ip_ibmf_hdl, &port->ip_ibmf_caps); + if (status != IBMF_SUCCESS) { + cmn_err(CE_NOTE, "hca_init, IBMF register failed (%d)", + status); + port->ip_ibmf_hdl = NULL; + ibdma_hca_fini(hca); + return (NULL); + } + + status = ibmf_setup_async_cb(port->ip_ibmf_hdl, + IBMF_QP_HANDLE_DEFAULT, ibdma_mad_recv_cb, port, 0); + if (status != IBMF_SUCCESS) { + cmn_err(CE_NOTE, "hca_init, IBMF cb setup failed (%d)", + status); + ibdma_hca_fini(hca); + return (NULL); + } + + status = ibt_modify_port_byguid(hca->ih_iou_guid, + ndx+1, IBT_PORT_SET_DEVMGT, 0); + if (status != IBT_SUCCESS) { + cmn_err(CE_NOTE, "hca_init, IBT modify port caps" + " error (%d)", status); + ibdma_hca_fini(hca); + return (NULL); + } + } + return (hca); +} + +/* + * ibdma_hca_fini() + */ +static void +ibdma_hca_fini(ibdma_hca_t *hca) +{ + int status; + int ndx; + ibdma_port_t *port; + ibdma_hdl_impl_t *hdl; + ibdma_hdl_impl_t *hdl_next; + + ASSERT(mutex_owned(&ibdma->ms_hca_list_lock)); + ASSERT(hca != NULL); + + rw_enter(&hca->ih_iou_rwlock, RW_WRITER); + + /* + * All handles should have been de-registered, but release + * any that are outstanding. + */ + hdl = list_head(&hca->ih_hdl_list); + while (hdl != NULL) { + hdl_next = list_next(&hca->ih_hdl_list, hdl); + list_remove(&hca->ih_hdl_list, hdl); + cmn_err(CE_NOTE, "hca_fini, unexpected ibdma user handle" + " exists"); + kmem_free(hdl, sizeof (*hdl)); + hdl = hdl_next; + } + list_destroy(&hca->ih_hdl_list); + + /* + * Un-register with the IBMF. + */ + for (ndx = 0; ndx < hca->ih_nports; ndx++) { + port = &hca->ih_port[ndx]; + port->ip_hcap = NULL; + + status = ibt_modify_port_byguid(hca->ih_iou_guid, + ndx+1, IBT_PORT_RESET_DEVMGT, 0); + if (status != IBT_SUCCESS) + cmn_err(CE_NOTE, "hca_fini, IBT modify port caps" + " error (%d)", status); + + if (port->ip_ibmf_hdl == NULL) + continue; + + status = ibmf_tear_down_async_cb(port->ip_ibmf_hdl, + IBMF_QP_HANDLE_DEFAULT, 0); + if (status != IBMF_SUCCESS) + cmn_err(CE_NOTE, "hca_fini, IBMF tear down cb" + " error (%d)", status); + + status = ibmf_unregister(&port->ip_ibmf_hdl, 0); + if (status != IBMF_SUCCESS) + cmn_err(CE_NOTE, "hca_fini, IBMF un-register" + " error (%d)", status); + port->ip_ibmf_hdl = NULL; + } + + status = ibt_close_hca(hca->ih_ibt_hdl); + if (status != IBT_SUCCESS) + cmn_err(CE_NOTE, "hca_fini close error (%d)", status); + + rw_exit(&hca->ih_iou_rwlock); + rw_destroy(&hca->ih_iou_rwlock); + kmem_free(hca, sizeof (ibdma_hca_t) + + (hca->ih_nports-1) * sizeof (ibdma_port_t)); +} + +/* DM IBMF MAD handlers */ +/* + * ibdma_create_resp_mad() + */ +static void +ibdma_create_resp_mad(ibmf_msg_t *msgp) +{ + /* + * Allocate send buffer fix up hdr for response. + */ + msgp->im_msgbufs_send.im_bufs_mad_hdr = + kmem_zalloc(IBDMA_MAD_SIZE, KM_SLEEP); + + msgp->im_msgbufs_send.im_bufs_cl_hdr = (uchar_t *) + msgp->im_msgbufs_send.im_bufs_mad_hdr + sizeof (ib_mad_hdr_t); + msgp->im_msgbufs_send.im_bufs_cl_hdr_len = IBDMA_DM_MAD_HDR_SIZE; + msgp->im_msgbufs_send.im_bufs_cl_data = + ((char *)msgp->im_msgbufs_send.im_bufs_cl_hdr + + IBDMA_DM_MAD_HDR_SIZE); + msgp->im_msgbufs_send.im_bufs_cl_data_len = + IBDMA_MAD_SIZE - sizeof (ib_mad_hdr_t) - IBDMA_DM_MAD_HDR_SIZE; + (void) memcpy(msgp->im_msgbufs_send.im_bufs_mad_hdr, + msgp->im_msgbufs_recv.im_bufs_mad_hdr, IBDMA_MAD_SIZE); + + /* + * We may want to support a GRH since this is a GMP; not + * required for current SRP device manager platforms. + */ +#if 0 + if (msgp->im_msg_flags & IBMF_MSG_FLAGS_GLOBAL_ADDRESS) { + ib_gid_t temp = msgp->im_global_addr.ig_recver_gid; + + msgp->im_global_addr.ig_recver_gid = + msgp->im_global_addr.ig_sender_gid; + msgp->im_global_addr.ig_sender_gid = temp; + } +#endif +} + +/* + * ibdma_mad_send_cb() + */ +/* ARGSUSED */ +static void +ibdma_mad_send_cb(ibmf_handle_t ibmf_hdl, ibmf_msg_t *msgp, void *arg) +{ + /* + * Just free the buffers and release the message. + */ + if (msgp->im_msgbufs_send.im_bufs_mad_hdr != NULL) { + kmem_free(msgp->im_msgbufs_send.im_bufs_mad_hdr, + IBDMA_MAD_SIZE); + msgp->im_msgbufs_send.im_bufs_mad_hdr = NULL; + } + if (ibmf_free_msg(ibmf_hdl, &msgp) != IBMF_SUCCESS) { + cmn_err(CE_NOTE, "mad_send_cb, IBMF message free error"); + } +} + +/* + * ibdma_mad_recv_cb() + */ +static void +ibdma_mad_recv_cb(ibmf_handle_t ibmf_hdl, ibmf_msg_t *msgp, void *args) +{ + int status; + ib_mad_hdr_t *in_mad; + ib_mad_hdr_t *out_mad; + ibdma_port_t *port = args; + + ASSERT(msgp != NULL); + ASSERT(port != NULL); + + if (msgp->im_msg_status != IBMF_SUCCESS) { + cmn_err(CE_NOTE, "mad_recv_cb, bad MAD receive status (%d)", + msgp->im_msg_status); + goto drop; + } + + in_mad = msgp->im_msgbufs_recv.im_bufs_mad_hdr; + + if (in_mad->MgmtClass != MAD_MGMT_CLASS_DEV_MGT) { +#ifdef DEBUG_IBDMA + cmn_err(CE_NOTE, "mad_recv_cb, MAD not of Dev Mgmt Class"); +#endif + goto drop; + } + + ibdma_create_resp_mad(msgp); + out_mad = msgp->im_msgbufs_send.im_bufs_mad_hdr; + + out_mad->R_Method = IB_DM_DEVMGT_METHOD_GET_RESP; + out_mad->Status = 0; + + if (in_mad->R_Method == MAD_METHOD_SET) { +#ifdef DEBUG_IBDMA + cmn_err(CE_NOTE, "mad_recv_cb, no attributes supported" + " for set"); +#endif + out_mad->Status = MAD_STATUS_UNSUPP_METHOD_ATTR; + goto send_resp; + } + + if (in_mad->R_Method != MAD_METHOD_GET) { +#ifdef DEBUG_IBDMA + cmn_err(CE_NOTE, "mad_recv_cb, no attributes supported" + " for set"); +#endif + out_mad->Status = MAD_STATUS_UNSUPP_METHOD; + goto send_resp; + } + + /* + * Process a GET method. + */ + switch (b2h16(in_mad->AttributeID)) { + + case IB_DM_ATTR_CLASSPORTINFO: + ibdma_get_class_portinfo(msgp); + break; + + case IB_DM_ATTR_IO_UNITINFO: + ibdma_get_io_unitinfo(port->ip_hcap, msgp); + break; + + case IB_DM_ATTR_IOC_CTRL_PROFILE: + ibdma_get_ioc_profile(port->ip_hcap, msgp); + break; + + case IB_DM_ATTR_SERVICE_ENTRIES: + ibdma_get_ioc_services(port->ip_hcap, msgp); + break; + + default: + out_mad->Status = MAD_STATUS_UNSUPP_METHOD_ATTR; + break; + } + +send_resp: + status = ibmf_msg_transport(ibmf_hdl, IBMF_QP_HANDLE_DEFAULT, + msgp, NULL, ibdma_mad_send_cb, NULL, 0); + if (status != IBMF_SUCCESS) { + cmn_err(CE_NOTE, "mad_recv_cb, send error (%d)", status); + ibdma_mad_send_cb(ibmf_hdl, msgp, NULL); + } + return; + +drop: + status = ibmf_free_msg(ibmf_hdl, &msgp); + if (status != IBMF_SUCCESS) { + cmn_err(CE_NOTE, "mad_recv_cb, error dropping (%d)", + status); + } +} + +/* + * ibdma_get_class_portinfo() + */ +static void +ibdma_get_class_portinfo(ibmf_msg_t *msg) +{ + ib_mad_classportinfo_t *cpip; + + cpip = (ib_mad_classportinfo_t *)msg->im_msgbufs_send.im_bufs_cl_data; + bzero(cpip, sizeof (*cpip)); + cpip->BaseVersion = MAD_CLASS_BASE_VERS_1; + cpip->ClassVersion = IB_DM_CLASS_VERSION_1; + cpip->RespTimeValue = h2b32(IBDMA_DM_RESP_TIME); +} + +/* + * ibdma_get_io_unitinfo() + */ +static void +ibdma_get_io_unitinfo(ibdma_hca_t *hca, ibmf_msg_t *msg) +{ + ib_dm_io_unitinfo_t *uip; + + uip = (ib_dm_io_unitinfo_t *)msg->im_msgbufs_send.im_bufs_cl_data; + rw_enter(&hca->ih_iou_rwlock, RW_READER); + bcopy(&hca->ih_iou, uip, sizeof (ib_dm_io_unitinfo_t)); + rw_exit(&hca->ih_iou_rwlock); +} + +/* + * ibdma_get_ioc_profile() + */ +static void +ibdma_get_ioc_profile(ibdma_hca_t *hca, ibmf_msg_t *msg) +{ + ib_dm_ioc_ctrl_profile_t *iocp; + uint32_t slot; + + ASSERT(msg != NULL); + + slot = b2h32(msg->im_msgbufs_recv.im_bufs_mad_hdr->AttributeModifier); + iocp = (ib_dm_ioc_ctrl_profile_t *) + msg->im_msgbufs_send.im_bufs_cl_data; + if (slot == 0 || slot > IBDMA_MAX_IOC) { + msg->im_msgbufs_send.im_bufs_mad_hdr->Status = + MAD_STATUS_INVALID_FIELD; + return; + } + + slot--; + rw_enter(&hca->ih_iou_rwlock, RW_READER); + if (ibdma_get_ioc_state(hca, slot) == IBDMA_IOC_PRESENT) { + bcopy(&hca->ih_ioc[slot].ii_profile, iocp, + sizeof (ib_dm_ioc_ctrl_profile_t)); + } else { + msg->im_msgbufs_send.im_bufs_mad_hdr->Status = + IB_DM_DEVMGT_MAD_STAT_NORESP; + } + rw_exit(&hca->ih_iou_rwlock); +} + +/* + * ibdma_get_ioc_services() + */ +static void +ibdma_get_ioc_services(ibdma_hca_t *hca, ibmf_msg_t *msg) +{ + ib_dm_srv_t *to_svcp; + ib_dm_srv_t *from_svcp; + uint32_t slot; + uint8_t hi; + uint8_t low; + + ASSERT(msg != NULL); + + slot = b2h32(msg->im_msgbufs_recv.im_bufs_mad_hdr->AttributeModifier); + hi = (slot >> 8) & 0x00FF; + low = slot & 0x00FF; + slot = (slot >> 16) & 0x0FFFF; + if (slot == 0 || slot > IBDMA_MAX_IOC) { + msg->im_msgbufs_send.im_bufs_mad_hdr->Status = + MAD_STATUS_INVALID_FIELD; + return; + } + + slot--; + + rw_enter(&hca->ih_iou_rwlock, RW_READER); + if (ibdma_get_ioc_state(hca, slot) != IBDMA_IOC_PRESENT) { + msg->im_msgbufs_send.im_bufs_mad_hdr->Status = + IB_DM_DEVMGT_MAD_STAT_NORESP; + rw_exit(&hca->ih_iou_rwlock); + return; + } + + if ((low > hi) || (hi - low > 4)) { + msg->im_msgbufs_send.im_bufs_mad_hdr->Status = + MAD_STATUS_INVALID_FIELD; + rw_exit(&hca->ih_iou_rwlock); + return; + } + + if (hi > hca->ih_ioc[slot].ii_profile.ioc_service_entries) { + msg->im_msgbufs_send.im_bufs_mad_hdr->Status = + MAD_STATUS_INVALID_FIELD; + rw_exit(&hca->ih_iou_rwlock); + return; + } + + to_svcp = (ib_dm_srv_t *)msg->im_msgbufs_send.im_bufs_cl_data; + from_svcp = hca->ih_ioc[slot].ii_srvcs + low; + bcopy(from_svcp, to_svcp, sizeof (ib_dm_srv_t) * (hi - low + 1)); + rw_exit(&hca->ih_iou_rwlock); +} + + +/* + * Client API internal helpers + */ + +/* + * ibdma_hdl_to_ioc() + */ +ibdma_hdl_impl_t * +ibdma_get_hdl_impl(ibdma_hdl_t hdl) +{ + ibdma_hca_t *hca; + ibdma_hdl_impl_t *hdl_tmp = hdl; + ibdma_hdl_impl_t *hdl_impl; + + ASSERT(mutex_owned(&ibdma->ms_hca_list_lock)); + + if (hdl_tmp == NULL) { + cmn_err(CE_NOTE, "get_hdl_impl, NULL handle"); + return (NULL); + } + + hca = ibdma_find_hca(hdl_tmp->ih_iou_guid); + if (hca == NULL) { + cmn_err(CE_NOTE, "get_hdl_impl, invalid handle, bad IOU"); + return (NULL); + } + + hdl_impl = list_head(&hca->ih_hdl_list); + while (hdl_impl != NULL) { + if (hdl_impl == hdl_tmp) { + break; + } + hdl_impl = list_next(&hca->ih_hdl_list, hdl_impl); + } + return (hdl_impl); +} + +/* + * ibdma_set_ioc_state() + * + * slot should be 0 based (not DM 1 based slot). + * + * I/O Unit write lock should be held outside of this function. + */ +static void +ibdma_set_ioc_state(ibdma_hca_t *hca, int slot, ibdma_ioc_state_t state) +{ + uint8_t cur; + uint16_t id; + + cur = hca->ih_iou.iou_ctrl_list[slot >> 1]; + if (slot & 1) { + cur = (cur & 0xF0) | state; + } else { + cur = (cur & 0x0F) | (state << 4); + } + hca->ih_iou.iou_ctrl_list[slot >> 1] = cur; + id = b2h16(hca->ih_iou.iou_changeid); + id++; + hca->ih_iou.iou_changeid = h2b16(id); +#ifdef DEBUG_IBDMA + cmn_err(CE_NOTE, "set_ioc_state, slot offset(%d), value(%d)", + slot, hca->ih_iou.iou_ctrl_list[slot >> 1]); +#endif +} + +/* + * ibdma_get_ioc_state() + * + * slot should be 0 based (not DM 1 based slot). + * + * I/O Unit read lock should be held outside of this function. + */ +static ibdma_ioc_state_t +ibdma_get_ioc_state(ibdma_hca_t *hca, int slot) +{ + uint8_t cur; + + if (slot >= IBDMA_MAX_IOC) + return (0xFF); + + cur = hca->ih_iou.iou_ctrl_list[slot >> 1]; + cur = slot & 1 ? cur & 0x0F : cur >> 4; + return (cur); +} + +/* CLIENT API Implementation */ +/* + * ibdma_ioc_register() + * + */ +ibdma_hdl_t +ibdma_ioc_register(ib_guid_t iou_guid, ib_dm_ioc_ctrl_profile_t *profile, + ib_dm_srv_t *services) +{ + int free_slot = -1; + int svc_entries; + int slot; + ibdma_hca_t *hca; + ibdma_hdl_impl_t *hdl; + + if (profile == NULL || services == NULL) { + cmn_err(CE_NOTE, "ioc_register, bad parameter"); + return (NULL); + } + + svc_entries = profile->ioc_service_entries; + if (svc_entries == 0) { + cmn_err(CE_NOTE, "ioc_register, bad profile no service"); + return (NULL); + } + + /* + * Find the associated I/O Unit. + */ + mutex_enter(&ibdma->ms_hca_list_lock); + hca = ibdma_find_hca(iou_guid); + if (hca == NULL) { + mutex_exit(&ibdma->ms_hca_list_lock); + cmn_err(CE_NOTE, "ioc_register, bad I/O Unit GUID (0x%llx)", + (u_longlong_t)iou_guid); + return (NULL); + } + + rw_enter(&hca->ih_iou_rwlock, RW_WRITER); + for (slot = 0; slot < IBDMA_MAX_IOC; slot++) { + if (hca->ih_ioc[slot].ii_inuse == 0) { + if (free_slot == -1) { + free_slot = slot; + } + continue; + } + + if (profile->ioc_guid == + hca->ih_ioc[slot].ii_profile.ioc_guid) { + rw_exit(&hca->ih_iou_rwlock); + mutex_exit(&ibdma->ms_hca_list_lock); +#ifdef DEBUG_IBDMA + cmn_err(CE_NOTE, "ioc_register, IOC previously" + " registered"); +#endif + return (NULL); + } + } + + if (free_slot < 0) { + rw_exit(&hca->ih_iou_rwlock); + cmn_err(CE_NOTE, "ioc_register, error - I/O Unit full"); + return (NULL); + } +#ifdef DEBUG_IBDMA + cmn_err(CE_NOTE, "ibdma_ioc_register, assigned to 0 based slot (%d)", + free_slot); +#endif + + hca->ih_ioc[free_slot].ii_inuse = 1; + hca->ih_ioc[free_slot].ii_slot = free_slot; + hca->ih_ioc[free_slot].ii_hcap = hca; + + /* + * Allocate local copy of profile and services. + */ + hca->ih_ioc[free_slot].ii_srvcs = + kmem_zalloc(sizeof (ib_dm_srv_t) * svc_entries, KM_SLEEP); + bcopy(profile, &hca->ih_ioc[free_slot].ii_profile, + sizeof (ib_dm_ioc_ctrl_profile_t)); + bcopy(services, hca->ih_ioc[free_slot].ii_srvcs, + sizeof (ib_dm_srv_t) * svc_entries); + + /* + * Update the profile copy with the I/O controller slot assigned. + * The slot occupies the lower 8 biths of the vendor ID/slot 32bit + * field. + */ + profile->ioc_vendorid |= h2b32(free_slot); + + ibdma_set_ioc_state(hca, free_slot, IBDMA_IOC_PRESENT); + + hdl = kmem_alloc(sizeof (*hdl), KM_SLEEP); + hdl->ih_iou_guid = hca->ih_iou_guid; + hdl->ih_ioc_ndx = (uint8_t)free_slot; + list_insert_tail(&hca->ih_hdl_list, hdl); + + rw_exit(&hca->ih_iou_rwlock); + mutex_exit(&ibdma->ms_hca_list_lock); + + return ((ibdma_hdl_t)hdl); +} + +/* + * ibdma_ioc_unregister() + * + */ +ibdma_status_t +ibdma_ioc_unregister(ibdma_hdl_t hdl) +{ + ibdma_ioc_t *ioc; + ibdma_hca_t *hca; + int slot; + ibdma_hdl_impl_t *hdl_tmp = hdl; + ibdma_hdl_impl_t *hdl_impl; + + if (hdl == NULL) { + cmn_err(CE_NOTE, "ioc_unregister, NULL handle"); + return (IBDMA_BAD_PARAM); + } + + mutex_enter(&ibdma->ms_hca_list_lock); + hca = ibdma_find_hca(hdl_tmp->ih_iou_guid); + if (hca == NULL) { + cmn_err(CE_NOTE, "ioc_unregsiter, invalid handle, IOU" + " not found"); + mutex_exit(&ibdma->ms_hca_list_lock); + return (IBDMA_BAD_PARAM); + } + + hdl_impl = list_head(&hca->ih_hdl_list); + while (hdl_impl != NULL) { + if (hdl_impl == hdl_tmp) { + break; + } + hdl_impl = list_next(&hca->ih_hdl_list, hdl_impl); + } + + if (hdl_impl == NULL) { + cmn_err(CE_NOTE, "ioc_unregsiter, invalid handle, not found"); + mutex_exit(&ibdma->ms_hca_list_lock); + return (IBDMA_BAD_PARAM); + } + + list_remove(&hca->ih_hdl_list, hdl_impl); + + if (hdl_impl->ih_ioc_ndx >= IBDMA_MAX_IOC) { + cmn_err(CE_NOTE, "ioc_unregister, corrupted handle"); + kmem_free(hdl_impl, sizeof (*hdl_impl)); + mutex_exit(&ibdma->ms_hca_list_lock); + return (IBDMA_BAD_PARAM); + } + ioc = &hca->ih_ioc[hdl_impl->ih_ioc_ndx]; + kmem_free(hdl_impl, sizeof (*hdl_impl)); + + if (ioc->ii_slot > IBDMA_MAX_IOC) { + cmn_err(CE_NOTE, "ioc_unregister, IOC corrupted, bad" + " slot in IOC"); + mutex_exit(&ibdma->ms_hca_list_lock); + return (IBDMA_BAD_PARAM); + } + + rw_enter(&ioc->ii_hcap->ih_iou_rwlock, RW_WRITER); + if (ioc->ii_inuse == 0) { + rw_exit(&ioc->ii_hcap->ih_iou_rwlock); + mutex_exit(&ibdma->ms_hca_list_lock); + cmn_err(CE_NOTE, "ioc_unregister, slot not in use (%d)", + ioc->ii_slot+1); + return (IBDMA_BAD_PARAM); + } + + ASSERT(ioc->ii_srvcs != NULL); + + slot = ioc->ii_slot; + hca = ioc->ii_hcap; + kmem_free(ioc->ii_srvcs, sizeof (ib_dm_srv_t) * + ioc->ii_profile.ioc_service_entries); + bzero(ioc, sizeof (ibdma_ioc_t)); + ibdma_set_ioc_state(hca, slot, IBDMA_IOC_NOT_INSTALLED); + + rw_exit(&hca->ih_iou_rwlock); + mutex_exit(&ibdma->ms_hca_list_lock); + + return (IBDMA_SUCCESS); +} + +/* + * ibdma_ioc_update() + * + */ +ibdma_status_t +ibdma_ioc_update(ibdma_hdl_t hdl, ib_dm_ioc_ctrl_profile_t *profile, + ib_dm_srv_t *services) +{ + ibdma_ioc_t *ioc; + ibdma_hca_t *hca; + ibdma_hdl_impl_t *hdl_tmp = hdl; + ibdma_hdl_impl_t *hdl_impl; + + if (hdl == NULL) { + cmn_err(CE_NOTE, "ioc_update, NULL handle"); + return (IBDMA_BAD_PARAM); + } + + if (profile == NULL || services == NULL) { + cmn_err(CE_NOTE, "ioc_update, NULL parameter"); + return (IBDMA_BAD_PARAM); + } + + mutex_enter(&ibdma->ms_hca_list_lock); + hca = ibdma_find_hca(hdl_tmp->ih_iou_guid); + if (hca == NULL) { + cmn_err(CE_NOTE, "ioc_update, invalid handle, IOU not found"); + mutex_exit(&ibdma->ms_hca_list_lock); + return (IBDMA_BAD_PARAM); + } + + hdl_impl = list_head(&hca->ih_hdl_list); + while (hdl_impl != NULL) { + if (hdl_impl == hdl_tmp) { + break; + } + hdl_impl = list_next(&hca->ih_hdl_list, hdl_impl); + } + + if (hdl_impl == NULL) { + cmn_err(CE_NOTE, "ioc_update, invalid handle, not found"); + mutex_exit(&ibdma->ms_hca_list_lock); + return (IBDMA_BAD_PARAM); + } + + if (hdl_impl->ih_ioc_ndx >= IBDMA_MAX_IOC) { + cmn_err(CE_NOTE, "ioc_update, corrupted handle"); + mutex_exit(&ibdma->ms_hca_list_lock); + return (IBDMA_BAD_PARAM); + } + ioc = &hca->ih_ioc[hdl_impl->ih_ioc_ndx]; + + if (ioc->ii_slot >= IBDMA_MAX_IOC || ioc->ii_hcap == NULL) { + cmn_err(CE_NOTE, "ioc_update, bad handle (%p)", + (void *)hdl); + mutex_exit(&ibdma->ms_hca_list_lock); + return (IBDMA_BAD_PARAM); + } + + rw_enter(&ioc->ii_hcap->ih_iou_rwlock, RW_WRITER); + if (ioc->ii_inuse == 0) { + rw_exit(&ioc->ii_hcap->ih_iou_rwlock); + mutex_exit(&ibdma->ms_hca_list_lock); + cmn_err(CE_NOTE, "ioc_udate slot not in use (%d)", + ioc->ii_slot+1); + return (IBDMA_BAD_PARAM); + } + + ASSERT(ioc->ii_srvcs != NULL); + + kmem_free(ioc->ii_srvcs, ioc->ii_profile.ioc_service_entries * + sizeof (ib_dm_srv_t)); + ioc->ii_srvcs = kmem_zalloc(profile->ioc_service_entries * + sizeof (ib_dm_srv_t), KM_SLEEP); + + bcopy(profile, &ioc->ii_profile, sizeof (ib_dm_ioc_ctrl_profile_t)); + bcopy(services, ioc->ii_srvcs, sizeof (ib_dm_srv_t) * + profile->ioc_service_entries); + /* + * Update the profile copy with the I/O controller slot assigned. + * The slot occupies the lower 8 biths of the vendor ID/slot 32bit + * field. + */ + profile->ioc_vendorid |= h2b32(ioc->ii_slot); + ibdma_set_ioc_state(ioc->ii_hcap, ioc->ii_slot, IBDMA_IOC_PRESENT); + rw_exit(&ioc->ii_hcap->ih_iou_rwlock); + mutex_exit(&ibdma->ms_hca_list_lock); + + return (IBDMA_SUCCESS); +} diff --git a/usr/src/uts/common/sys/ib/mgt/ibdma/ibdma.h b/usr/src/uts/common/sys/ib/mgt/ibdma/ibdma.h new file mode 100644 index 0000000000..0217c33610 --- /dev/null +++ b/usr/src/uts/common/sys/ib/mgt/ibdma/ibdma.h @@ -0,0 +1,101 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ + +/* + * Copyright 2009 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +#ifndef _IBDMA_H +#define _IBDMA_H + +/* + * ibdma.h + * + * Device Management Agent prototypes and structures shared with + * consumer (I/O Controller) via client API. + */ + +/* + * The Infiniband Device Management Agent manages an I/O Unit + * for each IB HCA, providing a common view to of the I/O Unit + * that presents protocol specific I/O Controllers. + * + * By default, the I/O Unit is unpopulated with I/O Controllers. Each + * underlying protocol transport registers their I/O Controller with + * the respective I/O Unit (HCA) providing their I/O Controller profile + * and the services they are making available. As services change, the + * the transport protocol calls back into the IB DMA to update their + * profile and services. + */ + +#ifdef __cplusplus +extern "C" { +#endif + +#include <sys/ib/mgt/ibmf/ibmf.h> +#include <sys/ib/mgt/ib_dm_attr.h> + +typedef enum ibdma_status_e { + IBDMA_SUCCESS = 0, /* good status */ + IBDMA_IOC_DUPLICATE, /* IOC GUID already exists */ + IBDMA_IOU_FULL, /* No slots available in IOU */ + IBDMA_BAD_IOC_PROFILE, /* IOC profile disparity */ + IBDMA_BAD_PARAM /* Invalid function parameter */ +} ibdma_status_t; + +/* + * I/O Controller Provider API. + * + * The DM Agent responds to I/O Unit requests on all IB fabric ports + * in the system, setting each ports "isDeviceManagementSupported" bit. + * + * I/O Controllers must register their IOC profile and associated + * services with the DM Agent. The DM Agent will assign a + * I/O Unit slot to the I/O Controller at that time. + */ +typedef void* ibdma_hdl_t; + +/* + * Register an IOC. + * + * Returns a handle used to un-register or update the IOC + * profile/services information. + */ +ibdma_hdl_t ibdma_ioc_register(ib_guid_t ioc_guid, + ib_dm_ioc_ctrl_profile_t *profile, ib_dm_srv_t *services); + +/* + * Un-Register an IOC. + */ +ibdma_status_t ibdma_ioc_unregister(ibdma_hdl_t hdl); + +/* + * Update a previously register IOC profile/services. + */ +ibdma_status_t ibdma_ioc_update(ibdma_hdl_t hdl, + ib_dm_ioc_ctrl_profile_t *profile, ib_dm_srv_t *services); + +#ifdef __cplusplus +} +#endif + +#endif /* _IBDMA_H */ diff --git a/usr/src/uts/common/sys/ib/mgt/ibdma/ibdma_impl.h b/usr/src/uts/common/sys/ib/mgt/ibdma/ibdma_impl.h new file mode 100644 index 0000000000..6c99379129 --- /dev/null +++ b/usr/src/uts/common/sys/ib/mgt/ibdma/ibdma_impl.h @@ -0,0 +1,152 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ + +/* + * Copyright 2009 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ +#ifndef _IBDMA_IMPL_H +#define _IBDMA_IMPL_H + +/* + * ibdma_impl.h + * + * Device Management Agent private prototypes and structures. + */ + +#ifdef __cpluplus +extern "C" { +#endif + +#include <sys/ib/ibtl/ibvti.h> /* IB verrbs interfaces */ +#include <sys/ib/mgt/ib_dm_attr.h> /* IB DM defines/structures */ +#include <sys/ib/mgt/ib_mad.h> /* IB MAD defines/structures */ + +enum { + IBDMA_MAD_SIZE = 256, + IBDMA_DM_MAD_HDR_SIZE = 40, + IBDMA_DM_RESP_TIME = 20, + IBDMA_MAX_IOC = 16 +}; + +/* + * Implementation of handle returned to consumer. + */ +typedef struct ibdma_hdl_impl_s { + list_node_t ih_node; + ib_guid_t ih_iou_guid; + uint8_t ih_ioc_ndx; +} ibdma_hdl_impl_t; + +/* + * Each I/O Controller slot for the I/O Unit. + */ +typedef struct ibdma_ioc_s { + uint8_t ii_inuse; + /* + * Just to map handle back to slot number and hca + */ + int ii_slot; + struct ibdma_hca_s *ii_hcap; + + /* + * Profile provided by the I/O Controller, must be stored + * in network order. Note that the profile indicates the + * number of service entries pointed to by ii_srvcs. + */ + ib_dm_ioc_ctrl_profile_t ii_profile; + ib_dm_srv_t *ii_srvcs; +} ibdma_ioc_t; + +/* + * The ibdma_hca_t structure is only used internally by the + * IB DM Agent. It is created when the associated HCA is + * opened as part of initialization or as a result of a + * notification via IBTF. It is destroyed when the HCA + * is closed as part of fini processing or as a result of + * a notification via IBTF. The structure is not directly + * accessed by IBMF call-backs or the consumer API. + */ +typedef struct ibdma_port_s { + ibmf_handle_t ip_ibmf_hdl; + ibmf_register_info_t ip_ibmf_reg; + ibmf_impl_caps_t ip_ibmf_caps; + struct ibdma_hca_s *ip_hcap; +} ibdma_port_t; + +typedef struct ibdma_hca_s { + list_node_t ih_node; + ibt_hca_hdl_t ih_ibt_hdl; + + /* + * Consumer handles associated with I/O Controllers + * that have registered with this I/O Unit. + */ + list_t ih_hdl_list; + + /* + * The I/O Unit that is presented to the IB Fabric. + * It is stored in network order. + */ + krwlock_t ih_iou_rwlock; + ib_guid_t ih_iou_guid; + ib_dm_io_unitinfo_t ih_iou; + ibdma_ioc_t ih_ioc[IBDMA_MAX_IOC]; + uint8_t ih_nports; + ibdma_port_t ih_port[1]; +} ibdma_hca_t; + + +/* + * The IBDMA module state information created and initialized + * at _init() and freed at _fini(). + */ +typedef struct ibdma_mod_state_s { + ibt_clnt_hdl_t ms_ibt_hdl; + + /* + * The HCA list lock is used protect the HCA list and + * is held during consumer routines (in place of a + * reference count) to ensure the HCA exists for the + * duration of it's use in the routine. + */ + kmutex_t ms_hca_list_lock; + list_t ms_hca_list; + uint_t ms_num_hcas; + +} ibdma_mod_state_t; + + +/* + * Client API internal helpers + */ +typedef enum ibdma_ioc_state_e { + IBDMA_IOC_NOT_INSTALLED = 0, + IBDMA_IOC_PRESENT = 1, + IBDMA_IOC_DOES_NOT_EXIST = 255, + IBDMA_HDL_MAGIC = 0x00931000 +} ibdma_ioc_state_t; + +static void +ibdma_set_ioc_state(ibdma_hca_t *hca, int slot, ibdma_ioc_state_t state); +static ibdma_ioc_state_t ibdma_get_ioc_state(ibdma_hca_t *hca, int slot); + +#endif /* _IBDMA_IMPL_H */ diff --git a/usr/src/uts/intel/Makefile.intel.shared b/usr/src/uts/intel/Makefile.intel.shared index bef9e45428..180e3efd00 100644 --- a/usr/src/uts/intel/Makefile.intel.shared +++ b/usr/src/uts/intel/Makefile.intel.shared @@ -319,6 +319,7 @@ DRV_KMODS += smbsrv DRV_KMODS += smp DRV_KMODS += sppp DRV_KMODS += sppptun +DRV_KMODS += srpt DRV_KMODS += st DRV_KMODS += sy DRV_KMODS += sysevent @@ -576,6 +577,7 @@ MISC_KMODS += hook MISC_KMODS += hpcsvc MISC_KMODS += ibcm MISC_KMODS += ibdm +MISC_KMODS += ibdma MISC_KMODS += ibmf MISC_KMODS += ibtl MISC_KMODS += idm diff --git a/usr/src/uts/intel/ibdma/Makefile b/usr/src/uts/intel/ibdma/Makefile new file mode 100644 index 0000000000..0450822478 --- /dev/null +++ b/usr/src/uts/intel/ibdma/Makefile @@ -0,0 +1,126 @@ +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# +# +# Copyright 2009 Sun Microsystems, Inc. All rights reserved. +# Use is subject to license terms. +# +# +# This makefile drives the production of the ibdm kernel module. +# +# intel architecture dependent +# + +# +# Path to the base of the uts directory tree (usually /usr/src/uts). +# +UTSBASE = ../.. + +# +# Define the module and object file sets. +# +MODULE = ibdma +OBJECTS = $(IBDMA_OBJS:%=$(OBJS_DIR)/%) +LINTS = $(IBDMA_OBJS:%.o=$(LINTS_DIR)/%.ln) +ROOTMODULE = $(ROOT_MISC_DIR)/$(MODULE) +WARLOCK_OUT = $(IBDMA_OBJS:%.o=%.ll) +WARLOCK_OK = $(MODULE).ok + +# +# Include common rules. +# +include $(UTSBASE)/intel/Makefile.intel + +# +# Define targets +# +ALL_TARGET = $(BINARY) +LINT_TARGET = $(MODULE).lint +INSTALL_TARGET = $(BINARY) $(ROOTMODULE) + +# +# Overrides. +# + +# +# lint pass one enforcement +# +CFLAGS += $(CCVERBOSE) + +# +# depends on misc/ibtl, drv/ib and misc/ibmf +# +LDFLAGS += -dy -Nmisc/ibtl -Nmisc/ibmf + +# +# For now, disable these lint checks; maintainers should endeavor +# to investigate and remove these for maximum lint coverage. +# Please do not carry these forward to new Makefiles. +# +LINTTAGS += -erroff=E_STATIC_UNUSED +LINTTAGS += -erroff=E_BAD_PTR_CAST_ALIGN + +# +# Default build targets. +# +.KEEP_STATE: + +def: $(DEF_DEPS) + +all: $(ALL_DEPS) + +clean: $(CLEAN_DEPS) + $(RM) $(WARLOCK_OUT) $(WARLOCK_OK) + +clobber: $(CLOBBER_DEPS) + $(RM) $(WARLOCK_OUT) $(WARLOCK_OK) + +lint: $(LINT_DEPS) + +modlintlib: $(MODLINTLIB_DEPS) + +clean.lint: $(CLEAN_LINT_DEPS) + +install: $(INSTALL_DEPS) + +# +# Include common targets. +# +include $(UTSBASE)/intel/Makefile.targ + + +# +# Defines for local commands. +# +WARLOCK = warlock +WLCC = wlcc +TOUCH = touch +TEST = test + +warlock: $(WARLOCK_OK) + + +$(WARLOCK_OK): $(WARLOCK_OUT) + $(TOUCH) $@ + +%.ll: $(UTSBASE)/common/io/ib/mgt/ibdma/%.c \ + $(UTSBASE)/common/sys/ib/mgt/ibdma/ibdma_impl.h \ + $(UTSBASE)/common/sys/ib/mgt/ibdma/ibdma.h + $(WLCC) $(CPPFLAGS) -DDEBUG -o $@ $< diff --git a/usr/src/uts/intel/srpt/Makefile b/usr/src/uts/intel/srpt/Makefile new file mode 100644 index 0000000000..b68438ceb3 --- /dev/null +++ b/usr/src/uts/intel/srpt/Makefile @@ -0,0 +1,94 @@ +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# +# +# Copyright 2009 Sun Microsystems, Inc. All rights reserved. +# Use is subject to license terms. +# +# This makefile drives the production of the srpt pseudo-driver for +# COMSTAR. + +# +# Path to the base of the uts directory tree (usually /usr/src/uts). +# + + +UTSBASE = ../.. + +ARCHDIR:sh = cd ..; basename `pwd` + +# +# Define the module and object file sets. +# +MODULE = srpt +OBJECTS = $(SRPT_OBJS:%=$(OBJS_DIR)/%) +LINTS = $(SRPT_OBJS:%.o=$(LINTS_DIR)/%.ln) +ROOTMODULE = $(ROOT_DRV_DIR)/$(MODULE) +CONF_SRCDIR = $(UTSBASE)/common/io/comstar/port/srpt + +# +# Include common rules. +# +include ../Makefile.$(ARCHDIR) + +# +# Define targets +# +ALL_TARGET = $(BINARY) $(SRC_CONFILE) +LINT_TARGET = $(MODULE).lint +INSTALL_TARGET = $(BINARY) $(ROOTMODULE) $(ROOT_CONFFILE) + +# +# Overrides and depends_on +# +MODSTUBS_DIR = $(OBJS_DIR) +LDFLAGS += -dy -Ndrv/stmf -Nmisc/ibtl -Nmisc/ibcm + +INC_PATH += -I$(UTSBASE)/common/io/comstar/port/srpt +INC_PATH += -I$(UTSBASE)/common/sys + +C99MODE= -xc99=%all +C99LMODE= -Xc99=%all + +# +# Default build targets. +# +.KEEP_STATE: + +def: $(DEF_DEPS) + +all: $(ALL_DEPS) + +clean: $(CLEAN_DEPS) + +clobber: $(CLOBBER_DEPS) + +lint: $(LINT_DEPS) + +modlintlib: $(MODLINTLIB_DEPS) + +clean.lint: $(CLEAN_LINT_DEPS) + +install: $(INSTALL_DEPS) + +# +# Include common targets. +# +include ../Makefile.targ diff --git a/usr/src/uts/sparc/Makefile.sparc.shared b/usr/src/uts/sparc/Makefile.sparc.shared index 0e39017b17..f60bad2e77 100644 --- a/usr/src/uts/sparc/Makefile.sparc.shared +++ b/usr/src/uts/sparc/Makefile.sparc.shared @@ -306,6 +306,7 @@ DRV_KMODS += ncall nsctl sdbc nskern sv DRV_KMODS += ii rdc rdcsrv rdcstub DRV_KMODS += iscsi DRV_KMODS += emlxs +DRV_KMODS += srpt $(CLOSED_BUILD)CLOSED_DRV_KMODS += ifp $(CLOSED_BUILD)CLOSED_DRV_KMODS += uata @@ -392,6 +393,7 @@ MISC_KMODS += kcf MISC_KMODS += ksocket MISC_KMODS += ibcm MISC_KMODS += ibdm +MISC_KMODS += ibdma MISC_KMODS += ibmf MISC_KMODS += ibtl MISC_KMODS += idm diff --git a/usr/src/uts/sparc/ibdma/Makefile b/usr/src/uts/sparc/ibdma/Makefile new file mode 100644 index 0000000000..ef3ee9c03e --- /dev/null +++ b/usr/src/uts/sparc/ibdma/Makefile @@ -0,0 +1,126 @@ +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# +# +# Copyright 2009 Sun Microsystems, Inc. All rights reserved. +# Use is subject to license terms. +# +# +# This makefile drives the production of the ibdma kernel module. +# +# sparc architecture dependent +# + +# +# Path to the base of the uts directory tree (usually /usr/src/uts). +# +UTSBASE = ../.. + +# +# Define the module and object file sets. +# +MODULE = ibdma +OBJECTS = $(IBDMA_OBJS:%=$(OBJS_DIR)/%) +LINTS = $(IBDMA_OBJS:%.o=$(LINTS_DIR)/%.ln) +ROOTMODULE = $(ROOT_MISC_DIR)/$(MODULE) +WARLOCK_OUT = $(IBDMA_OBJS:%.o=%.ll) +WARLOCK_OK = $(MODULE).ok + +# +# Include common rules. +# +include $(UTSBASE)/sparc/Makefile.sparc + +# +# Define targets +# +ALL_TARGET = $(BINARY) +LINT_TARGET = $(MODULE).lint +INSTALL_TARGET = $(BINARY) $(ROOTMODULE) + +# +# Overrides. +# +ALL_BUILDS = $(ALL_BUILDSONLY64) +DEF_BUILDS = $(DEF_BUILDSONLY64) + +# +# lint pass one enforcement +# +CFLAGS += $(CCVERBOSE) + +# +# depends on misc/ibtl, and misc/ibmf +# +LDFLAGS += -dy -Nmisc/ibtl -Nmisc/ibmf + +# +# For now, disable these lint checks; maintainers should endeavor +# to investigate and remove these for maximum lint coverage. +# Please do not carry these forward to new Makefiles. +# +LINTTAGS += -erroff=E_BAD_PTR_CAST_ALIGN + +# +# Default build targets. +# +.KEEP_STATE: + +def: $(DEF_DEPS) + +all: $(ALL_DEPS) + +clean: $(CLEAN_DEPS) + $(RM) $(WARLOCK_OUT) $(WARLOCK_OK) + +clobber: $(CLOBBER_DEPS) + $(RM) $(WARLOCK_OUT) $(WARLOCK_OK) + +lint: $(LINT_DEPS) + +modlintlib: $(MODLINTLIB_DEPS) lint32 + +clean.lint: $(CLEAN_LINT_DEPS) + +install: $(INSTALL_DEPS) + +# +# Include common targets. +# +include $(UTSBASE)/sparc/Makefile.targ + +# +# Defines for local commands. +# +WARLOCK = warlock +WLCC = wlcc +TOUCH = touch +TEST = test + +warlock: $(WARLOCK_OK) + + +$(WARLOCK_OK): $(WARLOCK_OUT) + $(TOUCH) $@ + +%.ll: $(UTSBASE)/common/io/ib/mgt/ibdma/%.c \ + $(UTSBASE)/common/sys/ib/mgt/ibdma/ibdma_impl.h \ + $(UTSBASE)/common/sys/ib/mgt/ibdma/ibdma.h + $(WLCC) $(CPPFLAGS) -DDEBUG -o $@ $< diff --git a/usr/src/uts/sparc/srpt/Makefile b/usr/src/uts/sparc/srpt/Makefile new file mode 100644 index 0000000000..b68438ceb3 --- /dev/null +++ b/usr/src/uts/sparc/srpt/Makefile @@ -0,0 +1,94 @@ +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# +# +# Copyright 2009 Sun Microsystems, Inc. All rights reserved. +# Use is subject to license terms. +# +# This makefile drives the production of the srpt pseudo-driver for +# COMSTAR. + +# +# Path to the base of the uts directory tree (usually /usr/src/uts). +# + + +UTSBASE = ../.. + +ARCHDIR:sh = cd ..; basename `pwd` + +# +# Define the module and object file sets. +# +MODULE = srpt +OBJECTS = $(SRPT_OBJS:%=$(OBJS_DIR)/%) +LINTS = $(SRPT_OBJS:%.o=$(LINTS_DIR)/%.ln) +ROOTMODULE = $(ROOT_DRV_DIR)/$(MODULE) +CONF_SRCDIR = $(UTSBASE)/common/io/comstar/port/srpt + +# +# Include common rules. +# +include ../Makefile.$(ARCHDIR) + +# +# Define targets +# +ALL_TARGET = $(BINARY) $(SRC_CONFILE) +LINT_TARGET = $(MODULE).lint +INSTALL_TARGET = $(BINARY) $(ROOTMODULE) $(ROOT_CONFFILE) + +# +# Overrides and depends_on +# +MODSTUBS_DIR = $(OBJS_DIR) +LDFLAGS += -dy -Ndrv/stmf -Nmisc/ibtl -Nmisc/ibcm + +INC_PATH += -I$(UTSBASE)/common/io/comstar/port/srpt +INC_PATH += -I$(UTSBASE)/common/sys + +C99MODE= -xc99=%all +C99LMODE= -Xc99=%all + +# +# Default build targets. +# +.KEEP_STATE: + +def: $(DEF_DEPS) + +all: $(ALL_DEPS) + +clean: $(CLEAN_DEPS) + +clobber: $(CLOBBER_DEPS) + +lint: $(LINT_DEPS) + +modlintlib: $(MODLINTLIB_DEPS) + +clean.lint: $(CLEAN_LINT_DEPS) + +install: $(INSTALL_DEPS) + +# +# Include common targets. +# +include ../Makefile.targ |