From b8075755af8aad9fe165974c494c48fe761cd07a Mon Sep 17 00:00:00 2001 From: Bill Pijewski Date: Wed, 26 Jan 2011 09:43:36 -0800 Subject: OS-147 vfsstat command to show VFS activity by zone OS-148 Update ziostat to coexist peacefully with vfsstat OS-149 New kstats to support vfsstat --- usr/src/uts/common/fs/vnode.c | 51 +++++++++++++++++++++++++++++++++++++++---- 1 file changed, 47 insertions(+), 4 deletions(-) (limited to 'usr/src/uts/common/fs/vnode.c') diff --git a/usr/src/uts/common/fs/vnode.c b/usr/src/uts/common/fs/vnode.c index 382369c7fc..5174c8bf5f 100644 --- a/usr/src/uts/common/fs/vnode.c +++ b/usr/src/uts/common/fs/vnode.c @@ -65,6 +65,7 @@ #include #include #include +#include /* Determine if this vnode is a file that is read-only */ #define ISROFILE(vp) \ @@ -189,6 +190,30 @@ static void (**vsd_destructor)(void *); } \ } +#define VOPSTATS_ZONE_IO_START(zonep, op, now) { \ + mutex_enter(&zonep->zone_vfs_lock); \ + uint64_t actv = zonep->zone_vfs_##op##_actv++; \ + hrtime_t delta = now - zonep->zone_vfs_##op##_lastupdate; \ + zonep->zone_vfs_##op##_lastupdate = now; \ + if (actv != 0) { \ + zonep->zone_vfs_##op##_lentime += delta * actv; \ + zonep->zone_vfs_##op##_time += delta; \ + } \ + mutex_exit(&zonep->zone_vfs_lock); \ +} + +#define VOPSTATS_ZONE_IO_DONE(zonep, op, len, now) { \ + mutex_enter(&zonep->zone_vfs_lock); \ + zonep->zone_vfs_n##op++; \ + zonep->zone_vfs_##op##_bytes += len; \ + uint64_t actv = zonep->zone_vfs_##op##_actv--; \ + hrtime_t delta = now - zonep->zone_vfs_##op##_lastupdate; \ + zonep->zone_vfs_##op##_lastupdate = now; \ + zonep->zone_vfs_##op##_lentime += delta * actv; \ + zonep->zone_vfs_##op##_time += delta; \ + mutex_exit(&zonep->zone_vfs_lock); \ +} + /* * If the filesystem does not support XIDs map credential * If the vfsp is NULL, perhaps we should also map? @@ -3222,12 +3247,21 @@ fop_read( { int err; ssize_t resid_start = uiop->uio_resid; + ssize_t len; + + if (vp->v_type == VREG || vp->v_type == VDIR || vp->v_type == VBLK) + VOPSTATS_ZONE_IO_START(curzone, read, gethrtime()); VOPXID_MAP_CR(vp, cr); err = (*(vp)->v_op->vop_read)(vp, uiop, ioflag, cr, ct); - VOPSTATS_UPDATE_IO(vp, read, - read_bytes, (resid_start - uiop->uio_resid)); + len = resid_start - uiop->uio_resid; + + VOPSTATS_UPDATE_IO(vp, read, read_bytes, len); + + if (vp->v_type == VREG || vp->v_type == VDIR || vp->v_type == VBLK) + VOPSTATS_ZONE_IO_DONE(curzone, read, len, gethrtime()); + return (err); } @@ -3241,12 +3275,21 @@ fop_write( { int err; ssize_t resid_start = uiop->uio_resid; + ssize_t len; + + if (vp->v_type == VREG || vp->v_type == VDIR || vp->v_type == VBLK) + VOPSTATS_ZONE_IO_START(curzone, write, gethrtime()); VOPXID_MAP_CR(vp, cr); err = (*(vp)->v_op->vop_write)(vp, uiop, ioflag, cr, ct); - VOPSTATS_UPDATE_IO(vp, write, - write_bytes, (resid_start - uiop->uio_resid)); + len = resid_start - uiop->uio_resid; + + VOPSTATS_UPDATE_IO(vp, write, write_bytes, len); + + if (vp->v_type == VREG || vp->v_type == VDIR || vp->v_type == VBLK) + VOPSTATS_ZONE_IO_DONE(curzone, write, len, gethrtime()); + return (err); } -- cgit v1.2.3 From 16460a24279ee5a749973d50d48c41070eb20074 Mon Sep 17 00:00:00 2001 From: Bill Pijewski Date: Thu, 27 Jan 2011 09:19:00 -0800 Subject: OS-163 VFS kstats should be of type KSTAT_TYPE_IO --- usr/src/cmd/stat/vfsstat/vfsstat.pl | 113 +++++++++++++---------------- usr/src/cmd/stat/ziostat/ziostat.pl | 7 +- usr/src/uts/common/fs/vnode.c | 63 ++++++++-------- usr/src/uts/common/os/zone.c | 138 ++++++------------------------------ usr/src/uts/common/sys/zone.h | 38 ++-------- 5 files changed, 109 insertions(+), 250 deletions(-) (limited to 'usr/src/uts/common/fs/vnode.c') diff --git a/usr/src/cmd/stat/vfsstat/vfsstat.pl b/usr/src/cmd/stat/vfsstat/vfsstat.pl index d908ac0253..180d8f0bea 100644 --- a/usr/src/cmd/stat/vfsstat/vfsstat.pl +++ b/usr/src/cmd/stat/vfsstat/vfsstat.pl @@ -73,43 +73,37 @@ if ( defined($ARGV[0]) ) { $main::opt_h = 0; -my $module = 'zones'; -chomp(my $zname = (`/sbin/zonename`)); - -my $NS_PER_SEC = 1000 * 1000 * 1000; my $BYTES_PER_MB = 1024 * 1024; my $BYTES_PER_KB = 1024; my $BYTES_PREFIX = $USE_MB ? "M" : "k"; my $BYTES_DIVISOR = $USE_MB ? $BYTES_PER_MB : $BYTES_PER_KB; -my $Modules = $Kstat->{$module}; +my $Modules = $Kstat->{'zone_vfs'}; -my $old_r_ops = 0; -my $old_w_ops = 0; -my $old_r_bytes = 0; -my $old_w_bytes = 0; -my $old_r_time = 0; -my $old_w_time = 0; -my $old_r_etime = 0; -my $old_w_etime = 0; -my $old_r_lentime = 0; -my $old_w_lentime = 0; +my $old_rops = 0; +my $old_wops = 0; +my $old_rbytes = 0; +my $old_wbytes = 0; +my $old_rtime = 0; +my $old_wtime = 0; +my $old_rlentime = 0; +my $old_wlentime = 0; +my $old_snaptime = 0; my $ii = 0; $Kstat->update(); while (1) { + printf(" r/s w/s %sr/s %sw/s wait_t ractv wactv " . + "read_t writ_t %%b zone\n", $BYTES_PREFIX, $BYTES_PREFIX); + foreach my $instance (sort keys(%$Modules)) { my $Instances = $Modules->{$instance}; foreach my $name (keys(%$Instances)) { $Stats = $Instances->{$name}; - - if ($name eq 'zone_vfs' && - $Stats->{'zonename'} eq $zname) { - print_stats(); - } + print_stats($name); } } @@ -123,75 +117,64 @@ while (1) { } sub print_stats { - my $r_ops = $Stats->{'nread'}; - my $w_ops = $Stats->{'nwrite'}; - my $r_bytes = $Stats->{'read_bytes'}; - my $w_bytes = $Stats->{'write_bytes'}; - - my $r_time = $Stats->{'read_time'}; - my $w_time = $Stats->{'write_time'}; - my $r_lentime = $Stats->{'read_lentime'}; - my $w_lentime = $Stats->{'write_lentime'}; - - my $r_etime = ($Stats->{'read_lastupdate'} - $old_r_etime) / - $NS_PER_SEC; - my $w_etime = ($Stats->{'write_lastupdate'} - $old_w_etime) / - $NS_PER_SEC; - - # An elapsed time of zero is not a good idea - if ($r_etime == 0) { - $r_etime = $interval; - } - if ($w_etime == 0) { - $w_etime = $interval; - } + my $zonename = $_[0]; + + my $rops = $Stats->{'reads'}; + my $wops = $Stats->{'writes'}; + my $rbytes = $Stats->{'nread'}; + my $wbytes = $Stats->{'nwritten'}; + + my $rtime = $Stats->{'rtime'}; + my $wtime = $Stats->{'wtime'}; + my $rlentime = $Stats->{'rlentime'}; + my $wlentime = $Stats->{'wlentime'}; - my $r_tps = ($r_ops - $old_r_ops) / $r_etime; - my $w_tps = ($w_ops - $old_w_ops) / $w_etime; + my $etime = $Stats->{'snaptime'} - + ($old_snaptime > 0 ? $old_snaptime : $Stats->{'crtime'}); + + my $r_tps = ($rops - $old_rops) / $etime; + my $w_tps = ($wops - $old_wops) / $etime; # XXX Need to investigate how to calculate this my $wait_t = 0.0; # Calculate average length of active queue - my $r_actv = ($r_lentime - $old_r_lentime) / $r_etime / $NS_PER_SEC; - my $w_actv = ($w_lentime - $old_w_lentime) / $w_etime / $NS_PER_SEC; + my $r_actv = ($rlentime - $old_rlentime) / $etime; + my $w_actv = ($wlentime - $old_wlentime) / $etime; # Calculate average service time my $read_t = $r_tps > 0 ? $r_actv * (1000 / $r_tps) : 0.0; my $writ_t = $w_tps > 0 ? $w_actv * (1000 / $w_tps) : 0.0; # Calculate the % time the VFS layer is active - my $r_b_pct = ($r_time - $old_r_time) / ($r_etime * $NS_PER_SEC * 100); - my $w_b_pct = ($w_time - $old_w_time) / ($w_etime * $NS_PER_SEC * 100); + my $r_b_pct = (($rtime - $old_rtime) / $etime) * 100; + my $w_b_pct = (($wtime - $old_wtime) / $etime) * 100; my $b_pct = ($r_b_pct + $w_b_pct) / 2; - printf(" r/s w/s %sr/s %sw/s wait_t ractv wactv " . - "read_t writ_t %%b zone\n", $BYTES_PREFIX, $BYTES_PREFIX); printf("%6.1f %6.1f %6.1f %6.1f %6.1f %5.1f %5.1f %6.1f %6.1f " . "%3d %s\n", - ($r_ops - $old_r_ops) / $r_etime, - ($w_ops - $old_w_ops) / $w_etime, - ($r_bytes - $old_r_bytes) / $r_etime / $BYTES_DIVISOR, - ($w_bytes - $old_w_bytes) / $w_etime / $BYTES_DIVISOR, + ($rops - $old_rops) / $etime, + ($wops - $old_wops) / $etime, + ($rbytes - $old_rbytes) / $etime / $BYTES_DIVISOR, + ($wbytes - $old_wbytes) / $etime / $BYTES_DIVISOR, $wait_t, $r_actv, $w_actv, $read_t, $writ_t, $b_pct, - $zname); + $zonename); # Save current calculations for next loop - $old_r_ops = $r_ops; - $old_w_ops = $w_ops; - $old_r_bytes = $r_bytes; - $old_w_bytes = $w_bytes; - $old_r_time = $r_time; - $old_w_time = $w_time; - $old_r_etime = $Stats->{'read_lastupdate'}; - $old_w_etime = $Stats->{'write_lastupdate'}; - $old_r_lentime = $r_lentime; - $old_w_lentime = $w_lentime; + $old_rops = $rops; + $old_wops = $wops; + $old_rbytes = $rbytes; + $old_wbytes = $wbytes; + $old_rtime = $rtime; + $old_wtime = $wtime; + $old_rlentime = $rlentime; + $old_wlentime = $wlentime; + $old_snaptime = $Stats->{'snaptime'}; } sub usage { diff --git a/usr/src/cmd/stat/ziostat/ziostat.pl b/usr/src/cmd/stat/ziostat/ziostat.pl index c3c2fed22d..4e78da4cf1 100755 --- a/usr/src/cmd/stat/ziostat/ziostat.pl +++ b/usr/src/cmd/stat/ziostat/ziostat.pl @@ -73,14 +73,13 @@ if ( defined($ARGV[0]) ) { $main::opt_h = 0; -my $module = 'zones'; my $BYTES_PER_MB = 1024 * 1024; my $BYTES_PER_KB = 1024; my $BYTES_PREFIX = $USE_MB ? "M" : "k"; my $BYTES_DIVISOR = $USE_MB ? $BYTES_PER_MB : $BYTES_PER_KB; -my $Modules = $Kstat->{$module}; +my $Modules = $Kstat->{'zone_io'}; my $old_wlentime = 0; my $old_wtime = 0; @@ -104,9 +103,7 @@ while (1) { foreach my $name (keys(%$Instances)) { $Stats = $Instances->{$name}; - if ($Stats->{'class'} eq 'zone_io') { - print_stats($name); - } + print_stats($name); } } diff --git a/usr/src/uts/common/fs/vnode.c b/usr/src/uts/common/fs/vnode.c index 5174c8bf5f..3c31a3f241 100644 --- a/usr/src/uts/common/fs/vnode.c +++ b/usr/src/uts/common/fs/vnode.c @@ -190,30 +190,6 @@ static void (**vsd_destructor)(void *); } \ } -#define VOPSTATS_ZONE_IO_START(zonep, op, now) { \ - mutex_enter(&zonep->zone_vfs_lock); \ - uint64_t actv = zonep->zone_vfs_##op##_actv++; \ - hrtime_t delta = now - zonep->zone_vfs_##op##_lastupdate; \ - zonep->zone_vfs_##op##_lastupdate = now; \ - if (actv != 0) { \ - zonep->zone_vfs_##op##_lentime += delta * actv; \ - zonep->zone_vfs_##op##_time += delta; \ - } \ - mutex_exit(&zonep->zone_vfs_lock); \ -} - -#define VOPSTATS_ZONE_IO_DONE(zonep, op, len, now) { \ - mutex_enter(&zonep->zone_vfs_lock); \ - zonep->zone_vfs_n##op++; \ - zonep->zone_vfs_##op##_bytes += len; \ - uint64_t actv = zonep->zone_vfs_##op##_actv--; \ - hrtime_t delta = now - zonep->zone_vfs_##op##_lastupdate; \ - zonep->zone_vfs_##op##_lastupdate = now; \ - zonep->zone_vfs_##op##_lentime += delta * actv; \ - zonep->zone_vfs_##op##_time += delta; \ - mutex_exit(&zonep->zone_vfs_lock); \ -} - /* * If the filesystem does not support XIDs map credential * If the vfsp is NULL, perhaps we should also map? @@ -3247,10 +3223,14 @@ fop_read( { int err; ssize_t resid_start = uiop->uio_resid; + zone_t *zonep = curzone; ssize_t len; - if (vp->v_type == VREG || vp->v_type == VDIR || vp->v_type == VBLK) - VOPSTATS_ZONE_IO_START(curzone, read, gethrtime()); + if (vp->v_type == VREG || vp->v_type == VDIR || vp->v_type == VBLK) { + mutex_enter(&zonep->zone_vfs_lock); + kstat_runq_enter(zonep->zone_vfs_kiop); + mutex_exit(&zonep->zone_vfs_lock); + } VOPXID_MAP_CR(vp, cr); @@ -3259,8 +3239,13 @@ fop_read( VOPSTATS_UPDATE_IO(vp, read, read_bytes, len); - if (vp->v_type == VREG || vp->v_type == VDIR || vp->v_type == VBLK) - VOPSTATS_ZONE_IO_DONE(curzone, read, len, gethrtime()); + if (vp->v_type == VREG || vp->v_type == VDIR || vp->v_type == VBLK) { + mutex_enter(&zonep->zone_vfs_lock); + zonep->zone_vfs_kiop->reads++; + zonep->zone_vfs_kiop->nread += len; + kstat_runq_exit(zonep->zone_vfs_kiop); + mutex_exit(&zonep->zone_vfs_lock); + } return (err); } @@ -3275,10 +3260,19 @@ fop_write( { int err; ssize_t resid_start = uiop->uio_resid; + zone_t *zonep = curzone; ssize_t len; - if (vp->v_type == VREG || vp->v_type == VDIR || vp->v_type == VBLK) - VOPSTATS_ZONE_IO_START(curzone, write, gethrtime()); + /* + * For the purposes of VFS kstat consumers, the "waitq" calculation is + * repurposed as the active queue for VFS write operations. There's no + * actual wait queue for VFS operations. + */ + if (vp->v_type == VREG || vp->v_type == VDIR || vp->v_type == VBLK) { + mutex_enter(&zonep->zone_vfs_lock); + kstat_waitq_enter(zonep->zone_vfs_kiop); + mutex_exit(&zonep->zone_vfs_lock); + } VOPXID_MAP_CR(vp, cr); @@ -3287,8 +3281,13 @@ fop_write( VOPSTATS_UPDATE_IO(vp, write, write_bytes, len); - if (vp->v_type == VREG || vp->v_type == VDIR || vp->v_type == VBLK) - VOPSTATS_ZONE_IO_DONE(curzone, write, len, gethrtime()); + if (vp->v_type == VREG || vp->v_type == VDIR || vp->v_type == VBLK) { + mutex_enter(&zonep->zone_vfs_lock); + zonep->zone_vfs_kiop->writes++; + zonep->zone_vfs_kiop->nwritten += len; + kstat_waitq_exit(zonep->zone_vfs_kiop); + mutex_exit(&zonep->zone_vfs_lock); + } return (err); } diff --git a/usr/src/uts/common/os/zone.c b/usr/src/uts/common/os/zone.c index 8ccbed8273..87cc04cca0 100644 --- a/usr/src/uts/common/os/zone.c +++ b/usr/src/uts/common/os/zone.c @@ -1833,85 +1833,6 @@ zone_swapresv_kstat_update(kstat_t *ksp, int rw) return (0); } -static int -zone_vfs_kstat_update(kstat_t *ksp, int rw) -{ - zone_t *zone = ksp->ks_private; - zone_vfs_kstat_t *zvp = ksp->ks_data; - - if (rw == KSTAT_WRITE) - return (EACCES); - - zvp->zv_nread.value.ui64 = zone->zone_vfs_nread; - zvp->zv_read_bytes.value.ui64 = zone->zone_vfs_read_bytes; - zvp->zv_read_actv.value.ui64 = zone->zone_vfs_read_actv; - zvp->zv_read_time.value.ui64 = zone->zone_vfs_read_time; - zvp->zv_read_lentime.value.ui64 = zone->zone_vfs_read_lentime; - zvp->zv_read_lastupdate.value.ui64 = zone->zone_vfs_read_lastupdate; - zvp->zv_nwrite.value.ui64 = zone->zone_vfs_nwrite; - zvp->zv_write_bytes.value.ui64 = zone->zone_vfs_write_bytes; - zvp->zv_write_actv.value.ui64 = zone->zone_vfs_write_actv; - zvp->zv_write_time.value.ui64 = zone->zone_vfs_write_time; - zvp->zv_write_lentime.value.ui64 = zone->zone_vfs_write_lentime; - zvp->zv_write_lastupdate.value.ui64 = zone->zone_vfs_write_lastupdate; - - return (0); -} - -static kstat_t * -zone_vfs_kstat_create(zone_t *zone, int (*updatefunc) (kstat_t *, int)) -{ - kstat_t *ksp; - zone_vfs_kstat_t *zvp; - - ksp = kstat_create_zone("zones", zone->zone_id, "zone_vfs", - "zone_stats", KSTAT_TYPE_NAMED, - sizeof (zone_vfs_kstat_t) / sizeof (kstat_named_t), - KSTAT_FLAG_VIRTUAL, zone->zone_id); - - if (ksp == NULL) - return (NULL); - - if (zone->zone_id != GLOBAL_ZONEID) - kstat_zone_add(ksp, GLOBAL_ZONEID); - - zvp = ksp->ks_data = kmem_alloc(sizeof (zone_vfs_kstat_t), KM_SLEEP); - - ksp->ks_data_size += strlen(zone->zone_name) + 1; - kstat_named_init(&zvp->zv_zonename, "zonename", KSTAT_DATA_STRING); - kstat_named_setstr(&zvp->zv_zonename, zone->zone_name); - - kstat_named_init(&zvp->zv_nread, "nread", - KSTAT_DATA_UINT64); - kstat_named_init(&zvp->zv_read_bytes, "read_bytes", - KSTAT_DATA_UINT64); - kstat_named_init(&zvp->zv_read_actv, "read_actv", - KSTAT_DATA_UINT64); - kstat_named_init(&zvp->zv_read_time, "read_time", - KSTAT_DATA_UINT64); - kstat_named_init(&zvp->zv_read_lentime, "read_lentime", - KSTAT_DATA_UINT64); - kstat_named_init(&zvp->zv_read_lastupdate, "read_lastupdate", - KSTAT_DATA_UINT64); - kstat_named_init(&zvp->zv_nwrite, "nwrite", - KSTAT_DATA_UINT64); - kstat_named_init(&zvp->zv_write_bytes, "write_bytes", - KSTAT_DATA_UINT64); - kstat_named_init(&zvp->zv_write_actv, "write_actv", - KSTAT_DATA_UINT64); - kstat_named_init(&zvp->zv_write_time, "write_time", - KSTAT_DATA_UINT64); - kstat_named_init(&zvp->zv_write_lentime, "write_lentime", - KSTAT_DATA_UINT64); - kstat_named_init(&zvp->zv_write_lastupdate, "write_lastupdate", - KSTAT_DATA_UINT64); - - ksp->ks_update = updatefunc; - ksp->ks_private = zone; - kstat_install(ksp); - return (ksp); -} - static kstat_t * zone_rctl_kstat_create_common(zone_t *zone, char *name, int (*updatefunc) (kstat_t *, int)) @@ -1948,8 +1869,8 @@ zone_kstat_create(zone_t *zone) zone->zone_nprocs_kstat = zone_rctl_kstat_create_common(zone, "nprocs", zone_nprocs_kstat_update); - zone->zone_io_ksp = kstat_create_zone("zones", zone->zone_id, - zone->zone_name, "zone_io", KSTAT_TYPE_IO, 1, + zone->zone_io_ksp = kstat_create_zone("zone_io", zone->zone_id, + zone->zone_name, "disk", KSTAT_TYPE_IO, 1, KSTAT_FLAG_PERSISTENT, zone->zone_id); if (zone->zone_io_ksp != NULL) { @@ -1957,11 +1878,22 @@ zone_kstat_create(zone_t *zone) kstat_install(zone->zone_io_ksp); zone->zone_io_kiop = zone->zone_io_ksp->ks_data; } else { - zone->zone_io_kiop = kmem_zalloc(sizeof (kstat_io_t), KM_SLEEP); + zone->zone_io_kiop = kmem_zalloc( + sizeof (kstat_io_t), KM_SLEEP); } - zone->zone_vfs_kstat = zone_vfs_kstat_create(zone, - zone_vfs_kstat_update); + zone->zone_vfs_ksp = kstat_create_zone("zone_vfs", zone->zone_id, + zone->zone_name, "vfs", KSTAT_TYPE_IO, 1, + KSTAT_FLAG_PERSISTENT, zone->zone_id); + + if (zone->zone_vfs_ksp != NULL) { + zone->zone_vfs_ksp->ks_lock = &zone->zone_vfs_lock; + kstat_install(zone->zone_vfs_ksp); + zone->zone_vfs_kiop = zone->zone_vfs_ksp->ks_data; + } else { + zone->zone_vfs_kiop = kmem_zalloc( + sizeof (kstat_io_t), KM_SLEEP); + } } static void @@ -1994,8 +1926,12 @@ zone_kstat_delete(zone_t *zone) kmem_free(zone->zone_io_kiop, sizeof (kstat_io_t)); } - zone_kstat_delete_common(&zone->zone_vfs_kstat, - sizeof (zone_vfs_kstat_t)); + if (zone->zone_vfs_ksp != NULL) { + kstat_delete(zone->zone_vfs_ksp); + zone->zone_vfs_ksp = NULL; + } else { + kmem_free(zone->zone_vfs_kiop, sizeof (kstat_io_t)); + } } /* @@ -2020,7 +1956,6 @@ zone_zsd_init(void) mutex_init(&zone0.zone_lock, NULL, MUTEX_DEFAULT, NULL); mutex_init(&zone0.zone_nlwps_lock, NULL, MUTEX_DEFAULT, NULL); mutex_init(&zone0.zone_mem_lock, NULL, MUTEX_DEFAULT, NULL); - mutex_init(&zone0.zone_vfs_lock, NULL, MUTEX_DEFAULT, NULL); zone0.zone_shares = 1; zone0.zone_nlwps = 0; zone0.zone_nlwps_ctl = INT_MAX; @@ -2056,19 +1991,6 @@ zone_zsd_init(void) zone0.zone_nprocs_kstat = NULL; zone0.zone_zfs_io_share = 1; - zone0.zone_vfs_nread = 0; - zone0.zone_vfs_read_bytes = 0; - zone0.zone_vfs_read_actv = 0; - zone0.zone_vfs_read_time = 0; - zone0.zone_vfs_read_lentime = 0; - zone0.zone_vfs_read_lastupdate = 0; - zone0.zone_vfs_nwrite = 0; - zone0.zone_vfs_write_bytes = 0; - zone0.zone_vfs_write_actv = 0; - zone0.zone_vfs_write_time = 0; - zone0.zone_vfs_write_lentime = 0; - zone0.zone_vfs_write_lastupdate = 0; - list_create(&zone0.zone_ref_list, sizeof (zone_ref_t), offsetof(zone_ref_t, zref_linkage)); list_create(&zone0.zone_zsd, sizeof (struct zsd_entry), @@ -4284,7 +4206,6 @@ zone_create(const char *zone_name, const char *zone_root, mutex_init(&zone->zone_lock, NULL, MUTEX_DEFAULT, NULL); mutex_init(&zone->zone_nlwps_lock, NULL, MUTEX_DEFAULT, NULL); mutex_init(&zone->zone_mem_lock, NULL, MUTEX_DEFAULT, NULL); - mutex_init(&zone->zone_vfs_lock, NULL, MUTEX_DEFAULT, NULL); cv_init(&zone->zone_cv, NULL, CV_DEFAULT, NULL); list_create(&zone->zone_ref_list, sizeof (zone_ref_t), offsetof(zone_ref_t, zref_linkage)); @@ -4345,20 +4266,7 @@ zone_create(const char *zone_name, const char *zone_root, zone->zone_max_lofi_ctl = UINT64_MAX; zone->zone_lockedmem_kstat = NULL; zone->zone_swapresv_kstat = NULL; - zone->zone_zfs_io_share= 1; - - zone->zone_vfs_nread = 0; - zone->zone_vfs_read_bytes = 0; - zone->zone_vfs_read_actv = 0; - zone->zone_vfs_read_time = 0; - zone->zone_vfs_read_lentime = 0; - zone->zone_vfs_read_lastupdate = 0; - zone->zone_vfs_nwrite = 0; - zone->zone_vfs_write_bytes = 0; - zone->zone_vfs_write_actv = 0; - zone->zone_vfs_write_time = 0; - zone->zone_vfs_write_lentime = 0; - zone->zone_vfs_write_lastupdate = 0; + zone->zone_zfs_io_share = 1; /* * Zsched initializes the rctls. diff --git a/usr/src/uts/common/sys/zone.h b/usr/src/uts/common/sys/zone.h index d32815be3f..b1d8f61474 100644 --- a/usr/src/uts/common/sys/zone.h +++ b/usr/src/uts/common/sys/zone.h @@ -383,22 +383,6 @@ typedef struct { hrtime_t zone_avg_cnt; } sys_zio_cntr_t; -typedef struct zone_vfs_kstat { - kstat_named_t zv_zonename; - kstat_named_t zv_nread; - kstat_named_t zv_read_bytes; - kstat_named_t zv_read_actv; - kstat_named_t zv_read_time; - kstat_named_t zv_read_lentime; - kstat_named_t zv_read_lastupdate; - kstat_named_t zv_nwrite; - kstat_named_t zv_write_bytes; - kstat_named_t zv_write_actv; - kstat_named_t zv_write_time; - kstat_named_t zv_write_lentime; - kstat_named_t zv_write_lastupdate; -} zone_vfs_kstat_t; - typedef struct zone { /* * zone_name is never modified once set. @@ -556,30 +540,18 @@ typedef struct zone { sys_zio_cntr_t zone_lwr_ops; /* - * kstats and counters for IO ops and bytes. + * kstats and counters for I/O ops and bytes. */ - kmutex_t zone_io_lock; /* protects IO statistics */ + kmutex_t zone_io_lock; /* protects I/O statistics */ kstat_t *zone_io_ksp; kstat_io_t *zone_io_kiop; /* - * kstats and counters for VFS operations. + * kstats and counters for VFS ops and bytes. */ kmutex_t zone_vfs_lock; /* protects VFS statistics */ - kstat_t *zone_vfs_kstat; - - uint64_t zone_vfs_nread; /* VFS read operations */ - uint64_t zone_vfs_read_bytes; /* VFS bytes read */ - uint64_t zone_vfs_read_actv; /* current active VFS reads */ - uint64_t zone_vfs_read_time; /* cumulative read svc time */ - uint64_t zone_vfs_read_lentime; /* cumul. read svc len*time */ - uint64_t zone_vfs_read_lastupdate; /* last VFS read update */ - uint64_t zone_vfs_nwrite; /* VFS write operations */ - uint64_t zone_vfs_write_bytes; /* VFS bytes written */ - uint64_t zone_vfs_write_actv; /* current active VFS writes */ - uint64_t zone_vfs_write_time; /* cumulative write svc time */ - uint64_t zone_vfs_write_lentime; /* cumul. write svc len*time */ - uint64_t zone_vfs_write_lastupdate; /* last VFS write update */ + kstat_t *zone_vfs_ksp; + kstat_io_t *zone_vfs_kiop; /* * Solaris Auditing per-zone audit context -- cgit v1.2.3 From 278c9a7a00cbf7a53131a08ae5e86c28de7d2c8e Mon Sep 17 00:00:00 2001 From: Bill Pijewski Date: Thu, 24 Mar 2011 21:20:59 -0700 Subject: OS-338 Kstat counters to show "slow" VFS operations --- usr/src/cmd/stat/vfsstat/vfsstat.pl | 13 +++-- usr/src/uts/common/fs/vnode.c | 64 +++++++++++++++++++---- usr/src/uts/common/os/zone.c | 100 ++++++++++++++++++++++++++++-------- usr/src/uts/common/sys/zone.h | 17 +++++- 4 files changed, 158 insertions(+), 36 deletions(-) (limited to 'usr/src/uts/common/fs/vnode.c') diff --git a/usr/src/cmd/stat/vfsstat/vfsstat.pl b/usr/src/cmd/stat/vfsstat/vfsstat.pl index 2fd135274c..ff02ef1d70 100644 --- a/usr/src/cmd/stat/vfsstat/vfsstat.pl +++ b/usr/src/cmd/stat/vfsstat/vfsstat.pl @@ -93,6 +93,7 @@ my $DATA_FMT = $USE_COMMA ? my $BYTES_PREFIX = $USE_MB ? "M" : "k"; my $BYTES_DIVISOR = $USE_MB ? 1024 * 1024 : 1024; my $INTERVAL_SUFFIX = $USE_INTERVAL ? "i" : "s"; +my $NANOSEC = 1000000000; my @fields = ( 'reads', 'writes', 'nread', 'nwritten', 'rtime', 'wtime', 'rlentime', 'wlentime', 'snaptime' ); @@ -179,16 +180,20 @@ sub print_stats { my $w_tps = ($data->{'writes'} - $old->{'writes'}) / $etime; # Calculate average length of active queue - my $r_actv = ($data->{'rlentime'} - $old->{'rlentime'}) / $etime; - my $w_actv = ($data->{'wlentime'} - $old->{'wlentime'}) / $etime; + my $r_actv = (($data->{'rlentime'} - $old->{'rlentime'}) / $NANOSEC) / + $etime; + my $w_actv = (($data->{'wlentime'} - $old->{'wlentime'}) / $NANOSEC) / + $etime; # Calculate average service time my $read_t = $r_tps > 0 ? $r_actv * (1000 / $r_tps) : 0.0; my $writ_t = $w_tps > 0 ? $w_actv * (1000 / $w_tps) : 0.0; # Calculate the % time the VFS layer is active - my $r_b_pct = (($data->{'rtime'} - $old->{'rtime'}) / $etime) * 100; - my $w_b_pct = (($data->{'wtime'} - $old->{'wtime'}) / $etime) * 100; + my $r_b_pct = ((($data->{'rtime'} - $old->{'rtime'}) / $NANOSEC) / + $etime) * 100; + my $w_b_pct = ((($data->{'wtime'} - $old->{'wtime'}) / $NANOSEC) / + $etime) * 100; if (! $HIDE_ZEROES || $reads != 0.0 || $writes != 0.0 || $nread != 0.0 || $nwritten != 0.0) { diff --git a/usr/src/uts/common/fs/vnode.c b/usr/src/uts/common/fs/vnode.c index 3c31a3f241..8567a1a10e 100644 --- a/usr/src/uts/common/fs/vnode.c +++ b/usr/src/uts/common/fs/vnode.c @@ -200,6 +200,10 @@ static void (**vsd_destructor)(void *); cr = crgetmapped(cr); \ } +#define VOP_LATENCY_10MS 10000000 +#define VOP_LATENCY_100MS 100000000 +#define VOP_LATENCY_1S 1000000000 + /* * Convert stat(2) formats to vnode types and vice versa. (Knows about * numerical order of S_IFMT and vnode types.) @@ -3221,14 +3225,19 @@ fop_read( cred_t *cr, caller_context_t *ct) { - int err; ssize_t resid_start = uiop->uio_resid; zone_t *zonep = curzone; + zone_vfs_kstat_t *zvp = zonep->zone_vfs_stats; + + hrtime_t start, lat; ssize_t len; + int err; if (vp->v_type == VREG || vp->v_type == VDIR || vp->v_type == VBLK) { + start = gethrtime(); + mutex_enter(&zonep->zone_vfs_lock); - kstat_runq_enter(zonep->zone_vfs_kiop); + kstat_runq_enter(&zonep->zone_vfs_rwstats); mutex_exit(&zonep->zone_vfs_lock); } @@ -3241,10 +3250,25 @@ fop_read( if (vp->v_type == VREG || vp->v_type == VDIR || vp->v_type == VBLK) { mutex_enter(&zonep->zone_vfs_lock); - zonep->zone_vfs_kiop->reads++; - zonep->zone_vfs_kiop->nread += len; - kstat_runq_exit(zonep->zone_vfs_kiop); + zonep->zone_vfs_rwstats.reads++; + zonep->zone_vfs_rwstats.nread += len; + kstat_runq_exit(&zonep->zone_vfs_rwstats); mutex_exit(&zonep->zone_vfs_lock); + + lat = gethrtime() - start; + + if (lat >= VOP_LATENCY_10MS) { + if (lat < VOP_LATENCY_100MS) + atomic_inc_64(&zvp->zv_10ms_ops.value.ui64); + else if (lat < VOP_LATENCY_1S) { + atomic_inc_64(&zvp->zv_10ms_ops.value.ui64); + atomic_inc_64(&zvp->zv_100ms_ops.value.ui64); + } else { + atomic_inc_64(&zvp->zv_10ms_ops.value.ui64); + atomic_inc_64(&zvp->zv_100ms_ops.value.ui64); + atomic_inc_64(&zvp->zv_1s_ops.value.ui64); + } + } } return (err); @@ -3258,10 +3282,13 @@ fop_write( cred_t *cr, caller_context_t *ct) { - int err; ssize_t resid_start = uiop->uio_resid; zone_t *zonep = curzone; + zone_vfs_kstat_t *zvp = zonep->zone_vfs_stats; + + hrtime_t start, lat; ssize_t len; + int err; /* * For the purposes of VFS kstat consumers, the "waitq" calculation is @@ -3269,8 +3296,10 @@ fop_write( * actual wait queue for VFS operations. */ if (vp->v_type == VREG || vp->v_type == VDIR || vp->v_type == VBLK) { + start = gethrtime(); + mutex_enter(&zonep->zone_vfs_lock); - kstat_waitq_enter(zonep->zone_vfs_kiop); + kstat_waitq_enter(&zonep->zone_vfs_rwstats); mutex_exit(&zonep->zone_vfs_lock); } @@ -3283,10 +3312,25 @@ fop_write( if (vp->v_type == VREG || vp->v_type == VDIR || vp->v_type == VBLK) { mutex_enter(&zonep->zone_vfs_lock); - zonep->zone_vfs_kiop->writes++; - zonep->zone_vfs_kiop->nwritten += len; - kstat_waitq_exit(zonep->zone_vfs_kiop); + zonep->zone_vfs_rwstats.writes++; + zonep->zone_vfs_rwstats.nwritten += len; + kstat_waitq_exit(&zonep->zone_vfs_rwstats); mutex_exit(&zonep->zone_vfs_lock); + + lat = gethrtime() - start; + + if (lat >= VOP_LATENCY_10MS) { + if (lat < VOP_LATENCY_100MS) + atomic_inc_64(&zvp->zv_10ms_ops.value.ui64); + else if (lat < VOP_LATENCY_1S) { + atomic_inc_64(&zvp->zv_10ms_ops.value.ui64); + atomic_inc_64(&zvp->zv_100ms_ops.value.ui64); + } else { + atomic_inc_64(&zvp->zv_10ms_ops.value.ui64); + atomic_inc_64(&zvp->zv_100ms_ops.value.ui64); + atomic_inc_64(&zvp->zv_1s_ops.value.ui64); + } + } } return (err); diff --git a/usr/src/uts/common/os/zone.c b/usr/src/uts/common/os/zone.c index c6d9e1ee83..ca02229671 100644 --- a/usr/src/uts/common/os/zone.c +++ b/usr/src/uts/common/os/zone.c @@ -1861,6 +1861,80 @@ zone_rctl_kstat_create_common(zone_t *zone, char *name, return (ksp); } +static int +zone_vfs_kstat_update(kstat_t *ksp, int rw) +{ + zone_t *zone = ksp->ks_private; + zone_vfs_kstat_t *zvp = ksp->ks_data; + kstat_io_t *kiop = &zone->zone_vfs_rwstats; + + if (rw == KSTAT_WRITE) + return (EACCES); + + /* + * Extract the VFS statistics from the kstat_io_t structure used by + * kstat_runq_enter() and related functions. Since the slow ops + * counters are updated directly by the VFS layer, there's no need to + * copy those statistics here. + * + * Note that kstat_runq_enter() and the related functions use + * gethrtime_unscaled(), so scale the time here. + */ + zvp->zv_nread.value.ui64 = kiop->nread; + zvp->zv_reads.value.ui64 = kiop->reads; + zvp->zv_rtime.value.ui64 = kiop->rtime; + zvp->zv_rlentime.value.ui64 = kiop->rlentime; + zvp->zv_nwritten.value.ui64 = kiop->nwritten; + zvp->zv_writes.value.ui64 = kiop->writes; + zvp->zv_wtime.value.ui64 = kiop->wtime; + zvp->zv_wlentime.value.ui64 = kiop->wlentime; + + scalehrtime((hrtime_t *)&zvp->zv_rtime.value.ui64); + scalehrtime((hrtime_t *)&zvp->zv_rlentime.value.ui64); + scalehrtime((hrtime_t *)&zvp->zv_wtime.value.ui64); + scalehrtime((hrtime_t *)&zvp->zv_wlentime.value.ui64); + + return (0); +} + +static kstat_t * +zone_vfs_kstat_create(zone_t *zone) +{ + kstat_t *ksp; + zone_vfs_kstat_t *zvp; + + if ((ksp = kstat_create_zone("zone_vfs", zone->zone_id, + zone->zone_name, "zone_vfs", KSTAT_TYPE_NAMED, + sizeof (zone_vfs_kstat_t) / sizeof (kstat_named_t), + KSTAT_FLAG_VIRTUAL, zone->zone_id)) == NULL) + return (NULL); + + if (zone->zone_id != GLOBAL_ZONEID) + kstat_zone_add(ksp, GLOBAL_ZONEID); + + zvp = ksp->ks_data = kmem_zalloc(sizeof (zone_vfs_kstat_t), KM_SLEEP); + ksp->ks_lock = &zone->zone_vfs_lock; + zone->zone_vfs_stats = zvp; + + kstat_named_init(&zvp->zv_nread, "nread", KSTAT_DATA_UINT64); + kstat_named_init(&zvp->zv_reads, "reads", KSTAT_DATA_UINT64); + kstat_named_init(&zvp->zv_rtime, "rtime", KSTAT_DATA_UINT64); + kstat_named_init(&zvp->zv_rlentime, "rlentime", KSTAT_DATA_UINT64); + kstat_named_init(&zvp->zv_nwritten, "nwritten", KSTAT_DATA_UINT64); + kstat_named_init(&zvp->zv_writes, "writes", KSTAT_DATA_UINT64); + kstat_named_init(&zvp->zv_wtime, "wtime", KSTAT_DATA_UINT64); + kstat_named_init(&zvp->zv_wlentime, "wlentime", KSTAT_DATA_UINT64); + kstat_named_init(&zvp->zv_10ms_ops, "10ms_ops", KSTAT_DATA_UINT64); + kstat_named_init(&zvp->zv_100ms_ops, "100ms_ops", KSTAT_DATA_UINT64); + kstat_named_init(&zvp->zv_1s_ops, "1s_ops", KSTAT_DATA_UINT64); + + ksp->ks_update = zone_vfs_kstat_update; + ksp->ks_private = zone; + + kstat_install(ksp); + return (ksp); +} + static kstat_t * zone_zfs_kstat_create(zone_t *zone) { @@ -1916,20 +1990,9 @@ zone_kstat_create(zone_t *zone) sizeof (kstat_io_t), KM_SLEEP); } - zone->zone_vfs_ksp = kstat_create_zone("zone_vfs", zone->zone_id, - zone->zone_name, "zone_vfs", KSTAT_TYPE_IO, 1, - KSTAT_FLAG_PERSISTENT, zone->zone_id); - - if (zone->zone_vfs_ksp != NULL) { - if (zone->zone_id != GLOBAL_ZONEID) - kstat_zone_add(zone->zone_vfs_ksp, GLOBAL_ZONEID); - - zone->zone_vfs_ksp->ks_lock = &zone->zone_vfs_lock; - kstat_install(zone->zone_vfs_ksp); - zone->zone_vfs_kiop = zone->zone_vfs_ksp->ks_data; - } else { - zone->zone_vfs_kiop = kmem_zalloc( - sizeof (kstat_io_t), KM_SLEEP); + if ((zone->zone_vfs_ksp = zone_vfs_kstat_create(zone)) == NULL) { + zone->zone_vfs_stats = kmem_zalloc( + sizeof (zone_vfs_kstat_t), KM_SLEEP); } if ((zone->zone_zfs_ksp = zone_zfs_kstat_create(zone)) == NULL) { @@ -1968,13 +2031,8 @@ zone_kstat_delete(zone_t *zone) kmem_free(zone->zone_io_kiop, sizeof (kstat_io_t)); } - if (zone->zone_vfs_ksp != NULL) { - kstat_delete(zone->zone_vfs_ksp); - zone->zone_vfs_ksp = NULL; - } else { - kmem_free(zone->zone_vfs_kiop, sizeof (kstat_io_t)); - } - + zone_kstat_delete_common(&zone->zone_vfs_ksp, + sizeof (zone_vfs_kstat_t)); zone_kstat_delete_common(&zone->zone_zfs_ksp, sizeof (zone_zfs_kstat_t)); } diff --git a/usr/src/uts/common/sys/zone.h b/usr/src/uts/common/sys/zone.h index 8a82a6c1ad..4ab70fe624 100644 --- a/usr/src/uts/common/sys/zone.h +++ b/usr/src/uts/common/sys/zone.h @@ -387,6 +387,20 @@ typedef struct { hrtime_t zone_avg_cnt; } sys_zio_cntr_t; +typedef struct { + kstat_named_t zv_nread; + kstat_named_t zv_reads; + kstat_named_t zv_rtime; + kstat_named_t zv_rlentime; + kstat_named_t zv_nwritten; + kstat_named_t zv_writes; + kstat_named_t zv_wtime; + kstat_named_t zv_wlentime; + kstat_named_t zv_10ms_ops; + kstat_named_t zv_100ms_ops; + kstat_named_t zv_1s_ops; +} zone_vfs_kstat_t; + typedef struct { kstat_named_t zz_throttle_cnt; kstat_named_t zz_throttle_time; @@ -560,7 +574,8 @@ typedef struct zone { */ kmutex_t zone_vfs_lock; /* protects VFS statistics */ kstat_t *zone_vfs_ksp; - kstat_io_t *zone_vfs_kiop; + kstat_io_t zone_vfs_rwstats; + zone_vfs_kstat_t *zone_vfs_stats; /* * kstats for ZFS observability. -- cgit v1.2.3 From 068cc3827b33c5b4a2930f53c1c71e51c8bb3fb4 Mon Sep 17 00:00:00 2001 From: Bill Pijewski Date: Tue, 20 Mar 2012 17:14:02 -0700 Subject: OS-1047 kstat for 10 sec FS ops --- usr/src/uts/common/fs/vnode.c | 12 ++++++++++++ usr/src/uts/common/os/zone.c | 1 + usr/src/uts/common/sys/zone.h | 1 + 3 files changed, 14 insertions(+) (limited to 'usr/src/uts/common/fs/vnode.c') diff --git a/usr/src/uts/common/fs/vnode.c b/usr/src/uts/common/fs/vnode.c index 8567a1a10e..67f21866ec 100644 --- a/usr/src/uts/common/fs/vnode.c +++ b/usr/src/uts/common/fs/vnode.c @@ -21,6 +21,7 @@ /* * Copyright (c) 1988, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2012, Joyent Inc. All rights reserved. */ /* Copyright (c) 1983, 1984, 1985, 1986, 1987, 1988, 1989 AT&T */ @@ -203,6 +204,7 @@ static void (**vsd_destructor)(void *); #define VOP_LATENCY_10MS 10000000 #define VOP_LATENCY_100MS 100000000 #define VOP_LATENCY_1S 1000000000 +#define VOP_LATENCY_10S 10000000000 /* * Convert stat(2) formats to vnode types and vice versa. (Knows about @@ -3263,10 +3265,15 @@ fop_read( else if (lat < VOP_LATENCY_1S) { atomic_inc_64(&zvp->zv_10ms_ops.value.ui64); atomic_inc_64(&zvp->zv_100ms_ops.value.ui64); + } else if (lat < VOP_LATENCY_10S) { + atomic_inc_64(&zvp->zv_10ms_ops.value.ui64); + atomic_inc_64(&zvp->zv_100ms_ops.value.ui64); + atomic_inc_64(&zvp->zv_1s_ops.value.ui64); } else { atomic_inc_64(&zvp->zv_10ms_ops.value.ui64); atomic_inc_64(&zvp->zv_100ms_ops.value.ui64); atomic_inc_64(&zvp->zv_1s_ops.value.ui64); + atomic_inc_64(&zvp->zv_10s_ops.value.ui64); } } } @@ -3325,10 +3332,15 @@ fop_write( else if (lat < VOP_LATENCY_1S) { atomic_inc_64(&zvp->zv_10ms_ops.value.ui64); atomic_inc_64(&zvp->zv_100ms_ops.value.ui64); + } else if (lat < VOP_LATENCY_10S) { + atomic_inc_64(&zvp->zv_10ms_ops.value.ui64); + atomic_inc_64(&zvp->zv_100ms_ops.value.ui64); + atomic_inc_64(&zvp->zv_1s_ops.value.ui64); } else { atomic_inc_64(&zvp->zv_10ms_ops.value.ui64); atomic_inc_64(&zvp->zv_100ms_ops.value.ui64); atomic_inc_64(&zvp->zv_1s_ops.value.ui64); + atomic_inc_64(&zvp->zv_10s_ops.value.ui64); } } } diff --git a/usr/src/uts/common/os/zone.c b/usr/src/uts/common/os/zone.c index 79f61ddcb9..3d2e7a6858 100644 --- a/usr/src/uts/common/os/zone.c +++ b/usr/src/uts/common/os/zone.c @@ -2051,6 +2051,7 @@ zone_vfs_kstat_create(zone_t *zone) kstat_named_init(&zvp->zv_10ms_ops, "10ms_ops", KSTAT_DATA_UINT64); kstat_named_init(&zvp->zv_100ms_ops, "100ms_ops", KSTAT_DATA_UINT64); kstat_named_init(&zvp->zv_1s_ops, "1s_ops", KSTAT_DATA_UINT64); + kstat_named_init(&zvp->zv_10s_ops, "10s_ops", KSTAT_DATA_UINT64); kstat_named_init(&zvp->zv_delay_cnt, "delay_cnt", KSTAT_DATA_UINT64); kstat_named_init(&zvp->zv_delay_time, "delay_time", KSTAT_DATA_UINT64); diff --git a/usr/src/uts/common/sys/zone.h b/usr/src/uts/common/sys/zone.h index a2b7217fd4..2f3b321cc1 100644 --- a/usr/src/uts/common/sys/zone.h +++ b/usr/src/uts/common/sys/zone.h @@ -403,6 +403,7 @@ typedef struct { kstat_named_t zv_10ms_ops; kstat_named_t zv_100ms_ops; kstat_named_t zv_1s_ops; + kstat_named_t zv_10s_ops; kstat_named_t zv_delay_cnt; kstat_named_t zv_delay_time; } zone_vfs_kstat_t; -- cgit v1.2.3