summaryrefslogtreecommitdiff
path: root/src/pmlogextract/pmlogextract.c
diff options
context:
space:
mode:
Diffstat (limited to 'src/pmlogextract/pmlogextract.c')
-rw-r--r--src/pmlogextract/pmlogextract.c2000
1 files changed, 2000 insertions, 0 deletions
diff --git a/src/pmlogextract/pmlogextract.c b/src/pmlogextract/pmlogextract.c
new file mode 100644
index 0000000..7c20292
--- /dev/null
+++ b/src/pmlogextract/pmlogextract.c
@@ -0,0 +1,2000 @@
+/*
+ * pmlogextract - extract desired metrics from PCP archive logs
+ *
+ * Copyright (c) 2014 Red Hat.
+ * Copyright (c) 1997-2002 Silicon Graphics, Inc. All Rights Reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation; either version 2 of the License, or (at your
+ * option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+ * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * for more details.
+ */
+
+#include <math.h>
+#include <ctype.h>
+#include <sys/stat.h>
+#include <assert.h>
+#include "pmapi.h"
+#include "impl.h"
+#include "logger.h"
+
+#ifdef PCP_DEBUG
+long totalmalloc;
+#endif
+static pmUnits nullunits;
+static int desperate;
+
+static pmLongOptions longopts[] = {
+ PMAPI_OPTIONS_HEADER("Options"),
+ { "config", 1, 'c', "FILE", "file to load configuration from" },
+ { "desperate", 0, 'd', 0, "desperate, save output after fatal error" },
+ { "first", 0, 'f', 0, "use timezone from first archive [default is last]" },
+ PMOPT_START,
+ { "samples", 1, 's', "NUM", "terminate after NUM log records have been written" },
+ PMOPT_FINISH,
+ { "", 1, 'v', "SAMPLES", "switch log volumes after this many samples" },
+ { "", 0, 'w', 0, "ignore day/month/year" },
+ PMOPT_TIMEZONE,
+ PMOPT_HOSTZONE,
+ PMAPI_OPTIONS_END
+};
+
+static pmOptions opts = {
+ .short_options = "c:D:dfS:s:T:v:wZ:z?",
+ .long_options = longopts,
+ .short_usage = "[options] input-archive output-archive",
+};
+
+const char *
+metricname(pmID pmid)
+{
+ static char *name = NULL;
+ if (name != NULL) {
+ free(name);
+ name = NULL;
+ }
+ if (pmNameID(pmid, &name) == 0)
+ return(name);
+ name = NULL;
+ return pmIDStr(pmid);
+}
+
+/*
+ * global constants
+ */
+#define LOG 0
+#define META 1
+#define LOG_META 2
+#define NUM_SEC_PER_DAY 86400
+
+#define NOT_WRITTEN 0
+#define MARK_FOR_WRITE 1
+#define WRITTEN 2
+
+/*
+ * reclist_t is in logger.h
+ * (list of pdu's to write out at start of time window)
+ */
+
+/*
+ * Input archive control is in logger.h
+ */
+
+
+/*
+ * PDU for pmResult (PDU_RESULT)
+ */
+typedef struct {
+ pmID pmid;
+ int numval; /* no. of vlist els to follow, or err */
+ int valfmt; /* insitu or pointer */
+ __pmValue_PDU vlist[1]; /* zero or more */
+} vlist_t;
+
+/*
+ * This struct is not needed?
+ */
+typedef struct {
+ /* __pmPDUHdr hdr; */
+ __pmPDU hdr;
+ __pmTimeval timestamp; /* when returned */
+ int numpmid; /* no. of PMIDs to follow */
+ __pmPDU data[1]; /* zero or more */
+} result_t;
+
+/*
+ * Mark record
+ */
+typedef struct {
+ __pmPDU len;
+ __pmPDU type;
+ __pmPDU from;
+ __pmTimeval timestamp; /* when returned */
+ int numpmid; /* zero PMIDs to follow */
+} mark_t;
+
+
+/*
+ * Global variables
+ */
+static int exit_status = 0;
+static int inarchvers = PM_LOG_VERS02; /* version of input archive */
+static int outarchvers = PM_LOG_VERS02; /* version of output archive */
+static int first_datarec = 1; /* first record flag */
+static int pre_startwin = 1; /* outside time win flag */
+static int written = 0; /* num log writes so far */
+int ml_numpmid = 0; /* num pmid in ml list */
+int ml_size = 0; /* actual size of ml array */
+mlist_t *ml = NULL; /* list of pmids with indoms */
+rlist_t *rl = NULL; /* list of pmResults */
+
+
+off_t new_log_offset; /* new log offset */
+off_t new_meta_offset; /* new meta offset */
+off_t old_log_offset; /* old log offset */
+off_t old_meta_offset; /* old meta offset */
+static off_t flushsize = 100000; /* bytes before flush */
+
+
+/* archive control stuff */
+char *outarchname = NULL; /* name of output archive */
+static __pmHashCtl mdesc_hash; /* pmids that have been written */
+static __pmHashCtl mindom_hash; /* indoms that have been written */
+static __pmLogCtl logctl; /* output archive control */
+inarch_t *inarch; /* input archive control(s) */
+int inarchnum; /* number of input archives */
+
+int ilog; /* index of earliest log */
+
+static reclist_t *rlog; /* log records to be written */
+static reclist_t *rdesc; /* meta desc records to be written */
+static reclist_t *rindom; /* meta indom records to be written */
+
+static __pmTimeval curlog; /* most recent timestamp in log */
+static __pmTimeval current; /* most recent timestamp overall */
+
+/* time window stuff */
+static struct timeval logstart_tval = {0,0}; /* extracted log start */
+static struct timeval logend_tval = {0,0}; /* extracted log end */
+static struct timeval winstart_tval = {0,0}; /* window start tval*/
+static struct timeval winend_tval = {0,0}; /* window end tval*/
+
+static __pmTimeval winstart = {-1,0}; /* window start time */
+static __pmTimeval winend = {-1,0}; /* window end time */
+static __pmTimeval logend = {-1,0}; /* log end time */
+
+/* command line args */
+char *configfile = NULL; /* -c arg - name of config file */
+int farg = 0; /* -f arg - use first timezone */
+int sarg = -1; /* -s arg - finish after X samples */
+char *Sarg = NULL; /* -S arg - window start */
+char *Targ = NULL; /* -T arg - window end */
+int varg = -1; /* -v arg - switch log vol every X */
+int warg = 0; /* -w arg - ignore day/month/year */
+int zarg = 0; /* -z arg - use archive timezone */
+char *tz = NULL; /* -Z arg - use timezone from user */
+
+/* cmd line args that could exist, but don't (needed for pmParseTimeWin) */
+char *Aarg = NULL; /* -A arg - non-existent */
+char *Oarg = NULL; /* -O arg - non-existent */
+
+/*--- START FUNCTIONS -------------------------------------------------------*/
+
+/*
+ * return -1, 0 or 1 as the __pmTimeval's compare
+ * a < b, a == b or a > b
+ */
+static int
+tvcmp(__pmTimeval a, __pmTimeval b)
+{
+ if (a.tv_sec < b.tv_sec)
+ return -1;
+ if (a.tv_sec > b.tv_sec)
+ return 1;
+ if (a.tv_usec < b.tv_usec)
+ return -1;
+ if (a.tv_usec > b.tv_usec)
+ return 1;
+ return 0;
+}
+
+static void
+abandon()
+{
+ char fname[MAXNAMELEN];
+ if (desperate == 0) {
+ fprintf(stderr, "Archive \"%s\" not created.\n", outarchname);
+ while (logctl.l_curvol >= 0) {
+ snprintf(fname, sizeof(fname), "%s.%d", outarchname, logctl.l_curvol);
+ unlink(fname);
+ logctl.l_curvol--;
+ }
+ snprintf(fname, sizeof(fname), "%s.meta", outarchname);
+ unlink(fname);
+ snprintf(fname, sizeof(fname), "%s.index", outarchname);
+ unlink(fname);
+ }
+ exit(1);
+}
+
+
+/*
+ * report that archive is corrupted
+ */
+static void
+_report(FILE *fp)
+{
+ off_t here;
+ struct stat sbuf;
+
+ here = lseek(fileno(fp), 0L, SEEK_CUR);
+ fprintf(stderr, "%s: Error occurred at byte offset %ld into a file of",
+ pmProgname, (long int)here);
+ if (fstat(fileno(fp), &sbuf) < 0)
+ fprintf(stderr, ": stat: %s\n", osstrerror());
+ else
+ fprintf(stderr, " %ld bytes.\n", (long int)sbuf.st_size);
+ fprintf(stderr, "The last record, and the remainder of this file will not be extracted.\n");
+ abandon();
+}
+
+
+/*
+ * switch output volumes
+ */
+static void
+newvolume(char *base, __pmTimeval *tvp)
+{
+ FILE *newfp;
+ int nextvol = logctl.l_curvol + 1;
+
+ if ((newfp = __pmLogNewFile(base, nextvol)) != NULL) {
+ struct timeval stamp;
+ fclose(logctl.l_mfp);
+ logctl.l_mfp = newfp;
+ logctl.l_label.ill_vol = logctl.l_curvol = nextvol;
+ __pmLogWriteLabel(logctl.l_mfp, &logctl.l_label);
+ fflush(logctl.l_mfp);
+ stamp.tv_sec = ntohl(tvp->tv_sec);
+ stamp.tv_usec = ntohl(tvp->tv_usec);
+ fprintf(stderr, "%s: New log volume %d, at ", pmProgname, nextvol);
+ __pmPrintStamp(stderr, &stamp);
+ fputc('\n', stderr);
+ }
+ else {
+ fprintf(stderr, "%s: Error: volume %d: %s\n",
+ pmProgname, nextvol, pmErrStr(-oserror()));
+ abandon();
+ }
+ flushsize = 100000;
+}
+
+
+/*
+ * construct new external label, and check label records from
+ * input archives
+ */
+static void
+newlabel(void)
+{
+ int i;
+ inarch_t *iap;
+ __pmLogLabel *lp = &logctl.l_label;
+
+ /* set outarch to inarch[0] to start off with */
+ iap = &inarch[0];
+
+ /* check version number */
+ inarchvers = iap->label.ll_magic & 0xff;
+ outarchvers = inarchvers;
+
+ if (inarchvers != PM_LOG_VERS02) {
+ fprintf(stderr,"%s: Error: illegal version number %d in archive (%s)\n",
+ pmProgname, inarchvers, iap->name);
+ abandon();
+ }
+
+ /* copy magic number, pid, host and timezone */
+ lp->ill_magic = iap->label.ll_magic;
+ lp->ill_pid = (int)getpid();
+ strncpy(lp->ill_hostname, iap->label.ll_hostname, PM_LOG_MAXHOSTLEN);
+ lp->ill_hostname[PM_LOG_MAXHOSTLEN-1] = '\0';
+ if (farg) {
+ /*
+ * use timezone from first archive ... this is the OLD default
+ */
+ strcpy(lp->ill_tz, iap->label.ll_tz);
+ }
+ else {
+ /*
+ * use timezone from last archive ... this is the NEW default
+ */
+ strcpy(lp->ill_tz, inarch[inarchnum-1].label.ll_tz);
+ }
+
+ /* reset outarch as appropriate, depending on other input archives */
+ for (i=0; i<inarchnum; i++) {
+ iap = &inarch[i];
+
+ /* Ensure all archives of the same version number */
+ if ((iap->label.ll_magic & 0xff) != inarchvers) {
+ fprintf(stderr,
+ "%s: Error: input archives with different version numbers\n"
+ "archive: %s version: %d\n"
+ "archive: %s version: %d\n",
+ pmProgname, inarch[0].name, inarchvers,
+ iap->name, (iap->label.ll_magic & 0xff));
+ abandon();
+ }
+
+ /* Ensure all archives of the same host */
+ if (strcmp(lp->ill_hostname, iap->label.ll_hostname) != 0) {
+ fprintf(stderr,"%s: Error: host name mismatch for input archives\n",
+ pmProgname);
+ fprintf(stderr, "archive: %s host: %s\n",
+ inarch[0].name, inarch[0].label.ll_hostname);
+ fprintf(stderr, "archive: %s host: %s\n",
+ iap->name, iap->label.ll_hostname);
+ abandon();
+ }
+
+ /* Ensure all archives of the same timezone */
+ if (strcmp(lp->ill_tz, iap->label.ll_tz) != 0) {
+ fprintf(stderr,
+ "%s: Warning: timezone mismatch for input archives\n",
+ pmProgname);
+ if (farg) {
+ fprintf(stderr, "archive: %s timezone: %s [will be used]\n",
+ inarch[0].name, lp->ill_tz);
+ fprintf(stderr, "archive: %s timezone: %s [will be ignored]\n",
+ iap->name, iap->label.ll_tz);
+ }
+ else {
+ fprintf(stderr, "archive: %s timezone: %s [will be used]\n",
+ inarch[inarchnum-1].name, lp->ill_tz);
+ fprintf(stderr, "archive: %s timezone: %s [will be ignored]\n",
+ iap->name, iap->label.ll_tz);
+ }
+ }
+ } /*for(i)*/
+}
+
+
+/*
+ *
+ */
+void
+writelabel_metati(int do_rewind)
+{
+ if (do_rewind) rewind(logctl.l_tifp);
+ logctl.l_label.ill_vol = PM_LOG_VOL_TI;
+ __pmLogWriteLabel(logctl.l_tifp, &logctl.l_label);
+
+ if (do_rewind) rewind(logctl.l_mdfp);
+ logctl.l_label.ill_vol = PM_LOG_VOL_META;
+ __pmLogWriteLabel(logctl.l_mdfp, &logctl.l_label);
+}
+
+
+/*
+ *
+ */
+void
+writelabel_data(void)
+{
+ logctl.l_label.ill_vol = 0;
+ __pmLogWriteLabel(logctl.l_mfp, &logctl.l_label);
+}
+
+
+/* --- Start of reclist functions --- */
+
+/*
+ * make a reclist_t record
+ */
+static reclist_t *
+mk_reclist_t(void)
+{
+ reclist_t *rec;
+
+ if ((rec = (reclist_t *)malloc(sizeof(reclist_t))) == NULL) {
+ fprintf(stderr, "%s: Error: cannot malloc space for record list.\n",
+ pmProgname);
+ abandon();
+ }
+#ifdef PCP_DEBUG
+ if (pmDebug & DBG_TRACE_APPL0) {
+ totalmalloc += sizeof(reclist_t);
+ fprintf(stderr, "mk_reclist_t: allocated %d\n", (int)sizeof(reclist_t));
+ }
+#endif
+ rec->pdu = NULL;
+ rec->desc.pmid = PM_ID_NULL;
+ rec->desc.type = PM_TYPE_NOSUPPORT;
+ rec->desc.indom = PM_IN_NULL;
+ rec->desc.sem = 0;
+ rec->desc.units = nullunits; /* struct assignment */
+ rec->written = NOT_WRITTEN;
+ rec->ptr = NULL;
+ rec->next = NULL;
+ return(rec);
+}
+
+/*
+ * find indom in indomreclist - if it isn't in the list then add it in
+ * with no pdu buffer
+ */
+static reclist_t *
+findnadd_indomreclist(int indom)
+{
+ reclist_t *curr;
+
+ if (rindom == NULL) {
+ rindom = mk_reclist_t();
+ rindom->desc.pmid = PM_ID_NULL;
+ rindom->desc.type = PM_TYPE_NOSUPPORT;
+ rindom->desc.indom = indom;
+ rindom->desc.sem = 0;
+ rindom->desc.units = nullunits; /* struct assignment */
+ return(rindom);
+ }
+ else {
+ curr = rindom;
+
+ /* find matching record or last record */
+ while (curr->next != NULL && curr->desc.indom != indom)
+ curr = curr->next;
+
+ if (curr->desc.indom == indom) {
+ /* we have found a matching record - return the pointer */
+ return(curr);
+ }
+ else {
+ /* we have not found a matching record - append new record */
+ curr->next = mk_reclist_t();
+ curr = curr->next;
+ curr->desc.pmid = PM_ID_NULL;
+ curr->desc.type = PM_TYPE_NOSUPPORT;
+ curr->desc.indom = indom;
+ curr->desc.sem = 0;
+ curr->desc.units = nullunits; /* struct assignment */
+ return(curr);
+ }
+ }
+
+}
+
+/*
+ * append a new record to the log record list
+ */
+void
+append_logreclist(int i)
+{
+ inarch_t *iap;
+ reclist_t *curr;
+
+ iap = &inarch[i];
+
+ if (rlog == NULL) {
+ rlog = mk_reclist_t();
+ rlog->pdu = iap->pb[LOG];
+ }
+ else {
+ curr = rlog;
+
+ /* find matching record or last record */
+ while (curr->next != NULL &&
+ curr->pdu[4] != iap->pb[LOG][4]) curr = curr->next;
+
+ if (curr->pdu[4] == iap->pb[LOG][4]) {
+ /* LOG: discard old record; insert new record */
+ __pmUnpinPDUBuf(curr->pdu);
+ curr->pdu = iap->pb[LOG];
+ }
+ else {
+ curr->next = mk_reclist_t();
+ curr = curr->next;
+ curr->pdu = iap->pb[LOG];
+ }
+ } /*else*/
+
+ iap->pb[LOG] = NULL;
+}
+
+/*
+ * append a new record to the desc meta record list if not seen
+ * before, else check the desc meta record is semantically the
+ * same as the last desc meta record for this pmid from this source
+ */
+void
+update_descreclist(int i)
+{
+ inarch_t *iap;
+ reclist_t *curr;
+ pmUnits pmu;
+ pmUnits *pmup;
+
+ iap = &inarch[i];
+
+ if (rdesc == NULL) {
+ /* first time */
+ curr = rdesc = mk_reclist_t();
+ }
+ else {
+ curr = rdesc;
+ /* find matching record or last record */
+ while (curr->next != NULL && curr->desc.pmid != ntoh_pmID(iap->pb[META][2]))
+ curr = curr->next;
+
+#ifdef PCP_DEBUG
+ if (pmDebug & DBG_TRACE_APPL1) {
+ fprintf(stderr, "update_descreclist: pmid: last/match %s", metricname(curr->desc.pmid));
+ fprintf(stderr, " new %s", metricname(ntoh_pmID(iap->pb[META][2])));
+ fputc('\n', stderr);
+ }
+#endif
+ }
+
+ if (curr->desc.pmid == ntoh_pmID(iap->pb[META][2])) {
+#ifdef PCP_DEBUG
+ if (pmDebug & DBG_TRACE_APPL1) {
+ fprintf(stderr, " type: old %s", pmTypeStr(curr->desc.type));
+ fprintf(stderr, " new %s", pmTypeStr(ntohl(iap->pb[META][3])));
+ fprintf(stderr, " indom: old %s", pmInDomStr(curr->desc.indom));
+ fprintf(stderr, " new %s", pmInDomStr(ntoh_pmInDom(iap->pb[META][4])));
+ fprintf(stderr, " sem: old %d", curr->desc.sem);
+ fprintf(stderr, " new %d", (int)ntohl(iap->pb[META][5]));
+ fprintf(stderr, " units: old %s", pmUnitsStr(&curr->desc.units));
+ pmup = (pmUnits *)&iap->pb[META][6];
+ pmu = ntoh_pmUnits(*pmup);
+ fprintf(stderr, " new %s", pmUnitsStr(&pmu));
+ fputc('\n', stderr);
+ }
+#endif
+ if (curr->desc.type != ntohl(iap->pb[META][3])) {
+ fprintf(stderr,
+ "%s: Error: metric %s: type changed from",
+ pmProgname, metricname(curr->desc.pmid));
+ fprintf(stderr, " %s", pmTypeStr(curr->desc.type));
+ fprintf(stderr, " to %s!\n", pmTypeStr(ntohl(iap->pb[META][3])));
+ abandon();
+ }
+ if (curr->desc.indom != ntoh_pmInDom(iap->pb[META][4])) {
+ fprintf(stderr,
+ "%s: Error: metric %s: indom changed from",
+ pmProgname, metricname(curr->desc.pmid));
+ fprintf(stderr, " %s", pmInDomStr(curr->desc.indom));
+ fprintf(stderr, " to %s!\n", pmInDomStr(ntoh_pmInDom(iap->pb[META][4])));
+ abandon();
+ }
+ if (curr->desc.sem != ntohl(iap->pb[META][5])) {
+ fprintf(stderr,
+ "%s: Error: metric %s: semantics changed from",
+ pmProgname, metricname(curr->desc.pmid));
+ fprintf(stderr, " %d", curr->desc.sem);
+ fprintf(stderr, " to %d!\n", (int)ntohl(iap->pb[META][5]));
+ abandon();
+ }
+ pmup = (pmUnits *)&iap->pb[META][6];
+ pmu = ntoh_pmUnits(*pmup);
+ if (curr->desc.units.dimSpace != pmu.dimSpace ||
+ curr->desc.units.dimTime != pmu.dimTime ||
+ curr->desc.units.dimCount != pmu.dimCount ||
+ curr->desc.units.scaleSpace != pmu.scaleSpace ||
+ curr->desc.units.scaleTime != pmu.scaleTime ||
+ curr->desc.units.scaleCount != pmu.scaleCount) {
+ fprintf(stderr,
+ "%s: Error: metric %s: units changed from",
+ pmProgname, metricname(curr->desc.pmid));
+ fprintf(stderr, " %s", pmUnitsStr(&curr->desc.units));
+ fprintf(stderr, " to %s!\n", pmUnitsStr(&pmu));
+ abandon();
+ }
+ /* not adding, so META: discard new record */
+ free(iap->pb[META]);
+ iap->pb[META] = NULL;
+ }
+ else {
+ /* append new record */
+ curr->next = mk_reclist_t();
+ curr = curr->next;
+ curr->pdu = iap->pb[META];
+ curr->desc.pmid = ntoh_pmID(iap->pb[META][2]);
+ curr->desc.type = ntohl(iap->pb[META][3]);
+ curr->desc.indom = ntoh_pmInDom(iap->pb[META][4]);
+ curr->desc.sem = ntohl(iap->pb[META][5]);
+ pmup =(pmUnits *)&iap->pb[META][6];
+ curr->desc.units = ntoh_pmUnits(*pmup);
+ curr->ptr = findnadd_indomreclist(curr->desc.indom);
+ iap->pb[META] = NULL;
+ }
+}
+
+/*
+ * append a new record to the indom meta record list
+ */
+void
+append_indomreclist(int i)
+{
+ inarch_t *iap;
+ reclist_t *curr;
+ reclist_t *rec;
+
+ iap = &inarch[i];
+
+ if (rindom == NULL) {
+ rindom = mk_reclist_t();
+ rindom->pdu = iap->pb[META];
+ rindom->stamp.tv_sec = ntohl(rindom->pdu[2]);
+ rindom->stamp.tv_usec = ntohl(rindom->pdu[3]);
+ rindom->desc.pmid = PM_ID_NULL;
+ rindom->desc.type = PM_TYPE_NOSUPPORT;
+ rindom->desc.indom = ntoh_pmInDom(iap->pb[META][4]);
+ rindom->desc.sem = 0;
+ rindom->desc.units = nullunits; /* struct assignment */
+ }
+ else {
+ curr = rindom;
+
+ /* find matching record or last record */
+ while (curr->next != NULL && curr->desc.indom != ntoh_pmInDom(iap->pb[META][4])) {
+ curr = curr->next;
+ }
+
+ if (curr->desc.indom == ntoh_pmInDom(iap->pb[META][4])) {
+ if (curr->pdu == NULL) {
+ /* insert new record */
+ curr->pdu = iap->pb[META];
+ curr->stamp.tv_sec = ntohl(curr->pdu[2]);
+ curr->stamp.tv_usec = ntohl(curr->pdu[3]);
+ }
+ else {
+ /* do NOT discard old record; insert new record */
+ rec = mk_reclist_t();
+ rec->pdu = iap->pb[META];
+ rec->stamp.tv_sec = ntohl(rec->pdu[2]);
+ rec->stamp.tv_usec = ntohl(rec->pdu[3]);
+ rec->desc.pmid = PM_ID_NULL;
+ rec->desc.type = PM_TYPE_NOSUPPORT;
+ rec->desc.indom = ntoh_pmInDom(iap->pb[META][4]);
+ rec->desc.sem = 0;
+ rec->desc.units = nullunits; /* struct assignment */
+ rec->next = curr->next;
+ curr->next = rec;
+ }
+ }
+ else {
+ /* append new record */
+ curr->next = mk_reclist_t();
+ curr = curr->next;
+ curr->pdu = iap->pb[META];
+ curr->stamp.tv_sec = ntohl(curr->pdu[2]);
+ curr->stamp.tv_usec = ntohl(curr->pdu[3]);
+ curr->desc.pmid = PM_ID_NULL;
+ curr->desc.type = PM_TYPE_NOSUPPORT;
+ curr->desc.indom = ntoh_pmInDom(iap->pb[META][4]);
+ curr->desc.sem = 0;
+ curr->desc.units = nullunits; /* struct assignment */
+ }
+ } /*else*/
+
+ iap->pb[META] = NULL;
+}
+
+/*
+ * write out one desc/indom record
+ */
+void
+write_rec(reclist_t *rec)
+{
+ int sts;
+
+ if (rec->written == MARK_FOR_WRITE) {
+ if (rec->pdu == NULL) {
+ fprintf(stderr, "%s: Fatal Error!\n", pmProgname);
+ fprintf(stderr," record is marked for write, but pdu is NULL\n");
+ abandon();
+ }
+
+#ifdef PCP_DEBUG
+ if (pmDebug & DBG_TRACE_LOGMETA) {
+ __pmLogHdr *h;
+ int len;
+ int type;
+ h = (__pmLogHdr *)rec->pdu;
+ len = ntohl(h->len);
+ type = ntohl(h->type);
+ fprintf(stderr, "write_rec: record len=%d, type=%d @ offset=%d\n",
+ len, type, (int)(ftell(logctl.l_mdfp) - sizeof(__pmLogHdr)));
+ if (type == TYPE_DESC) {
+ pmDesc *dp;
+ pmDesc desc;
+ int *namelen;
+ char *name; /* just first name for diag */
+ dp = (pmDesc *)((void *)rec->pdu + sizeof(__pmLogHdr));
+ desc.type = ntohl(dp->type);
+ desc.sem = ntohl(dp->sem);
+ desc.indom = ntoh_pmInDom(dp->indom);
+ desc.units = ntoh_pmUnits(dp->units);
+ desc.pmid = ntoh_pmID(dp->pmid);
+ namelen = (int *)((void *)rec->pdu + sizeof(__pmLogHdr) + sizeof(pmDesc) + sizeof(int));
+ len = ntohl(*namelen);
+ name = (char *)((void *)rec->pdu + sizeof(__pmLogHdr) + sizeof(pmDesc) + sizeof(int) + sizeof(int));
+ fprintf(stderr, "PMID: %s name: %*.*s\n", pmIDStr(desc.pmid), len, len, name);
+ __pmPrintDesc(stderr, &desc);
+ }
+ else if (type == TYPE_INDOM) {
+ __pmTimeval *tvp;
+ __pmTimeval when;
+ int k = 2;
+ pmInDom indom;
+ int numinst;
+ int *instlist;
+ int inst;
+
+ tvp = (__pmTimeval *)&rec->pdu[k];
+ when.tv_sec = ntohl(tvp->tv_sec);
+ when.tv_usec = ntohl(tvp->tv_usec);
+ k += sizeof(__pmTimeval)/sizeof(rec->pdu[0]);
+ indom = ntoh_pmInDom((unsigned int)rec->pdu[k++]);
+ fprintf(stderr, "INDOM: %s when: ", pmInDomStr(indom));
+ __pmPrintTimeval(stderr, &when);
+ numinst = ntohl(rec->pdu[k++]);
+ fprintf(stderr, " numinst: %d", numinst);
+ if (numinst > 0) {
+ int i;
+ instlist = (int *)&rec->pdu[k];
+ for (i = 0; i < numinst; i++) {
+ inst = ntohl(instlist[i]);
+ fprintf(stderr, " [%d] %d", i, inst);
+ }
+ }
+ fputc('\n', stderr);
+ }
+ else {
+ fprintf(stderr, "Botch: bad type\n");
+ }
+ }
+#endif
+
+ /* write out the pdu ; exit if write failed */
+ if ((sts = _pmLogPut(logctl.l_mdfp, rec->pdu)) < 0) {
+ fprintf(stderr, "%s: Error: _pmLogPut: meta data : %s\n",
+ pmProgname, pmErrStr(sts));
+ abandon();
+ }
+ /* META: free PDU buffer */
+ free(rec->pdu);
+ rec->pdu = NULL;
+ rec->written = WRITTEN;
+ }
+ else {
+ fprintf(stderr,
+ "%s : Warning: attempting to write out meta record (%d,%d)\n",
+ pmProgname, rec->desc.pmid, rec->desc.indom);
+ fprintf(stderr, " when it is not marked for writing (%d)\n",
+ rec->written);
+ }
+}
+
+void
+write_metareclist(pmResult *result, int *needti)
+{
+ int i;
+ reclist_t *curr_desc; /* current desc record */
+ reclist_t *curr_indom; /* current indom record */
+ reclist_t *othr_indom; /* other indom record */
+ pmID pmid;
+ pmInDom indom;
+ struct timeval *this; /* ptr to timestamp in result */
+
+ this = &result->timestamp;
+
+ /* if pmid in result matches a pmid in desc then write desc
+ */
+ for (i=0; i<result->numpmid; i++) {
+ pmid = result->vset[i]->pmid;
+ indom = PM_IN_NULL;
+ curr_indom = NULL;
+
+ curr_desc = rdesc;
+ while (curr_desc != NULL && curr_desc->desc.pmid != pmid)
+ curr_desc = curr_desc->next;
+
+ if (curr_desc == NULL) {
+ /* descriptor has not been found - this is bad
+ */
+ fprintf(stderr, "%s: Error: meta data (TYPE_DESC) for pmid %s has not been found.\n", pmProgname, pmIDStr(pmid));
+ abandon();
+ }
+ else {
+ /* descriptor has been found
+ */
+ if (curr_desc->written == WRITTEN) {
+ /* descriptor has been written before (no need to write again)
+ * but still need to check indom
+ */
+ indom = curr_desc->desc.indom;
+ curr_indom = curr_desc->ptr;
+ }
+ else if (curr_desc->pdu == NULL) {
+ /* descriptor is in list, has not been written, but no pdu
+ * - this is bad
+ */
+ fprintf(stderr, "%s: Error: missing pdu for pmid %s\n",
+ pmProgname, pmIDStr(pmid));
+ abandon();
+ }
+ else {
+ /* descriptor is in list, has not been written, and has pdu
+ * write!
+ */
+ curr_desc->written = MARK_FOR_WRITE;
+ write_rec(curr_desc);
+ indom = curr_desc->desc.indom;
+ curr_indom = curr_desc->ptr;
+ }
+ }
+
+ /* descriptor has been found and written,
+ * now go and find & write the indom
+ */
+ if (indom != PM_INDOM_NULL) {
+ /* there may be more than one indom in the list, so we need
+ * to traverse the entire list
+ * - we can safely ignore all indoms after the current timestamp
+ * - we want the latest indom at, or before the current timestamp
+ */
+ othr_indom = NULL;
+ while (curr_indom != NULL && curr_indom->desc.indom == indom) {
+ if (curr_indom->stamp.tv_sec < this->tv_sec ||
+ (curr_indom->stamp.tv_sec == this->tv_sec &&
+ curr_indom->stamp.tv_usec <= this->tv_usec))
+ {
+ /* indom is in list, indom has pdu
+ * and timestamp in pdu suits us
+ */
+ if (othr_indom == NULL) {
+ othr_indom = curr_indom;
+ }
+ else if (othr_indom->stamp.tv_sec < curr_indom->stamp.tv_sec ||
+ (othr_indom->stamp.tv_sec == curr_indom->stamp.tv_sec &&
+ othr_indom->stamp.tv_usec <= curr_indom->stamp.tv_usec))
+ {
+ /* we already have a perfectly good indom,
+ * but curr_indom has a better timestamp
+ */
+ othr_indom = curr_indom;
+ }
+ }
+ curr_indom = curr_indom->next;
+ } /*while()*/
+
+ if (othr_indom != NULL && othr_indom->pdu != NULL && othr_indom->written != WRITTEN) {
+ othr_indom->written = MARK_FOR_WRITE;
+ othr_indom->pdu[2] = htonl(this->tv_sec);
+ othr_indom->pdu[3] = htonl(this->tv_usec);
+
+ /* make sure to set needti, when writing out the indom
+ */
+ *needti = 1;
+ write_rec(othr_indom);
+ }
+ }
+ } /*for(i)*/
+}
+
+/* --- End of reclist functions --- */
+
+/*
+ * create a mark record
+ */
+__pmPDU *
+_createmark(void)
+{
+ mark_t *markp;
+
+ /*
+ * add space for trailer in case __pmLogPutResult2() is called with
+ * this PDU buffer
+ */
+ markp = (mark_t *)malloc(sizeof(mark_t)+sizeof(int));
+ if (markp == NULL) {
+ fprintf(stderr, "%s: Error: mark_t malloc: %s\n",
+ pmProgname, osstrerror());
+ abandon();
+ }
+#ifdef PCP_DEBUG
+ if (pmDebug & DBG_TRACE_APPL0) {
+ totalmalloc += sizeof(mark_t);
+ fprintf(stderr, "_createmark : allocated %d\n", (int)sizeof(mark_t));
+ }
+#endif
+
+ markp->len = (int)sizeof(mark_t);
+ markp->type = markp->from = 0;
+ markp->timestamp = current;
+ markp->timestamp.tv_usec += 1000; /* + 1msec */
+ if (markp->timestamp.tv_usec > 1000000) {
+ markp->timestamp.tv_usec -= 1000000;
+ markp->timestamp.tv_sec++;
+ }
+ markp->numpmid = 0;
+ return((__pmPDU *)markp);
+}
+
+void
+checklogtime(__pmTimeval *this, int i)
+{
+ if ((curlog.tv_sec == 0 && curlog.tv_usec == 0) ||
+ (curlog.tv_sec > this->tv_sec ||
+ (curlog.tv_sec == this->tv_sec && curlog.tv_usec > this->tv_usec))) {
+ ilog = i;
+ curlog.tv_sec = this->tv_sec;
+ curlog.tv_usec = this->tv_usec;
+ }
+}
+
+
+/*
+ * pick next meta record - if all meta is at EOF return -1
+ * (normally this function returns 0)
+ */
+static int
+nextmeta(void)
+{
+ int i;
+ int j;
+ int want;
+ int numeof = 0;
+ int sts;
+ pmID pmid; /* pmid for TYPE_DESC */
+ pmInDom indom; /* indom for TYPE_INDOM */
+ __pmLogCtl *lcp;
+ __pmContext *ctxp;
+ inarch_t *iap; /* pointer to input archive control */
+
+ for (i=0; i<inarchnum; i++) {
+ iap = &inarch[i];
+
+ /* if at the end of meta file then skip this archive
+ */
+ if (iap->eof[META]) {
+ ++numeof;
+ continue;
+ }
+
+ /* we should never already have a meta record
+ */
+ if (iap->pb[META] != NULL) {
+ fprintf(stderr, "%s: Fatal Error!\n", pmProgname);
+ fprintf(stderr, " iap->pb[META] is not NULL\n");
+ abandon();
+ }
+ if ((ctxp = __pmHandleToPtr(iap->ctx)) == NULL) {
+ fprintf(stderr, "%s: botch: __pmHandleToPtr(%d) returns NULL!\n", pmProgname, iap->ctx);
+ abandon();
+ }
+ lcp = ctxp->c_archctl->ac_log;
+
+againmeta:
+ /* get next meta record */
+
+ if ((sts = _pmLogGet(lcp, PM_LOG_VOL_META, &iap->pb[META])) < 0) {
+ iap->eof[META] = 1;
+ ++numeof;
+ if (sts != PM_ERR_EOL) {
+ fprintf(stderr, "%s: Error: _pmLogGet[meta %s]: %s\n",
+ pmProgname, iap->name, pmErrStr(sts));
+ _report(lcp->l_mdfp);
+ }
+ PM_UNLOCK(ctxp->c_lock);
+ continue;
+ }
+
+ /* pmDesc entries, if not seen before & wanted,
+ * then append to desc list
+ */
+ if (ntohl(iap->pb[META][1]) == TYPE_DESC) {
+ pmid = ntoh_pmID(iap->pb[META][2]);
+
+ /* if ml is defined, then look for pmid in the list
+ * if pmid is not in the list then discard it immediately
+ */
+ want = 0;
+ if (ml == NULL)
+ want = 1;
+ else {
+ for (j=0; j<ml_numpmid; j++) {
+ if (pmid == ml[j].idesc->pmid)
+ want = 1;
+ }
+ }
+
+ if (want) {
+ if (__pmHashSearch((int)pmid, &mdesc_hash) == NULL)
+ __pmHashAdd((int)pmid, NULL, &mdesc_hash);
+ /*
+ * update the desc list (add first time, check on subsequent
+ * sightings of desc for this pmid from this source
+ * update_descreclist() sets pb[META] to NULL
+ */
+ update_descreclist(i);
+ }
+ else {
+ /* not wanted */
+ free(iap->pb[META]);
+ iap->pb[META] = NULL;
+ goto againmeta;
+ }
+ }
+ else if (ntohl(iap->pb[META][1]) == TYPE_INDOM) {
+ /* if ml is defined, then look for instance domain in the list
+ * if indom is not in the list then discard it immediately
+ */
+ indom = ntoh_pmInDom(iap->pb[META][4]);
+ want = 0;
+ if (ml == NULL)
+ want = 1;
+ else {
+ for (j=0; j<ml_numpmid; j++) {
+ if (indom == ml[j].idesc->indom)
+ want = 1;
+ }
+ }
+
+ if (want) {
+ if (__pmHashSearch((int)indom, &mindom_hash) == NULL) {
+ /* meta record has never been seen ... add it to the list */
+ __pmHashAdd((int)indom, NULL, &mindom_hash);
+ }
+ /* add to indom list */
+ /* append_indomreclist() sets pb[META] to NULL
+ * append_indomreclist() may unpin the pdu buffer
+ */
+ append_indomreclist(i);
+ }
+ else {
+ /* META: don't want this meta */
+ free(iap->pb[META]);
+ iap->pb[META] = NULL;
+ goto againmeta;
+ }
+ }
+ else {
+ fprintf(stderr, "%s: Error: unrecognised meta data type: %d\n",
+ pmProgname, (int)ntohl(iap->pb[META][1]));
+ abandon();
+ }
+
+ PM_UNLOCK(ctxp->c_lock);
+ }
+
+ if (numeof == inarchnum) return(-1);
+ return(0);
+}
+
+
+/*
+ * read in next log record for every archive
+ */
+static int
+nextlog(void)
+{
+ int i;
+ int eoflog = 0; /* number of log files at eof */
+ int sts;
+ __pmTimeval curtime;
+ __pmLogCtl *lcp;
+ __pmContext *ctxp;
+ inarch_t *iap;
+
+
+ for (i=0; i<inarchnum; i++) {
+ iap = &inarch[i];
+
+ /* if at the end of log file then skip this archive
+ */
+ if (iap->eof[LOG]) {
+ ++eoflog;
+ continue;
+ }
+
+ /* if we already have a log record then skip this archive
+ */
+ if (iap->_Nresult != NULL) {
+ continue;
+ }
+
+ /* if mark has been written out, then log is at EOF
+ */
+ if (iap->mark) {
+ iap->eof[LOG] = 1;
+ ++eoflog;
+ continue;
+ }
+
+ if ((ctxp = __pmHandleToPtr(iap->ctx)) == NULL) {
+ fprintf(stderr, "%s: botch: __pmHandleToPtr(%d) returns NULL!\n", pmProgname, iap->ctx);
+ abandon();
+ }
+ lcp = ctxp->c_archctl->ac_log;
+
+againlog:
+ if ((sts=__pmLogRead(lcp, PM_MODE_FORW, NULL, &iap->_result, PMLOGREAD_NEXT)) < 0) {
+ if (sts != PM_ERR_EOL) {
+ fprintf(stderr, "%s: Error: __pmLogRead[log %s]: %s\n",
+ pmProgname, iap->name, pmErrStr(sts));
+ _report(lcp->l_mfp);
+ }
+ /* if the first data record has not been written out, then
+ * do not generate a mark record, and you may as well ignore
+ * this archive
+ */
+ if (first_datarec) {
+ iap->mark = 1;
+ iap->eof[LOG] = 1;
+ ++eoflog;
+ }
+ else {
+ iap->mark = 1;
+ iap->pb[LOG] = _createmark();
+ }
+ PM_UNLOCK(ctxp->c_lock);
+ continue;
+ }
+ assert(iap->_result != NULL);
+
+
+ /* set current log time - this is only done so that we can
+ * determine whether to keep or discard the log
+ */
+ curtime.tv_sec = iap->_result->timestamp.tv_sec;
+ curtime.tv_usec = iap->_result->timestamp.tv_usec;
+
+ /* if log time is greater than (or equal to) the current window
+ * start time, then we may want it
+ * (irrespective of the current window end time)
+ */
+ if (tvcmp(curtime, winstart) < 0) {
+ /* log is not in time window - discard result and get next record
+ */
+ pmFreeResult(iap->_result);
+ iap->_result = NULL;
+ goto againlog;
+ }
+ else {
+ /* log is within time window - check whether we want this record
+ */
+ if (iap->_result->numpmid == 0) {
+ /* mark record, process this one as is
+ */
+ iap->_Nresult = iap->_result;
+ }
+ else if (ml == NULL) {
+ /* ml is NOT defined, we want everything
+ */
+ iap->_Nresult = iap->_result;
+ }
+ else {
+ /* ml is defined, need to search metric list for wanted pmid's
+ * (searchmlist may return a NULL pointer - this is fine)
+ */
+ iap->_Nresult = searchmlist(iap->_result);
+ }
+
+ if (iap->_Nresult == NULL) {
+ /* dont want any of the metrics in _result, try again
+ */
+ pmFreeResult(iap->_result);
+ iap->_result = NULL;
+ goto againlog;
+ }
+ }
+
+ PM_UNLOCK(ctxp->c_lock);
+ } /*for(i)*/
+
+ /* if we are here, then each archive control struct should either
+ * be at eof, or it should have a _result, or it should have a mark PDU
+ * (if we have a _result, we may want all/some/none of the pmid's in it)
+ */
+
+ if (eoflog == inarchnum) return(-1);
+ return 0;
+}
+
+/*
+ * parse command line arguments
+ */
+int
+parseargs(int argc, char *argv[])
+{
+ int c;
+ int sts;
+ char *endnum;
+ struct stat sbuf;
+
+ while ((c = pmgetopt_r(argc, argv, &opts)) != EOF) {
+ switch (c) {
+
+ case 'c': /* config file */
+ configfile = opts.optarg;
+ if (stat(configfile, &sbuf) < 0) {
+ pmprintf("%s: %s - invalid file\n", pmProgname, configfile);
+ opts.errors++;
+ }
+ break;
+
+ case 'D': /* debug flag */
+ sts = __pmParseDebug(opts.optarg);
+ if (sts < 0) {
+ pmprintf("%s: unrecognized debug flag specification (%s)\n",
+ pmProgname, opts.optarg);
+ opts.errors++;
+ }
+ else
+ pmDebug |= sts;
+ break;
+
+ case 'd': /* desperate to save output archive, even after error */
+ desperate = 1;
+ break;
+
+ case 'f': /* use timezone from first archive */
+ farg = 1;
+ break;
+
+ case 's': /* number of samples to write out */
+ sarg = (int)strtol(opts.optarg, &endnum, 10);
+ if (*endnum != '\0' || sarg < 0) {
+ pmprintf("%s: -s requires numeric argument\n", pmProgname);
+ opts.errors++;
+ }
+ break;
+
+ case 'S': /* start time for extracting */
+ Sarg = opts.optarg;
+ break;
+
+ case 'T': /* end time for extracting */
+ Targ = opts.optarg;
+ break;
+
+ case 'v': /* number of samples per volume */
+ varg = (int)strtol(opts.optarg, &endnum, 10);
+ if (*endnum != '\0' || varg < 0) {
+ pmprintf("%s: -v requires numeric argument\n", pmProgname);
+ opts.errors++;
+ }
+ break;
+
+ case 'w': /* ignore day/month/year */
+ warg++;
+ break;
+
+ case 'Z': /* use timezone from command line */
+ if (zarg) {
+ pmprintf("%s: at most one of -Z and/or -z allowed\n",
+ pmProgname);
+ opts.errors++;
+ }
+ tz = opts.optarg;
+ break;
+
+ case 'z': /* use timezone from archive */
+ if (tz != NULL) {
+ pmprintf("%s: at most one of -Z and/or -z allowed\n",
+ pmProgname);
+ opts.errors++;
+ }
+ zarg++;
+ break;
+
+ case '?':
+ default:
+ opts.errors++;
+ break;
+ }
+ }
+
+ if (warg) {
+ if (Sarg == NULL || Targ == NULL) {
+ fprintf(stderr, "%s: Warning: -w flag requires that both -S and -T are specified.\nIgnoring -w flag.\n", pmProgname);
+ warg = 0;
+ }
+ }
+
+
+ if (opts.errors == 0 && opts.optind > argc - 2) {
+ pmprintf("%s: Error: insufficient arguments\n", pmProgname);
+ opts.errors++;
+ }
+
+ return -opts.errors;
+}
+
+int
+parseconfig(void)
+{
+ int errflag = 0;
+
+ if ((yyin = fopen(configfile, "r")) == NULL) {
+ fprintf(stderr, "%s: Cannot open config file \"%s\": %s\n",
+ pmProgname, configfile, osstrerror());
+ exit(1);
+ }
+
+ if (yyparse() != 0)
+ exit(1);
+
+ fclose(yyin);
+ yyin = NULL;
+
+ return(-errflag);
+}
+
+/*
+ * we are within time window ... return 0
+ * we are outside of time window & mk new window ... return 1
+ * we are outside of time window & exit ... return -1
+ */
+static int
+checkwinend(__pmTimeval now)
+{
+ int i;
+ int sts;
+ __pmTimeval tmptime;
+ inarch_t *iap;
+ __pmPDU *markpdu; /* mark b/n time windows */
+
+ if (winend.tv_sec < 0 || tvcmp(now, winend) <= 0)
+ return(0);
+
+ /* we have reached the end of a window
+ * - if warg is not set, then we have finished (break)
+ * - otherwise, calculate start and end of next window,
+ * set pre_startwin, discard logs before winstart,
+ * and write out mark
+ */
+ if (!warg)
+ return(-1);
+
+ winstart.tv_sec += NUM_SEC_PER_DAY;
+ winend.tv_sec += NUM_SEC_PER_DAY;
+ pre_startwin = 1;
+
+ /* if start of next window is later than max termination
+ * then bail out here
+ */
+ if (tvcmp(winstart, logend) > 0)
+ return(-1);
+
+ ilog = -1;
+ for (i=0; i<inarchnum; i++) {
+ iap = &inarch[i];
+ if (iap->_Nresult != NULL) {
+ tmptime.tv_sec = iap->_Nresult->timestamp.tv_sec;
+ tmptime.tv_usec = iap->_Nresult->timestamp.tv_usec;
+ if (tvcmp(tmptime, winstart) < 0) {
+ /* free _result and _Nresult
+ */
+ if (iap->_result != iap->_Nresult) {
+ free(iap->_Nresult);
+ }
+ if (iap->_result != NULL) {
+ pmFreeResult(iap->_result);
+ iap->_result = NULL;
+ }
+ iap->_Nresult = NULL;
+ iap->pb[LOG] = NULL;
+ }
+ }
+ if (iap->pb[LOG] != NULL) {
+ tmptime.tv_sec = ntohl(iap->pb[LOG][3]);
+ tmptime.tv_usec = ntohl(iap->pb[LOG][4]);
+ if (tvcmp(tmptime, winstart) < 0) {
+ /* free PDU buffer ... it is probably a mark
+ * and has not been pinned
+ */
+ free(iap->pb[LOG]);
+ iap->pb[LOG] = NULL;
+ }
+ }
+ } /*for(i)*/
+
+ /* must create "mark" record and write it out */
+ /* (need only one mark record) */
+ markpdu = _createmark();
+ if ((sts = __pmLogPutResult2(&logctl, markpdu)) < 0) {
+ fprintf(stderr, "%s: Error: __pmLogPutResult2: log data: %s\n",
+ pmProgname, pmErrStr(sts));
+ abandon();
+ }
+ written++;
+ free(markpdu);
+ return(1);
+}
+
+
+/*
+ *
+ */
+void
+writerlist(rlist_t **rlready, __pmTimeval mintime)
+{
+ int sts;
+ int needti = 0; /* need to flush/update */
+ __pmTimeval titime = {0,0};/* time of last temporal index write */
+ __pmTimeval restime; /* time of result */
+ rlist_t *elm; /* element of rlready to be written out */
+ __pmPDU *pb; /* pdu buffer */
+ unsigned long peek_offset;
+
+ while (*rlready != NULL) {
+ restime.tv_sec = (*rlready)->res->timestamp.tv_sec;
+ restime.tv_usec = (*rlready)->res->timestamp.tv_usec;
+
+ if (tvcmp(restime, mintime) > 0) {
+#if 0
+fprintf(stderr, "writelist: restime %d.%06d mintime %d.%06d ", restime.tv_sec, restime.tv_usec, mintime.tv_sec, mintime.tv_usec);
+fprintf(stderr, " break!\n");
+#endif
+ break;
+ }
+
+ /* get the first element from the list
+ */
+ elm = *rlready;
+ *rlready = elm->next;
+
+ /* if this is the first record (for output archive) then do some
+ * admin stuff
+ */
+ if (first_datarec) {
+ first_datarec = 0;
+ logctl.l_label.ill_start.tv_sec = elm->res->timestamp.tv_sec;
+ logctl.l_label.ill_start.tv_usec = elm->res->timestamp.tv_usec;
+ logctl.l_state = PM_LOG_STATE_INIT;
+ writelabel_data();
+ }
+
+ /* if we are in a pre_startwin state, and we are writing
+ * something out, then we are not in a pre_startwin state any more
+ * (it also means that there may be some discrete metrics to be
+ * written out)
+ */
+ if (pre_startwin)
+ pre_startwin = 0;
+
+
+ /* convert log record to a pdu
+ */
+ sts = __pmEncodeResult(PDU_OVERRIDE2, elm->res, &pb);
+ if (sts < 0) {
+ fprintf(stderr, "%s: Error: __pmEncodeResult: %s\n",
+ pmProgname, pmErrStr(sts));
+ abandon();
+ }
+
+ /* switch volumes if required */
+ if (varg > 0) {
+ if (written > 0 && (written % varg) == 0) {
+ newvolume(outarchname, (__pmTimeval *)&pb[3]);
+ }
+ }
+ /*
+ * Even without a -v option, we may need to switch volumes
+ * if the data file exceeds 2^31-1 bytes
+ */
+ peek_offset = ftell(logctl.l_mfp);
+ peek_offset += ((__pmPDUHdr *)pb)->len - sizeof(__pmPDUHdr) + 2*sizeof(int);
+ if (peek_offset > 0x7fffffff) {
+ newvolume(outarchname, (__pmTimeval *)&pb[3]);
+ }
+
+ /* write out the descriptor and instance domain pdu's first
+ */
+ write_metareclist(elm->res, &needti);
+
+ /* write out log record */
+ old_log_offset = ftell(logctl.l_mfp);
+ assert(old_log_offset >= 0);
+ if ((sts = __pmLogPutResult2(&logctl, pb)) < 0) {
+ fprintf(stderr, "%s: Error: __pmLogPutResult2: log data: %s\n",
+ pmProgname, pmErrStr(sts));
+ abandon();
+ }
+ written++;
+
+
+ /* check whether we need to write TI (temporal index) */
+ if (old_log_offset == 0 ||
+ old_log_offset == sizeof(__pmLogLabel)+2*sizeof(int) ||
+ ftell(logctl.l_mfp) > flushsize)
+ needti = 1;
+
+ /* make sure that we do not write out the temporal index more
+ * than once for the same timestamp
+ */
+ if (needti && tvcmp(titime, restime) >= 0)
+ needti = 0;
+
+ /* flush/update */
+ if (needti) {
+ titime = restime;
+
+ fflush(logctl.l_mfp);
+ fflush(logctl.l_mdfp);
+
+ if (old_log_offset == 0)
+ old_log_offset = sizeof(__pmLogLabel)+2*sizeof(int);
+
+ new_log_offset = ftell(logctl.l_mfp);
+ assert(new_log_offset >= 0);
+ new_meta_offset = ftell(logctl.l_mdfp);
+ assert(new_meta_offset >= 0);
+
+ fseek(logctl.l_mfp, (long)old_log_offset, SEEK_SET);
+ fseek(logctl.l_mdfp, (long)old_meta_offset, SEEK_SET);
+
+ __pmLogPutIndex(&logctl, &restime);
+
+ fseek(logctl.l_mfp, (long)new_log_offset, SEEK_SET);
+ fseek(logctl.l_mdfp, (long)new_meta_offset, SEEK_SET);
+
+ old_log_offset = ftell(logctl.l_mfp);
+ assert(old_log_offset >= 0);
+ old_meta_offset = ftell(logctl.l_mdfp);
+ assert(old_meta_offset >= 0);
+
+ flushsize = ftell(logctl.l_mfp) + 100000;
+ }
+
+ /* free PDU buffer */
+ __pmUnpinPDUBuf(pb);
+ pb = NULL;
+
+ elm->res = NULL;
+ elm->next = NULL;
+ free(elm);
+
+ } /*while(*rlready)*/
+}
+
+
+/*
+ * mark record has been created and assigned to iap->pb[LOG]
+ * write it out
+ */
+void
+writemark(inarch_t *iap)
+{
+ int sts;
+ mark_t *p = (mark_t *)iap->pb[LOG];
+
+ if (!iap->mark) {
+ fprintf(stderr, "%s: Fatal Error!\n", pmProgname);
+ fprintf(stderr, " writemark called, but mark not set\n");
+ abandon();
+ }
+
+ if (p == NULL) {
+ fprintf(stderr, "%s: Fatal Error!\n", pmProgname);
+ fprintf(stderr, " writemark called, but no pdu\n");
+ abandon();
+ }
+
+ p->timestamp.tv_sec = htonl(p->timestamp.tv_sec);
+ p->timestamp.tv_usec = htonl(p->timestamp.tv_usec);
+
+ if ((sts = __pmLogPutResult2(&logctl, iap->pb[LOG])) < 0) {
+ fprintf(stderr, "%s: Error: __pmLogPutResult2: log data: %s\n",
+ pmProgname, pmErrStr(sts));
+ abandon();
+ }
+ written++;
+ free(iap->pb[LOG]);
+ iap->pb[LOG] = NULL;
+}
+
+/*--- END FUNCTIONS ---------------------------------------------------------*/
+
+int
+main(int argc, char **argv)
+{
+ int i;
+ int j;
+ int sts;
+ int stslog; /* sts from nextlog() */
+ int stsmeta; /* sts from nextmeta() */
+
+ char *msg;
+
+ __pmTimeval now = {0,0}; /* the current time */
+ __pmTimeval mintime = {0,0};
+ __pmTimeval tmptime = {0,0};
+
+ __pmTimeval tstamp; /* temporary timestamp */
+ inarch_t *iap; /* ptr to archive control */
+ rlist_t *rlready; /* list of results ready for writing */
+ struct timeval unused;
+
+
+ rlog = NULL; /* list of log records to write */
+ rdesc = NULL; /* list of meta desc records to write */
+ rindom = NULL; /* list of meta indom records to write */
+ rlready = NULL;
+
+
+ /* process cmd line args */
+ if (parseargs(argc, argv) < 0) {
+ pmUsageMessage(&opts);
+ exit(1);
+ }
+
+
+ /* input archive names are argv[opts.optind] ... argv[argc-2]) */
+ /* output archive name is argv[argc-1]) */
+
+ /* output archive */
+ outarchname = argv[argc-1];
+
+ /* input archive(s) */
+ inarchnum = argc - 1 - opts.optind;
+ inarch = (inarch_t *) malloc(inarchnum * sizeof(inarch_t));
+ if (inarch == NULL) {
+ fprintf(stderr, "%s: Error: mallco inarch: %s\n",
+ pmProgname, osstrerror());
+ exit(1);
+ }
+#ifdef PCP_DEBUG
+ if (pmDebug & DBG_TRACE_APPL0) {
+ totalmalloc += (inarchnum * sizeof(inarch_t));
+ fprintf(stderr, "main : allocated %d\n",
+ (int)(inarchnum * sizeof(inarch_t)));
+ }
+#endif
+
+
+ for (i=0; i<inarchnum; i++, opts.optind++) {
+ iap = &inarch[i];
+
+ iap->name = argv[opts.optind];
+
+ iap->pb[LOG] = iap->pb[META] = NULL;
+ iap->eof[LOG] = iap->eof[META] = 0;
+ iap->mark = 0;
+ iap->_result = NULL;
+ iap->_Nresult = NULL;
+
+ if ((iap->ctx = pmNewContext(PM_CONTEXT_ARCHIVE, iap->name)) < 0) {
+ fprintf(stderr, "%s: Error: cannot open archive \"%s\": %s\n",
+ pmProgname, iap->name, pmErrStr(iap->ctx));
+ exit(1);
+ }
+
+ if ((sts = pmUseContext(iap->ctx)) < 0) {
+ fprintf(stderr, "%s: Error: cannot use context (%s): %s\n",
+ pmProgname, iap->name, pmErrStr(sts));
+ exit(1);
+ }
+
+ if ((sts = pmGetArchiveLabel(&iap->label)) < 0) {
+ fprintf(stderr, "%s: Error: cannot get archive label record (%s): %s\n", pmProgname, iap->name, pmErrStr(sts));
+ exit(1);
+ }
+
+ if ((sts = pmGetArchiveEnd(&unused)) < 0) {
+ fprintf(stderr, "%s: Error: cannot get end of archive (%s): %s\n",
+ pmProgname, iap->name, pmErrStr(sts));
+ exit(1);
+ }
+
+ if (i == 0) {
+ /* start time */
+ logstart_tval.tv_sec = iap->label.ll_start.tv_sec;
+ logstart_tval.tv_usec = iap->label.ll_start.tv_usec;
+
+ /* end time */
+ logend_tval.tv_sec = unused.tv_sec;
+ logend_tval.tv_usec = unused.tv_usec;
+ }
+ else {
+ /* get the earlier start time */
+ if (logstart_tval.tv_sec > iap->label.ll_start.tv_sec ||
+ (logstart_tval.tv_sec == iap->label.ll_start.tv_sec &&
+ logstart_tval.tv_usec > iap->label.ll_start.tv_usec)) {
+ logstart_tval.tv_sec = iap->label.ll_start.tv_sec;
+ logstart_tval.tv_usec = iap->label.ll_start.tv_usec;
+ }
+
+ /* get the later end time */
+ if (logend_tval.tv_sec < unused.tv_sec ||
+ (logend_tval.tv_sec == unused.tv_sec &&
+ logend_tval.tv_usec < unused.tv_usec)) {
+ logend_tval.tv_sec = unused.tv_sec;
+ logend_tval.tv_usec = unused.tv_usec;
+ }
+ }
+ } /*for(i)*/
+
+ logctl.l_label.ill_start.tv_sec = logstart_tval.tv_sec;
+ logctl.l_label.ill_start.tv_usec = logstart_tval.tv_usec;
+
+ /* process config file
+ * - this includes a list of metrics and their instances
+ */
+ if (configfile && parseconfig() < 0)
+ exit(1);
+
+ if (zarg) {
+ /* use TZ from metrics source (input-archive) */
+ if ((sts = pmNewZone(inarch[0].label.ll_tz)) < 0) {
+ fprintf(stderr, "%s: Cannot set context timezone: %s\n",
+ pmProgname, pmErrStr(sts));
+ exit_status = 1;
+ goto cleanup;
+ }
+ printf("Note: timezone set to local timezone of host \"%s\" from archive\n\n", inarch[0].label.ll_hostname);
+ }
+ else if (tz != NULL) {
+ /* use TZ as specified by user */
+ if ((sts = pmNewZone(tz)) < 0) {
+ fprintf(stderr, "%s: Cannot set timezone to \"%s\": %s\n",
+ pmProgname, tz, pmErrStr(sts));
+ exit_status = 1;
+ goto cleanup;
+ }
+ printf("Note: timezone set to \"TZ=%s\"\n\n", tz);
+ }
+ else {
+ char *tz;
+ tz = __pmTimezone();
+ /* use TZ from local host */
+ if ((sts = pmNewZone(tz)) < 0) {
+ fprintf(stderr, "%s: Cannot set local host's timezone: %s\n",
+ pmProgname, pmErrStr(sts));
+ exit_status = 1;
+ goto cleanup;
+ }
+ }
+
+
+ /* create output log - must be done before writing label */
+ if ((sts = __pmLogCreate("", outarchname, outarchvers, &logctl)) < 0) {
+ fprintf(stderr, "%s: Error: __pmLogCreate: %s\n",
+ pmProgname, pmErrStr(sts));
+ exit(1);
+ }
+
+ /* This must be done after log is created:
+ * - checks that archive version, host, and timezone are ok
+ * - set archive version, host, and timezone of output archive
+ */
+ newlabel();
+
+ /* write label record */
+ writelabel_metati(0);
+
+
+ /* set winstart and winend timevals */
+ sts = pmParseTimeWindow(Sarg, Targ, Aarg, Oarg,
+ &logstart_tval, &logend_tval,
+ &winstart_tval, &winend_tval, &unused, &msg);
+ if (sts < 0) {
+ fprintf(stderr, "%s: Invalid time window specified: %s\n",
+ pmProgname, msg);
+ abandon();
+ }
+ winstart.tv_sec = winstart_tval.tv_sec;
+ winstart.tv_usec = winstart_tval.tv_usec;
+ winend.tv_sec = winend_tval.tv_sec;
+ winend.tv_usec = winend_tval.tv_usec;
+ logend.tv_sec = logend_tval.tv_sec;
+ logend.tv_usec = logend_tval.tv_usec;
+
+ if (warg) {
+ if (winstart.tv_sec + NUM_SEC_PER_DAY < winend.tv_sec) {
+ fprintf(stderr, "%s: Warning: -S and -T must specify a time window within\nthe same day, for -w to be used. Ignoring -w flag.\n", pmProgname);
+ warg = 0;
+ }
+ }
+
+ ilog = -1;
+ written = 0;
+ curlog.tv_sec = 0;
+ curlog.tv_usec = 0;
+ current.tv_sec = 0;
+ current.tv_usec = 0;
+ first_datarec = 1;
+ pre_startwin = 1;
+
+ /* get all meta data first
+ * nextmeta() should return 0 (will return -1 when all meta is eof)
+ */
+ do {
+ stsmeta = nextmeta();
+ } while (stsmeta >= 0);
+
+
+ /* get log record - choose one with earliest timestamp
+ * write out meta data (required by this log record)
+ * write out log
+ * do ti update if necessary
+ */
+ while (sarg == -1 || written < sarg) {
+ ilog = -1;
+ curlog.tv_sec = 0;
+ curlog.tv_usec = 0;
+ old_meta_offset = ftell(logctl.l_mdfp);
+ assert(old_meta_offset >= 0);
+
+ /* nextlog() resets ilog, and curlog (to the smallest timestamp)
+ */
+ stslog = nextlog();
+
+ if (stslog < 0)
+ break;
+
+ /* find the _Nresult (or mark pdu) with the earliest timestamp;
+ * set ilog
+ * (this is a bit more complex when tflag is specified)
+ */
+ mintime.tv_sec = mintime.tv_usec = 0;
+ for (i=0; i<inarchnum; i++) {
+ if (inarch[i]._Nresult != NULL) {
+ tstamp.tv_sec = inarch[i]._Nresult->timestamp.tv_sec;
+ tstamp.tv_usec = inarch[i]._Nresult->timestamp.tv_usec;
+ checklogtime(&tstamp, i);
+
+ if (ilog == i) {
+ tmptime = curlog;
+ if (mintime.tv_sec <= 0 || tvcmp(mintime, tmptime) > 0)
+ mintime = tmptime;
+ }
+ }
+ else if (inarch[i].pb[LOG] != NULL) {
+ tstamp.tv_sec = inarch[i].pb[LOG][3]; /* no swab needed */
+ tstamp.tv_usec = inarch[i].pb[LOG][4]; /* no swab needed */
+ checklogtime(&tstamp, i);
+
+ if (ilog == i) {
+ tmptime = curlog;
+ if (mintime.tv_sec <= 0 || tvcmp(mintime, tmptime) > 0)
+ mintime = tmptime;
+ }
+ }
+ }
+
+ /* now == the earliest timestamp of the archive(s)
+ * and/or mark records
+ * mintime == now or timestamp of the earliest mark
+ * (whichever is smaller)
+ */
+ now = curlog;
+
+ /* note - mark (after last archive) will be created, but this
+ * break, will prevent it from being written out
+ */
+ if (tvcmp(now, logend) > 0)
+ break;
+
+ sts = checkwinend(now);
+ if (sts < 0)
+ break;
+ if (sts > 0)
+ continue;
+
+ current = curlog;
+
+ /* prepare to write out log record
+ */
+ if (ilog < 0 || ilog >= inarchnum) {
+ fprintf(stderr, "%s: Fatal Error!\n", pmProgname);
+ fprintf(stderr, " log file index = %d\n", ilog);
+ abandon();
+ }
+
+
+ iap = &inarch[ilog];
+ if (iap->mark)
+ writemark(iap);
+ else {
+ /* result is to be written out, but there is no _Nresult
+ */
+ if (iap->_Nresult == NULL) {
+ fprintf(stderr, "%s: Fatal Error!\n", pmProgname);
+ fprintf(stderr, " pick == LOG and _Nresult = NULL\n");
+ abandon();
+ }
+ insertresult(&rlready, iap->_Nresult);
+#if 0
+{
+ rlist_t *rp;
+ int i;
+
+ fprintf(stderr, "rlready");
+ for (i = 0, rp = rlready; rp != NULL; i++, rp = rp->next) {
+ fprintf(stderr, " [%d] t=%d.%06d numpmid=%d", i, (int)rp->res->timestamp.tv_sec, (int)rp->res->timestamp.tv_usec, rp->res->numpmid);
+ }
+ fprintf(stderr, " now=%d.%06d\n", now.tv_sec, now.tv_usec);
+}
+#endif
+
+ writerlist(&rlready, curlog);
+
+ /* writerlist frees elm (elements of rlready) but does not
+ * free _result & _Nresult
+ */
+
+ /* free _result & _Nresult
+ * _Nresult may contain space that was allocated
+ * in __pmStuffValue this space has PM_VAL_SPTR format,
+ * and has to be freed first
+ * (in order to avoid memory leaks)
+ */
+ if (iap->_result != iap->_Nresult && iap->_Nresult != NULL) {
+ pmValueSet *vsetp;
+ for (i=0; i<iap->_Nresult->numpmid; i++) {
+ vsetp = iap->_Nresult->vset[i];
+ if (vsetp->valfmt == PM_VAL_SPTR) {
+ for (j=0; j<vsetp->numval; j++) {
+ free(vsetp->vlist[j].value.pval);
+ }
+ }
+ }
+ free(iap->_Nresult);
+ }
+ if (iap->_result != NULL) {
+ pmFreeResult(iap->_result);
+ iap->_result = NULL;
+ }
+ iap->_Nresult = NULL;
+ }
+ } /*while()*/
+
+ if (first_datarec) {
+ fprintf(stderr, "%s: Warning: no qualifying records found.\n",
+ pmProgname);
+cleanup:
+ abandon();
+ }
+ else {
+ /* write the last time stamp */
+ fflush(logctl.l_mfp);
+ fflush(logctl.l_mdfp);
+
+ if (old_log_offset == 0)
+ old_log_offset = sizeof(__pmLogLabel)+2*sizeof(int);
+
+ new_log_offset = ftell(logctl.l_mfp);
+ assert(new_log_offset >= 0);
+ new_meta_offset = ftell(logctl.l_mdfp);
+ assert(new_meta_offset >= 0);
+
+#if 0
+ fprintf(stderr, "*** last tstamp: \n\tmintime=%d.%06d \n\ttmptime=%d.%06d \n\tlogend=%d.%06d \n\twinend=%d.%06d \n\tcurrent=%d.%06d\n",
+ mintime.tv_sec, mintime.tv_usec, tmptime.tv_sec, tmptime.tv_usec, logend.tv_sec, logend.tv_usec, winend.tv_sec, winend.tv_usec, current.tv_sec, current.tv_usec);
+#endif
+
+ fseek(logctl.l_mfp, old_log_offset, SEEK_SET);
+ __pmLogPutIndex(&logctl, &current);
+
+
+ /* need to fix up label with new start-time */
+ writelabel_metati(1);
+ }
+#ifdef PCP_DEBUG
+ if (pmDebug & DBG_TRACE_APPL0) {
+ fprintf(stderr, "main : total allocated %ld\n", totalmalloc);
+ }
+#endif
+
+ exit(exit_status);
+}
generated by cgit v1.2.3 (git 2.39.1) at 2024-06-02 05:29:30 +0000