6894057 cmd format Failed to initialize libfdisk

6 files changed, 49 insertions, 11 deletions

diff --git a/usr/src/cmd/boot/bootadm/bootadm.c b/usr/src/cmd/boot/bootadm/bootadm.c
index 1803a13102..84f1509da6 100644
--- a/usr/src/cmd/boot/bootadm/bootadm.c
+++ b/usr/src/cmd/boot/bootadm/bootadm.c
@@ -4924,8 +4924,11 @@ get_partition(char *device)
 				break;
 			case FDISK_ENOLOGDRIVE:
 				break;
+			case FDISK_EBADMAGIC:
+				/*FALLTHROUGH*/
 			default:
 				free(wholedisk);
+				libfdisk_fini(&epp);
 				return (partno);
 				break;
 		}
diff --git a/usr/src/cmd/boot/installgrub/installgrub.c b/usr/src/cmd/boot/installgrub/installgrub.c
index ed552b44ce..6a60554e1b 100644
--- a/usr/src/cmd/boot/installgrub/installgrub.c
+++ b/usr/src/cmd/boot/installgrub/installgrub.c
@@ -311,13 +311,15 @@ get_start_sector(int fd)
 	 */
 	if ((rval = libfdisk_init(&epp, device_p0, NULL, FDISK_READ_DISK))
 	    != FDISK_SUCCESS) {
+		libfdisk_fini(&epp);
 		switch (rval) {
 			/*
-			 * The first 2 cases are not an error per-se, just that
+			 * The first 3 cases are not an error per-se, just that
 			 * there is no Solaris logical partition
 			 */
 			case FDISK_EBADLOGDRIVE:
 			case FDISK_ENOLOGDRIVE:
+			case FDISK_EBADMAGIC:
 				(void) fprintf(stderr, NOSOLPAR);
 				exit(-1);
 				/*NOTREACHED*/
@@ -341,12 +343,12 @@ get_start_sector(int fd)
 	}
 
 	rval = fdisk_get_solaris_part(epp, &pno, &secnum, &numsec);
+	libfdisk_fini(&epp);
 	if (rval != FDISK_SUCCESS) {
 		/* No solaris logical partition */
 		(void) fprintf(stderr, NOSOLPAR);
 		exit(-1);
 	}
-	libfdisk_fini(&epp);
 
 	start_sect = secnum;
 	partition = pno - 1;
diff --git a/usr/src/cmd/fdisk/fdisk.c b/usr/src/cmd/fdisk/fdisk.c
index 3d0dadc8bb..fcf854766b 100644
--- a/usr/src/cmd/fdisk/fdisk.c
+++ b/usr/src/cmd/fdisk/fdisk.c
@@ -1181,31 +1181,37 @@ main(int argc, char *argv[])
 	    != FDISK_SUCCESS) {
 		switch (rval) {
 			/*
-			 * FDISK_EBADLOGDRIVE and FDISK_ENOLOGDRIVE can
-			 * be considered as soft errors and hence
-			 * we do not exit
+			 * FDISK_EBADLOGDRIVE, FDISK_ENOLOGDRIVE and
+			 * FDISK_EBADMAGIC can be considered as
+			 * soft errors and hence we do not exit
 			 */
 			case FDISK_EBADLOGDRIVE:
 				break;
 			case FDISK_ENOLOGDRIVE:
 				break;
+			case FDISK_EBADMAGIC:
+				break;
 			case FDISK_ENOVGEOM:
 				fprintf(stderr, "Could not get virtual"
 				    " geometry for this device\n");
+				libfdisk_fini(&epp);
 				exit(1);
 				break;
 			case FDISK_ENOPGEOM:
 				fprintf(stderr, "Could not get physical"
 				    " geometry for this device\n");
+				libfdisk_fini(&epp);
 				exit(1);
 				break;
 			case FDISK_ENOLGEOM:
 				fprintf(stderr, "Could not get label"
 				    " geometry for this device\n");
+				libfdisk_fini(&epp);
 				exit(1);
 				break;
 			default:
 				perror("Failed to initialise libfdisk.\n");
+				libfdisk_fini(&epp);
 				exit(1);
 				break;
 		}
diff --git a/usr/src/cmd/format/menu_fdisk.c b/usr/src/cmd/format/menu_fdisk.c
index dab5436643..404904140c 100644
--- a/usr/src/cmd/format/menu_fdisk.c
+++ b/usr/src/cmd/format/menu_fdisk.c
@@ -952,31 +952,37 @@ extpart_init(ext_part_t **epp)
 	    FDISK_SUCCESS) {
 		switch (rval) {
 			/*
-			 * FDISK_EBADLOGDRIVE and FDISK_ENOLOGDRIVE can
-			 * be considered as soft errors and hence
-			 * we do not exit
+			 * FDISK_EBADLOGDRIVE, FDISK_ENOLOGDRIVE
+			 * and FDISK_EBADMAGIC can be considered
+			 * as soft errors and hence we do not exit.
 			 */
 			case FDISK_EBADLOGDRIVE:
 				break;
 			case FDISK_ENOLOGDRIVE:
 				break;
+			case FDISK_EBADMAGIC:
+				break;
 			case FDISK_ENOVGEOM:
 				err_print("Could not get virtual geometry for"
 				    " this device\n");
+				libfdisk_fini(epp);
 				fullabort();
 				break;
 			case FDISK_ENOPGEOM:
 				err_print("Could not get physical geometry for"
 				    " this device\n");
+				libfdisk_fini(epp);
 				fullabort();
 				break;
 			case FDISK_ENOLGEOM:
 				err_print("Could not get label geometry for "
 				    " this device\n");
+				libfdisk_fini(epp);
 				fullabort();
 				break;
 			default:
 				err_print("Failed to initialise libfdisk.\n");
+				libfdisk_fini(epp);
 				fullabort();
 				break;
 		}
diff --git a/usr/src/lib/libdiskmgt/common/partition.c b/usr/src/lib/libdiskmgt/common/partition.c
index b10cfc658a..ed30cde80f 100644
--- a/usr/src/lib/libdiskmgt/common/partition.c
+++ b/usr/src/lib/libdiskmgt/common/partition.c
@@ -652,9 +652,15 @@ get_parts(disk_t *disk, struct ipart *iparts, char *opath, int opath_len)
 				case FDISK_EBADLOGDRIVE:
 				case FDISK_ENOLOGDRIVE:
 					free(device);
+					libfdisk_fini(&epp);
 					continue;
+				case FDISK_EBADMAGIC:
+					free(device);
+					libfdisk_fini(&epp);
+					return (ENOTTY);
 				default:
 					free(device);
+					libfdisk_fini(&epp);
 					return (ENODEV);
 			}
 		}
diff --git a/usr/src/lib/libfdisk/common/libfdisk.c b/usr/src/lib/libfdisk/common/libfdisk.c
index db6916679b..0664bff1fa 100644
--- a/usr/src/lib/libfdisk/common/libfdisk.c
+++ b/usr/src/lib/libfdisk/common/libfdisk.c
@@ -155,6 +155,7 @@ libfdisk_init(ext_part_t **epp, char *devstr, struct ipart *parttab, int opflag)
 	ext_part_t *temp;
 	struct stat sbuf;
 	int rval = FDISK_SUCCESS;
+	int found_bad_magic = 0;
 
 	if ((temp = calloc(1, sizeof (ext_part_t))) == NULL) {
 		return (ENOMEM);
@@ -206,7 +207,15 @@ libfdisk_init(ext_part_t **epp, char *devstr, struct ipart *parttab, int opflag)
 	if ((temp->mtable = parttab) == NULL) {
 		if ((rval = fdisk_init_master_part_table(temp)) !=
 		    FDISK_SUCCESS) {
-			return (rval);
+			/*
+			 * When we have no fdisk magic 0xAA55 on the disk,
+			 * we return FDISK_EBADMAGIC after successfully
+			 * obtaining the disk geometry.
+			 */
+			if (rval != FDISK_EBADMAGIC)
+				return (rval);
+			else
+				found_bad_magic = 1;
 		}
 	}
 
@@ -218,6 +227,10 @@ libfdisk_init(ext_part_t **epp, char *devstr, struct ipart *parttab, int opflag)
 
 	*epp = temp;
 
+	if (found_bad_magic != 0) {
+		return (FDISK_EBADMAGIC);
+	}
+
 	if (opflag & FDISK_READ_DISK) {
 		rval = fdisk_read_extpart(*epp);
 	}
@@ -591,7 +604,7 @@ fdisk_validate_logical_drive(ext_part_t *epp, uint32_t begsec,
  * Procedure to walk through the extended partitions and build a Singly
  * Linked List out of the data.
  */
-int
+static int
 fdisk_read_extpart(ext_part_t *epp)
 {
 	struct ipart *fdp, *ext_fdp;
@@ -754,13 +767,15 @@ fdisk_read_master_part_table(ext_part_t *epp)
 	if (read(epp->dev_fd, buf, sectsize) < sectsize) {
 		return (EIO);
 	}
-	bcopy(&buf[FDISK_PART_TABLE_START], epp->mtable, cpcnt);
 
 	/*LINTED*/
 	if (LE_16((*(uint16_t *)&buf[510])) != MBB_MAGIC) {
+		bzero(epp->mtable, cpcnt);
 		return (FDISK_EBADMAGIC);
 	}
 
+	bcopy(&buf[FDISK_PART_TABLE_START], epp->mtable, cpcnt);
+
 	return (FDISK_SUCCESS);
 }