3035 LZ4 compression support in ZFS and GRUB

Reviewed by: Matthew Ahrens <mahrens@delphix.com>
Reviewed by: Christopher Siden <christopher.siden@delphix.com>
Reviewed by: George Wilson <george.wilson@delphix.com>
Approved by: Christopher Siden <csiden@delphix.com>

6 files changed, 360 insertions, 9 deletions

diff --git a/usr/src/grub/grub-0.97/stage2/Makefile.am b/usr/src/grub/grub-0.97/stage2/Makefile.am
index 44144e7353..8f8d203ce2 100644
--- a/usr/src/grub/grub-0.97/stage2/Makefile.am
+++ b/usr/src/grub/grub-0.97/stage2/Makefile.am
@@ -19,7 +19,7 @@ noinst_LIBRARIES = libgrub.a
 libgrub_a_SOURCES = boot.c builtins.c char_io.c cmdline.c common.c \
 	disk_io.c fsys_ext2fs.c fsys_fat.c fsys_ffs.c fsys_iso9660.c \
 	fsys_jfs.c fsys_minix.c fsys_reiserfs.c fsys_ufs.c fsys_ufs2.c \
-	fsys_zfs.c zfs_fletcher.c zfs_lzjb.c zfs_sha256.c \
+	fsys_zfs.c zfs_fletcher.c zfs_lzjb.c zfs_lz4.c zfs_sha256.c \
 	fsys_vstafs.c fsys_xfs.c gunzip.c md5.c serial.c stage2.c \
 	terminfo.c tparm.c graphics.c
 libgrub_a_CFLAGS = $(GRUB_CFLAGS) -I$(top_srcdir)/lib \
@@ -101,7 +101,7 @@ pre_stage2_exec_SOURCES = asm.S bios.c boot.c builtins.c char_io.c \
 	cmdline.c common.c console.c disk_io.c fsys_ext2fs.c \
 	fsys_fat.c fsys_ffs.c fsys_iso9660.c fsys_jfs.c fsys_minix.c \
 	fsys_reiserfs.c fsys_ufs.c fsys_ufs2.c fsys_vstafs.c fsys_xfs.c \
-	fsys_zfs.c zfs_fletcher.c zfs_lzjb.c zfs_sha256.c \
+	fsys_zfs.c zfs_fletcher.c zfs_lzjb.c zfs_lz4.c zfs_sha256.c \
 	gunzip.c hercules.c md5.c serial.c smp-imps.c stage2.c terminfo.c \
 	tparm.c graphics.c
 pre_stage2_exec_CFLAGS = $(STAGE2_COMPILE) $(FSYS_CFLAGS)
@@ -196,7 +196,8 @@ ufs2_stage1_5_exec_LDFLAGS = $(STAGE1_5_LINK)
 
 # For zfs_stage1_5 target.
 zfs_stage1_5_exec_SOURCES = start.S asm.S common.c char_io.c disk_io.c \
-	stage1_5.c fsys_zfs.c zfs_fletcher.c zfs_lzjb.c zfs_sha256.c bios.c
+	stage1_5.c fsys_zfs.c zfs_fletcher.c zfs_lzjb.c zfs_lz4.c \
+	zfs_sha256.c bios.c
 zfs_stage1_5_exec_CFLAGS = $(STAGE1_5_COMPILE) -DFSYS_ZFS=1 \
 	-DNO_BLOCK_FILES=1
 zfs_stage1_5_exec_CCASFLAGS = $(STAGE1_5_COMPILE) -DFSYS_ZFS=1 \
diff --git a/usr/src/grub/grub-0.97/stage2/Makefile.solaris b/usr/src/grub/grub-0.97/stage2/Makefile.solaris
index e0c125a5c7..94b168f43d 100644
--- a/usr/src/grub/grub-0.97/stage2/Makefile.solaris
+++ b/usr/src/grub/grub-0.97/stage2/Makefile.solaris
@@ -1,8 +1,7 @@
 #
 # Copyright 2005 Sun Microsystems, Inc.  All rights reserved.
 # Use is subject to license terms.
-#
-# ident	"%Z%%M%	%I%	%E% SMI"
+# Copyright (c) 2013 by Saso Kiselkov. All rights reserved.
 #
 include ../../Makefile.grub
 include ../Makefile.solaris.defs
@@ -75,6 +74,7 @@ LIBGRUB_OBJS	= libgrub_a-boot.o		\
 		  libgrub_a-fsys_reiserfs.o	\
 		  libgrub_a-zfs_sha256.o	\
 		  libgrub_a-zfs_lzjb.o		\
+		  libgrub_a-zfs_lz4.o		\
 		  libgrub_a-zfs_fletcher.o	\
 		  libgrub_a-fsys_zfs.o		\
 		  libgrub_a-fsys_ufs.o		\
@@ -164,6 +164,7 @@ DISKLESS_OBJS		= diskless_exec-bios.o		\
 			  diskless_exec-fsys_reiserfs.o	\
 			  diskless_exec-zfs_sha256.o	\
 			  diskless_exec-zfs_lzjb.o	\
+			  diskless_exec-zfs_lz4.o	\
 			  diskless_exec-zfs_fletcher.o	\
 			  diskless_exec-fsys_zfs.o	\
 			  diskless_exec-fsys_ufs.o	\
@@ -375,6 +376,7 @@ PRE_STAGE2_OBJS		= pre_stage2_exec-bios.o \
 			  pre_stage2_exec-fsys_reiserfs.o \
 			  pre_stage2_exec-zfs_sha256.o \
 			  pre_stage2_exec-zfs_lzjb.o \
+			  pre_stage2_exec-zfs_lz4.o \
 			  pre_stage2_exec-zfs_fletcher.o \
 			  pre_stage2_exec-fsys_zfs.o \
 			  pre_stage2_exec-fsys_ufs.o \
@@ -533,6 +535,7 @@ ZFS_STAGE1_5_OBJS	= zfs_stage1_5_exec-bios.o \
 			  zfs_stage1_5_exec-disk_io.o \
 			  zfs_stage1_5_exec-zfs_sha256.o \
 			  zfs_stage1_5_exec-zfs_lzjb.o \
+			  zfs_stage1_5_exec-zfs_lz4.o \
 			  zfs_stage1_5_exec-zfs_fletcher.o \
 			  zfs_stage1_5_exec-fsys_zfs.o \
 			  zfs_stage1_5_exec-stage1_5.o
@@ -607,7 +610,7 @@ SRC_FILES	= Makefile.am Makefile.in apic.h apm.S asm.S bios.c boot.c \
 		  dir.h disk_inode.h disk_inode_ffs.h disk_io.c fat.h \
 		  filesys.h freebsd.h fs.h fsys_ext2fs.c fsys_fat.c \
 		  fsys_ffs.c fsys_iso9660.c fsys_jfs.c fsys_minix.c \
-		  zfs_sha256.c zfs_lzjb.c zfs_fletcher.c fsys_zfs.c \
+		  zfs_sha256.c zfs_lzjb.c zfs_lz4.c zfs_fletcher.c fsys_zfs.c \
 		  fsys_reiserfs.c fsys_ufs.c fsys_ufs2.c fsys_vstafs.c \
 		  fsys_xfs.c graphics.c graphics.h gunzip.c hercules.c \
 		  hercules.h i386-elf.h imgact_aout.h iso9660.h jfs.h \
diff --git a/usr/src/grub/grub-0.97/stage2/fsys_zfs.c b/usr/src/grub/grub-0.97/stage2/fsys_zfs.c
index 3d1129c4ea..bc90f7719a 100644
--- a/usr/src/grub/grub-0.97/stage2/fsys_zfs.c
+++ b/usr/src/grub/grub-0.97/stage2/fsys_zfs.c
@@ -24,6 +24,7 @@
 
 /*
  * Copyright (c) 2012 by Delphix. All rights reserved.
+ * Copyright (c) 2013 by Saso Kiselkov. All rights reserved.
  */
 
 /*
@@ -74,7 +75,18 @@ decomp_entry_t decomp_table[ZIO_COMPRESS_FUNCTIONS] =
 	{"on", lzjb_decompress}, 	/* ZIO_COMPRESS_ON */
 	{"off", 0},			/* ZIO_COMPRESS_OFF */
 	{"lzjb", lzjb_decompress},	/* ZIO_COMPRESS_LZJB */
-	{"empty", 0}			/* ZIO_COMPRESS_EMPTY */
+	{"empty", 0},			/* ZIO_COMPRESS_EMPTY */
+	{"gzip-1", 0},			/* ZIO_COMPRESS_GZIP_1 */
+	{"gzip-2", 0},			/* ZIO_COMPRESS_GZIP_2 */
+	{"gzip-3", 0},			/* ZIO_COMPRESS_GZIP_3 */
+	{"gzip-4", 0},			/* ZIO_COMPRESS_GZIP_4 */
+	{"gzip-5", 0},			/* ZIO_COMPRESS_GZIP_5 */
+	{"gzip-6", 0},			/* ZIO_COMPRESS_GZIP_6 */
+	{"gzip-7", 0},			/* ZIO_COMPRESS_GZIP_7 */
+	{"gzip-8", 0},			/* ZIO_COMPRESS_GZIP_8 */
+	{"gzip-9", 0},			/* ZIO_COMPRESS_GZIP_9 */
+	{"zle", 0},			/* ZIO_COMPRESS_ZLE */
+	{"lz4", lz4_decompress}		/* ZIO_COMPRESS_LZ4 */
 };
 
 static int zio_read_data(blkptr_t *bp, void *buf, char *stack);
@@ -412,8 +424,13 @@ zio_read(blkptr_t *bp, void *buf, char *stack)
 		return (ERR_FSYS_CORRUPT);
 	}
 
-	if (comp != ZIO_COMPRESS_OFF)
-		decomp_table[comp].decomp_func(buf, retbuf, psize, lsize);
+	if (comp != ZIO_COMPRESS_OFF) {
+		if (decomp_table[comp].decomp_func(buf, retbuf, psize,
+		    lsize) != 0) {
+			grub_printf("zio_read decompression failed\n");
+			return (ERR_FSYS_CORRUPT);
+		}
+	}
 
 	return (0);
 }
@@ -945,6 +962,7 @@ get_default_bootfsobj(dnode_phys_t *mosmdn, uint64_t *obj, char *stack)
  * to be listed here since grub opens pools in read-only mode.
  */
 static const char *spa_feature_names[] = {
+	"org.illumos:lz4_compress",
 	NULL
 };
 
diff --git a/usr/src/grub/grub-0.97/stage2/fsys_zfs.h b/usr/src/grub/grub-0.97/stage2/fsys_zfs.h
index 11236c33de..2f77a5b6a7 100644
--- a/usr/src/grub/grub-0.97/stage2/fsys_zfs.h
+++ b/usr/src/grub/grub-0.97/stage2/fsys_zfs.h
@@ -24,6 +24,7 @@
 
 /*
  * Copyright (c) 2012 by Delphix. All rights reserved.
+ * Copyright (c) 2013 by Saso Kiselkov. All rights reserved.
  */
 
 #ifndef _FSYS_ZFS_H
@@ -213,6 +214,7 @@ extern void fletcher_4_native(const void *, uint64_t, zio_cksum_t *);
 extern void fletcher_4_byteswap(const void *, uint64_t, zio_cksum_t *);
 extern void zio_checksum_SHA256(const void *, uint64_t, zio_cksum_t *);
 extern int lzjb_decompress(void *, void *, size_t, size_t);
+extern int lz4_decompress(void *, void *, size_t, size_t);
 
 #endif	/* FSYS_ZFS */
 
diff --git a/usr/src/grub/grub-0.97/stage2/zfs-include/zio.h b/usr/src/grub/grub-0.97/stage2/zfs-include/zio.h
index 298017aea4..3b893f451e 100644
--- a/usr/src/grub/grub-0.97/stage2/zfs-include/zio.h
+++ b/usr/src/grub/grub-0.97/stage2/zfs-include/zio.h
@@ -20,6 +20,9 @@
  * Copyright 2010 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
+/*
+ * Copyright 2013 by Saso Kiselkov. All rights reserved.
+ */
 
 #ifndef _ZIO_H
 #define	_ZIO_H
@@ -73,6 +76,17 @@ enum zio_compress {
 	ZIO_COMPRESS_OFF,
 	ZIO_COMPRESS_LZJB,
 	ZIO_COMPRESS_EMPTY,
+	ZIO_COMPRESS_GZIP_1,
+	ZIO_COMPRESS_GZIP_2,
+	ZIO_COMPRESS_GZIP_3,
+	ZIO_COMPRESS_GZIP_4,
+	ZIO_COMPRESS_GZIP_5,
+	ZIO_COMPRESS_GZIP_6,
+	ZIO_COMPRESS_GZIP_7,
+	ZIO_COMPRESS_GZIP_8,
+	ZIO_COMPRESS_GZIP_9,
+	ZIO_COMPRESS_ZLE,
+	ZIO_COMPRESS_LZ4,
 	ZIO_COMPRESS_FUNCTIONS
 };
 
diff --git a/usr/src/grub/grub-0.97/stage2/zfs_lz4.c b/usr/src/grub/grub-0.97/stage2/zfs_lz4.c
new file mode 100644
index 0000000000..42c03f9135
--- /dev/null
+++ b/usr/src/grub/grub-0.97/stage2/zfs_lz4.c
@@ -0,0 +1,313 @@
+/*
+ * LZ4 - Fast LZ compression algorithm
+ * Header File
+ * Copyright (C) 2011-2013, Yann Collet.
+ * BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * You can contact the author at :
+ * - LZ4 homepage : http://fastcompression.blogspot.com/p/lz4.html
+ * - LZ4 source repository : http://code.google.com/p/lz4/
+ */
+
+#include "fsys_zfs.h"
+#include <string.h>
+
+static int LZ4_uncompress_unknownOutputSize(const char *source, char *dest,
+					    int isize, int maxOutputSize);
+
+int
+lz4_decompress(void *s_start, void *d_start, size_t s_len, size_t d_len)
+{
+	const uint8_t *src = s_start;
+	uint32_t bufsiz = (src[0] << 24) | (src[1] << 16) | (src[2] << 8) |
+	    src[3];
+
+	/* invalid compressed buffer size encoded at start */
+	if (bufsiz + 4 > s_len)
+		return (1);
+
+	/*
+	 * Returns 0 on success (decompression function returned non-negative)
+	 * and non-zero on failure (decompression function returned negative).
+	 */
+	return (LZ4_uncompress_unknownOutputSize(s_start + 4, d_start, bufsiz,
+	    d_len) < 0);
+}
+
+/*
+ * CPU Feature Detection
+ */
+
+/* 32 or 64 bits ? */
+#if (defined(__x86_64__) || defined(__x86_64) || defined(__amd64__) || \
+	defined(__amd64) || defined(__ppc64__) || defined(_WIN64) || \
+	defined(__LP64__) || defined(_LP64))
+#define	LZ4_ARCH64	1
+#else
+#define	LZ4_ARCH64	0
+#endif
+
+/*
+ * Little Endian or Big Endian?
+ * Note: overwrite the below #define if you know your architecture endianess.
+ */
+#if (defined(__BIG_ENDIAN__) || defined(__BIG_ENDIAN) || \
+	defined(_BIG_ENDIAN) || defined(_ARCH_PPC) || defined(__PPC__) || \
+	defined(__PPC) || defined(PPC) || defined(__powerpc__) || \
+	defined(__powerpc) || defined(powerpc) || \
+	((defined(__BYTE_ORDER__)&&(__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__))))
+#define	LZ4_BIG_ENDIAN	1
+#else
+	/*
+	 * Little Endian assumed. PDP Endian and other very rare endian format
+	 * are unsupported.
+	 */
+#endif
+
+/*
+ * Compiler Options
+ */
+#if __STDC_VERSION__ >= 199901L	/* C99 */
+/* "restrict" is a known keyword */
+#else
+/* Disable restrict */
+#define	restrict
+#endif
+
+#define	GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
+
+#define	lz4_bswap16(x) ((unsigned short int) ((((x) >> 8) & 0xffu) \
+	| (((x) & 0xffu) << 8)))
+
+#if (GCC_VERSION >= 302) || (__INTEL_COMPILER >= 800) || defined(__clang__)
+#define	expect(expr, value)    (__builtin_expect((expr), (value)))
+#else
+#define	expect(expr, value)    (expr)
+#endif
+
+#define	likely(expr)	expect((expr) != 0, 1)
+#define	unlikely(expr)	expect((expr) != 0, 0)
+
+/* Basic types */
+#define	BYTE	uint8_t
+#define	U16	uint16_t
+#define	U32	uint32_t
+#define	S32	int32_t
+#define	U64	uint64_t
+
+typedef struct _U16_S {
+	U16 v;
+} U16_S;
+typedef struct _U32_S {
+	U32 v;
+} U32_S;
+typedef struct _U64_S {
+	U64 v;
+} U64_S;
+
+#define	A64(x)	(((U64_S *)(x))->v)
+#define	A32(x)	(((U32_S *)(x))->v)
+#define	A16(x)	(((U16_S *)(x))->v)
+
+/*
+ * Constants
+ */
+#define	MINMATCH 4
+
+#define	COPYLENGTH 8
+#define	LASTLITERALS 5
+
+#define	ML_BITS 4
+#define	ML_MASK ((1U<<ML_BITS)-1)
+#define	RUN_BITS (8-ML_BITS)
+#define	RUN_MASK ((1U<<RUN_BITS)-1)
+
+/*
+ * Architecture-specific macros
+ */
+#if LZ4_ARCH64
+#define	STEPSIZE 8
+#define	UARCH U64
+#define	AARCH A64
+#define	LZ4_COPYSTEP(s, d)	A64(d) = A64(s); d += 8; s += 8;
+#define	LZ4_COPYPACKET(s, d)	LZ4_COPYSTEP(s, d)
+#define	LZ4_SECURECOPY(s, d, e)	if (d < e) LZ4_WILDCOPY(s, d, e)
+#define	HTYPE U32
+#define	INITBASE(base)		const BYTE* const base = ip
+#else
+#define	STEPSIZE 4
+#define	UARCH U32
+#define	AARCH A32
+#define	LZ4_COPYSTEP(s, d)	A32(d) = A32(s); d += 4; s += 4;
+#define	LZ4_COPYPACKET(s, d)	LZ4_COPYSTEP(s, d); LZ4_COPYSTEP(s, d);
+#define	LZ4_SECURECOPY		LZ4_WILDCOPY
+#define	HTYPE const BYTE*
+#define	INITBASE(base)		const int base = 0
+#endif
+
+#if (defined(LZ4_BIG_ENDIAN) && !defined(BIG_ENDIAN_NATIVE_BUT_INCOMPATIBLE))
+#define	LZ4_READ_LITTLEENDIAN_16(d, s, p) \
+	{ U16 v = A16(p); v = lz4_bswap16(v); d = (s) - v; }
+#define	LZ4_WRITE_LITTLEENDIAN_16(p, i) \
+	{ U16 v = (U16)(i); v = lz4_bswap16(v); A16(p) = v; p += 2; }
+#else
+#define	LZ4_READ_LITTLEENDIAN_16(d, s, p) { d = (s) - A16(p); }
+#define	LZ4_WRITE_LITTLEENDIAN_16(p, v)  { A16(p) = v; p += 2; }
+#endif
+
+/* Macros */
+#define	LZ4_WILDCOPY(s, d, e) do { LZ4_COPYPACKET(s, d) } while (d < e);
+
+/* Decompression functions */
+
+static int
+LZ4_uncompress_unknownOutputSize(const char *source,
+    char *dest, int isize, int maxOutputSize)
+{
+	/* Local Variables */
+	const BYTE *restrict ip = (const BYTE *) source;
+	const BYTE *const iend = ip + isize;
+	const BYTE *restrict ref;
+
+	BYTE *restrict op = (BYTE *) dest;
+	BYTE *const oend = op + maxOutputSize;
+	BYTE *cpy;
+
+	size_t dec[] = { 0, 3, 2, 3, 0, 0, 0, 0 };
+
+	/* Main Loop */
+	while (ip < iend) {
+		BYTE token;
+		int length;
+
+		/* get runlength */
+		token = *ip++;
+		if ((length = (token >> ML_BITS)) == RUN_MASK) {
+			int s = 255;
+			while ((ip < iend) && (s == 255)) {
+				s = *ip++;
+				length += s;
+			}
+		}
+		/* copy literals */
+		cpy = op + length;
+		if ((cpy > oend - COPYLENGTH) ||
+		    (ip + length > iend - COPYLENGTH)) {
+			if (cpy > oend)
+				/*
+				 * Error: request to write beyond destination
+				 * buffer.
+				 */
+				goto _output_error;
+			if (ip + length > iend)
+				/*
+				 * Error : request to read beyond source
+				 * buffer.
+				 */
+				goto _output_error;
+			memcpy(op, ip, length);
+			op += length;
+			ip += length;
+			if (ip < iend)
+				/* Error : LZ4 format violation */
+				goto _output_error;
+			/* Necessarily EOF, due to parsing restrictions. */
+			break;
+		}
+		LZ4_WILDCOPY(ip, op, cpy);
+		ip -= (op - cpy);
+		op = cpy;
+
+		/* get offset */
+		LZ4_READ_LITTLEENDIAN_16(ref, cpy, ip);
+		ip += 2;
+		if (ref < (BYTE * const) dest)
+			/*
+			 * Error: offset creates reference outside of
+			 * destination buffer.
+			 */
+			goto _output_error;
+
+		/* get matchlength */
+		if ((length = (token & ML_MASK)) == ML_MASK) {
+			while (ip < iend) {
+				int s = *ip++;
+				length += s;
+				if (s == 255)
+					continue;
+				break;
+			}
+		}
+		/* copy repeated sequence */
+		if unlikely(op - ref < STEPSIZE) {
+#if LZ4_ARCH64
+			size_t dec2table[] = { 0, 0, 0, -1, 0, 1, 2, 3 };
+			size_t dec2 = dec2table[op - ref];
+#else
+			const int dec2 = 0;
+#endif
+			*op++ = *ref++;
+			*op++ = *ref++;
+			*op++ = *ref++;
+			*op++ = *ref++;
+			ref -= dec[op - ref];
+			A32(op) = A32(ref);
+			op += STEPSIZE - 4;
+			ref -= dec2;
+		} else {
+			LZ4_COPYSTEP(ref, op);
+		}
+		cpy = op + length - (STEPSIZE - 4);
+		if (cpy > oend - COPYLENGTH) {
+			if (cpy > oend)
+				/*
+				 * Error: request to write outside of
+				 * destination buffer.
+				 */
+				goto _output_error;
+			LZ4_SECURECOPY(ref, op, (oend - COPYLENGTH));
+			while (op < cpy)
+				*op++ = *ref++;
+			op = cpy;
+			if (op == oend)
+				/*
+				 * Check EOF (should never happen, since last
+				 * 5 bytes are supposed to be literals).
+				 */
+				break;
+			continue;
+		}
+		LZ4_SECURECOPY(ref, op, cpy);
+		op = cpy;	/* correction */
+	}
+
+	/* end of decoding */
+	return (int)(((char *)op) - dest);
+
+	/* write overflow error detected */
+	_output_error:
+	return (int)(-(((char *)ip) - source));
+}